starred/adfilt
1
0
Fork 0
mirror of https://github.com/DandelionSprout/adfilt.git synced 2026-09-27 22:04:25 +01:00
Code Issues Packages Projects Releases 4 Wiki Activity
Files
83199158366d1020e8ee172d6269176e44822deb
adfilt/AdGuard Home Compilation List/AGHtest.py
T

1442 lines
29 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import requests
import re
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription.notlist', 'https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2020.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', 'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt', 'https://filters.adtidy.org/extension/ublock/filters/16.txt', 'https://www.i-dont-care-about-cookies.eu/abp/', 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', 'https://easylist-downloads.adblockplus.org/abp-filters-anti-cv.txt', 'https://easylist-downloads.adblockplus.org/antiadblockfilters.txt', 'https://easylist-downloads.adblockplus.org/advblock.txt', 'https://easylist-downloads.adblockplus.org/Liste_AR.txt', 'https://easylist-downloads.adblockplus.org/easylistspanish.txt']
UNSUPPORTED_AGH = ['##', '@#', '#?#', '#%#', '!+', 'domain=', 'generichide', '$ghide', ',ghide', '$csp', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '$domain', ',domain', '[Adblock Plus 2.0]', 'CV-', '$csp']
UNSUPPORTED_IP = ['##', '@#', '#?#', '#%#', 'domain=', 'generichide', '$csp', 'badfilter', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '!+']
OUTPUT = 'xyzzyx.txt'
OUTPUT_AGH = 'AdGuardHomeCompilationList.txt'
OUTPUT_IP = 'AdGuardHomeCompilationListIPs.txt'
# function that downloads the filter list
def download_filters() -> str:
text = ''
for url in SOURCES:
r = requests.get(url)
text += r.text
return text
def is_supported_agh(line) -> bool:
for token in UNSUPPORTED_AGH:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_agh(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])third-party",
"",
line
)
line = re.sub(
r"([$,])~third-party",
"",
line
)
line = re.sub(
r"([$,])3p",
"",
line
)
line = re.sub(
r"([$,])first-party",
"",
line
)
line = re.sub(
r"([$,])1p",
"",
line
)
line = re.sub(
r"([$,])image",
"",
line
)
line = re.sub(
r"([$,])media",
"",
line
)
line = re.sub(
r"([$,])script",
"",
line
)
line = re.sub(
r"([$,])popup",
"",
line
)
line = re.sub(
r"([$,])popunder",
"",
line
)
line = re.sub(
r"([$,])document",
"",
line
)
line = re.sub(
r"([$,])doc",
"",
line
)
line = re.sub(
r"([$,])subdocument",
"",
line
)
line = re.sub(
r"([$,])object",
"",
line
)
line = re.sub(
r"([$,])~object-subrequest",
"",
line
)
line = re.sub(
r"([$,])frame",
"",
line
)
line = re.sub(
r"([$,])all",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r",domain=in-addr\.arpa",
"",
line
)
line = re.sub(
r".*\^[*$,][ac-hj-z?].*",
"",
line
)
line = re.sub(
r".*\.js$",
"",
line
)
line = re.sub(
r"^(\.[a-z0-9.]{4,60}\.)$",
r"||*\1",
line
)
line = re.sub(
r"^[&+=?^_%].*",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[0-9].*",
"",
line
)
line = re.sub(
r"^!#.*",
"",
line
)
line = re.sub(
r"^\|[a-z0-9].*",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*",
"",
line
)
line = re.sub(
r"^Amasty_.*",
"",
line
)
line = re.sub(
r"^\*.*",
"",
line
)
line = re.sub(
r"^\\.*",
"",
line
)
line = re.sub(
r"^:[a-z0-9?!].*",
"",
line
)
line = re.sub(
r"^@[a-z0-9].*",
"",
line
)
line = re.sub(
r"@?@?\|\|.*/.*",
r"",
line
)
line = re.sub(
r"\^,badfilter",
r"^badfilter",
line
)
line = re.sub(
r"\^badfilter",
r"^$badfilter",
line
)
line = re.sub(
r".*\^\*.*",
r"",
line
)
line = re.sub(
r".*\*\^[a-z0-9?].*",
r"",
line
)
line = re.sub(
r".*\^&.*",
r"",
line
)
line = re.sub(
r".*\|.*/.*",
r"",
line
)
line = re.sub(
r".*\*[&?@].*",
r"",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"^@@/.*",
r"",
line
)
line = re.sub(
r"^[,=^?~].*",
r"",
line
)
line = re.sub(
r"^! \|.*",
r"",
line
)
line = re.sub(
r"^\.php\?$",
r"",
line
)
line = re.sub(
r"^\|\|goo\.gl\^$",
r"",
line
)
line = re.sub(
r"^\|\|9anime\.\*\^$",
r"",
line
)
line = re.sub(
r"^\|\|play-asia\.com\^$",
r"",
line
)
line = re.sub(
r"^\|\|dmcdn\.net\^$",
r"",
line
)
line = re.sub(
r"^\|\|canyoublockit\.com\^$",
r"",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\3.\n@@://\1.\2.\3.*in-addr.arpa^",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\n@@://\1.\2.*in-addr.arpa^",
line
)
line = re.sub(
r"^! http.*",
r"",
line
)
line = re.sub(
r"^://.*/.*",
r"",
line
)
line = re.sub(
r".*104\\\.154\\\.\..*",
r"",
line
)
line = re.sub(
r"^/.*[a-z0-9]\\?/[a-z0-9\(].*",
r"",
line
)
line = re.sub(
r"\|\|.*_.*",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\\/.*",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\/.*",
r"",
line
)
line = re.sub(
r"^/[a-z0-9.*_/-]{1,70}/$",
r"",
line
)
line = re.sub(
r"^[a-z0-9@|].*\?.*",
r"",
line
)
line = re.sub(
r"^! Checksum:.*",
r"",
line
)
line = re.sub(
r".*\(update frequency\)",
r"",
line
)
line = re.sub(
r"^! 20(1|2).*",
r"",
line
)
line = re.sub(
r"^! Ref:.*",
r"",
line
)
line = re.sub(
r"^! #",
r"",
line
)
line = re.sub(
r"\*\^$",
r"*",
line
)
line = re.sub(
r"^\|\|\*\.",
r".",
line
)
line = re.sub(
r"\.\*$",
r".",
line
)
line = re.sub(
r"^[0-9]{1,5}$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[/?_+=&]$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-z0-9.$-].*",
r"",
line
)
line = re.sub(
r"^[@.].*[a-z0-9*]/.*",
r"",
line
)
line = re.sub(
r"^#\..*",
r"",
line
)
line = re.sub(
r".*[$,]jsinject$",
r"",
line
)
line = re.sub(
r".* - .*",
r"",
line
)
line = re.sub(
r"^![a-z0-9].*#.*",
r"",
line
)
line = re.sub(
r"^!\*(\*.*|$)",
r"",
line
)
line = re.sub(
r"^! ! \+.*",
r"",
line
)
line = re.sub(
r"^! AG_.*",
r"",
line
)
line = re.sub(
r"^\|.*:[0-9].*",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-zA-Z0-9.$*-].*",
r"",
line
)
line = re.sub(
r"^/.*[_?=/;~@%#+,].*",
r"",
line
)
line = re.sub(
r"^[|:/.-].*[%:].*",
r"",
line
)
line = re.sub(
r"^/[a-zA-Z0-9-]{1,100}[|^]$",
r"",
line
)
if is_supported_agh(line) and not line == '':
text += line + '\r\n'
return text
# ——— IP version ———
def is_supported_ip(line) -> bool:
for token in UNSUPPORTED_IP:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_ip(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])third-party",
"",
line
)
line = re.sub(
r"([$,])~third-party",
"",
line
)
line = re.sub(
r"([$,])3p",
"",
line
)
line = re.sub(
r"([$,])first-party",
"",
line
)
line = re.sub(
r"([$,])1p",
"",
line
)
line = re.sub(
r"([$,])image",
"",
line
)
line = re.sub(
r"([$,])media",
"",
line
)
line = re.sub(
r"([$,])script",
"",
line
)
line = re.sub(
r"([$,])popup",
"",
line
)
line = re.sub(
r"([$,])popunder",
"",
line
)
line = re.sub(
r"([$,])document",
"",
line
)
line = re.sub(
r"([$,])subdocument",
"",
line
)
line = re.sub(
r"([$,])~subdocument",
"",
line
)
line = re.sub(
r"([$,])object",
"",
line
)
line = re.sub(
r"([$,])~object-subrequest",
"",
line
)
line = re.sub(
r"([$,])frame",
"",
line
)
line = re.sub(
r"([$,])all",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r".*\^[*$,].*",
"",
line
)
line = re.sub(
r".*\.js$",
"",
line
)
line = re.sub(
r"^[&+-.=?^_%].*",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[0-9].*",
"",
line
)
line = re.sub(
r"^\|[a-z0-9].*",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*",
"",
line
)
line = re.sub(
r"^Amasty_.*",
"",
line
)
line = re.sub(
r"^\*.*",
r"",
line
)
line = re.sub(
r"^\\.*",
r"",
line
)
line = re.sub(
r"^:.*",
r"",
line
)
line = re.sub(
r"^@[a-z0-9].*",
r"",
line
)
line = re.sub(
r"[@]?[@]?\|\|.*[a-zа-яみ].*",
r"",
line
)
line = re.sub(
r"! (Version: .*[a-z].*)",
r"# \1",
line
)
line = re.sub(
r"^![a-z #].*",
r"",
line
)
line = re.sub(
r"^!-.*",
r"",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"\[Adblock Plus 2\.0\]",
r"",
line
)
line = re.sub(
r"\|\|",
r"",
line
)
line = re.sub(
r"\^$",
r"",
line
)
line = re.sub(
r"^([0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?)\.$",
r"\1.0/24",
line
)
line = re.sub(
r"^([0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?)\.$",
r"\1.0.0/16",
line
)
line = re.sub(
r"^@@.*[a-z].*",
r"",
line
)
line = re.sub(
r"\[Adblock Plus 3\.4\]",
"! Title: IP Entries from Adblock Lists\n! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, AdGuard Base Filter, AdGuard French Filter, EasyList Germany, ABP Anti-Circumvention Filters, RU AdList, Liste AR, EasyList Spanish \n! Expires: 14 days\n! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.\n! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.",
line
)
line = re.sub(
r"# Version: (.*)-.*",
r"! Version: \1-Alpha",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"^/.*[a-zA-Z0-9_=.,*?|^@&~+;-]$",
r"",
line
)
line = re.sub(
r"^/.*[\]]$",
r"",
line
)
line = re.sub(
r"^://.*/.*",
r"",
line
)
line = re.sub(
r".*104\\\.154\\\.\..*",
r"",
line
)
line = re.sub(
r"^/.*[a-z0-9]\\?/[a-z0-9\(].*",
r"",
line
)
line = re.sub(
r"^/\^[h.].*",
r"",
line
)
line = re.sub(
r"^/\\.*",
r"",
line
)
line = re.sub(
r"^/\(https\?:\\/\\/\)([0-9][0-9]?[0-9]?)\\.([0-9][0-9]?[0-9]?)\\.([0-9][0-9]?[0-9]?)\\.\..*/",
r"\1.\2.\3.0/24",
line
)
line = re.sub(
r"^/\(https\?:\\/\\/\)([0-9][0-9]?[0-9]?)\\.([0-9][0-9]?[0-9]?)\\.\..*/",
r"\1.\2.0.0/16",
line
)
line = re.sub(
r"^/.*",
r"",
line
)
line = re.sub(
r"^@@.*",
r"",
line
)
line = re.sub(
r"^![a-zA-Z!*].*",
r"",
line
)
line = re.sub(
r"^[A-Za-z].*",
r"",
line
)
if is_supported_ip(line) and not line == '':
text += line + '\r\n'
return text
if __name__ == "__main__":
print('Starting the script')
text = download_filters()
lines = text.splitlines(False)
print('Total number of rules: ' + str(len(lines)))
agh_filter = prepare_agh(lines)
ip_filter = prepare_ip(lines)
with open(OUTPUT, "w") as text_file:
text_file.write(text)
with open(OUTPUT_AGH, "w") as text_file:
text_file.write(agh_filter)
with open(OUTPUT_IP, "w") as text_file:
text_file.write(ip_filter)
print('The list versions have been generated.')
# ————————————————————————————————————————————
import requests
import re
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription-Notifications.notlist', 'https://easylist-downloads.adblockplus.org/fanboy-notifications.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/sections/push-notifications.txt']
UNSUPPORTED_AGH = ['##', '@#', '#?#', '#%#', '!+', 'domain=', 'generichide', '$ghide', ',ghide', '$csp', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '$domain', ',domain', '[Adblock Plus 2.0]', 'CV-']
OUTPUT = 'xyzzyxnotif.txt'
OUTPUT_AGH = 'AdGuardHomeCompilationList-Notifications.txt'
# function that downloads the filter list
def download_filters() -> str:
text = ''
for url in SOURCES:
r = requests.get(url)
text += r.text
return text
def is_supported_agh(line) -> bool:
for token in UNSUPPORTED_AGH:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_agh(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])third-party",
"",
line
)
line = re.sub(
r"([$,])~third-party",
"",
line
)
line = re.sub(
r"([$,])3p",
"",
line
)
line = re.sub(
r"([$,])first-party",
"",
line
)
line = re.sub(
r"([$,])1p",
"",
line
)
line = re.sub(
r"([$,])image",
"",
line
)
line = re.sub(
r"([$,])media",
"",
line
)
line = re.sub(
r"([$,])script",
"",
line
)
line = re.sub(
r"([$,])popup",
"",
line
)
line = re.sub(
r"([$,])popunder",
"",
line
)
line = re.sub(
r"([$,])document",
"",
line
)
line = re.sub(
r"([$,])doc",
"",
line
)
line = re.sub(
r"([$,])subdocument",
"",
line
)
line = re.sub(
r"([$,])object",
"",
line
)
line = re.sub(
r"([$,])~object-subrequest",
"",
line
)
line = re.sub(
r"([$,])frame",
"",
line
)
line = re.sub(
r"([$,])all",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r",domain=in-addr\.arpa",
"",
line
)
line = re.sub(
r".*\^[*$,][ac-hj-z?].*",
"",
line
)
line = re.sub(
r".*\.js$",
"",
line
)
line = re.sub(
r"^(\.[a-z0-9.]{4,60}\.)$",
r"||*\1",
line
)
line = re.sub(
r"^[&+=?^_%].*",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[0-9].*",
"",
line
)
line = re.sub(
r"^!#.*",
"",
line
)
line = re.sub(
r"^\|[a-z0-9].*",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*",
"",
line
)
line = re.sub(
r"^Amasty_.*",
"",
line
)
line = re.sub(
r"^\*.*",
"",
line
)
line = re.sub(
r"^\\.*",
"",
line
)
line = re.sub(
r"^:[a-z0-9?!].*",
"",
line
)
line = re.sub(
r"^@[a-z0-9].*",
"",
line
)
line = re.sub(
r"@?@?\|\|.*/.*",
r"",
line
)
line = re.sub(
r"\^,badfilter",
r"^badfilter",
line
)
line = re.sub(
r"\^badfilter",
r"^$badfilter",
line
)
line = re.sub(
r".*\^\*.*",
r"",
line
)
line = re.sub(
r".*\*\^[a-z0-9?].*",
r"",
line
)
line = re.sub(
r".*\^&.*",
r"",
line
)
line = re.sub(
r".*\|.*/.*",
r"",
line
)
line = re.sub(
r".*\*[&?@].*",
r"",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"^@@/.*",
r"",
line
)
line = re.sub(
r"^[,=^?~].*",
r"",
line
)
line = re.sub(
r"^! \|.*",
r"",
line
)
line = re.sub(
r"^\.php\?$",
r"",
line
)
line = re.sub(
r"^\|\|goo\.gl\^$",
r"",
line
)
line = re.sub(
r"^\|\|9anime\.\*\^$",
r"",
line
)
line = re.sub(
r"^\|\|play-asia\.com\^$",
r"",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\3.\n@@://\1.\2.\3.*in-addr.arpa^",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\n@@://\1.\2.*in-addr.arpa^",
line
)
line = re.sub(
r"^! http.*",
r"",
line
)
line = re.sub(
r"^://.*/.*",
r"",
line
)
line = re.sub(
r".*104\\\.154\\\.\..*",
r"",
line
)
line = re.sub(
r"^/.*[a-z0-9]\\?/[a-z0-9\(].*",
r"",
line
)
line = re.sub(
r"\|\|.*_.*",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\\/.*",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\/.*",
r"",
line
)
line = re.sub(
r"^/[a-z0-9.*_/-]{1,70}/$",
r"",
line
)
line = re.sub(
r"^[a-z0-9@|].*\?.*",
r"",
line
)
line = re.sub(
r"^! Checksum:.*",
r"",
line
)
line = re.sub(
r".*\(update frequency\)",
r"",
line
)
line = re.sub(
r"^! 20(1|2).*",
r"",
line
)
line = re.sub(
r"^! Ref:.*",
r"",
line
)
line = re.sub(
r"^! #",
r"",
line
)
line = re.sub(
r"\*\^$",
r"*",
line
)
line = re.sub(
r"^\|\|\*\.",
r".",
line
)
line = re.sub(
r"\.\*$",
r".",
line
)
line = re.sub(
r"^[0-9]{1,5}$",
r"",
line
)
line = re.sub(
r"! Disabling of push-notifications",
r"! Title: AdGuard Annoyances Web Push\n! Homepage: https://github.com/AdguardTeam/AdguardFilters/tree/master/AnnoyancesFilter/sections\n! Sourced from: https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/sections/push-notifications.txt\n! Disabling of push-notifications",
line
)
line = re.sub(
r"^[a-z0-9.-].*[/?_+=&]$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-z0-9.$-].*",
r"",
line
)
line = re.sub(
r"^/.*[_?=/;~@%#+].*",
r"",
line
)
if is_supported_agh(line) and not line == '':
text += line + '\r\n'
return text
if __name__ == "__main__":
print('Starting the script')
text = download_filters()
lines = text.splitlines(False)
print('Total number of rules: ' + str(len(lines)))
agh_filter = prepare_agh(lines)
with open(OUTPUT, "w") as text_file:
text_file.write(text)
with open(OUTPUT_AGH, "w") as text_file:
text_file.write(agh_filter)
print('The notifications list version has been generated.')
Reference in New Issue View Git Blame Copy Permalink