From 0326606f1cea37cec2d6a9a0cb992d4b60c62207 Mon Sep 17 00:00:00 2001 From: Imre Kristoffer Eilertsen Date: Wed, 3 Feb 2021 18:12:11 +0100 Subject: [PATCH] Delete xyzAM_prepare_filters.py --- .../xyzAM_prepare_filters.py | 671 ------------------ 1 file changed, 671 deletions(-) delete mode 100644 Alternate versions Anti-Malware List/xyzAM_prepare_filters.py diff --git a/Alternate versions Anti-Malware List/xyzAM_prepare_filters.py b/Alternate versions Anti-Malware List/xyzAM_prepare_filters.py deleted file mode 100644 index ae94a2ebc..000000000 --- a/Alternate versions Anti-Malware List/xyzAM_prepare_filters.py +++ /dev/null @@ -1,671 +0,0 @@ -import requests -import re - -SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Dandelion%20Sprout\'s%20Anti-Malware%20List.txt'] - -UNSUPPORTED_ABP = ['$important', ',important' '$redirect=', ',redirect=', - ':style', '##+js', '.*#' , ':xpath', ':matches-css', 'dk,no##'] -UNSUPPORTED_TPL = ['##', '#@#', '#?#', r'\.no\.$'] -UNSUPPORTED_PRIVOXY = ['##', '#@#', '#?#', '@@', '!#'] -UNSUPPORTED_HOSTS = ['##', '#@#', '#?#', '@@', '!#', '[Adblock Plus 3.2]', '*'] - -OUTPUT = 'xyzzyx.txt' -OUTPUT_AG = 'AntiMalwareAdGuard.txt' -OUTPUT_ABP = 'AntiMalwareABP.txt' -OUTPUT_TPL = 'AntiMalwareTPL.tpl' -OUTPUT_PRIVOXY = 'AntiMalwarePrivoxy.action' -OUTPUT_HOSTS = 'AntiMalwareHosts.txt' -OUTPUT_DOMAINS = 'AntiMalwareDomains.txt' - -# function that downloads the filter list -def download_filters() -> str: - text = '' - for url in SOURCES: - r = requests.get(url) - text += r.text - return text - -# function that prepares the filter list for AdGuard -def prepare_ag(lines) -> str: - text = '' - - for line in lines: - - - # until this is done: https://github.com/AdguardTeam/CoreLibs/issues/152 - line = re.sub( - "\$doc", - "$empty,important", - line - ) - - line = re.sub( - "Dandelion Sprout's Anti-Malware List", - "Dandelion Sprout's Anti-Malware List (for AdGuard)", - line - ) - - line = re.sub( - r"! Version: [0-9][0-9][0-9][0-9].*", - "", - line - ) - - line = re.sub( - "\[Adblock Plus 3.2\]", - "[AdGuard ≥6]", - line - ) - - line = re.sub( - r"! Redirect:.*", - "", - line - ) - - line = re.sub( - r"=~ Warning.*", - "", - line - ) - - line = re.sub( - r",domain$", - "", - line - ) - - line = re.sub( - r"\|~ Warning.*", - "", - line - ) - - text += line + '\r\n' - - return text - -def is_supported_abp(line) -> bool: - for token in UNSUPPORTED_ABP: - if token in line: - return False - - return True - -# function that prepares the filter list for ABP -def prepare_abp(lines) -> str: - text = '' - - # remove or modifiy entries with unsupported modifiers - for line in lines: - - # remove $document modifier from the rule - line = re.sub( - "\$doc,", - "$", - line - ) - - # remove $important modifier from the rule - line = re.sub( - r",important", - "", - line - ) - - line = re.sub( - r"\$important", - "", - line - ) - - line = re.sub( - "Dandelion Sprout's Anti-Malware List", - "Dandelion Sprout's Anti-Malware List (for AdBlock and Adblock Plus)", - line - ) - - line = re.sub( - r"!#if.*", - "", - line - ) - - line = re.sub( - r"!#endif", - "", - line - ) - - line = re.sub( - r"^no##.*", - "", - line - ) - - line = re.sub( - r"! Redirect:.*", - "", - line - ) - - line = re.sub( - r"=~ Warning.*", - "", - line - ) - - line = re.sub( - r"\$domain$", - "", - line - ) - - line = re.sub( - r"\|~ Warning.*", - "", - line - ) - - if is_supported_abp(line): - text += line + '\r\n' - - return text - -# Attempts to achieve Internet Explorer TPL support -def is_supported_tpl(line) -> bool: - for token in UNSUPPORTED_TPL: - if token in line: - return False - - return True - -# function that prepares the filter list for ABP -def prepare_tpl(lines) -> str: - text = '' - - # remove or modifiy entries with unsupported modifiers - for line in lines: - - line = re.sub( - "! ", - "# ", - line - ) - - line = re.sub( - "\[Adblock Plus 3.2\]", - "msFilterList", - line - ) - - line = re.sub( - r"^([@][@][|][|])", - "+d ", - line - ) - - line = re.sub( - r"^([|][|])", - "-d ", - line - ) - - line = re.sub( - r"^/", - "- ", - line - ) - - # TO-DO: Figure out how to make this NOT apply to lines that have the character "!" in them. - line = re.sub( - "/", - " ", - line - ) - - line = re.sub( - "\^", - "", - line - ) - - line = re.sub( - r"\$.*", - "", - line - ) - - line = re.sub( - r"! Redirect:.*", - "", - line - ) - - line = re.sub( - "-d \* ", - "- ", - line - ) - - line = re.sub( - "-d \.", - "- ", - line - ) - - line = re.sub( - "com\* ", - "com ", - line - ) - - line = re.sub( - r"([-][d].*[.][*].*)", - "", - line - ) - - line = re.sub( - r"@@\*\..*", - "", - line - ) - - line = re.sub( - "@@_", - "+d _", - line - ) - - line = re.sub( - r"^([+][d][\s][a-z].*[\s][a-z].*)", - "", - line - ) - - line = re.sub( - r"^([-][d].*[o|k|m]\.$)", - "", - line - ) - - line = re.sub( - r"!#if.*", - "", - line - ) - - line = re.sub( - "!#endif", - "", - line - ) - - line = re.sub( - "Expires: ", - "expires = ", - line - ) - - line = re.sub( - r"^\.([d|i|n]).*", - "", - line - ) - - line = re.sub( - "^\.", - "- ", - line - ) - - line = re.sub( - r"^@@ .*", - "", - line - ) - - line = re.sub( - " 12 hours", - " 1", - line - ) - - line = re.sub( - " 5 days", - " 5", - line - ) - - line = re.sub( - "# Redirect:.*", - "", - line - ) - - line = re.sub( - r"\+d\s[a-z0-9].*\s[a-z0-9*].*", - "", - line - ) - - line = re.sub( - "-d.*-\*$", - "", - line - ) - - line = re.sub( - "Dandelion Sprout's Anti-Malware List", - "Dandelion Sprout's Anti-Malware List (Internet Explorer TPL)", - line - ) - - line = re.sub( - r"^- [a-z][a-z]$", - "", - line - ) - - line = re.sub( - r"^- loan$", - "", - line - ) - - line = re.sub( - r"^- agency$", - "", - line - ) - - line = re.sub( - r"^-d .*\*\..*", - "", - line - ) - - if is_supported_tpl(line): - text += line + '\r\n' - - return text - -# ————— Privoxy version ————— - -# Attempts to achieve Internet Explorer TPL support -def is_supported_privoxy(line) -> bool: - for token in UNSUPPORTED_PRIVOXY: - if token in line: - return False - - return True - -def prepare_privoxy(lines) -> str: - text = '' - - # remove or modifiy entries with unsupported modifiers - for line in lines: - - line = re.sub( - "! ", - "# ", - line - ) - - line = re.sub( - r"\$.*", - "", - line - ) - - line = re.sub( - "\|\|", - ".", - line - ) - - line = re.sub( - "\^", - "", - line - ) - - line = re.sub( - "\[Adblock Plus 3.2\]", - "{+block}", - line - ) - - line = re.sub( - r"^\!.*", - "#", - line - ) - - line = re.sub( - "# 🇬🇧: ", - "{+block{", - line - ) - - # Note the use of parenthesises and "\1" combined. - line = re.sub( - r"(\{\+block{.*)", - r"\1}}", - line - ) - - line = re.sub( - r"^\.\.", - ".", - line - ) - - line = re.sub( - "Dandelion Sprout's Anti-Malware List", - "Dandelion Sprout's Anti-Malware List (for Privoxy)", - line - ) - - line = re.sub( - r"# Placeholder line.*", - "{-block}\n# Manually updated Privoxy version of the whitelisted domains from the other versions of this list.\n.coolcmd.tk\n.budterence.tk\n.intr0.tk\n.google.tk\n.transportnews.tk\n.unicorncardlist.tk\n.c0d3c.tk\n.google.ga\n.filtri-dns.ga\n.google.ml\n.deimos.gq\n.1hos.cf\n.intr0.cf\n.ivoid.cd\n.domainvoider.cf\n.google.cf\n.rths.cf\n.anonytext.tk\n.tokelau-info.tk\n.fakaofo.tk\n.nukunonu.tk\n.anpigabon.ga\n.dgdi.ga\n.voitures.ga\n.mobili.ml\n.inege.gq\n.tvgelive.gq\n.comprarcarros.gq\n.voitures.cf\n.assembleenationale-rca.cf\n.cps-rca.cf\n.acap.cf", - line - ) - - line = re.sub( - r"^(\# —.*)", - r"{+block}\n\1", - line - ) - - line = re.sub( - "-Beta", - "-Alpha", - line - ) - - if is_supported_privoxy(line): - text += line + '\r\n' - - return text - -# ————— /hosts file version ————— - -# Attempts to achieve Internet Explorer TPL support -def is_supported_hosts(line) -> bool: - for token in UNSUPPORTED_HOSTS: - if token in line: - return False - - return True - -def prepare_hosts(lines) -> str: - text = '' - - # remove or modifiy entries with unsupported modifiers - for line in lines: - - line = re.sub( - "! ", - "# ", - line - ) - - line = re.sub( - r"\^.*", - "", - line - ) - - line = re.sub( - r"^\!.*", - "#", - line - ) - - line = re.sub( - "Dandelion Sprout's Anti-Malware List", - "Dandelion Sprout's Anti-Malware List («hosts» file version)", - line - ) - - line = re.sub( - r"# Placeholder line.*", - "", - line - ) - - line = re.sub( - "-Beta", - "-Alpha", - line - ) - - line = re.sub( - r"\|\|.*/.*", - "", - line - ) - - line = re.sub( - "\|\|", - "127.0.0.1 ", - line - ) - - line = re.sub( - r"127\.0\.0\.1 \..*", - "", - line - ) - - if is_supported_hosts(line): - text += line + '\r\n' - - return text - -# ————— Domains list version ————— - -def is_supported_domains(line) -> bool: - for token in UNSUPPORTED_HOSTS: - if token in line: - return False - - return True - -def prepare_domains(lines) -> str: - text = '' - - # remove or modifiy entries with unsupported modifiers - for line in lines: - - line = re.sub( - "! ", - "# ", - line - ) - - line = re.sub( - r"\^.*", - "", - line - ) - - line = re.sub( - r"^\!.*", - "#", - line - ) - - line = re.sub( - "Dandelion Sprout's Anti-Malware List", - "Dandelion Sprout's Anti-Malware List (Domains list version)", - line - ) - - line = re.sub( - r"# Placeholder line.*", - "", - line - ) - - line = re.sub( - r"\|\|.*/.*", - "", - line - ) - - line = re.sub( - r"\|\|\..*", - "", - line - ) - - line = re.sub( - "\|\|", - "", - line - ) - - if is_supported_domains(line): - text += line + '\r\n' - - return text - -if __name__ == "__main__": - print('Starting the script') - text = download_filters() - lines = text.splitlines(False) - print('Total number of rules: ' + str(len(lines))) - - ag_filter = prepare_ag(lines) - abp_filter = prepare_abp(lines) - tpl_filter = prepare_tpl(lines) - privoxy_filter = prepare_privoxy(lines) - hosts_filter = prepare_hosts(lines) - domains_filter = prepare_domains(lines) - - with open(OUTPUT, "w") as text_file: - text_file.write(text) - - with open(OUTPUT_AG, "w") as text_file: - text_file.write(ag_filter) - - with open(OUTPUT_ABP, "w") as text_file: - text_file.write(abp_filter) - - with open(OUTPUT_TPL, "w") as text_file: - text_file.write(tpl_filter) - - with open(OUTPUT_PRIVOXY, "w") as text_file: - text_file.write(privoxy_filter) - - with open(OUTPUT_HOSTS, "w") as text_file: - text_file.write(hosts_filter) - - with open(OUTPUT_DOMAINS, "w") as text_file: - text_file.write(domains_filter) - - print('The script has finished its work')