starred/adfilt
1
0
Fork 0
mirror of https://github.com/DandelionSprout/adfilt.git synced 2026-09-24 18:40:23 +01:00
Code Issues Packages Projects Releases 4 Wiki Activity
Files
master
adfilt/AdGuard Home Compilation List/AGHtest.py
T
Imre Eilertsen 2f5ef93604 Add files via upload
2025-06-29 21:57:25 +02:00

1600 lines
34 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import requests
import re
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription.notlist', 'https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2020.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2021.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2022.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2023.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2024.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2025.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/quick-fixes.txt', 'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/BaseFilter/sections/adservers.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/BaseFilter/sections/general_extensions.txt', 'https://filters.adtidy.org/extension/ublock/filters/16.txt', 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', 'https://raw.githubusercontent.com/abp-filters/abp-filters-anti-cv/master/english.txt', 'https://raw.githubusercontent.com/abp-filters/abp-filters-anti-cv/master/french.txt', 'https://easylist-downloads.adblockplus.org/antiadblockfilters.txt', 'https://easylist-downloads.adblockplus.org/advblock.txt', 'https://easylist-downloads.adblockplus.org/Liste_AR.txt', 'https://easylist-downloads.adblockplus.org/easylistspanish.txt']
UNSUPPORTED_AGH = ['##', '@#', '#?#', '#%#', '!+', 'domain=', 'generichide', '$ghide', ',ghide', '$csp', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '$domain', ',domain', '[Adblock Plus 2.0]', 'CV-', '$csp', '$:']
UNSUPPORTED_IP = ['##', '@#', '#?#', '#%#', 'domain=', 'generichide', '$csp', 'badfilter', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '!+', '$:']
OUTPUT = 'xyzzyx.txt'
OUTPUT_AGH = 'AdGuardHomeCompilationList.txt'
OUTPUT_IP = 'AdGuardHomeCompilationListIPs.txt'
# function that downloads the filter list
def download_filters() -> str:
text = ''
for url in SOURCES:
r = requests.get(url)
text += r.text
return text
def is_supported_agh(line) -> bool:
for token in UNSUPPORTED_AGH:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_agh(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])(1p|3p|all|doc$|document|first-party|frame|image|media|network|object|popunder|popup|script|subdocument|third-party|~object-subrequest|~third-party)",
r"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r",domain=~?in-addr\.arpa",
"",
line
)
line = re.sub(
r"^.*\^[*$,][ac-hj-z?].*$",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[:@][a-z0-9?!].*$",
"",
line
)
line = re.sub(
r"^@?@?\|.*[/_#,].*$",
r"",
line
)
line = re.sub(
r"\^,?badfilter",
r"^$badfilter",
line
)
line = re.sub(
r"^.*\^\*.*$",
r"",
line
)
line = re.sub(
r"^.*\*\^[a-z0-9?].*$",
r"",
line
)
line = re.sub(
r"^.*\^&.*$",
r"",
line
)
line = re.sub(
r"^.*\|.*/.*$",
r"",
line
)
line = re.sub(
r"^.*\*[&?@].*$",
r"",
line
)
line = re.sub(
r"^(!| |\$|@@)$",
r"",
line
)
line = re.sub(
r"^([&+=?^_%,~#*]|@@/|\\.|;[a-z0-9]).*$",
r"",
line
)
line = re.sub(
r"^! (AG_|Checksum:|Ref:|http|\||\[|#|!).*$",
r"",
line
)
line = re.sub(
r"^\|\|9anime\.\*(\^|\|)$",
r"",
line
)
line = re.sub(
r"/\^https\?:\\/\\/([0-9]{1,3})\\\.([0-9]{1,3})\\\.([0-9]{1,3})\\.\(\\d\)\{1,3\}\.\*\/",
r"://\1.\2.\3.\n@@://\1.\2.\3.*in-addr.arpa^",
line
)
line = re.sub(
r"\|\|([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.$",
r"://\1.\2.\3.\n@@://\1.\2.\3.*in-addr.arpa^",
line
)
line = re.sub(
r"^(\|\||(:/)?/)?([0-9]{1,3})\.([0-9]{1,3})\.$",
r"",
line
)
line = re.sub(
r"^://.*/.*$",
r"",
line
)
line = re.sub(
r"^/.*[a-z0-9]\\?/[a-z0-9\(].*$",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\\/.*$",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\/.*$",
r"",
line
)
line = re.sub(
r"^/[a-z0-9.*_/-]{1,70}/$",
r"",
line
)
line = re.sub(
r"^[a-z0-9@|].*\?.*$",
r"",
line
)
line = re.sub(
r"^.*\(update frequency\)",
r"",
line
)
line = re.sub(
r"^! 20(1|2).*$",
r"",
line
)
line = re.sub(
r"\*(\^|\|)$",
r"*",
line
)
line = re.sub(
r"^\|\|\*\.",
r".",
line
)
line = re.sub(
r"\.\*$",
r".",
line
)
line = re.sub(
r"^[0-9]{1,5}$",
r"",
line
)
line = re.sub(
r"^[a-zA-Z0-9.-].*[/?_+=&]$",
r"",
line
)
line = re.sub(
r"^[@.].*[a-z0-9*]/.*$",
r"",
line
)
line = re.sub(
r"^.*[$,]jsinject$",
r"",
line
)
line = re.sub(
r"^.* - .*$",
r"",
line
)
line = re.sub(
r"^![a-z0-9].*#.*$",
r"",
line
)
line = re.sub(
r"^!\*(\*.*|$)",
r"",
line
)
line = re.sub(
r"^! ! \+.*$",
r"",
line
)
line = re.sub(
r"^\|.*:[0-9].*$",
r"",
line
)
line = re.sub(
r"^[a-zA-Z0-9.-].*[a-z0-9.-][/?_+=&][a-zA-Z0-9.$*-].*$",
r"",
line
)
line = re.sub(
r"^/.*[_=/;~@%#+,!].*$",
r"",
line
)
line = re.sub(
r"^[@|:/.-].*[%&?_=].*$",
r"",
line
)
line = re.sub(
r"^[|:/.-].*:.*$",
r"",
line
)
line = re.sub(
r"^(/|:?\|?\||\.)[a-zA-Z0-9-]{1,100}[|^]$",
r"",
line
)
line = re.sub(
r"^[.:/|a-zA-Z0-9-].*[*.](png|jpe?g|gif|js|swf|gz|exe|php)($|\||\^)",
r"",
line
)
line = re.sub(
r"^\..*=.*$",
r"",
line
)
line = re.sub(
r"^[a-z0-9./|-].*\^=.*$",
r"",
line
)
line = re.sub(
r"\^,~",
r"^$~",
line
)
line = re.sub(
r"^.*\.,badfilter$",
r"",
line
)
line = re.sub(
r"(.) {1,}$",
r"\1",
line
)
line = re.sub(
r"^! \*\*\* easylistgermany:easylistgermany/easylistgermany_allowlist_popup\.txt \*\*\*$",
r"! Title: ABP Filters",
line
)
line = re.sub(
r"^!-------------{1,}!$",
r"",
line
)
line = re.sub(
r"^[a-z0-9/-]{1,5}$",
r"",
line
)
line = re.sub(
r"^\.[a-zA-Z0-9]{3,}\|$",
r"",
line
)
line = re.sub(
r"^! E[nN][dD].*$",
r"",
line
)
line = re.sub(
r"^! Expires: [0-9] .*$",
r"",
line
)
line = re.sub(
r"^! (Good|Bad): .*$",
r"",
line
)
line = re.sub(
r"^.* end ⬆️$",
r"",
line
)
line = re.sub(
r"^! This section contains the list of .*$",
r"",
line
)
line = re.sub(
r"^! Note, .*$",
r"",
line
)
line = re.sub(
r"^.*github\.com/NanoMeow.*$",
r"",
line
)
line = re.sub(
r" -{4,}(!?)$",
r" ---\1",
line
)
line = re.sub(
r"! ?-{3,}!?$",
r"",
line
)
line = re.sub(
r" ={4,}$",
r" ===",
line
)
line = re.sub(
r"^!-{4,}([ A-ZÉ])",
r"!---\1",
line
)
line = re.sub(
r"([a-z(])-{4,}!$",
r"\1---!",
line
)
line = re.sub(
r"^! Last modified: %timestamp%$",
r"",
line
)
line = re.sub(
r"\*\*\*! Title: ",
r"***\n! Title: ",
line
)
line = re.sub(
r"^! \*\*\* .*hide.*$",
r"",
line
)
line = re.sub(
r"^! Smart ?JSON.*$",
r"",
line
)
line = re.sub(
r" {2,}",
r" ",
line
)
line = re.sub(
r"^! \*\*\* easylistspanish:easylistspanish(_adult/adult_thirdparty|/easylistspanish_allowlist_|_adult/adult_allowlist).*$",
r"",
line
)
line = re.sub(
r"^! \*\*\* easylist:easylist_adult/adult_specific_block_popup\.txt \*\*\*",
r"",
line
)
line = re.sub(
r"^! \*\*\* easylist:easylist_adult/adult_allowlist.*$",
r"",
line
)
line = re.sub(
r"^! \*\*\* easylistgermany:easylistgermany/(easylistgermany_allowlist|easylistgermany_specific_block_popup\.).*$",
r"",
line
)
line = re.sub(
r"! \*\*\* antiadblockfilters.*(czech|dutch|finnish|hebrew|indonesian|latvian|romanian|slovak).*$",
r"",
line
)
line = re.sub(
r"^\|\|51\.89\.187\.1(3[7-9]|4[0-3])(\^|\|)$",
r"",
line
)
line = re.sub(
r"^/[a-zA-Z]{1,8}$",
r"",
line
)
line = re.sub(
r"^!.* Adblock Plus($| .*)",
r"",
line
)
line = re.sub(
r"^.*\.html?(\^|\|)$",
r"",
line
)
line = re.sub(
r"^!.* property.*$",
r"",
line
)
line = re.sub(
r"^!.*([Éé]lément|CSS|General hid|JS).*$",
r"",
line
)
line = re.sub(
r"^.*#\$\?#.*$",
r"",
line
)
# $badfilter-ed entries, as Sublime Text kept complaining about "Ran out of stack space" when I tried to semi-automate it with RegEx.
line = re.sub(
r"^(/adblockpopup\.|/ads\.css|/propads\.)(\$badfilter)?$",
r"",
line
)
line = re.sub(
r"^\|.*[a-zA-Z0-9]\|[|a-zA-Z0-9].*$",
r"",
line
)
line = re.sub(
r"^.*Reek's Anti-Adblock Killer.*$",
r"",
line
)
line = re.sub(
r"^(:/)?/[b-z]([a-z]{1,2})?\.$",
r"",
line
)
# We are in the year 2025 (or later).
line = re.sub(
r"http://creativecommons\.org/licenses/",
r"https://creativecommons.org/licenses/",
line
)
line = re.sub(
r"^/([a-z0-9]{1,8}\.)$",
r"://\1",
line
)
line = re.sub(
r"([a-zA-Z0-9.,?!'\":; ])(->|=>)([a-zA-Z0-9.,?!'\":; ]|$)",
r"\1→\3",
line
)
line = re.sub(
r"^.*\$\[.*$",
r"",
line
)
line = re.sub(
r"^[a-zA-Z0-9:/|].*\$\$.*$",
r"",
line
)
line = re.sub(
r"^[0-9].*[: ].*$",
r"",
line
)
line = re.sub(
r"^([0-9.]{7,15})(\^|\|)$",
r"\1",
line
)
line = re.sub(
r"^([.:/|-].*\.(com|org|net|uk|eu|at|it|io|fi|de|nl|agency|es|pl|is|br|gr|ru|su|pro|xyz|win|ch))$",
r"\1^",
line
)
line = re.sub(
r"^[.:/|].*\.(wasm|comm)(\^|\|)$",
r"",
line
)
line = re.sub(
r"\.\^$",
r"^",
line
)
line = re.sub(
r"^(\.|:?/?/|\|\|?)[a-z0-9-]{1,}(\^|\|)$",
r"",
line
)
line = re.sub(
r"^![!#].*$",
"",
line
)
line = re.sub(
r"^(@@|\.|(:/)?/|\|\|?|-)[a-z0-9]\*\.[a-z]{2,}(\^|\|)?$",
"",
line
)
line = re.sub(
r"^/([a-zA-Z0-9].*$)",
r"|\1",
line
)
line = re.sub(
r"^://([a-zA-Z0-9].*$)",
r"|\1",
line
)
line = re.sub(
r"^\|\|((\d{1,3}\.){3}\d{1,3})\^$",
r"\1",
line
)
line = re.sub(
r"^@.*(gatewaypundit|wltreport).*$",
r"",
line
)
line = re.sub(
r"^[|:/._a-z0-9-].*akamaiedge.*$",
r"",
line
)
line = re.sub(
r"^([|:/._a-z0-9-].*(akamai|azure[we]|\.cdn77\.org|amazonaws\.).*[^|])$",
r"\1$dnstype=~CNAME",
line
)
line = re.sub(
r"^! {0,}$",
r"",
line
)
line = re.sub(
r"^((\d{1,3}\.){3}[12]?\d)$",
r"|\1|",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?com(([,: ]|\)).*)?$",
r"\1COM\3",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?com(([,: ]|\)).*)?$",
r"\1COM\3",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?com(([,: ]|\)).*)?$",
r"\1COM\3",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?net(([,: ]|\)).*)?$",
r"\1NET\3",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?org(([,: ]|\)).*)?$",
r"\1ORG\3",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?org(([,: ]|\)).*)?$",
r"\1ORG\3",
line
)
line = re.sub(
r"^(!.*([, ]|\()[a-z0-9-]{2,}) ?\. ?org(([,: ]|\)).*)?$",
r"\1ORG\3",
line
)
line = re.sub(
r"^.*\$:.*$",
r"",
line
)
line = re.sub(
r"^.*hid[ie].*Adblock Plus.*$",
r"",
line
)
if is_supported_agh(line) and not line == '':
text += line + '\n'
return text
# ——— IP version ———
def is_supported_ip(line) -> bool:
for token in UNSUPPORTED_IP:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_ip(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"^.*\)([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})\\.\..*$",
r"\1.\2.\3.0/24",
line
)
line = re.sub(
r"([$,])(1p|3p|all|doc|document|first-party|frame|image|media|network|object|popunder|popup|script|subdocument|third-party|~object-subrequest|~third-party)",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r"^.*\^[*$,].*$",
"",
line
)
line = re.sub(
r"^.*\.js$",
"",
line
)
line = re.sub(
r"^([&+-.=?^_%:*/@]|\\.).*$",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^\|[a-z0-9].*$",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*$",
"",
line
)
line = re.sub(
r"^@[a-z0-9].*$",
r"",
line
)
line = re.sub(
r"@?@?\|\|.*[a-zа-яみ].*$",
r"",
line
)
line = re.sub(
r"! (Version: .*[a-z].*)",
r"# \1",
line
)
line = re.sub(
r"^!([a-z #]|\[).*$",
r"",
line
)
line = re.sub(
r"^!-.*$",
r"",
line
)
line = re.sub(
r"\|\|",
r"",
line
)
line = re.sub(
r"(\^|\|)$",
r"",
line
)
line = re.sub(
r"^(([0-9]{1,3}\.){3})$",
r"\1.0/24",
line
)
line = re.sub(
r"^([0-9]{1,3}\.){2}$",
r"",
line
)
line = re.sub(
r"^@@.*[a-z].*$",
r"",
line
)
line = re.sub(
r"\[Adblock Plus 3\.[0-9]{1,3}\]",
"! Title: IP Entries from Adblock Lists\n! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, AdGuard Base Filter, AdGuard French Filter, EasyList Germany, ABP Anti-Circumvention Filters, RU AdList, Liste AR, EasyList Spanish \n! Expires: 14 days\n! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.\n! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.",
line
)
line = re.sub(
r"# Version: (.*)-.*$",
r"! Version: \1-Alpha",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"^/.*[a-zA-Z0-9_=.,*?|^@&~+;-]$",
r"",
line
)
line = re.sub(
r"^/.*[\]]$",
r"",
line
)
line = re.sub(
r"^://.*/.*$",
r"",
line
)
line = re.sub(
r"^.*104\\\.154\\\.\..*$",
r"",
line
)
line = re.sub(
r"^/\^[h.].*$",
r"",
line
)
line = re.sub(
r"^/\\.*$",
r"",
line
)
line = re.sub(
r"^/\(https\?:\\/\\/\)([0-9][0-9]?[0-9]?)\\.([0-9][0-9]?[0-9]?)\\.\..*/",
r"",
line
)
line = re.sub(
r"^![a-zA-Z!*].*$",
r"",
line
)
line = re.sub(
r"^[A-Za-z].*$",
r"",
line
)
line = re.sub(
r"^ $",
r"",
line
)
line = re.sub(
r"^#[a-z0-9.#].*$",
"",
line
)
line = re.sub(
r"^.*dmcdn\..*$",
r"",
line
)
line = re.sub(
r"^[a-z.#%$[].*$",
r"",
line
)
line = re.sub(
r"(.) {1,}$",
r"\1",
line
)
line = re.sub(
r"^[a-zA-Z0-9].*[#$|a-z].*$",
r"",
line
)
line = re.sub(
r"\.\.",
r".",
line
)
line = re.sub(
r"^~.*$",
r"",
line
)
if is_supported_ip(line) and not line == '':
text += line + '\n'
return text
if __name__ == "__main__":
print('Starting the script')
text = download_filters()
lines = text.splitlines(False)
print('Total number of rules: ' + str(len(lines)))
agh_filter = prepare_agh(lines)
ip_filter = prepare_ip(lines)
with open(OUTPUT, "w", encoding="utf-8", newline='\n') as text_file:
text_file.write(text)
with open(OUTPUT_AGH, "w", encoding="utf-8", newline='\n') as text_file:
text_file.write(agh_filter)
with open(OUTPUT_IP, "w", encoding="utf-8", newline='\n') as text_file:
text_file.write(ip_filter)
print('The list versions have been generated.')
# ————————————————————————————————————————————
import requests
import re
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription-Notifications.notlist', 'https://easylist-downloads.adblockplus.org/fanboy-notifications.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/Popups/sections/push-notifications_general.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/Popups/sections/push-notifications_specific.txt']
UNSUPPORTED_AGH = ['##', '@#', '#?#', '#%#', '!+', 'domain=', 'generichide', '$ghide', ',ghide', '$csp', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '$domain', ',domain', '[Adblock Plus 2.0]', 'CV-']
OUTPUT = 'xyzzyxnotif.txt'
OUTPUT_AGH = 'AdGuardHomeCompilationList-Notifications.txt'
# function that downloads the filter list
def download_filters() -> str:
text = ''
for url in SOURCES:
r = requests.get(url)
text += r.text
return text
def is_supported_agh(line) -> bool:
for token in UNSUPPORTED_AGH:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_agh(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])third-party",
"",
line
)
line = re.sub(
r"([$,])~third-party",
"",
line
)
line = re.sub(
r"([$,])3p",
"",
line
)
line = re.sub(
r"([$,])first-party",
"",
line
)
line = re.sub(
r"([$,])1p",
"",
line
)
line = re.sub(
r"([$,])image",
"",
line
)
line = re.sub(
r"([$,])media",
"",
line
)
line = re.sub(
r"([$,])script",
"",
line
)
line = re.sub(
r"([$,])popup",
"",
line
)
line = re.sub(
r"([$,])popunder",
"",
line
)
line = re.sub(
r"([$,])document",
"",
line
)
line = re.sub(
r"([$,])doc",
"",
line
)
line = re.sub(
r"([$,])subdocument",
"",
line
)
line = re.sub(
r"([$,])object",
"",
line
)
line = re.sub(
r"([$,])~object-subrequest",
"",
line
)
line = re.sub(
r"([$,])frame",
"",
line
)
line = re.sub(
r"([$,])all",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r",domain=~?in-addr\.arpa",
"",
line
)
line = re.sub(
r"/\^https\?:\\/\\/([0-9]{1,3})\\\.([0-9]{1,3})\\\.([0-9]{1,3})\\.\(\\d\)\{1,3\}\.\*\/",
"\1.\2.\3.0/24",
line
)
line = re.sub(
r"^.*\^[*$,][ac-hj-z?].*$",
"",
line
)
line = re.sub(
r"^.*\.js$",
"",
line
)
line = re.sub(
r"^(\.[a-z0-9.]{4,60}\.)$",
r"||*\1",
line
)
line = re.sub(
r"^[&+=?^_%].*$",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[0-9].*$",
"",
line
)
line = re.sub(
r"^!#.*$",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*$",
"",
line
)
line = re.sub(
r"^Amasty_.*$",
"",
line
)
line = re.sub(
r"^\*.*$",
"",
line
)
line = re.sub(
r"^\\.*$",
"",
line
)
line = re.sub(
r"^:[a-z0-9?!].*$",
"",
line
)
line = re.sub(
r"^@[a-z0-9].*$",
"",
line
)
line = re.sub(
r"@?@?\|\|.*/.*$",
r"",
line
)
line = re.sub(
r"\^,badfilter",
r"^badfilter",
line
)
line = re.sub(
r"\^badfilter",
r"^$badfilter",
line
)
line = re.sub(
r"^.*\^\*.*$",
r"",
line
)
line = re.sub(
r"^.*\*\^[a-z0-9?].*$",
r"",
line
)
line = re.sub(
r"^.*\^&.*$",
r"",
line
)
line = re.sub(
r"^.*\|.*/.*$",
r"",
line
)
line = re.sub(
r"^.*\*[&?@].*$",
r"",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"^@@/.*$",
r"",
line
)
line = re.sub(
r"^[,=^?~].*$",
r"",
line
)
line = re.sub(
r"^! \|.*$",
r"",
line
)
line = re.sub(
r"\.php(\||\?)$",
r"",
line
)
line = re.sub(
r"^\|\|goo\.gl(\^|\|)$",
r"",
line
)
line = re.sub(
r"^\|\|9anime\.\*(\^|\|)$",
r"",
line
)
line = re.sub(
r"^\|\|play-asia\.com(\^|\|)$",
r"",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\3.\n@@://\1.\2.\3.*in-addr.arpa^",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\n@@://\1.\2.*in-addr.arpa^",
line
)
line = re.sub(
r"^! http.*$",
r"",
line
)
line = re.sub(
r"^://.*/.*$",
r"",
line
)
line = re.sub(
r"^.*104\\\.154\\\.\..*$",
r"",
line
)
line = re.sub(
r"^/.*[a-z0-9]\\?/[a-z0-9\(].*$",
r"",
line
)
line = re.sub(
r"\|\|.*_.*$",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\\/.*$",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\/.*$",
r"",
line
)
line = re.sub(
r"^/[a-z0-9.*_/-]{1,70}/$",
r"",
line
)
line = re.sub(
r"^[a-z0-9@|].*\?.*$",
r"",
line
)
line = re.sub(
r"^! Checksum:.*$",
r"",
line
)
line = re.sub(
r"^.*\(update frequency\)",
r"",
line
)
line = re.sub(
r"^! 20(1|2).*$",
r"",
line
)
line = re.sub(
r"^! Ref:.*$",
r"",
line
)
line = re.sub(
r"^! #",
r"",
line
)
line = re.sub(
r"\*(\^|\|)$",
r"*",
line
)
line = re.sub(
r"^\|\|\*\.",
r".",
line
)
line = re.sub(
r"\.\*$",
r".",
line
)
line = re.sub(
r"^[0-9]{1,5}$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[/?_+=&]$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-z0-9.$-].*$",
r"",
line
)
line = re.sub(
r"^/.*[_?=/;~@%#+,].*$",
r"",
line
)
line = re.sub(
r"\^,~",
r"^$~",
line
)
line = re.sub(
r"^.*\.,badfilter$",
r"",
line
)
line = re.sub(
r"^.*dmcdn\..*$",
r"",
line
)
line = re.sub(
r"([a-zA-Z0-9.,;'?!#_-]) $",
r"\1",
line
)
line = re.sub(
r"^(! This section contains )",
r"! Title: AdGuard Popups filter - Push notifications, specific\n\1",
line
)
line = re.sub(
r"^.*/,stylesheet$",
r"",
line
)
line = re.sub(
r"^%.*$",
r"",
line
)
line = re.sub(
r"http://creativecommons\.org/licenses/",
r"https://creativecommons.org/licenses/",
line
)
line = re.sub(
r"^/([a-zA-Z0-9].*$)",
r"|\1",
line
)
line = re.sub(
r"^! {0,}$",
r"",
line
)
line = re.sub(
r"^! !SECTION.*$",
r"",
line
)
line = re.sub(
r"^.*\$:.*$",
r"",
line
)
line = re.sub(
r"^.*hid[ie].*Adblock Plus.*$",
r"",
line
)
if is_supported_agh(line) and not line == '':
text += line + '\n'
return text
if __name__ == "__main__":
print('Starting the script')
text = download_filters()
lines = text.splitlines(False)
print('Total number of rules: ' + str(len(lines)))
agh_filter = prepare_agh(lines)
with open(OUTPUT, "w", encoding="utf-8", newline='\n') as text_file:
text_file.write(text)
with open(OUTPUT_AGH, "w", encoding="utf-8", newline='\n') as text_file:
text_file.write(agh_filter)
print('The notifications list version has been generated.')
Reference in New Issue View Git Blame Copy Permalink