starred/adfilt
1
0
Fork 0
mirror of https://github.com/DandelionSprout/adfilt.git synced 2026-10-04 09:41:07 +01:00
Code Issues Packages Projects Releases 4 Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
Imre Kristoffer Eilertsen
2020-10-02 11:41:29 +02:00
committed by GitHub
parent c5f6edb761
commit cff1a7bcf8
3 changed files with 3276 additions and 748 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AdGuard Home Compilation List/AGHtest.py
+594 -7
View File
@@ -1,7 +1,7 @@
import requests
import re
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription.notlist', 'https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2020.txt', 'https://raw.githubusercontent.com/NanoAdblocker/NanoFilters/master/NanoMirror/NanoDefender.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', 'https://filters.adtidy.org/extension/ublock/filters/16.txt', 'https://www.i-dont-care-about-cookies.eu/abp/', 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', 'https://easylist-downloads.adblockplus.org/abp-filters-anti-cv.txt', 'https://easylist-downloads.adblockplus.org/antiadblockfilters.txt', 'https://easylist-downloads.adblockplus.org/advblock.txt', 'https://easylist-downloads.adblockplus.org/Liste_AR.txt', 'https://easylist-downloads.adblockplus.org/easylistspanish.txt', 'https://easylist-downloads.adblockplus.org/fanboy-notifications.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/sections/push-notifications.txt']
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription.notlist', 'https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters-2020.txt', 'https://raw.githubusercontent.com/NanoAdblocker/NanoFilters/master/NanoMirror/NanoDefender.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', 'https://filters.adtidy.org/extension/ublock/filters/2_without_easylist.txt', 'https://filters.adtidy.org/extension/ublock/filters/16.txt', 'https://www.i-dont-care-about-cookies.eu/abp/', 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', 'https://easylist-downloads.adblockplus.org/abp-filters-anti-cv.txt', 'https://easylist-downloads.adblockplus.org/antiadblockfilters.txt', 'https://easylist-downloads.adblockplus.org/advblock.txt', 'https://easylist-downloads.adblockplus.org/Liste_AR.txt', 'https://easylist-downloads.adblockplus.org/easylistspanish.txt']
UNSUPPORTED_AGH = ['##', '@#', '#?#', '#%#', '!+', 'domain=', 'generichide', '$ghide', ',ghide', '$csp', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '$domain', ',domain', '[Adblock Plus 2.0]', 'CV-']
UNSUPPORTED_IP = ['##', '@#', '#?#', '#%#', 'domain=', 'generichide', '$csp', 'badfilter', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '!+']
@@ -169,7 +169,7 @@ def prepare_agh(lines) -> str:
)
line = re.sub(
r"^[&+-.=?^_%].*",
r"^[&+=?^_%].*",
"",
line
)
@@ -295,7 +295,7 @@ def prepare_agh(lines) -> str:
)
line = re.sub(
r"^[,=^?~.-].*",
r"^[,=^?~].*",
r"",
line
)
@@ -469,8 +469,74 @@ def prepare_agh(lines) -> str:
)
line = re.sub(
r"! Disabling of push-notifications",
r"! Title: AdGuard Annoyances Web Push\n! Homepage: https://github.com/AdguardTeam/AdguardFilters/tree/master/AnnoyancesFilter/sections\n! Sourced from: https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/sections/push-notifications.txt\n! Disabling of push-notifications",
r"^[a-z0-9.-].*[/?_+=&]$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-z0-9.$-].*",
r"",
line
)
line = re.sub(
r"^[@.].*[a-z0-9*]/.*",
r"",
line
)
line = re.sub(
r"^#\..*",
r"",
line
)
line = re.sub(
r".*[$,]jsinject$",
r"",
line
)
line = re.sub(
r".* - .*",
r"",
line
)
line = re.sub(
r"^![a-z0-9].*#.*",
r"",
line
)
line = re.sub(
r"^!\*(\*.*|$)",
r"",
line
)
line = re.sub(
r"^! ! \+.*",
r"",
line
)
line = re.sub(
r"^! AG_.*",
r"",
line
)
line = re.sub(
r"^\|.*:[0-9].*",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-zA-Z0-9.$*-].*",
r"",
line
)
@@ -753,7 +819,7 @@ def prepare_ip(lines) -> str:
line = re.sub(
r"\[Adblock Plus 3\.4\]",
"! Title: IP Entries from Adblock Lists\n! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, Nano Defender Integration, AdGuard French Filter, EasyList Germany, ABP Anti-Circumvention Filters, RU AdList, Liste AR, EasyList Spanish \n! Expires: 14 days\n! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.\n! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.",
"! Title: IP Entries from Adblock Lists\n! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, Nano Defender Integration, AdGuard Base Filter, AdGuard French Filter, EasyList Germany, ABP Anti-Circumvention Filters, RU AdList, Liste AR, EasyList Spanish \n! Expires: 14 days\n! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.\n! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.",
line
)
@@ -835,6 +901,18 @@ def prepare_ip(lines) -> str:
line
)
line = re.sub(
r"^![a-zA-Z!*].*",
r"",
line
)
line = re.sub(
r"^[A-Za-z].*",
r"",
line
)
if is_supported_ip(line) and not line == '':
text += line + '\r\n'
@@ -858,4 +936,513 @@ if __name__ == "__main__":
with open(OUTPUT_IP, "w") as text_file:
text_file.write(ip_filter)
print('The list versions have been generated.')
print('The list versions have been generated.')
# ————————————————————————————————————————————
import requests
import re
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription-Notifications.notlist', 'https://easylist-downloads.adblockplus.org/fanboy-notifications.txt', 'https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/sections/push-notifications.txt']
UNSUPPORTED_AGH = ['##', '@#', '#?#', '#%#', '!+', 'domain=', 'generichide', '$ghide', ',ghide', '$csp', 'xmlhttprequest', '$xhr', '$stylesheet', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#', '$domain', ',domain', '[Adblock Plus 2.0]', 'CV-']
OUTPUT = 'xyzzyxnotif.txt'
OUTPUT_AGH = 'AdGuardHomeCompilationList-Notifications.txt'
# function that downloads the filter list
def download_filters() -> str:
text = ''
for url in SOURCES:
r = requests.get(url)
text += r.text
return text
def is_supported_agh(line) -> bool:
for token in UNSUPPORTED_AGH:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_agh(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])third-party",
"",
line
)
line = re.sub(
r"([$,])~third-party",
"",
line
)
line = re.sub(
r"([$,])3p",
"",
line
)
line = re.sub(
r"([$,])first-party",
"",
line
)
line = re.sub(
r"([$,])1p",
"",
line
)
line = re.sub(
r"([$,])image",
"",
line
)
line = re.sub(
r"([$,])media",
"",
line
)
line = re.sub(
r"([$,])script",
"",
line
)
line = re.sub(
r"([$,])popup",
"",
line
)
line = re.sub(
r"([$,])popunder",
"",
line
)
line = re.sub(
r"([$,])document",
"",
line
)
line = re.sub(
r"([$,])doc",
"",
line
)
line = re.sub(
r"([$,])subdocument",
"",
line
)
line = re.sub(
r"([$,])object",
"",
line
)
line = re.sub(
r"([$,])~object-subrequest",
"",
line
)
line = re.sub(
r"([$,])frame",
"",
line
)
line = re.sub(
r"([$,])all",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r",domain=in-addr\.arpa",
"",
line
)
line = re.sub(
r".*\^[*$,][ac-hj-z?].*",
"",
line
)
line = re.sub(
r".*\.js$",
"",
line
)
line = re.sub(
r"^(\.[a-z0-9.]{4,60}\.)$",
r"||*\1",
line
)
line = re.sub(
r"^[&+=?^_%].*",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[0-9].*",
"",
line
)
line = re.sub(
r"^!#.*",
"",
line
)
line = re.sub(
r"^\|[a-z0-9].*",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*",
"",
line
)
line = re.sub(
r"^Amasty_.*",
"",
line
)
line = re.sub(
r"^\*.*",
"",
line
)
line = re.sub(
r"^\\.*",
"",
line
)
line = re.sub(
r"^:[a-z0-9?!].*",
"",
line
)
line = re.sub(
r"^@[a-z0-9].*",
"",
line
)
line = re.sub(
r"@?@?\|\|.*/.*",
r"",
line
)
line = re.sub(
r"\^,badfilter",
r"^badfilter",
line
)
line = re.sub(
r"\^badfilter",
r"^$badfilter",
line
)
line = re.sub(
r".*\^\*.*",
r"",
line
)
line = re.sub(
r".*\*\^[a-z0-9?].*",
r"",
line
)
line = re.sub(
r".*\^&.*",
r"",
line
)
line = re.sub(
r".*\|.*/.*",
r"",
line
)
line = re.sub(
r".*\*[&?@].*",
r"",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"^@@/.*",
r"",
line
)
line = re.sub(
r"^[,=^?~].*",
r"",
line
)
line = re.sub(
r"^! \|.*",
r"",
line
)
line = re.sub(
r"^\.php\?$",
r"",
line
)
line = re.sub(
r"^\|\|goo\.gl\^$",
r"",
line
)
line = re.sub(
r"^\|\|9anime\.\*\^$",
r"",
line
)
line = re.sub(
r"^\|\|play-asia\.com\^$",
r"",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\3.\n@@://\1.\2.\3.*in-addr.arpa^",
line
)
line = re.sub(
r"\|\|([0-9][0-9]?[0-9]?)\.([0-9][0-9]?[0-9]?)\.$",
r"://\1.\2.\n@@://\1.\2.*in-addr.arpa^",
line
)
line = re.sub(
r"^! http.*",
r"",
line
)
line = re.sub(
r"^/([a-z0-9.-]{1,60}\.$)",
r"://\1",
line
)
line = re.sub(
r"^/.*[a-zA-Z0-9_=.,*?!|^@&~+;-]$",
r"",
line
)
line = re.sub(
r"^/.*[\]]$",
r"",
line
)
line = re.sub(
r"^://.*/.*",
r"",
line
)
line = re.sub(
r".*104\\\.154\\\.\..*",
r"",
line
)
line = re.sub(
r"^/.*[a-z0-9]\\?/[a-z0-9\(].*",
r"",
line
)
line = re.sub(
r"\|\|.*_.*",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\\/.*",
r"",
line
)
line = re.sub(
r"^/.*[a-z]\/.*",
r"",
line
)
line = re.sub(
r"^/[a-z0-9.*_/-]{1,70}/$",
r"",
line
)
line = re.sub(
r"^[a-z0-9@|].*\?.*",
r"",
line
)
line = re.sub(
r"^! Checksum:.*",
r"",
line
)
line = re.sub(
r".*\(update frequency\)",
r"",
line
)
line = re.sub(
r"^! 20(1|2).*",
r"",
line
)
line = re.sub(
r"^! Ref:.*",
r"",
line
)
line = re.sub(
r"^! #",
r"",
line
)
line = re.sub(
r"\*\^$",
r"*",
line
)
line = re.sub(
r"^\|\|\*\.",
r".",
line
)
line = re.sub(
r"\.\*$",
r".",
line
)
line = re.sub(
r"^[0-9]{1,5}$",
r"",
line
)
line = re.sub(
r"! Disabling of push-notifications",
r"! Title: AdGuard Annoyances Web Push\n! Homepage: https://github.com/AdguardTeam/AdguardFilters/tree/master/AnnoyancesFilter/sections\n! Sourced from: https://raw.githubusercontent.com/AdguardTeam/AdguardFilters/master/AnnoyancesFilter/sections/push-notifications.txt\n! Disabling of push-notifications",
line
)
line = re.sub(
r"^[a-z0-9.-].*[/?_+=&]$",
r"",
line
)
line = re.sub(
r"^[a-z0-9.-].*[a-z0-9.-][/?_+=&][a-z0-9.$-].*",
r"",
line
)
if is_supported_agh(line) and not line == '':
text += line + '\r\n'
return text
if __name__ == "__main__":
print('Starting the script')
text = download_filters()
lines = text.splitlines(False)
print('Total number of rules: ' + str(len(lines)))
agh_filter = prepare_agh(lines)
with open(OUTPUT, "w") as text_file:
text_file.write(text)
with open(OUTPUT_AGH, "w") as text_file:
text_file.write(agh_filter)
print('The notifications list version has been generated.')
AdGuard Home Compilation List/AdGuardHomeCompilationList.txt
+2668 -734
View File
File diff suppressed because it is too large Load Diff
AdGuard Home Compilation List/AdGuardHomeCompilationListIPs.txt
+14 -7
View File
@@ -1,9 +1,9 @@
! Title: IP Entries from Adblock Lists
! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, Nano Defender Integration, AdGuard French Filter, EasyList Germany, ABP Anti-Circumvention Filters, RU AdList, Liste AR, EasyList Spanish
! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, Nano Defender Integration, AdGuard Base Filter, AdGuard French Filter, EasyList Germany, ABP Anti-Circumvention Filters, RU AdList, Liste AR, EasyList Spanish
! Expires: 14 days
! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.
! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.
! Version: 01October2020v1-Alpha
! Version: 02October2020v1-Alpha
104.197.0.0/16
104.198.0.0/16
130.211.0.0/16
@@ -39,6 +39,17 @@
193.238.46.6
35.224.227.218
35.238.205.163
35.193.89.147
35.188.14.213
35.184.169.188
78.140.130.88
23.236.55.237
130.211.230.53
178.17.164.58
78.140.130.91
203.94.227.140
203.94.248.40
35.239.57.233
108.163.194.100
138.197.81.16
145.239.218.240
@@ -57,8 +68,6 @@
46.249.58.117
74.117.179.63
77.247.181.115
78.140.130.88
78.140.130.91
85.116.37.142
85.170.167.205
89.185.33.51
@@ -84,7 +93,6 @@
194.182.75.43
194.88.104.61
217.23.1.125
23.236.55.237
25.115.187.96
25.131.85.202
25.52.111.246
@@ -120,5 +128,4 @@
37.59.72.131
46.151.209.69
104.154.237.193
35.188.59.75
35.193.89.147
35.188.59.75