starred/adfilt
1
0
Fork 0
mirror of https://github.com/DandelionSprout/adfilt.git synced 2026-10-02 22:04:28 +01:00
Code Issues Packages Projects Releases 4 Wiki Activity

This took an obscene amount of time to successfully craft

Browse Source
This commit is contained in:
Imre Kristoffer Eilertsen
2019-12-05 14:32:42 +01:00
committed by GitHub
parent 3b7cd9fe38
commit 5fd6073966
3 changed files with 6113 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AdGuard Home Compilation List/AGHtest.py
+352 -3
View File
@@ -1,12 +1,14 @@
import requests
import re
SOURCES = ['https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', 'https://raw.githubusercontent.com/NanoAdblocker/NanoFilters/master/NanoMirror/NanoDefender.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', 'https://easylist-downloads.adblockplus.org/liste_fr.txt', 'https://www.i-dont-care-about-cookies.eu/abp/', 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', 'https://easylist-downloads.adblockplus.org/abp-filters-anti-cv.txt']
SOURCES = ['https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/TopDescription.notlist', 'https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', 'https://raw.githubusercontent.com/NanoAdblocker/NanoFilters/master/NanoMirror/NanoDefender.txt', 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', 'https://easylist-downloads.adblockplus.org/liste_fr.txt', 'https://www.i-dont-care-about-cookies.eu/abp/', 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', 'https://easylist-downloads.adblockplus.org/abp-filters-anti-cv.txt']
UNSUPPORTED_AGH = ['/', '##', '@#', '#?#', 'domain=', 'generichide', '$csp', 'badfilter', 'xmlhttprequest', '$xhr', '$stylesheet', '~image', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#']
UNSUPPORTED_AGH = ['##', '@#', '#?#', 'domain=', 'generichide', '$csp', 'badfilter', 'xmlhttprequest', '$xhr', '$stylesheet', '~image', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#']
UNSUPPORTED_IP = ['##', '@#', '#?#', 'domain=', 'generichide', '$csp', 'badfilter', 'xmlhttprequest', '$xhr', '$stylesheet', '~image', '$elemhide', '$inline-script', '$other', '$~object', 'redirect=', '#$#']
OUTPUT = 'xyzzyx.txt'
OUTPUT_AGH = 'AdGuardHomeCompilationList.txt'
OUTPUT_IP = 'AdGuardHomeCompilationListIPs.txt'
# function that downloads the filter list
def download_filters() -> str:
@@ -190,11 +192,354 @@ def prepare_agh(lines) -> str:
line
)
line = re.sub(
r"^Amasty_.*",
"",
line
)
line = re.sub(
r"^\*.*",
"",
line
)
line = re.sub(
r"^\\.*",
"",
line
)
line = re.sub(
r"^:.*",
"",
line
)
line = re.sub(
r"^@[a-z0-9].*",
"",
line
)
line = re.sub(
r"^[/\\].*",
r"",
line
)
line = re.sub(
r"@?@?\|\|.*/.*",
r"",
line
)
if is_supported_agh(line) and not line == '':
text += line + '\r\n'
return text
# ——— IP version ———
def is_supported_ip(line) -> bool:
for token in UNSUPPORTED_IP:
if token in line:
return False
return True
# function that prepares the filter list for AdGuard Home
def prepare_ip(lines) -> str:
text = ''
previous_line = None
for line in lines:
if line == previous_line:
continue
line = re.sub(
r"([$,])third-party",
"",
line
)
line = re.sub(
r"([$,])~third-party",
"",
line
)
line = re.sub(
r"([$,])3p",
"",
line
)
line = re.sub(
r"([$,])first-party",
"",
line
)
line = re.sub(
r"([$,])1p",
"",
line
)
line = re.sub(
r"([$,])image",
"",
line
)
line = re.sub(
r"([$,])media",
"",
line
)
line = re.sub(
r"([$,])script",
"",
line
)
line = re.sub(
r"([$,])popup",
"",
line
)
line = re.sub(
r"([$,])popunder",
"",
line
)
line = re.sub(
r"([$,])document",
"",
line
)
line = re.sub(
r"([$,])subdocument",
"",
line
)
line = re.sub(
r"([$,])~subdocument",
"",
line
)
line = re.sub(
r"([$,])object",
"",
line
)
line = re.sub(
r"([$,])~object-subrequest",
"",
line
)
line = re.sub(
r"([$,])frame",
"",
line
)
line = re.sub(
r"([$,])all",
"",
line
)
line = re.sub(
r",important",
"$important",
line
)
line = re.sub(
r".*\^[*$,].*",
"",
line
)
line = re.sub(
r".*\.js$",
"",
line
)
line = re.sub(
r"^[&+-.=?^_%].*",
"",
line
)
line = re.sub(
r"^([a-z])",
r"||\1",
line
)
line = re.sub(
r"^[0-9].*",
"",
line
)
line = re.sub(
r"^\|[a-z0-9].*",
"",
line
)
line = re.sub(
r"^;[a-z0-9].*",
"",
line
)
line = re.sub(
r"^Amasty_.*",
"",
line
)
line = re.sub(
r"^\*.*",
r"",
line
)
line = re.sub(
r"^\\.*",
r"",
line
)
line = re.sub(
r"^:.*",
r"",
line
)
line = re.sub(
r"^@[a-z0-9].*",
r"",
line
)
line = re.sub(
r"[@]?[@]?\|\|.*[a-zみ].*",
r"",
line
)
line = re.sub(
r"! (Version: .*[a-z].*)",
r"# \1",
line
)
line = re.sub(
r"^![a-z #].*",
r"",
line
)
line = re.sub(
r"^!-.*",
r"",
line
)
line = re.sub(
r"^!$",
r"",
line
)
line = re.sub(
r"\[Adblock Plus 2\.0\]",
r"",
line
)
line = re.sub(
r"\[Nano Adblocker\]",
r"",
line
)
line = re.sub(
r"\|\|",
r"",
line
)
line = re.sub(
r"\^$",
r"",
line
)
line = re.sub(
r"^([0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?)\.$",
r"\1.0/24",
line
)
line = re.sub(
r"^([0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?)\.$",
r"\1.0.0/16",
line
)
line = re.sub(
r"^[/\\].*",
r"",
line
)
line = re.sub(
r"^@@.*[a-z].*",
r"",
line
)
line = re.sub(
r"\[Adblock Plus 3\.4\]",
"! Title: IP Entries from Adblock Lists\n! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, Nano Defender Integration, Liste FR, I Don't Care About Cookies, EasyList Germany, ABP Anti-Circumvention Filters\n! Expires: 14 days\n! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.\n! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.",
line
)
line = re.sub(
r"# Version: (.*)-.*",
r"! Version: \1-Alpha",
line
)
line = re.sub(
r"^!$",
r"",
line
)
if is_supported_ip(line) and not line == '':
text += line + '\r\n'
return text
if __name__ == "__main__":
print('Starting the script')
text = download_filters()
@@ -202,6 +547,7 @@ if __name__ == "__main__":
print('Total number of rules: ' + str(len(lines)))
agh_filter = prepare_agh(lines)
ip_filter = prepare_ip(lines)
with open(OUTPUT, "w") as text_file:
text_file.write(text)
@@ -209,4 +555,7 @@ if __name__ == "__main__":
with open(OUTPUT_AGH, "w") as text_file:
text_file.write(agh_filter)
print('The domains-based list versions have been generated.')
with open(OUTPUT_IP, "w") as text_file:
text_file.write(ip_filter)
print('The list versions have been generated.')
AdGuard Home Compilation List/AdGuardHomeCompilationList.txt
+5621 -17
View File
File diff suppressed because it is too large Load Diff
AdGuard Home Compilation List/AdGuardHomeCompilationListIPs.txt
+140
View File
@@ -0,0 +1,140 @@
! Title: IP Entries from Adblock Lists
! Contains transformated entries from: EasyList, uBlock Filters, uBlock Filters - Badware Risks, Nano Defender Integration, Liste FR, I Don't Care About Cookies, EasyList Germany, ABP Anti-Circumvention Filters
! Expires: 14 days
! Licence: In accordance with the Dandelicence, the borrowed entries are considered to have been changed and reduced enough from their original lists, that they're counted as transformative work, meaning that creditation and seperate paragraphs are not necessary unless one of the lists' makers were to ask for such.
! Description: This was made as a proof-of-concept to see if the IP-based entries of major adblock lists, could be used to create an IP adblocker list for IP blockers, despite how IP lists are normally only meant to block malware, E-mail spam, or port scanners.
! Version: 05December2019v1-Alpha
104.154.237.93
174.142.194.177
194.71.107.25
213.163.70.183
35.184.98.90
46.165.197.153
46.165.197.231
50.7.243.123
63.225.61.4
64.20.60.123
74.117.182.77
78.138.126.253
78.140.131.214
104.154.0.0/16
104.197.0.0/16
104.198.0.0/16
130.211.0.0/16
216.21.0.0/16
35.184.0.0/16
35.188.0.0/16
35.193.0.0/16
167.206.10.148
130.211.0.0/16
142.91.0.0/16
185.147.34.126
216.21.13.0/24
5.45.79.15
195.228.74.26
206.217.206.137
212.150.34.117
76.76.5.113
80.77.113.200
85.17.210.202
89.248.172.46
104.197.10.19
104.197.10.88
104.197.127.140
104.197.188.104
104.197.241.137
104.197.34.33
104.197.4.220
178.17.164.58
96.9.176.245
104.198.198.188
104.198.198.188
104.197.199.227
104.197.199.227
35.226.75.50
193.238.46.6
35.224.227.218
104.154.237.0/24
104.238.188.0/24
104.238.189.0/24
108.163.194.100
138.197.81.16
145.239.218.240
146.148.85.61
163.172.16.147
178.162.205.67
178.175.132.130
178.33.220.81
188.72.206.0/24
198.16.64.66
213.206.98.70
31.7.60.66
35.184.0.0/16
35.184.169.188
35.188.0.0/16
35.193.89.147
37.46.192.170
46.101.101.204
46.101.125.0/24
46.249.58.117
74.117.179.63
77.247.181.115
78.140.130.88
78.140.130.91
85.116.37.142
85.170.167.205
89.185.33.51
91.210.165.104
95.211.211.158
185.165.169.108
35.194.16.41
35.194.26.233
35.224.227.218
35.225.234.73
35.239.57.233
45.32.105.134
77.162.125.199
104.154.237.0/24
104.238.188.0/24
104.238.189.0/24
107.191.46.37
108.61.209.24
109.206.180.190
109.234.38.0/24
109.236.83.0/24
138.68.82.192
148.163.101.199
159.65.203.16
162.210.198.134
162.243.200.63
162.243.87.218
164.132.171.89
178.175.132.130
188.165.253.6
194.182.75.43
194.88.104.61
217.23.1.125
23.236.55.237
25.115.187.96
25.131.85.202
25.52.111.246
34.250.229.112
35.224.227.218
37.187.248.215
37.46.192.170
45.32.147.99
45.77.61.11
46.101.101.204
46.101.125.0/24
46.101.139.152
5.135.12.64
5.39.67.191
52.51.50.245
54.200.58.62
62.212.73.45
85.170.167.205
91.210.165.104
94.23.210.144
94.23.251.46
85.114.133.62
35.194.16.41