diff --git a/NorwegianExperimentalList alternate versions/nordic_prepare_filters_domains.py b/NorwegianExperimentalList alternate versions/nordic_prepare_filters_domains.py new file mode 100644 index 000000000..71d2f71d9 --- /dev/null +++ b/NorwegianExperimentalList alternate versions/nordic_prepare_filters_domains.py @@ -0,0 +1,212 @@ +import requests +import re + +SOURCES = ['https://gitlab.com/DandelionSprout/adfilt/raw/master/NorwegianExperimentalList%20alternate%20versions/DandelionSproutsNorskeFiltreDomains.txt'] + +UNSUPPORTED_HOSTS = ['*', '# ———'] +UNSUPPORTED_LS = ['*', '# ———', '# Translated title', '# Version', 'Don\'t be worried', 'General-info'] +UNSUPPORTED_DNSMASQ = ['*', '# '] + +OUTPUT = 'xyzzyxdomains.txt' +OUTPUT_HOSTS = 'AdawayHosts' +OUTPUT_PRIVOXY = 'NordicFiltersPrivoxy.txt' +OUTPUT_LS = 'LittleSnitchNorwegianRules.lsrules' +OUTPUT_DNSMASQ = 'NordicFiltersDnsmasq.conf' + +# function that downloads the filter list +def download_filters() -> str: + text = '' + for url in SOURCES: + r = requests.get(url) + text += r.text + return text + +def is_supported_hosts(line) -> bool: + for token in UNSUPPORTED_HOSTS: + if token in line: + return False + + return True + +# function that prepares the filter list for ABP +def prepare_hosts(lines) -> str: + text = '' + + # remove or modifiy entries with unsupported modifiers + for line in lines: + + # remove $document modifier from the rule + line = re.sub( + r"^(?!#)", + "127.0.0.1 ", + line + ) + + line = re.sub( + r"^([\s])", + "", + line + ) + + if is_supported_hosts(line): + text += line + '\r\n' + + return text + +# function that prepares the filter list for ABP +def prepare_privoxy(lines) -> str: + text = '' + + # remove or modifiy entries with unsupported modifiers + for line in lines: + + # remove $document modifier from the rule + + line = re.sub( + r"^(?!#)", + ".", + line + ) + + line = re.sub( + r"^\.$", + "", + line + ) + + if is_supported_hosts(line): + text += line + '\r\n' + + return text + +def is_supported_ls(line) -> bool: + for token in UNSUPPORTED_LS: + if token in line: + return False + + return True + +# function that prepares the filter list for ABP +def prepare_ls(lines) -> str: + text = '' + + # remove or modifiy entries with unsupported modifiers + for line in lines: + + # remove $document modifier from the rule + + line = re.sub( + r"^(?!#)", + "{ \"action\": \"deny\", \"process\": \"any\", \"remote-hosts\": \"", + line + ) + + line = re.sub( + r"$(?<=\.com$)|(?<=\.no$)|(?<=\.dk$)|(?<=\.is$)|(?<=\.org$)|(?<=\.net$)|(?<=\.tc$)|(?<=\.it$)|(?<=\.online$)|(?<=\.top$)|(?<=\.tv$)|(?<=\.eu$)|(?<=\.us$)|(?<=\.info$)|(?<=\.club$)", + "\" },", + line + ) + + line = re.sub( + r"\.co$", + ".co\" },", + line + ) + + line = re.sub( + r"{ \"action\": \"deny\", \"process\": \"any\", \"remote-hosts\": \"$", + "", + line + ) + + line = re.sub( + r"^# Title:.*", + "{ \"title\": \"👒 Dandelion Sprout's Nordic List for LS\",", + line + ) + + line = re.sub( + r"^# Description:.*", + "\"description\": \"This list aims to block Norwegian and Danish scam sellers, ad servers, and a small handful of tracking servers.\",\n\"rules\": [", + line + ) + + if is_supported_ls(line): + text += line + '\r\n' + + return text + +def is_supported_dnsmasq(line) -> bool: + for token in UNSUPPORTED_DNSMASQ: + if token in line: + return False + + return True + +# function that prepares the filter list for ABP +def prepare_dnsmasq(lines) -> str: + text = '' + + + # remove or modifiy entries with unsupported modifiers + for line in lines: + + # remove $document modifier from the rule + + line = re.sub( + r"^(?!#)", + "address=/", + line + ) + + line = re.sub( + r"$(?<=\.com$)|(?<=\.no$)|(?<=\.dk$)|(?<=\.is$)|(?<=\.org$)|(?<=\.net$)|(?<=\.tc$)|(?<=\.it$)|(?<=\.online$)|(?<=\.top$)|(?<=\.tv$)|(?<=\.eu$)|(?<=\.us$)|(?<=\.info$)|(?<=\.club$)", + "/127.0.0.1", + line + ) + + line = re.sub( + r"\.co$", + ".co/127.0.0.1", + line + ) + + line = re.sub( + r"address=/$", + "", + line + ) + + if is_supported_dnsmasq(line): + text += line + '\r\n' + + return text + + +if __name__ == "__main__": + print('Starting the script') + text = download_filters() + lines = text.splitlines(False) + print('Total number of rules: ' + str(len(lines))) + + hosts_filter = prepare_hosts(lines) + privoxy_filter = prepare_privoxy(lines) + ls_filter = prepare_ls(lines) + dnsmasq_filter = prepare_dnsmasq(lines) + + with open(OUTPUT, "w") as text_file: + text_file.write(text) + + with open(OUTPUT_HOSTS, "w") as text_file: + text_file.write(hosts_filter) + + with open(OUTPUT_PRIVOXY, "w") as text_file: + text_file.write(privoxy_filter) + + with open(OUTPUT_LS, "w") as text_file: + text_file.write(ls_filter) + + with open(OUTPUT_DNSMASQ, "w") as text_file: + text_file.write(dnsmasq_filter) + + print('The script has finished its work')