From 45fa325d4249f73d18a62809d0441796a24c6f61 Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Wed, 4 Nov 2015 11:47:55 -0300 Subject: [PATCH 1/8] Delete .DS_Store from Data If the Data folder is opened in OSX it creates a .DS_Store file and the python script tries to use it a source and returns errors, deleting it solves this issue. --- updateHostsFile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/updateHostsFile.py b/updateHostsFile.py index c73113e3a..b0a5df0d6 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -73,6 +73,9 @@ SOURCES = os.listdir(DATA_PATH) README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md') README_FILE = os.path.join(BASEDIR_PATH, 'readme.md') TARGET_HOST = '0.0.0.0' +DS_STORE = BASEDIR_PATH + '/data/.DS_Store' +if os.path.isfile(DS_STORE): + os.remove(DS_STORE) # Exclusions EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end From 091a18ee8d01a4d07ab5cd0b717f42f5ea795170 Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Thu, 5 Nov 2015 09:38:08 -0300 Subject: [PATCH 2/8] Update updateHostsFile.py --- updateHostsFile.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index b0a5df0d6..484ccdb81 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -16,6 +16,7 @@ import string import subprocess import sys import tempfile +import glob # zip files are not used actually, support deleted # StringIO is not needed in Python 3 # Python 3 works differently with urlopen @@ -63,19 +64,19 @@ def writeData(f, data): else: f.write(str(data).encode('UTF-8')) +# This function doesn't list hidden files +def listdir_nohidden(path): + return glob.glob(os.path.join(path, '*')) # Project Settings BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__)) DATA_PATH = os.path.join(BASEDIR_PATH, 'data') DATA_FILENAMES = 'hosts' UPDATE_URL_FILENAME = 'update.info' -SOURCES = os.listdir(DATA_PATH) +SOURCES = listdir_nohidden(DATA_PATH) README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md') README_FILE = os.path.join(BASEDIR_PATH, 'readme.md') TARGET_HOST = '0.0.0.0' -DS_STORE = BASEDIR_PATH + '/data/.DS_Store' -if os.path.isfile(DS_STORE): - os.remove(DS_STORE) # Exclusions EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end From ad5605c4c091b92f65f40d518d9da0750b849cb6 Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Thu, 5 Nov 2015 10:37:20 -0300 Subject: [PATCH 3/8] Ignore whitelist file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1baeb4f38..3a446e358 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ myhosts +whitelist From 461b06710c9530227249d6cd5b30278d4f881008 Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Thu, 5 Nov 2015 10:44:42 -0300 Subject: [PATCH 4/8] Exclude domains contained in a whitelist --- updateHostsFile.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/updateHostsFile.py b/updateHostsFile.py index 484ccdb81..fc460aa1f 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -77,10 +77,18 @@ SOURCES = listdir_nohidden(DATA_PATH) README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md') README_FILE = os.path.join(BASEDIR_PATH, 'readme.md') TARGET_HOST = '0.0.0.0' +WHITELIST_FILE = os.path.join(BASEDIR_PATH, 'whitelist') # Exclusions EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end +# Exclutions from whitelist file +EXCLUSIONS = [] +if os.path.isfile(WHITELIST_FILE): + with open(WHITELIST_FILE, "r") as ins: + for line in ins: + EXCLUSIONS.append(line) + # Common domains to exclude COMMON_EXCLUSIONS = ['hulu.com'] @@ -90,6 +98,7 @@ numberOfRules = 0 def main(): promptForUpdate() + excludeFromFile() promptForExclusions() mergeFile = createInitialFile() finalFile = removeDups(mergeFile) @@ -99,6 +108,14 @@ def main(): promptForMove(finalFile) +# Exclusion from file +def excludeFromFile(): + for domain in EXCLUSIONS: + if (domain != '' and not domain.startswith("#")): + domainRegex = re.compile("www\d{0,3}[.]|https?") + if not (domainRegex.match(domain)): + excludeDomain(domain) + # Prompt the User def promptForUpdate(): response = query_yes_no("Do you want to update all data sources?") From 58e5277b254bfc3105e98cc7a616cadd3b8e210a Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Thu, 5 Nov 2015 10:47:55 -0300 Subject: [PATCH 5/8] Create whitelist --- whitelist | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 whitelist diff --git a/whitelist b/whitelist new file mode 100644 index 000000000..8d48e12ae --- /dev/null +++ b/whitelist @@ -0,0 +1,2 @@ +# Domains added below will be ignored, For example: +# your-domain-name.com From 6a07a22b61cf70c6f494229fb29777a85be5be6b Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Sat, 7 Nov 2015 17:01:55 -0300 Subject: [PATCH 6/8] Fixed: Exclude domains in the whitelist It still can be improved but now it's working as it should. Not tested with Python 3. Working with Python 2.7.10 --- updateHostsFile.py | 47 +++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index fc460aa1f..f151df340 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -78,16 +78,11 @@ README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md') README_FILE = os.path.join(BASEDIR_PATH, 'readme.md') TARGET_HOST = '0.0.0.0' WHITELIST_FILE = os.path.join(BASEDIR_PATH, 'whitelist') +HOSTS_FILE = os.path.join(BASEDIR_PATH, 'hosts') # Exclusions EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end - -# Exclutions from whitelist file EXCLUSIONS = [] -if os.path.isfile(WHITELIST_FILE): - with open(WHITELIST_FILE, "r") as ins: - for line in ins: - EXCLUSIONS.append(line) # Common domains to exclude COMMON_EXCLUSIONS = ['hulu.com'] @@ -98,11 +93,11 @@ numberOfRules = 0 def main(): promptForUpdate() - excludeFromFile() promptForExclusions() mergeFile = createInitialFile() finalFile = removeDups(mergeFile) finalizeFile(finalFile) + excludeFromFile() updateReadme(numberOfRules) printSuccess('Success! Your shiny new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.') @@ -110,11 +105,37 @@ def main(): # Exclusion from file def excludeFromFile(): - for domain in EXCLUSIONS: - if (domain != '' and not domain.startswith("#")): - domainRegex = re.compile("www\d{0,3}[.]|https?") - if not (domainRegex.match(domain)): - excludeDomain(domain) + global numberOfRules + if os.path.isfile(WHITELIST_FILE): + with open(WHITELIST_FILE, "r") as ins: + for line in ins: + EXCLUSIONS.append(line) + f = open(HOSTS_FILE) + output = [] + for line in f: + write = 'true' + for domain in EXCLUSIONS: + if domain in line: + write = 'false' + numberOfRules -= 1 + break + if (write == 'true'): + output.append(line) + f.close() + f = open(HOSTS_FILE, 'w') + f.writelines(output) + f.close() + f = open(HOSTS_FILE) + output = [] + for line in f: + if 'unique entries' not in line: + output.append(line) + else: + output.append('# Merging these sources produced ' + "{:,}".format( numberOfRules ) + ' unique entries\n') + f.close() + f = open(HOSTS_FILE, 'w') + f.writelines(output) + f.close() # Prompt the User def promptForUpdate(): @@ -131,7 +152,7 @@ def promptForExclusions(): if (response == "yes"): displayExclusionOptions() else: - print ('OK, we won\'t exclude any domains.') + print ('OK, we\'ll only exclude domains in the whitelist.') def promptForMoreCustomExclusions(): response = query_yes_no("Do you have more domains you want to enter?") From e1d41823bd5cfc0e48fdd27b6fa71dc1e2fefcf4 Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Sat, 7 Nov 2015 17:54:36 -0300 Subject: [PATCH 7/8] Fixed: Exclude domains in the whitelist (2) Now it probably works with Python 3 and the code is much cleaner. --- updateHostsFile.py | 49 ++++++++++------------------------------------ 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index f151df340..b6be9c8b2 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -78,12 +78,10 @@ README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md') README_FILE = os.path.join(BASEDIR_PATH, 'readme.md') TARGET_HOST = '0.0.0.0' WHITELIST_FILE = os.path.join(BASEDIR_PATH, 'whitelist') -HOSTS_FILE = os.path.join(BASEDIR_PATH, 'hosts') # Exclusions EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end EXCLUSIONS = [] - # Common domains to exclude COMMON_EXCLUSIONS = ['hulu.com'] @@ -97,46 +95,11 @@ def main(): mergeFile = createInitialFile() finalFile = removeDups(mergeFile) finalizeFile(finalFile) - excludeFromFile() updateReadme(numberOfRules) printSuccess('Success! Your shiny new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.') promptForMove(finalFile) -# Exclusion from file -def excludeFromFile(): - global numberOfRules - if os.path.isfile(WHITELIST_FILE): - with open(WHITELIST_FILE, "r") as ins: - for line in ins: - EXCLUSIONS.append(line) - f = open(HOSTS_FILE) - output = [] - for line in f: - write = 'true' - for domain in EXCLUSIONS: - if domain in line: - write = 'false' - numberOfRules -= 1 - break - if (write == 'true'): - output.append(line) - f.close() - f = open(HOSTS_FILE, 'w') - f.writelines(output) - f.close() - f = open(HOSTS_FILE) - output = [] - for line in f: - if 'unique entries' not in line: - output.append(line) - else: - output.append('# Merging these sources produced ' + "{:,}".format( numberOfRules ) + ' unique entries\n') - f.close() - f = open(HOSTS_FILE, 'w') - f.writelines(output) - f.close() - # Prompt the User def promptForUpdate(): response = query_yes_no("Do you want to update all data sources?") @@ -244,6 +207,10 @@ def createInitialFile(): def removeDups(mergeFile): global numberOfRules + if os.path.isfile(WHITELIST_FILE): + with open(WHITELIST_FILE, "r") as ins: + for line in ins: + EXCLUSIONS.append(line) # Another mode is required to read and write the file in Python 3 finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+b') @@ -252,6 +219,7 @@ def removeDups(mergeFile): hostnames = set() hostnames.add("localhost") for line in mergeFile.readlines(): + write = 'true' # Explicit encoding line = line.decode("UTF-8") # Testing the first character doesn't require startswith @@ -264,8 +232,11 @@ def removeDups(mergeFile): if matchesExclusions(strippedRule): continue hostname, normalizedRule = normalizeRule(strippedRule) # normalize rule - - if normalizedRule and (hostname not in hostnames): + for exclude in EXCLUSIONS: + if (exclude in line): + write = 'false' + break + if normalizedRule and (hostname not in hostnames) and (write == 'true'): writeData(finalFile, normalizedRule) hostnames.add(hostname) numberOfRules += 1 From d1dba2066330592ffb8d6fc4a6b677e0859b599a Mon Sep 17 00:00:00 2001 From: FrancoGag Date: Mon, 9 Nov 2015 17:09:59 -0300 Subject: [PATCH 8/8] Renamed removeDups() to removeDupsAndExcl() Renamed removeDups() to reflect that now the function also excludes domains and keywords in the whitelist. --- updateHostsFile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index b6be9c8b2..de8502e16 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -93,7 +93,7 @@ def main(): promptForUpdate() promptForExclusions() mergeFile = createInitialFile() - finalFile = removeDups(mergeFile) + finalFile = removeDupsAndExcl(mergeFile) finalizeFile(finalFile) updateReadme(numberOfRules) printSuccess('Success! Your shiny new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.') @@ -205,7 +205,7 @@ def createInitialFile(): return mergeFile -def removeDups(mergeFile): +def removeDupsAndExcl(mergeFile): global numberOfRules if os.path.isfile(WHITELIST_FILE): with open(WHITELIST_FILE, "r") as ins: