From 6f6693c17adb640b017c9cbf684284516ca32e19 Mon Sep 17 00:00:00 2001 From: Zaur Molotnikov Date: Mon, 26 Oct 2015 23:16:55 +0100 Subject: [PATCH 1/2] Migrated to Python 3 --- updateHostsFile.py | 76 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index 63af4e7e0..76dc39149 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +#This will be changed back, when Python 2 support patch is applied # Script by Ben Limmer # https://github.com/l1m5 @@ -13,9 +14,13 @@ import string import subprocess import sys import tempfile -import urllib2 -import zipfile -import StringIO +# zip files are not used actually, support deleted +# StringIO is not needed in Python 3 +# Python 3 works differently with urlopen +import urllib +from urllib import request + +# In Python 3 "print" is a function, braces are added everywhere # Project Settings BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -54,7 +59,7 @@ def promptForUpdate(): if (response == "yes"): updateAllSources() else: - print 'OK, we\'ll stick with what we\'ve got locally.' + print ('OK, we\'ll stick with what we\'ve got locally.') def promptForExclusions(): response = query_yes_no("Do you want to exclude any domains?\n" + @@ -63,7 +68,7 @@ def promptForExclusions(): if (response == "yes"): displayExclusionOptions() else: - print 'OK, we won\'t exclude any domains.' + print ('OK, we won\'t exclude any domains.') def promptForMoreCustomExclusions(): response = query_yes_no("Do you have more domains you want to enter?") @@ -117,19 +122,15 @@ def updateAllSources(): updateURL = getUpdateURLFromFile(source) if (updateURL == None): continue; - print 'Updating source ' + source + ' from ' + updateURL - updatedFile = urllib2.urlopen(updateURL) + print ('Updating source ' + source + ' from ' + updateURL) + # This is migration to Python 3, it works differently with urlopen + updatedFile = urllib.request.urlopen(updateURL) - updatedFile = updatedFile.read() + # zip file support remove - if '.zip' in updateURL: - updatedZippedFile = zipfile.ZipFile(StringIO.StringIO(updatedFile)) - for name in updatedZippedFile.namelist(): - if name in ('hosts', 'hosts.txt'): - updatedFile = updatedZippedFile.open(name).read() - break - - updatedFile = string.replace( updatedFile, '\r', '' ) #get rid of carriage-return symbols + # in Python 3 one has to specify encoding explicitly + updatedFile = updatedFile.read().decode("UTF-8") + updatedFile = updatedFile.replace('\r', '') #get rid of carriage-return symbols dataFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'w') dataFile.write(updatedFile) @@ -150,24 +151,29 @@ def getUpdateURLFromFile(source): # File Logic def createInitialFile(): - mergeFile = tempfile.NamedTemporaryFile() + mergeFile = tempfile.NamedTemporaryFile() for source in SOURCES: curFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'r') - mergeFile.write('\n# Begin ' + source + '\n') - mergeFile.write(curFile.read()) - mergeFile.write('\n# End ' + source + '\n') + # Python 3 requires explicit support for encoding + mergeFile.write(bytes('\n# Begin ' + source + '\n', 'UTF-8')) + mergeFile.write(bytes(curFile.read(), 'UTF-8')) + mergeFile.write(bytes('\n# End ' + source + '\n', 'UTF-8')) return mergeFile def removeDups(mergeFile): global numberOfRules - finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'w+b') + # Another mode is required to read and write the file in Python 3 + finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+') mergeFile.seek(0) # reset file pointer hostnames = set() hostnames.add("localhost") for line in mergeFile.readlines(): - if line[0].startswith("#") or re.match(r'^\s*$', line[0]): + # Explicit encoding + line = line.decode("UTF-8") + # Testing the first character doesn't require startswith + if line[0] == '#' or re.match(r'^\s*$', line[0]): finalFile.write(line) #maintain the comments for readability continue strippedRule = stripRule(line) #strip comments @@ -194,7 +200,7 @@ def normalizeRule(rule): return hostname, "%s %s #%s\n" % (TARGET_HOST, hostname, suffix) else: return hostname, "%s %s\n" % (TARGET_HOST, hostname) - print '==>%s<==' % rule + print ('==>%s<==' % rule) return None, None def finalizeFile(finalFile): @@ -206,8 +212,9 @@ def finalizeFile(finalFile): def stripRule(line): splitLine = line.split() if (len(splitLine) < 2) : + # This is due to the diffrences between bytes and string type in Python 3 printFailure('A line in the hostfile is going to cause problems because it is nonstandard\n' + - 'The line reads ' + line + ' please check your data files. Maybe you have a comment without a #?') + 'The line reads ' + str(line) + ' please check your data files. Maybe you have a comment without a #?') sys.exit() return splitLine[0] + ' ' + splitLine[1] @@ -237,10 +244,10 @@ def updateReadme(numberOfRules): def moveHostsFileIntoPlace(finalFile): if (os.name == 'posix'): - print 'Moving the file requires administrative privileges. You might need to enter your password.' + print ('Moving the file requires administrative privileges. You might need to enter your password.') if(subprocess.call(["/usr/bin/sudo", "cp", os.path.abspath(finalFile.name), "/etc/hosts"])): printFailure("Moving the file failed.") - print 'Flushing the DNS Cache to utilize new hosts file...' + print ('Flushing the DNS Cache to utilize new hosts file...') if (platform.system() == 'Darwin'): if(subprocess.call(["/usr/bin/sudo", "killall", "-HUP", "mDNSResponder"])): printFailure("Flushing the DNS Cache failed.") @@ -248,8 +255,8 @@ def moveHostsFileIntoPlace(finalFile): if(subprocess.call(["/usr/bin/sudo", "/etc/rc.d/init.d/nscd", "restart"])): printFailure("Flushing the DNS Cache failed.") elif (os.name == 'nt'): - print 'Automatically moving the hosts file in place is not yet supported.' - print 'Please move the generated file to %SystemRoot%\system32\drivers\etc\hosts' + print ('Automatically moving the hosts file in place is not yet supported.') + print ('Please move the generated file to %SystemRoot%\system32\drivers\etc\hosts') # End File Logic @@ -278,7 +285,8 @@ def query_yes_no(question, default="yes"): while 1: sys.stdout.write(colorize(question, colors.PROMPT) + prompt) - choice = raw_input().lower() + # input works instead of raw_input in Python 3 + choice = input().lower() if default is not None and choice == '': return default elif choice in valid.keys(): @@ -290,11 +298,11 @@ def query_yes_no(question, default="yes"): def isValidDomainFormat(domain): if (domain == ''): - print "You didn\'t enter a domain. Try again." + print ("You didn\'t enter a domain. Try again.") return False domainRegex = re.compile("www\d{0,3}[.]|https?") if (domainRegex.match(domain)): - print "The domain " + domain + " is not valid. Do not include www.domain.com or http(s)://domain.com. Try again." + print ("The domain " + domain + " is not valid. Do not include www.domain.com or http(s)://domain.com. Try again.") return False else: return True @@ -310,10 +318,10 @@ def colorize(text, color): return color + text + colors.ENDC def printSuccess(text): - print colorize(text, colors.SUCCESS) + print (colorize(text, colors.SUCCESS)) def printFailure(text): - print colorize(text, colors.FAIL) + print (colorize(text, colors.FAIL)) # End Helper Functions if __name__ == "__main__": From d5166167509d51c44c2c7fd8941e64cad952fa73 Mon Sep 17 00:00:00 2001 From: Zaur Molotnikov Date: Mon, 26 Oct 2015 23:46:48 +0100 Subject: [PATCH 2/2] Cross-python implementation --- updateHostsFile.py | 112 +++++++++++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 35 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index 76dc39149..e057cafc9 100755 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 -#This will be changed back, when Python 2 support patch is applied +#!/usr/bin/env python # Script by Ben Limmer # https://github.com/l1m5 @@ -7,6 +6,9 @@ # This simple Python script will combine all the host files you provide # as sources into one, unique host file to keep you internet browsing happy. +# Making Python 2 compatible with Python 3 +from __future__ import absolute_import, division, print_function, unicode_literals + import os import platform import re @@ -17,11 +19,51 @@ import tempfile # zip files are not used actually, support deleted # StringIO is not needed in Python 3 # Python 3 works differently with urlopen -import urllib -from urllib import request + +# Supporting urlopen in Python 2 and Python 3 +try: + from urllib.parse import urlparse, urlencode + from urllib.request import urlopen, Request + from urllib.error import HTTPError +except ImportError: + from urlparse import urlparse + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError + +# This function handles both Python 2 and Python 3 +def getFileByUrl(url): + try: + f = urlopen(url) + return f.read().decode("UTF-8") + except: + print ("Problem getting file: ", url); + raise + # In Python 3 "print" is a function, braces are added everywhere +# Detecting Python 3 for version-dependent implementations +Python3=False; +cur_version = sys.version_info +if cur_version >= (3, 0): + Python3=True; + +# This function works in both Python 2 and Python 3 +def myInput(msg=""): + if Python3: + return input(msg); + else: + return raw_input(msg); + + +# Cross-python writing function +def writeData(f, data): + if Python3: + f.write(bytes(data, 'UTF-8')) + else: + f.write(str(data).encode('UTF-8')) + + # Project Settings BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__)) DATA_PATH = os.path.join(BASEDIR_PATH, 'data') @@ -96,10 +138,11 @@ def displayExclusionOptions(): response = query_yes_no("Do you want to exclude any other domains?") if (response == "yes"): gatherCustomExclusions() - + def gatherCustomExclusions(): while True: - domainFromUser = raw_input("Enter the domain you want to exclude (e.g. facebook.com): ") + # Cross-python Input + domainFromUser = myInput("Enter the domain you want to exclude (e.g. facebook.com): ") if (isValidDomainFormat(domainFromUser)): excludeDomain(domainFromUser) if (promptForMoreCustomExclusions() == False): @@ -123,17 +166,13 @@ def updateAllSources(): if (updateURL == None): continue; print ('Updating source ' + source + ' from ' + updateURL) - # This is migration to Python 3, it works differently with urlopen - updatedFile = urllib.request.urlopen(updateURL) - - # zip file support remove - - # in Python 3 one has to specify encoding explicitly - updatedFile = updatedFile.read().decode("UTF-8") + # Cross-python call + updatedFile = getFileByUrl(updateURL); updatedFile = updatedFile.replace('\r', '') #get rid of carriage-return symbols - dataFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'w') - dataFile.write(updatedFile) + # This is cross-python code + dataFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'wb') + writeData(dataFile, updatedFile); dataFile.close() def getUpdateURLFromFile(source): @@ -154,17 +193,18 @@ def createInitialFile(): mergeFile = tempfile.NamedTemporaryFile() for source in SOURCES: curFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'r') - # Python 3 requires explicit support for encoding - mergeFile.write(bytes('\n# Begin ' + source + '\n', 'UTF-8')) - mergeFile.write(bytes(curFile.read(), 'UTF-8')) - mergeFile.write(bytes('\n# End ' + source + '\n', 'UTF-8')) + #Done in a cross-python way + writeData(mergeFile, '\n# Begin ' + source + '\n') + writeData(mergeFile, curFile.read()) + writeData(mergeFile, '\n# End ' + source + '\n') + return mergeFile def removeDups(mergeFile): global numberOfRules # Another mode is required to read and write the file in Python 3 - finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+') + finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+b') mergeFile.seek(0) # reset file pointer hostnames = set() @@ -174,15 +214,17 @@ def removeDups(mergeFile): line = line.decode("UTF-8") # Testing the first character doesn't require startswith if line[0] == '#' or re.match(r'^\s*$', line[0]): - finalFile.write(line) #maintain the comments for readability + # Cross-python write + writeData(finalFile, line) continue + strippedRule = stripRule(line) #strip comments if matchesExclusions(strippedRule): continue hostname, normalizedRule = normalizeRule(strippedRule) # normalize rule if normalizedRule and (hostname not in hostnames): - finalFile.write(normalizedRule) + writeData(finalFile, normalizedRule) hostnames.add(hostname) numberOfRules += 1 @@ -223,18 +265,18 @@ def writeOpeningHeader(finalFile): finalFile.seek(0) #reset file pointer fileContents = finalFile.read(); #save content finalFile.seek(0) #write at the top - finalFile.write('# This file is a merged collection of hosts from reputable sources,\n') - finalFile.write('# with a dash of crowd sourcing via Github\n#\n') - finalFile.write('# Project home page: https://github.com/StevenBlack/hosts\n#\n') - finalFile.write('# Current sources:\n') + writeData(finalFile, '# This file is a merged collection of hosts from reputable sources,\n') + writeData(finalFile, '# with a dash of crowd sourcing via Github\n#\n') + writeData(finalFile, '# Project home page: https://github.com/StevenBlack/hosts\n#\n') + writeData(finalFile, '# Current sources:\n') for source in SOURCES: - finalFile.write('# ' + source + '\n') - finalFile.write('#\n') - finalFile.write('# Merging these sources produced ' + "{:,}".format( numberOfRules ) + ' unique entries\n') - finalFile.write('# ===============================================================\n') - finalFile.write('\n') - finalFile.write('127.0.0.1 localhost\n') - finalFile.write('\n') + writeData(finalFile, '# ' + source + '\n') + writeData(finalFile, '#\n') + writeData(finalFile, '# Merging these sources produced ' + "{:,}".format( numberOfRules ) + ' unique entries\n') + writeData(finalFile, '# ===============================================================\n') + writeData(finalFile, '\n') + writeData(finalFile, '127.0.0.1 localhost\n') + writeData(finalFile, '\n') finalFile.write(fileContents) def updateReadme(numberOfRules): @@ -285,8 +327,8 @@ def query_yes_no(question, default="yes"): while 1: sys.stdout.write(colorize(question, colors.PROMPT) + prompt) - # input works instead of raw_input in Python 3 - choice = input().lower() + # Changed to be cross-python + choice = myInput().lower() if default is not None and choice == '': return default elif choice in valid.keys():