From 01db3264597b44600adab73526720c080eee6970 Mon Sep 17 00:00:00 2001 From: Ankit Pati Date: Fri, 16 Feb 2018 08:26:16 +0530 Subject: [PATCH 1/3] Add Option to Minimise hosts File This option (--minimise, -m) differs from compress (--compress, -c) in that it keeps each host on a different line, while still removing unnecessary newlines and comments. This is necessary because many implementations of URL blockers that rely on hosts files do not conform to the standard which allows multiple hosts on a single line. --- updateHostsFile.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/updateHostsFile.py b/updateHostsFile.py index 81a6ef2ef..53625c94e 100644 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -63,6 +63,7 @@ def get_defaults(): "extensionspath": path_join_robust(BASEDIR_PATH, "extensions"), "extensions": [], "compress": False, + "minimise": False, "outputsubfolder": "", "hostfilename": "hosts", "targetip": "0.0.0.0", @@ -126,6 +127,11 @@ def main(): "putting multiple domains in " "each line. Improve the " "performances under Windows.") + parser.add_argument("--minimise", "-m", dest="minimise", + default=False, action="store_true", + help="Minimise the hosts file " + "ignoring non-necessary lines " + "(empty lines and comments).") global settings @@ -186,6 +192,12 @@ def main(): compressed_file = tempfile.NamedTemporaryFile() remove_dups_and_excl(merge_file, exclusion_regexes, compressed_file) compress_file(compressed_file, settings["targetip"], final_file) + elif settings["minimise"]: + final_file = open(path_join_robust(settings["outputpath"], "hosts"), + "w+b" if PY3 else "w+") + minimised_file = tempfile.NamedTemporaryFile() + remove_dups_and_excl(merge_file, exclusion_regexes, minimised_file) + minimise_file(minimised_file, settings["targetip"], final_file) else: final_file = remove_dups_and_excl(merge_file, exclusion_regexes) @@ -686,6 +698,37 @@ def compress_file(input_file, target_ip, output_file): input_file.close() +def minimise_file(input_file, target_ip, output_file): + """ + Reduce the file dimension removing non-necessary lines (empty lines and + comments). + + Parameters + ---------- + input_file : file + The file object that contains the hostnames that we are reducing. + target_ip : str + The target IP address. + output_file : file + The file object that will contain the reduced hostnames. + """ + + input_file.seek(0) # reset file pointer + write_data(output_file, '\n') + + lines = [] + for line in input_file.readlines(): + line = line.decode("UTF-8") + + if line.startswith(target_ip): + lines.append(line[:line.find('#')].strip() + '\n') + + for line in lines: + write_data(output_file, line) + + input_file.close() + + def remove_dups_and_excl(merge_file, exclusion_regexes, output_file=None): """ Remove duplicates and remove hosts that we are excluding. From 9a8b507f6ee6507be837d5fbc348d5d87b346c03 Mon Sep 17 00:00:00 2001 From: Ankit Pati Date: Sat, 17 Feb 2018 07:20:23 +0530 Subject: [PATCH 2/3] update defaults dictionary for minimise switch --- testUpdateHostsFile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testUpdateHostsFile.py b/testUpdateHostsFile.py index c5564c4ed..c1968c882 100644 --- a/testUpdateHostsFile.py +++ b/testUpdateHostsFile.py @@ -101,6 +101,7 @@ class TestGetDefaults(Base): "extensionspath": "foo" + self.sep + "extensions", "extensions": [], "compress": False, + "minimise": False, "outputsubfolder": "", "hostfilename": "hosts", "targetip": "0.0.0.0", From dae2172f3b9f9de3a290bd1d89748514be235024 Mon Sep 17 00:00:00 2001 From: Ankit Pati Date: Sat, 17 Feb 2018 07:34:12 +0530 Subject: [PATCH 3/3] update docs after adding minimise switch --- readme_template.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/readme_template.md b/readme_template.md index c3250e11b..fad36de4f 100644 --- a/readme_template.md +++ b/readme_template.md @@ -109,6 +109,11 @@ ignoring non-necessary lines (empty lines and comments) and putting multiple domains in each line. Reducing the number of lines of the hosts file improves the performances under Windows (with DNS Client service enabled). +`--minimise`, or `-m`: `false` (default) or `true`, like `--compress`, but puts +each domain on a separate line. This is necessary because many implementations +of URL blockers that rely on `hosts` files do not conform to the standard which +allows multiple hosts on a single line. + ## How do I control which sources are unified? Add one or more *additional* sources, each in a subfolder of the `data/`