From bc8e4c92b85f911a25d0678468c9bad0ce806168 Mon Sep 17 00:00:00 2001 From: Max G Date: Sat, 11 Jul 2020 00:23:31 +0300 Subject: [PATCH 1/5] Implemented connectivity check (Fixed #1038) --- updateHostsFile.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index 8eaff1717..64d3a6a20 100644 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -242,7 +242,8 @@ def main(): update_sources = prompt_for_update(freshen=settings["freshen"], update_auto=auto) if update_sources: - update_all_sources(source_data_filename, settings["hostfilename"]) + if await_network_connection(): + update_all_sources(source_data_filename, settings["hostfilename"]) gather_exclusions = prompt_for_exclusions(skip_prompt=auto) @@ -325,6 +326,34 @@ def main(): flush_cache=settings["flushdnscache"], prompt_flush=not auto ) +def await_network_connection(retries=3, delay=10): + """ + Check if we have a proper name resolution and connectivity + By pinging a common domain + + Parameters + ---------- + retries : int + Number of times to retry again (on ping failure) + delay : int + How many seconds to wait before retrying again (on ping failure) + + Returns + ------- + -> bool + True : A ping succeded + False : All retries have failed + """ + while retries: + if 0 == subprocess.call(['ping', 'one.one.one.one', '-c', '1'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL): + return True + + print(f'No internet connection! Retrying in {delay} seconds') + time.sleep(delay) + retries -= 1 + return False + # Prompt the User def prompt_for_update(freshen, update_auto): From 0f727b8bedce24cf50ed2fd6d6b31611dca7257b Mon Sep 17 00:00:00 2001 From: Max G Date: Sat, 11 Jul 2020 13:57:42 +0300 Subject: [PATCH 2/5] Revert "Implemented connectivity check (Fixed #1038)" This reverts commit bc8e4c92b85f911a25d0678468c9bad0ce806168. --- updateHostsFile.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index 64d3a6a20..8eaff1717 100644 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -242,8 +242,7 @@ def main(): update_sources = prompt_for_update(freshen=settings["freshen"], update_auto=auto) if update_sources: - if await_network_connection(): - update_all_sources(source_data_filename, settings["hostfilename"]) + update_all_sources(source_data_filename, settings["hostfilename"]) gather_exclusions = prompt_for_exclusions(skip_prompt=auto) @@ -326,34 +325,6 @@ def main(): flush_cache=settings["flushdnscache"], prompt_flush=not auto ) -def await_network_connection(retries=3, delay=10): - """ - Check if we have a proper name resolution and connectivity - By pinging a common domain - - Parameters - ---------- - retries : int - Number of times to retry again (on ping failure) - delay : int - How many seconds to wait before retrying again (on ping failure) - - Returns - ------- - -> bool - True : A ping succeded - False : All retries have failed - """ - while retries: - if 0 == subprocess.call(['ping', 'one.one.one.one', '-c', '1'], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL): - return True - - print(f'No internet connection! Retrying in {delay} seconds') - time.sleep(delay) - retries -= 1 - return False - # Prompt the User def prompt_for_update(freshen, update_auto): From 0a96dcad34f3aae9b11df7d53a25fc620551ac5c Mon Sep 17 00:00:00 2001 From: Max G Date: Sat, 11 Jul 2020 13:58:44 +0300 Subject: [PATCH 3/5] Use 'with' to not leave a resource open --- updateHostsFile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index 8eaff1717..a73da4f25 100644 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -1483,9 +1483,9 @@ def get_file_by_url(url): """ try: - f = urlopen(url) - soup = BeautifulSoup(f.read(), "lxml").get_text() - return "\n".join(list(map(domain_to_idna, soup.split("\n")))) + with urlopen(url) as f: + soup = BeautifulSoup(f.read(), "lxml").get_text() + return "\n".join(list(map(domain_to_idna, soup.split("\n")))) except Exception: print("Problem getting file: ", url) From f26168194b98e0a3336b79ec4f798f8c69daaeb3 Mon Sep 17 00:00:00 2001 From: Max G Date: Sat, 11 Jul 2020 14:09:51 +0300 Subject: [PATCH 4/5] Retry to get file from URL once failed (Resolves #1038) --- testUpdateHostsFile.py | 8 ++++---- updateHostsFile.py | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/testUpdateHostsFile.py b/testUpdateHostsFile.py index 62616dbc9..77e3ac44a 100644 --- a/testUpdateHostsFile.py +++ b/testUpdateHostsFile.py @@ -1617,14 +1617,14 @@ class GetFileByUrl(BaseStdout): url = b"www.google.com" expected = "www.google.com" - actual = get_file_by_url(url) + actual = get_file_by_url(url, delay=0) self.assertEqual(actual, expected) @mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_fail) def test_read_url_fail(self, _): url = b"www.google.com" - self.assertIsNone(get_file_by_url(url)) + self.assertIsNone(get_file_by_url(url, delay=0)) expected = "Problem getting file:" output = sys.stdout.getvalue() @@ -1634,7 +1634,7 @@ class GetFileByUrl(BaseStdout): @mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_read_fail) def test_read_url_read_fail(self, _): url = b"www.google.com" - self.assertIsNone(get_file_by_url(url)) + self.assertIsNone(get_file_by_url(url, delay=0)) expected = "Problem getting file:" output = sys.stdout.getvalue() @@ -1644,7 +1644,7 @@ class GetFileByUrl(BaseStdout): @mock.patch("updateHostsFile.urlopen", side_effect=mock_url_open_decode_fail) def test_read_url_decode_fail(self, _): url = b"www.google.com" - self.assertIsNone(get_file_by_url(url)) + self.assertIsNone(get_file_by_url(url, delay=0)) expected = "Problem getting file:" output = sys.stdout.getvalue() diff --git a/updateHostsFile.py b/updateHostsFile.py index a73da4f25..130cf8aa6 100644 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -1461,7 +1461,7 @@ def maybe_copy_example_file(file_path): shutil.copyfile(example_file_path, file_path) -def get_file_by_url(url): +def get_file_by_url(url, retries=3, delay=10): """ Get a file data located at a particular URL. @@ -1482,12 +1482,17 @@ def get_file_by_url(url): format we have to encode or decode data before parsing it to UTF-8. """ - try: - with urlopen(url) as f: - soup = BeautifulSoup(f.read(), "lxml").get_text() - return "\n".join(list(map(domain_to_idna, soup.split("\n")))) - except Exception: - print("Problem getting file: ", url) + while retries: + try: + with urlopen(url) as f: + soup = BeautifulSoup(f.read(), "lxml").get_text() + return "\n".join(list(map(domain_to_idna, soup.split("\n")))) + except Exception as e: + if 'failure in name resolution' in str(e): + print('No internet connection! Retrying in {} seconds'.format(delay)) + time.sleep(delay) + retries -= 1 + return print("Problem getting file: ", url) def write_data(f, data): From 2abefc7cfad3a6b23ffe1be386dbefeb6a0b36f1 Mon Sep 17 00:00:00 2001 From: Max G Date: Mon, 13 Jul 2020 18:05:35 +0300 Subject: [PATCH 5/5] Fixed retrying --- updateHostsFile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/updateHostsFile.py b/updateHostsFile.py index 130cf8aa6..8a9c8dd30 100644 --- a/updateHostsFile.py +++ b/updateHostsFile.py @@ -1492,7 +1492,9 @@ def get_file_by_url(url, retries=3, delay=10): print('No internet connection! Retrying in {} seconds'.format(delay)) time.sleep(delay) retries -= 1 - return print("Problem getting file: ", url) + continue + break + print("Problem getting file: ", url) def write_data(f, data):