Merge pull request #1342 from codeswhite/patch-network-retry

Implemented connectivity check (Fixed #1038)
This commit is contained in:
Steven Black
2020-07-13 13:41:19 -04:00
committed by GitHub
2 changed files with 18 additions and 11 deletions
+14 -7
View File
@@ -1469,7 +1469,7 @@ def maybe_copy_example_file(file_path):
shutil.copyfile(example_file_path, file_path)
def get_file_by_url(url):
def get_file_by_url(url, retries=3, delay=10):
"""
Get a file data located at a particular URL.
@@ -1490,12 +1490,19 @@ def get_file_by_url(url):
format we have to encode or decode data before parsing it to UTF-8.
"""
try:
f = urlopen(url)
soup = BeautifulSoup(f.read(), "lxml").get_text()
return "\n".join(list(map(domain_to_idna, soup.split("\n"))))
except Exception:
print("Problem getting file: ", url)
while retries:
try:
with urlopen(url) as f:
soup = BeautifulSoup(f.read(), "lxml").get_text()
return "\n".join(list(map(domain_to_idna, soup.split("\n"))))
except Exception as e:
if 'failure in name resolution' in str(e):
print('No internet connection! Retrying in {} seconds'.format(delay))
time.sleep(delay)
retries -= 1
continue
break
print("Problem getting file: ", url)
def write_data(f, data):