Repository: facebookresearch/ParlAI

Issue:
unimportant off-by-one in build_data.download
https://github.com/facebookresearch/ParlAI/blob/f261852f688dc1ae9e2382518c757da14a603cac/parlai/core/build_data.py#L75-L80

This reads like there are 5 retries, but there are really 6. Should change this code to be more readable.


Discussion:

Resolving patch:
diff --git a/parlai/core/build_data.py b/parlai/core/build_data.py
--- a/parlai/core/build_data.py
+++ b/parlai/core/build_data.py
@@ -145,7 +145,7 @@ def mark_done(path, version_string=None):
             write.write('\n' + version_string)
 
 
-def download(url, path, fname, redownload=False):
+def download(url, path, fname, redownload=False, num_retries=5):
     """
     Download file using `requests`.
 
@@ -155,12 +155,12 @@ def download(url, path, fname, redownload=False):
     outfile = os.path.join(path, fname)
     download = not os.path.isfile(outfile) or redownload
     print("[ downloading: " + url + " to " + outfile + " ]")
-    retry = 5
+    retry = num_retries
     exp_backoff = [2 ** r for r in reversed(range(retry))]
 
     pbar = tqdm.tqdm(unit='B', unit_scale=True, desc='Downloading {}'.format(fname))
 
-    while download and retry >= 0:
+    while download and retry > 0:
         resume_file = outfile + '.part'
         resume = os.path.isfile(resume_file)
         if resume:
@@ -210,26 +210,24 @@ def download(url, path, fname, redownload=False):
             ):
                 retry -= 1
                 pbar.clear()
-                if retry >= 0:
-                    print('Connection error, retrying. (%d retries left)' % retry)
+                if retry > 0:
+                    pl = 'y' if retry == 1 else 'ies'
+                    print(f'Connection error, retrying. ({retry} retr{pl} left)')
                     time.sleep(exp_backoff[retry])
                 else:
                     print('Retried too many times, stopped retrying.')
             finally:
                 if response:
                     response.close()
-    if retry < 0:
-        raise RuntimeWarning('Connection broken too many times. Stopped retrying.')
+    if retry <= 0:
+        raise RuntimeError('Connection broken too many times. Stopped retrying.')
 
     if download and retry > 0:
         pbar.update(done - pbar.n)
         if done < total_size:
-            raise RuntimeWarning(
-                'Received less data than specified in '
-                + 'Content-Length header for '
-                + url
-                + '.'
-                + ' There may be a download problem.'
+            raise RuntimeError(
+                f'Received less data than specified in Content-Length header for '
+                f'{url}. There may be a download problem.'
             )
         move(resume_file, outfile)
 


Test patch:
diff --git a/tests/test_build_data.py b/tests/test_build_data.py
--- a/tests/test_build_data.py
+++ b/tests/test_build_data.py
@@ -6,6 +6,8 @@
 import os
 from parlai.core import build_data
 import unittest
+import unittest.mock
+import requests
 import parlai.utils.testing as testing_utils
 import multiprocessing
 from parlai.core.params import ParlaiParser
@@ -70,6 +72,16 @@ def test_download_multiprocess_chunks(self):
         self.assertIn(200, output_statuses, 'unexpected error code')
         self.assertIn(403, output_statuses, 'unexpected error code')
 
+    def test_connectionerror_download(self):
+        with unittest.mock.patch('requests.Session.get') as Session:
+            Session.side_effect = requests.exceptions.ConnectTimeout
+            with testing_utils.tempdir() as tmpdir:
+                with self.assertRaises(RuntimeError):
+                    build_data.download(
+                        'http://test.com/bad', tmpdir, 'foo', num_retries=3
+                    )
+            assert Session.call_count == 3
+
 
 if __name__ == '__main__':
     multiprocessing.set_start_method('spawn')