Repository: facebookresearch/ParlAI

Issue:
Humanize and throttle download progress output for better user experience
Currently everything is shown in bytes and doesn't really provide any meaningful readable information to user. Further, output to stdout at every chunk downloaded is harmful for overall performance. We should humanize and throttle download progress.


Discussion:

Resolving patch:
diff --git a/parlai/core/build_data.py b/parlai/core/build_data.py
--- a/parlai/core/build_data.py
+++ b/parlai/core/build_data.py
@@ -14,6 +14,8 @@
 import requests
 import shutil
 
+from parlai.core.utils import ProgressLogger
+
 
 def built(path, version_string=None):
     """Checks if '.built' flag has been set for that task.
@@ -40,20 +42,6 @@ def mark_done(path, version_string=None):
         if version_string:
             write.write('\n' + version_string)
 
-
-def log_progress(curr, total, width=40):
-    """Displays a bar showing the current progress."""
-    done = min(curr * width // total, width)
-    remain = width - done
-    progress = '[{}{}] {} / {}'.format(
-        ''.join(['|'] * done),
-        ''.join(['.'] * remain),
-        curr,
-        total
-    )
-    print(progress, end='\r')
-
-
 def download(url, path, fname, redownload=False):
     """Downloads file using `requests`. If ``redownload`` is set to false, then
     will not download tar file again if it is present (default ``True``)."""
@@ -62,6 +50,9 @@ def download(url, path, fname, redownload=False):
 
     retry = 5
     exp_backoff = [2 ** r for r in reversed(range(retry))]
+
+    logger = ProgressLogger()
+
     while download and retry >= 0:
         resume_file = outfile + '.part'
         resume = os.path.isfile(resume_file)
@@ -99,7 +90,7 @@ def download(url, path, fname, redownload=False):
                             if total_size < done:
                                 # don't freak out if content-length was too small
                                 total_size = done
-                            log_progress(done, total_size)
+                            logger.log(done, total_size)
                     break
             except requests.exceptions.ConnectionError:
                 retry -= 1
diff --git a/parlai/core/utils.py b/parlai/core/utils.py
--- a/parlai/core/utils.py
+++ b/parlai/core/utils.py
@@ -231,7 +231,7 @@ def parse(txt):
         history['dialog'].clear()
         history['labels'] = []
         history['episode_done'] = False
- 
+
     if useReplies != 'none':
         if useReplies == 'model':
             if reply != '':
@@ -262,3 +262,45 @@ def __exit__(self, exc_type, exc_value, exc_traceback):
 def no_lock():
     """Builds a nolock for other classes to use for no-op locking."""
     return single_nolock
+
+
+class ProgressLogger(object):
+    """
+    Throttles and display progress in human readable form.
+    Default throttle speed is 1 sec
+    """
+    def __init__(self, throttle=1, should_humanize=True):
+        self.latest = time.time()
+        self.throttle_speed = throttle
+        self.should_humanize = should_humanize
+
+    def humanize(self, num, suffix='B'):
+        for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
+            if abs(num) < 1024.0:
+                return "%3.1f%s%s" % (num, unit, suffix)
+            num /= 1024.0
+        return "%.1f%s%s" % (num, 'Yi', suffix)
+
+    def log(self, curr, total, width=40):
+        """Displays a bar showing the current progress."""
+        curr_time = time.time()
+        if curr_time - self.latest < self.throttle_speed:
+            return
+        else:
+            self.latest = curr_time
+
+        self.latest = curr_time
+        done = min(curr * width // total, width)
+        remain = width - done
+
+        if self.should_humanize:
+            curr = self.humanize(curr)
+            total = self.humanize(total)
+
+        progress = '[{}{}] {} / {}'.format(
+            ''.join(['|'] * done),
+            ''.join(['.'] * remain),
+            curr,
+            total
+        )
+        print(progress, end='\r')


Test patch: