Repository: conan-io/conan

Issue:
[POC] Check tqdm progress bars
In order to provide feedback for #3536 related to tqdm, we may include a progress bar somewhere to check that it plays well with User interface, CI systems,... 

As [@memsharded suggested](https://github.com/conan-io/conan/issues/3536#issuecomment-420934410), the decompress job could be a good point to test it.




Discussion:

Resolving patch:
diff --git a/conans/client/output.py b/conans/client/output.py
--- a/conans/client/output.py
+++ b/conans/client/output.py
@@ -100,6 +100,9 @@ def rewrite_line(self, line):
         self._stream.flush()
         self._color = tmp_color
 
+    def flush(self):
+        self._stream.flush()
+
 
 class ScopedOutput(ConanOutput):
     def __init__(self, scope, output):
diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py
--- a/conans/client/remote_manager.py
+++ b/conans/client/remote_manager.py
@@ -29,6 +29,7 @@
 from conans.util.env_reader import get_env
 from conans.search.search import filter_packages
 from conans.client.cmd.uploader import UPLOAD_POLICY_SKIP
+from conans.util import progress_bar
 
 
 class RemoteManager(object):
@@ -184,7 +185,7 @@ def get_recipe(self, conan_reference, remote):
         duration = time.time() - t1
         log_recipe_download(conan_reference, duration, remote.name, zipped_files)
 
-        unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
+        unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
         # Make sure that the source dir is deleted
         rm_conandir(self._client_cache.source(conan_reference))
         touch_folder(dest_folder)
@@ -202,7 +203,8 @@ def get_recipe_sources(self, conan_reference, export_folder, export_sources_fold
         duration = time.time() - t1
         log_recipe_sources_download(conan_reference, duration, remote.name, zipped_files)
 
-        unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME)
+        unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME,
+                            output=self._output)
         c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD)
         if os.path.exists(c_src_path):
             merge_directories(c_src_path, export_sources_folder)
@@ -219,7 +221,7 @@ def get_package(self, package_reference, dest_folder, remote, output, recorder):
             zipped_files = self._call_remote(remote, "get_package", package_reference, dest_folder)
             duration = time.time() - t1
             log_package_download(package_reference, duration, remote, zipped_files)
-            unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
+            unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME, output=self._output)
             # Issue #214 https://github.com/conan-io/conan/issues/214
             touch_folder(dest_folder)
             if get_env("CONAN_READ_ONLY_CACHE", False):
@@ -366,21 +368,22 @@ def compress_files(files, symlinks, name, dest_dir):
     return tgz_path
 
 
-def unzip_and_get_files(files, destination_dir, tgz_name):
+def unzip_and_get_files(files, destination_dir, tgz_name, output):
     """Moves all files from package_files, {relative_name: tmp_abs_path}
     to destination_dir, unzipping the "tgz_name" if found"""
 
     tgz_file = files.pop(tgz_name, None)
     check_compressed_files(tgz_name, files)
     if tgz_file:
-        uncompress_file(tgz_file, destination_dir)
+        uncompress_file(tgz_file, destination_dir, output=output)
         os.remove(tgz_file)
 
 
-def uncompress_file(src_path, dest_folder):
+def uncompress_file(src_path, dest_folder, output):
     t1 = time.time()
     try:
-        with open(src_path, 'rb') as file_handler:
+        with progress_bar.open_binary(src_path, desc="Decompressing %s" % os.path.basename(src_path),
+                                      output=output) as file_handler:
             tar_extract(file_handler, dest_folder)
     except Exception as e:
         error_msg = "Error while downloading/extracting files to %s\n%s\n" % (dest_folder, str(e))
diff --git a/conans/util/progress_bar.py b/conans/util/progress_bar.py
new file mode 100644
--- /dev/null
+++ b/conans/util/progress_bar.py
@@ -0,0 +1,72 @@
+
+import os
+from tqdm import tqdm
+from contextlib import contextmanager
+
+TIMEOUT_BEAT_SECONDS = 30
+TIMEOUT_BEAT_CHARACTER = '.'
+
+
+class _FileReaderWithProgressBar(object):
+
+    tqdm_defaults = {'unit': 'B',
+                     'unit_scale': True,
+                     'unit_divisor': 1024,
+                     'ascii': False,  # Fancy output (forces unicode progress bar)
+                     }
+
+    def __init__(self, fileobj, output, desc=None):
+        pb_kwargs = self.tqdm_defaults.copy()
+        self._ori_output = output
+
+        # If there is no terminal, just print a beat every TIMEOUT_BEAT seconds.
+        if not output.is_terminal:
+            output = _NoTerminalOutput(output)
+            pb_kwargs['mininterval'] = TIMEOUT_BEAT_SECONDS
+
+        self._output = output
+        self._fileobj = fileobj
+        self.seek(0, os.SEEK_END)
+        self._pb = tqdm(total=self.tell(), desc=desc, file=output, **pb_kwargs)
+        self.seek(0)
+
+    def seek(self, *args, **kwargs):
+        return self._fileobj.seek(*args, **kwargs)
+
+    def tell(self):
+        return self._fileobj.tell()
+
+    def read(self, size):
+        prev = self.tell()
+        ret = self._fileobj.read(size)
+        self._pb.update(self.tell() - prev)
+        return ret
+
+    def pb_close(self):
+        self._pb.close()
+
+    def pb_write(self, message):
+        """ Allow to write messages to output without interfering with the progress bar """
+        tqdm.write(message, file=self._ori_output)
+
+
+class _NoTerminalOutput(object):
+    """ Helper class: Replace every message sent to it with a fixed one """
+    def __init__(self, output):
+        self._output = output
+
+    def write(self, *args, **kwargs):
+        self._output.write(TIMEOUT_BEAT_CHARACTER)
+
+    def flush(self):
+        self._output.flush()
+
+
+@contextmanager
+def open_binary(path, output, **kwargs):
+    with open(path, mode='rb') as f:
+        file_wrapped = _FileReaderWithProgressBar(f, output=output, **kwargs)
+        yield file_wrapped
+        file_wrapped.pb_close()
+        if not output.is_terminal:
+            output.write("\n")


Test patch: