Repository: huggingface/huggingface_hub

Issue:
http-based upload function should have a progress bar and show upload speed
**Is your feature request related to a problem? Please describe.**
When uploading big model files, it'd be good to have feedback and UI elements to show the progress and the upload speed. This is implemented in LFS-based upload function but not in http-based upload functions.

**Describe the solution you'd like**
Plenty of solutions (such as [this one](https://gist.github.com/tyhoff/b757e6af83c1fd2b7b83057adf02c139)) on the web that uses `tqdm` to track the progress of `requests.put` (i think that's what is used under the hood?). 


Discussion:
Hi @sijunhe , thanks for posting. I looked a bit at the feature request and it's not _that_ easy to implement the progress bars. What is possible to do is to have a progress bar when uploading a lot of LFS files (examples: thousands of images in a dataset). Bar shows progress for each uploaded file.

However in a lot of cases you would like to see the individual progress of a single file (let's say a 10GB model weights for example). In this case, `requests` does not provide a convenient way to monitor it. When uploading a very huge file (>5GB) the upload is done in multiple parts. But 5GB per chunk is already a lot (a 10GB file would display a bar with 2 steps...).
**EDIT:** there seem to be a possibility to monitor huge uploads with `requests`: https://stackoverflow.com/a/13911048. Would you like to check if it works and maybe try to implement it in `huggingface_hub`. I can really help with the integration once it's confirmed that it works (and that it's indeed the feature you were thinking about). For integration, everything is under `_commit_api.py` and `lfs.py`.
maybe related to `class SliceFileObj`?

Resolving patch:
diff --git a/src/huggingface_hub/_commit_api.py b/src/huggingface_hub/_commit_api.py
--- a/src/huggingface_hub/_commit_api.py
+++ b/src/huggingface_hub/_commit_api.py
@@ -6,24 +6,20 @@
 import os
 import warnings
 from collections import defaultdict
-from concurrent import futures
-from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path, PurePosixPath
-from typing import Any, BinaryIO, Dict, Iterable, List, Optional, Union
+from typing import Any, BinaryIO, Dict, Iterable, Iterator, List, Optional, Union
+
+from tqdm.contrib.concurrent import thread_map
 
 import requests
 
 from .constants import ENDPOINT
 from .lfs import UploadInfo, _validate_batch_actions, lfs_upload, post_lfs_batch_info
-from .utils import (
-    build_hf_headers,
-    chunk_iterable,
-    hf_raise_for_status,
-    logging,
-    validate_hf_hub_args,
-)
+from .utils import build_hf_headers, chunk_iterable, hf_raise_for_status, logging
+from .utils import tqdm as hf_tqdm
+from .utils import tqdm_stream_file, validate_hf_hub_args
 from .utils._deprecation import _deprecate_method
 from .utils._typing import Literal
 
@@ -141,11 +137,17 @@ def _upload_info(self) -> UploadInfo:
         return self.upload_info
 
     @contextmanager
-    def as_file(self):
+    def as_file(self, with_tqdm: bool = False) -> Iterator[BinaryIO]:
         """
         A context manager that yields a file-like object allowing to read the underlying
         data behind `path_or_fileobj`.
 
+        Args:
+            with_tqdm (`bool`, *optional*, defaults to `False`):
+                If True, iterating over the file object will display a progress bar. Only
+                works if the file-like object is a path to a file. Pure bytes and buffers
+                are not supported.
+
         Example:
 
         ```python
@@ -157,12 +159,28 @@ def as_file(self):
 
         >>> with operation.as_file() as file:
         ...     content = file.read()
-        ```
 
+        >>> with operation.as_file(with_tqdm=True) as file:
+        ...     while True:
+        ...         data = file.read(1024)
+        ...         if not data:
+        ...              break
+        config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]
+
+        >>> with operation.as_file(with_tqdm=True) as file:
+        ...     requests.put(..., data=file)
+        config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]
+        ```
         """
-        if isinstance(self.path_or_fileobj, str):
-            with open(self.path_or_fileobj, "rb") as file:
-                yield file
+        if isinstance(self.path_or_fileobj, str) or isinstance(
+            self.path_or_fileobj, Path
+        ):
+            if with_tqdm:
+                with tqdm_stream_file(self.path_or_fileobj) as file:
+                    yield file
+            else:
+                with open(self.path_or_fileobj, "rb") as file:
+                    yield file
         elif isinstance(self.path_or_fileobj, bytes):
             yield io.BytesIO(self.path_or_fileobj)
         elif isinstance(self.path_or_fileobj, io.BufferedIOBase):
@@ -292,44 +310,46 @@ def upload_lfs_files(
             raise ValueError(f"LFS batch endpoint returned errors:\n{message}")
 
         batch_actions += batch_actions_chunk
-
-    # Step 2: upload files concurrently according to these instructions
     oid2addop = {add_op.upload_info.sha256.hex(): add_op for add_op in additions}
-    with ThreadPoolExecutor(max_workers=num_threads) as pool:
-        logger.debug(
-            f"Uploading {len(batch_actions)} LFS files to the Hub using up to"
-            f" {num_threads} threads concurrently"
-        )
-        # Upload the files concurrently, stopping on the first exception
-        futures2operation: Dict[futures.Future, CommitOperationAdd] = {
-            pool.submit(
-                _upload_lfs_object,
-                operation=oid2addop[batch_action["oid"]],
-                lfs_batch_action=batch_action,
-                token=token,
-            ): oid2addop[batch_action["oid"]]
-            for batch_action in batch_actions
-        }
-        completed, pending = futures.wait(
-            futures2operation,
-            return_when=futures.FIRST_EXCEPTION,
-        )
 
-        for pending_future in pending:
-            # Cancel pending futures
-            # Uploads already in progress can't be stopped unfortunately,
-            # as `requests` does not support cancelling / aborting a request
-            pending_future.cancel()
+    # Step 2: ignore files that have already been uploaded
+    filtered_actions = []
+    for action in batch_actions:
+        if action.get("actions") is None:
+            logger.debug(
+                f"Content of file {oid2addop[action['oid']].path_in_repo} is already"
+                " present upstream - skipping upload."
+            )
+        else:
+            filtered_actions.append(action)
 
-        for future in completed:
-            operation = futures2operation[future]
-            try:
-                future.result()
-            except Exception as exc:
-                # Raise the first exception encountered
-                raise RuntimeError(
-                    f"Error while uploading {operation.path_in_repo} to the Hub"
-                ) from exc
+    if len(filtered_actions) == 0:
+        logger.debug("No LFS files to upload.")
+        return
+
+    # Step 3: upload files concurrently according to these instructions
+    def _inner_upload_lfs_object(batch_action):
+        try:
+            operation = oid2addop[batch_action["oid"]]
+            return _upload_lfs_object(
+                operation=operation, lfs_batch_action=batch_action, token=token
+            )
+        except Exception as exc:
+            raise RuntimeError(
+                f"Error while uploading '{operation.path_in_repo}' to the Hub."
+            ) from exc
+
+    logger.debug(
+        f"Uploading {len(filtered_actions)} LFS files to the Hub using up to"
+        f" {num_threads} threads concurrently"
+    )
+    thread_map(
+        _inner_upload_lfs_object,
+        filtered_actions,
+        desc=f"Upload {len(filtered_actions)} LFS files",
+        max_workers=num_threads,
+        tqdm_class=hf_tqdm,
+    )
 
 
 def _upload_lfs_object(
@@ -366,8 +386,7 @@ def _upload_lfs_object(
         return
     upload_action = lfs_batch_action["actions"].get("upload")
     verify_action = lfs_batch_action["actions"].get("verify")
-
-    with operation.as_file() as fileobj:
+    with operation.as_file(with_tqdm=True) as fileobj:
         logger.debug(f"Uploading {operation.path_in_repo} as LFS file...")
         lfs_upload(
             fileobj=fileobj,
diff --git a/src/huggingface_hub/utils/__init__.py b/src/huggingface_hub/utils/__init__.py
--- a/src/huggingface_hub/utils/__init__.py
+++ b/src/huggingface_hub/utils/__init__.py
@@ -85,4 +85,5 @@
     disable_progress_bars,
     enable_progress_bars,
     tqdm,
+    tqdm_stream_file,
 )
diff --git a/src/huggingface_hub/utils/tqdm.py b/src/huggingface_hub/utils/tqdm.py
--- a/src/huggingface_hub/utils/tqdm.py
+++ b/src/huggingface_hub/utils/tqdm.py
@@ -54,7 +54,11 @@
        do_something()
     ```
 """
+import io
 import warnings
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator, Optional, Union
 
 from tqdm.auto import tqdm as old_tqdm
 
@@ -118,3 +122,52 @@ def __init__(self, *args, **kwargs):
         if are_progress_bars_disabled():
             kwargs["disable"] = True
         super().__init__(*args, **kwargs)
+
+
+@contextmanager
+def tqdm_stream_file(path: Union[Path, str]) -> Iterator[io.BufferedReader]:
+    """
+    Open a file as binary and wrap the `read` method to display a progress bar when it's
+    streamed.
+
+    First implemented in `transformers` in 2019 but removed when switched to git-lfs.
+    Today used in `huggingface_hub` to show progress bar when uploading an LFS file to
+    the Hub. See github.com/huggingface/transformers/pull/2078#discussion_r354739608 for
+    implementation details.
+
+    Note: currently implementation handles only files stored on disk as it is the most
+          common use case. Could be extended to stream any `BinaryIO` object but we might
+          have to debug some corner cases.
+
+    Example:
+    ```py
+    >>> with tqdm_stream_file("config.json") as f:
+    >>>     requests.put(url, data=f)
+    config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]
+    ```
+    """
+    if isinstance(path, str):
+        path = Path(path)
+
+    with path.open("rb") as f:
+        total_size = path.stat().st_size
+        pbar = tqdm(
+            unit="B",
+            unit_scale=True,
+            total=total_size,
+            initial=0,
+            desc=path.name,
+        )
+
+        f_read = f.read
+
+        def _inner_read(size: Optional[int] = -1) -> bytes:
+            data = f_read(size)
+            pbar.update(len(data))
+            return data
+
+        f.read = _inner_read  # type: ignore
+
+        yield f
+
+        pbar.close()


Test patch:
diff --git a/tests/test_utils_tqdm.py b/tests/test_utils_tqdm.py
--- a/tests/test_utils_tqdm.py
+++ b/tests/test_utils_tqdm.py
@@ -1,4 +1,7 @@
+import tempfile
+import time
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 
 import pytest
@@ -9,6 +12,7 @@
     disable_progress_bars,
     enable_progress_bars,
     tqdm,
+    tqdm_stream_file,
 )
 
 
@@ -114,3 +118,22 @@ def test_tqdm_enabled(self) -> None:
         captured = self.capsys.readouterr()
         self.assertEqual(captured.out, "")
         self.assertIn("10/10", captured.err)  # tqdm log
+
+    def test_tqdm_stream_file(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            filepath = Path(tmpdir) / "config.json"
+            with filepath.open("w") as f:
+                f.write("#" * 1000)
+
+            with tqdm_stream_file(filepath) as f:
+                while True:
+                    data = f.read(100)
+                    if not data:
+                        break
+                    time.sleep(0.001)  # Simulate a delay between each chunk
+
+            captured = self.capsys.readouterr()
+            self.assertEqual(captured.out, "")
+            self.assertIn("config.json: 100%", captured.err)  # log file name
+            self.assertIn("|█████████", captured.err)  # tqdm bar
+            self.assertIn("1.00k/1.00k", captured.err)  # size in B