Repository: huggingface/huggingface_hub
Pull request #980 (2022-08-09 18:19:02)

Add utility to load files from cache
This PR adds the utility we use in Transformers to check if a file is in the cache when there is a connection error, while making it handle any repo type. I haven't done anything for it to be automatically used in case of connection error in `hf_hub_download`, as I don't know if it's something you want automatically baked in the library or not (in Transformers, we use this to load the files from the cache when there is any connection error, for instance a downtime on hf.co).

See [this comment](https://github.com/huggingface/transformers/pull/18438#discussion_r941304410) where we were talking of moving is here.

HuggingFaceDocBuilderDev: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/huggingface_hub/pr_980). All of your documentation changes will be reflected on that endpoint.

Patch:
diff --git a/setup.cfg b/setup.cfg
index 5204109c..6e107fef 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -48,5 +48,8 @@ multi_line_output = 3
 use_parentheses = True
 
 [flake8]
-ignore = E203, E501, E741, W503, W605
+ignore = E203, E501, E741, W503
 max-line-length = 88
+
+[tool:pytest]
+addopts = -Werror::FutureWarning
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 0245e511..1cd4a871 100644
--- a/setup.py
+++ b/setup.py
@@ -37,7 +37,6 @@ extras["tensorflow"] = ["tensorflow", "pydot", "graphviz"]
 
 extras["testing"] = [
     "pytest",
-    "pytest-cov",
     "datasets",
     "soundfile",
 ]
diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py
index db0f2228..62180344 100644
--- a/src/huggingface_hub/__init__.py
+++ b/src/huggingface_hub/__init__.py
@@ -141,7 +141,12 @@ __getattr__, __dir__, __all__ = _attach(
             "from_pretrained_fastai",
             "push_to_hub_fastai",
         ],
-        "file_download": ["cached_download", "hf_hub_download", "hf_hub_url"],
+        "file_download": [
+            "cached_download",
+            "hf_hub_download",
+            "hf_hub_url",
+            "try_to_load_from_cache",
+        ],
         "hf_api": [
             "CommitOperation",
             "CommitOperationAdd",
diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py
index a7f31ba2..4f18090b 100644
--- a/src/huggingface_hub/file_download.py
+++ b/src/huggingface_hub/file_download.py
@@ -1188,3 +1188,54 @@ def hf_hub_download(
         pass
 
     return pointer_path
+
+
+def try_to_load_from_cache(
+    repo_id: str,
+    filename: str,
+    cache_dir: Union[str, Path, None] = None,
+    revision: Optional[str] = None,
+    repo_type: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Explores the cache to return the latest cached file for a given revision if found.
+
+    This function will not raise any exception if the file in not cached.
+
+    Returns:
+        Local path (string) of file or `None` if no cached file is found.
+    """
+    if revision is None:
+        revision = "main"
+    if repo_type is None:
+        repo_type = "model"
+    if repo_type not in REPO_TYPES:
+        raise ValueError(
+            f"Invalid repo type: {repo_type}. Accepted repo types are:"
+            f" {str(REPO_TYPES)}"
+        )
+    if cache_dir is None:
+        cache_dir = HUGGINGFACE_HUB_CACHE
+
+    object_id = repo_id.replace("/", "--")
+    repo_cache = os.path.join(cache_dir, f"{repo_type}s--{object_id}")
+    if not os.path.isdir(repo_cache):
+        # No cache for this model
+        return None
+    for subfolder in ["refs", "snapshots"]:
+        if not os.path.isdir(os.path.join(repo_cache, subfolder)):
+            return None
+
+    # Resolve refs (for instance to convert main to the associated commit sha)
+    cached_refs = os.listdir(os.path.join(repo_cache, "refs"))
+    if revision in cached_refs:
+        with open(os.path.join(repo_cache, "refs", revision)) as f:
+            revision = f.read()
+
+    cached_shas = os.listdir(os.path.join(repo_cache, "snapshots"))
+    if revision not in cached_shas:
+        # No cache for this revision and we won't try to return a random revision
+        return None
+
+    cached_file = os.path.join(repo_cache, "snapshots", revision, filename)
+    return cached_file if os.path.isfile(cached_file) else None
diff --git a/src/huggingface_hub/utils/endpoint_helpers.py b/src/huggingface_hub/utils/endpoint_helpers.py
index 4010c8cd..cc97f047 100644
--- a/src/huggingface_hub/utils/endpoint_helpers.py
+++ b/src/huggingface_hub/utils/endpoint_helpers.py
@@ -54,7 +54,7 @@ def _filter_emissions(
                 if emission:
                     emission = str(emission)
                     if any(char.isdigit() for char in emission):
-                        emission = re.search("\d+\.\d+|\d+", emission).group(0)
+                        emission = re.search(r"\d+\.\d+|\d+", emission).group(0)
                         emissions.append((i, float(emission)))
     filtered_results = []
     for idx, emission in emissions:


Test patch:
diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
index 73a31980..e95320c2 100644
--- a/.github/workflows/python-tests.yml
+++ b/.github/workflows/python-tests.yml
@@ -42,15 +42,12 @@ jobs:
       - run: |
           if [[ "${{ matrix.test_repository }}" == "Repository only" ]]
           then
-            pytest --cov -Werror::FutureWarning --log-cli-level=INFO -sv ./tests/test_repository.py -k RepositoryTest
+            pytest --log-cli-level=INFO -sv ./tests/test_repository.py -k RepositoryTest
 
           else
-            pytest --cov -Werror::FutureWarning --log-cli-level=INFO -sv ./tests/ -k 'not RepositoryTest'
+            pytest --log-cli-level=INFO -sv ./tests/ -k 'not RepositoryTest'
           fi
 
-      - name: Upload coverage reports to Codecov with GitHub Action
-        uses: codecov/codecov-action@v3
-
   build_pytorch:
     runs-on: ubuntu-latest
     strategy:
@@ -70,10 +67,7 @@ jobs:
         pip install --upgrade pip
         pip install .[testing,torch]
 
-    - run: pytest --cov -Werror::FutureWarning -sv ./tests/test_hubmixin*
-
-    - name: Upload coverage reports to Codecov with GitHub Action
-      uses: codecov/codecov-action@v3
+    - run: pytest -sv ./tests/test_hubmixin*
 
   build_tensorflow:
     runs-on: ubuntu-latest
@@ -95,12 +89,9 @@ jobs:
         pip install --upgrade pip
         pip install .[testing,tensorflow]
 
-    - run: pytest --cov -Werror::FutureWarning -sv ./tests/test_tf*
-
-    - run: pytest --cov -Werror::FutureWarning -sv ./tests/test_keras*
+    - run: pytest -sv ./tests/test_tf*
 
-    - name: Upload coverage reports to Codecov with GitHub Action
-      uses: codecov/codecov-action@v3
+    - run: pytest -sv ./tests/test_keras*
 
   build_fastai:
     runs-on: ubuntu-latest
@@ -121,10 +112,7 @@ jobs:
         pip install --upgrade pip
         pip install .[testing,fastai]
 
-    - run: pytest --cov -Werror::FutureWarning -sv ./tests/test_fastai*
-
-    - name: Upload coverage reports to Codecov with GitHub Action
-      uses: codecov/codecov-action@v3
+    - run: pytest -sv ./tests/test_fastai*
 
   tests_lfs:
     runs-on: ubuntu-latest
@@ -146,7 +134,4 @@ jobs:
         pip install --upgrade pip
         pip install .[testing]
 
-    - run: RUN_GIT_LFS_TESTS=1 pytest --cov -Werror::FutureWarning -sv ./tests/ -k "HfLargefilesTest"
-
-    - name: Upload coverage reports to Codecov with GitHub Action
-      uses: codecov/codecov-action@v3
+    - run: RUN_GIT_LFS_TESTS=1 pytest -sv ./tests/ -k "HfLargefilesTest"
diff --git a/tests/test_file_download.py b/tests/test_file_download.py
index 0681ddc3..7a9dcf0a 100644
--- a/tests/test_file_download.py
+++ b/tests/test_file_download.py
@@ -24,6 +24,7 @@ from huggingface_hub.file_download import (
     filename_to_url,
     hf_hub_download,
     hf_hub_url,
+    try_to_load_from_cache,
 )
 from huggingface_hub.utils import (
     EntryNotFoundError,
@@ -186,3 +187,28 @@ class CachedDownloadTests(unittest.TestCase):
         )
         metadata = filename_to_url(filepath, legacy_cache_layout=True)
         self.assertEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')
+
+    def test_try_to_load_from_cache(self):
+        # Make sure the file is cached
+        filepath = hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME)
+
+        new_file_path = try_to_load_from_cache(DUMMY_MODEL_ID, filename=CONFIG_NAME)
+        self.assertEqual(filepath, new_file_path)
+
+        new_file_path = try_to_load_from_cache(
+            DUMMY_MODEL_ID, filename=CONFIG_NAME, revision="main"
+        )
+        self.assertEqual(filepath, new_file_path)
+
+        # If file is not cached, returns None
+        self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename="conf.json"))
+        # Same for uncached revisions
+        self.assertIsNone(
+            try_to_load_from_cache(
+                DUMMY_MODEL_ID,
+                filename=CONFIG_NAME,
+                revision="aaa",
+            )
+        )
+        # Same for uncached models
+        self.assertIsNone(try_to_load_from_cache("bert-base", filename=CONFIG_NAME))