Repository: huggingface/huggingface_hub
Pull request #1307 (2023-01-24 10:40:52)

Fix caching error when offline and no refs dir
Will not fix the root issue of https://github.com/huggingface/huggingface_hub/issues/1305 but fixes the non-explicit `FileNotFoundError` that was raised (in favor of the documented and expected `LocalEntryNotFoundError`).

cc @NielsRogge for some reason downloading a file in your space failed due to a connection issue (infra issue ? `HF_HUB_OFFLINE` set ?). Instead of the proper `LocalEntryNotFoundError` (that is handled nicely in `transformers`) you had a `FlieNotFoundError` because the cache in your space was not initialized yet. This also explains why it's working locally on your computer (since I guess cache structure is already good). This PR should fix it.


Patch:
diff --git a/docs/source/how-to-cache.mdx b/docs/source/how-to-cache.mdx
index b72b5384..f3b98615 100644
--- a/docs/source/how-to-cache.mdx
+++ b/docs/source/how-to-cache.mdx
@@ -87,6 +87,46 @@ That `README.md` file is actually a symlink linking to the blob that has the has
 By creating the skeleton this way we open the mechanism to file sharing: if the same file was fetched in
 revision `bbbbbb`, it would have the same hash and the file would not need to be re-downloaded.
 
+### .no_exist (advanced)
+
+In addition to the `blobs`, `refs` and `snapshots` folders, you might also find a `.no_exist` folder
+in your cache. This folder keeps track of files that you've tried to download once but don't exist
+on the Hub. Its structure is the same as the `snapshots` folder with 1 subfolder per known revision:
+
+```
+<CACHE_DIR>/<REPO_NAME>/.no_exist/aaaaaa/config_that_does_not_exist.json
+```
+
+Unlike the `snapshots` folder, files are simple empty files (no symlinks). In this example,
+the file `"config_that_does_not_exist.json"` does not exist on the Hub for the revision `"aaaaaa"`.
+As it only stores empty files, this folder is neglectable is term of disk usage.
+
+So now you might wonder, why is this information even relevant?
+In some cases, a framework tries to load optional files for a model. Saving the non-existence
+of optional files makes it faster to load a model as it saves 1 HTTP call per possible optional file.
+This is for example the case in `transformers` where each tokenizer can support additional files.
+The first time you load the tokenizer on your machine, it will cache which optional files exists (and
+which doesn't) to make the loading time faster for the next initializations.
+
+To test if a file is cached locally (without making any HTTP request), you can use the [`try_to_load_from_cache`]
+helper. It will either return the filepath (if exists and cached), the object `_CACHED_NO_EXIST` (if non-existence
+is cached) or `None` (if we don't know).
+
+```python
+from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST
+
+filepath = try_to_load_from_cache()
+if isinstance(filepath, str):
+    # file exists and is cached
+    ...
+elif filepath is _CACHED_NO_EXIST:
+    # non-existence of file is cached
+    ...
+else:
+    # file is not cached
+    ...
+```
+
 ### In practice
 
 In practice, your cache should look like the following tree:
diff --git a/docs/source/package_reference/cache.mdx b/docs/source/package_reference/cache.mdx
index 2fb86bf0..709564f1 100644
--- a/docs/source/package_reference/cache.mdx
+++ b/docs/source/package_reference/cache.mdx
@@ -6,7 +6,11 @@ for a detailed presentation of caching at HF.
 
 ## Helpers
 
-## cached_assets_path
+### try_to_load_from_cache
+
+[[autodoc]] huggingface_hub.try_to_load_from_cache
+
+### cached_assets_path
 
 [[autodoc]] huggingface_hub.cached_assets_path
 
diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py
index a41519a0..e103f6ed 100644
--- a/src/huggingface_hub/__init__.py
+++ b/src/huggingface_hub/__init__.py
@@ -94,6 +94,7 @@ _SUBMOD_ATTRS = {
     ],
     "file_download": [
         "HfFileMetadata",
+        "_CACHED_NO_EXIST",
         "cached_download",
         "get_hf_file_metadata",
         "hf_hub_download",
@@ -336,6 +337,7 @@ if TYPE_CHECKING:  # pragma: no cover
     from .fastai_utils import _save_pretrained_fastai  # noqa: F401
     from .fastai_utils import from_pretrained_fastai  # noqa: F401
     from .fastai_utils import push_to_hub_fastai  # noqa: F401
+    from .file_download import _CACHED_NO_EXIST  # noqa: F401
     from .file_download import HfFileMetadata  # noqa: F401
     from .file_download import cached_download  # noqa: F401
     from .file_download import get_hf_file_metadata  # noqa: F401
diff --git a/src/huggingface_hub/file_download.py b/src/huggingface_hub/file_download.py
index aae1549c..1d5c1f06 100644
--- a/src/huggingface_hub/file_download.py
+++ b/src/huggingface_hub/file_download.py
@@ -1177,18 +1177,24 @@ def hf_hub_download(
                 "We have no connection or you passed local_files_only, so"
                 " force_download is not an accepted option."
             )
+
+        # Try to get "commit_hash" from "revision"
+        commit_hash = None
         if REGEX_COMMIT_HASH.match(revision):
             commit_hash = revision
         else:
             ref_path = os.path.join(storage_folder, "refs", revision)
-            with open(ref_path) as f:
-                commit_hash = f.read()
-
-        pointer_path = os.path.join(
-            storage_folder, "snapshots", commit_hash, relative_filename
-        )
-        if os.path.exists(pointer_path):
-            return pointer_path
+            if os.path.isfile(ref_path):
+                with open(ref_path) as f:
+                    commit_hash = f.read()
+
+        # Return pointer file if exists
+        if commit_hash is not None:
+            pointer_path = os.path.join(
+                storage_folder, "snapshots", commit_hash, relative_filename
+            )
+            if os.path.exists(pointer_path):
+                return pointer_path
 
         # If we couldn't find an appropriate file on disk,
         # raise an error.
@@ -1327,6 +1333,23 @@ def try_to_load_from_cache(
             - The exact path to the cached file if it's found in the cache
             - A special value `_CACHED_NO_EXIST` if the file does not exist at the given commit hash and this fact was
               cached.
+
+    Example:
+
+    ```python
+    from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST
+
+    filepath = try_to_load_from_cache()
+    if isinstance(filepath, str):
+        # file exists and is cached
+        ...
+    elif filepath is _CACHED_NO_EXIST:
+        # non-existence of file is cached
+        ...
+    else:
+        # file is not cached
+        ...
+    ```
     """
     if revision is None:
         revision = "main"
@@ -1345,25 +1368,32 @@ def try_to_load_from_cache(
     if not os.path.isdir(repo_cache):
         # No cache for this model
         return None
-    for subfolder in ["refs", "snapshots"]:
-        if not os.path.isdir(os.path.join(repo_cache, subfolder)):
-            return None
 
-    # Resolve refs (for instance to convert main to the associated commit sha)
-    cached_refs = os.listdir(os.path.join(repo_cache, "refs"))
-    if revision in cached_refs:
-        with open(os.path.join(repo_cache, "refs", revision)) as f:
-            revision = f.read()
+    refs_dir = os.path.join(repo_cache, "refs")
+    snapshots_dir = os.path.join(repo_cache, "snapshots")
+    no_exist_dir = os.path.join(repo_cache, ".no_exist")
 
-    if os.path.isfile(os.path.join(repo_cache, ".no_exist", revision, filename)):
+    # Resolve refs (for instance to convert main to the associated commit sha)
+    if os.path.isdir(refs_dir):
+        cached_refs = os.listdir(refs_dir)
+        if revision in cached_refs:
+            with open(os.path.join(refs_dir, revision)) as f:
+                revision = f.read()
+
+    # Check if file is cached as "no_exist"
+    if os.path.isfile(os.path.join(no_exist_dir, revision, filename)):
         return _CACHED_NO_EXIST
 
-    cached_shas = os.listdir(os.path.join(repo_cache, "snapshots"))
+    # Check if revision folder exists
+    if not os.path.exists(snapshots_dir):
+        return None
+    cached_shas = os.listdir(snapshots_dir)
     if revision not in cached_shas:
         # No cache for this revision and we won't try to return a random revision
         return None
 
-    cached_file = os.path.join(repo_cache, "snapshots", revision, filename)
+    # Check if file exists in cache
+    cached_file = os.path.join(snapshots_dir, revision, filename)
     return cached_file if os.path.isfile(cached_file) else None
 
 


Test patch:
diff --git a/tests/test_file_download.py b/tests/test_file_download.py
index a82cb372..1e23d276 100644
--- a/tests/test_file_download.py
+++ b/tests/test_file_download.py
@@ -318,6 +318,24 @@ class CachedDownloadTests(unittest.TestCase):
         self.assertEqual(filepath.name, CONFIG_NAME)
         self.assertEqual(Path(filepath).parent.parent.name, "snapshots")
 
+    def test_hf_hub_download_offline_no_refs(self):
+        """Regression test for #1305.
+
+        If "refs/" dir did not exists on "local_files_only" (or connection broken), a
+        non-explicit `FileNotFoundError` was raised (for the "/refs/revision" file) instead
+        of the documented `LocalEntryNotFoundError` (for the actual searched file).
+
+        See https://github.com/huggingface/huggingface_hub/issues/1305.
+        """
+        with SoftTemporaryDirectory() as cache_dir:
+            with self.assertRaises(LocalEntryNotFoundError):
+                hf_hub_download(
+                    DUMMY_MODEL_ID,
+                    filename=CONFIG_NAME,
+                    local_files_only=True,
+                    cache_dir=cache_dir,
+                )
+
     def test_hf_hub_url_with_empty_subfolder(self):
         """
         Check subfolder arg is processed correctly when empty string is passed to
@@ -348,7 +366,7 @@ class CachedDownloadTests(unittest.TestCase):
         metadata = filename_to_url(filepath, legacy_cache_layout=True)
         self.assertEqual(metadata[1], f'"{DUMMY_MODEL_ID_PINNED_SHA1}"')
 
-    def test_try_to_load_from_cache(self):
+    def test_try_to_load_from_cache_exist(self):
         # Make sure the file is cached
         filepath = hf_hub_download(DUMMY_MODEL_ID, filename=CONFIG_NAME)
 
@@ -389,6 +407,53 @@ class CachedDownloadTests(unittest.TestCase):
         # If file non-existence is not cached, returns None
         self.assertIsNone(try_to_load_from_cache(DUMMY_MODEL_ID, filename="dummy2"))
 
+    def test_try_to_load_from_cache_specific_commit_id_exist(self):
+        """Regression test for #1306.
+
+        See https://github.com/huggingface/huggingface_hub/pull/1306."""
+        with SoftTemporaryDirectory() as cache_dir:
+            # Cache file from specific commit id (no "refs/"" folder)
+            commit_id = HfApi().model_info(DUMMY_MODEL_ID).sha
+            filepath = hf_hub_download(
+                DUMMY_MODEL_ID,
+                filename=CONFIG_NAME,
+                revision=commit_id,
+                cache_dir=cache_dir,
+            )
+
+            # Must be able to retrieve it "offline"
+            attempt = try_to_load_from_cache(
+                DUMMY_MODEL_ID,
+                filename=CONFIG_NAME,
+                revision=commit_id,
+                cache_dir=cache_dir,
+            )
+            self.assertEqual(filepath, attempt)
+
+    def test_try_to_load_from_cache_specific_commit_id_no_exist(self):
+        """Regression test for #1306.
+
+        See https://github.com/huggingface/huggingface_hub/pull/1306."""
+        with SoftTemporaryDirectory() as cache_dir:
+            # Cache file from specific commit id (no "refs/"" folder)
+            commit_id = HfApi().model_info(DUMMY_MODEL_ID).sha
+            with self.assertRaises(EntryNotFoundError):
+                hf_hub_download(
+                    DUMMY_MODEL_ID,
+                    filename="missing_file",
+                    revision=commit_id,
+                    cache_dir=cache_dir,
+                )
+
+            # Must be able to retrieve it "offline"
+            attempt = try_to_load_from_cache(
+                DUMMY_MODEL_ID,
+                filename="missing_file",
+                revision=commit_id,
+                cache_dir=cache_dir,
+            )
+            self.assertEqual(attempt, _CACHED_NO_EXIST)
+
     def test_get_hf_file_metadata_basic(self) -> None:
         """Test getting metadata from a file on the Hub."""
         url = hf_hub_url(