Repository: huggingface/diffusers

Issue:
Store cached models in location compatible with `huggingface-cli`
Reference: https://github.com/huggingface/huggingface_hub/issues/1259

`huggingface-cli scan-cache` doesn't see cached diffusers models. Are there any drawbacks to changing the cache folder?


Discussion:
Yes we could indeed change the cache here! This was a copy-paste back then from the `transformers` repo! Would you like to open a PR to change the default cache @pcuenca ? Otherwise happy to look into it in a couple of days.
I support the idea to get everything under `$HF_HOME/hub/` so that users can benefit from `huggingface_hub` tool. Also beneficial if a user downloads some weights outside of `diffusers` for some reason, the cache will be shared. This is not the case at the moment.

Also note that if there are some stuff cached but not from the Hub (preprocessed data, community weights,...) it would make sense to use `assets` cache ([see docs](https://huggingface.co/docs/huggingface_hub/how-to-cache#caching-assets)). I can help with that if needed :)
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/diffusers/blob/main/CONTRIBUTING.md) are likely to be ignored.
cc @pcuenca would you like to tackle this one? Otherwise, I should be able to find some time
Yeah, I'll take a look today.

Resolving patch:
diff --git a/src/diffusers/utils/constants.py b/src/diffusers/utils/constants.py
--- a/src/diffusers/utils/constants.py
+++ b/src/diffusers/utils/constants.py
@@ -13,11 +13,10 @@
 # limitations under the License.
 import os
 
+from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home
 
-hf_cache_home = os.path.expanduser(
-    os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface"))
-)
-default_cache_path = os.path.join(hf_cache_home, "diffusers")
+
+default_cache_path = HUGGINGFACE_HUB_CACHE
 
 
 CONFIG_NAME = "config.json"
diff --git a/src/diffusers/utils/hub_utils.py b/src/diffusers/utils/hub_utils.py
--- a/src/diffusers/utils/hub_utils.py
+++ b/src/diffusers/utils/hub_utils.py
@@ -16,6 +16,7 @@
 
 import os
 import sys
+import traceback
 from pathlib import Path
 from typing import Dict, Optional, Union
 from uuid import uuid4
@@ -24,7 +25,7 @@
 from huggingface_hub.utils import is_jinja_available
 
 from .. import __version__
-from .constants import HUGGINGFACE_CO_RESOLVE_ENDPOINT
+from .constants import DIFFUSERS_CACHE, HUGGINGFACE_CO_RESOLVE_ENDPOINT
 from .import_utils import (
     ENV_VARS_TRUE_VALUES,
     _flax_version,
@@ -129,3 +130,72 @@ def create_model_card(args, model_name):
 
     card_path = os.path.join(args.output_dir, "README.md")
     model_card.save(card_path)
+
+
+# Old default cache path, potentially to be migrated.
+# This logic was more or less taken from `transformers`, with the following differences:
+# - Diffusers doesn't use custom environment variables to specify the cache path.
+# - There is no need to migrate the cache format, just move the files to the new location.
+hf_cache_home = os.path.expanduser(
+    os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface"))
+)
+old_diffusers_cache = os.path.join(hf_cache_home, "diffusers")
+
+
+def move_cache(old_cache_dir: Optional[str] = None, new_cache_dir: Optional[str] = None) -> None:
+    if new_cache_dir is None:
+        new_cache_dir = DIFFUSERS_CACHE
+    if old_cache_dir is None:
+        old_cache_dir = old_diffusers_cache
+
+    old_cache_dir = Path(old_cache_dir).expanduser()
+    new_cache_dir = Path(new_cache_dir).expanduser()
+    for old_blob_path in old_cache_dir.glob("**/blobs/*"):  #  move file blob by blob
+        if old_blob_path.is_file() and not old_blob_path.is_symlink():
+            new_blob_path = new_cache_dir / old_blob_path.relative_to(old_cache_dir)
+            new_blob_path.parent.mkdir(parents=True, exist_ok=True)
+            os.replace(old_blob_path, new_blob_path)
+            try:
+                os.symlink(new_blob_path, old_blob_path)
+            except OSError:
+                logger.warning(
+                    "Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded."
+                )
+    # At this point, old_cache_dir contains symlinks to the new cache (it can still be used).
+
+
+cache_version_file = os.path.join(DIFFUSERS_CACHE, "version_diffusers_cache.txt")
+if not os.path.isfile(cache_version_file):
+    cache_version = 0
+else:
+    with open(cache_version_file) as f:
+        cache_version = int(f.read())
+
+if cache_version < 1:
+    old_cache_is_not_empty = os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0
+    if old_cache_is_not_empty:
+        logger.warning(
+            "The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your "
+            "existing cached models. This is a one-time operation, you can interrupt it or run it "
+            "later by calling `diffusers.utils.hub_utils.move_cache()`."
+        )
+        try:
+            move_cache()
+        except Exception as e:
+            trace = "\n".join(traceback.format_tb(e.__traceback__))
+            logger.error(
+                f"There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease "
+                "file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole "
+                "message and we will do our best to help."
+            )
+
+if cache_version < 1:
+    try:
+        os.makedirs(DIFFUSERS_CACHE, exist_ok=True)
+        with open(cache_version_file, "w") as f:
+            f.write("1")
+    except Exception:
+        logger.warning(
+            f"There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure "
+            "the directory exists and can be written to."
+        )


Test patch: