# coding=utf-8 # Copyright 2019-present, the HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pathlib import Path from typing import Union from ..constants import HF_ASSETS_CACHE def cached_assets_path( library_name: str, namespace: str = "default", subfolder: str = "default", *, assets_dir: Union[str, Path, None] = None, ): """Return a folder path to cache arbitrary files. `huggingface_hub` provides a canonical folder path to store assets. This is the recommended way to integrate cache in a downstream library as it will benefit from the builtins tools to scan and delete the cache properly. The distinction is made between files cached from the Hub and assets. Files from the Hub are cached in a git-aware manner and entirely managed by `huggingface_hub`. See [related documentation](https://huggingface.co/docs/huggingface_hub/how-to-cache). All other files that a downstream library caches are considered to be "assets" (files downloaded from external sources, extracted from a .tar archive, preprocessed for training,...). Once the folder path is generated, it is guaranteed to exist and to be a directory. The path is based on 3 levels of depth: the library name, a namespace and a subfolder. Those 3 levels grants flexibility while allowing `huggingface_hub` to expect folders when scanning/deleting parts of the assets cache. Within a library, it is expected that all namespaces share the same subset of subfolder names but this is not a mandatory rule. The downstream library has then full control on which file structure to adopt within its cache. Namespace and subfolder are optional (would default to a `"default/"` subfolder) but library name is mandatory as we want every downstream library to manage its own cache. Expected tree: ```text assets/ └── datasets/ │ ├── SQuAD/ │ │ ├── downloaded/ │ │ ├── extracted/ │ │ └── processed/ │ ├── Helsinki-NLP--tatoeba_mt/ │ ├── downloaded/ │ ├── extracted/ │ └── processed/ └── transformers/ ├── default/ │ ├── something/ ├── bert-base-cased/ │ ├── default/ │ └── training/ hub/ └── models--julien-c--EsperBERTo-small/ ├── blobs/ │ ├── (...) │ ├── (...) ├── refs/ │ └── (...) └── [ 128] snapshots/ ├── 2439f60ef33a0d46d85da5001d52aeda5b00ce9f/ │ ├── (...) └── bbc77c8132af1cc5cf678da3f1ddf2de43606d48/ └── (...) ``` Args: library_name (`str`): Name of the library that will manage the cache folder. Example: `"dataset"`. namespace (`str`, *optional*, defaults to "default"): Namespace to which the data belongs. Example: `"SQuAD"`. subfolder (`str`, *optional*, defaults to "default"): Subfolder in which the data will be stored. Example: `extracted`. assets_dir (`str`, `Path`, *optional*): Path to the folder where assets are cached. This must not be the same folder where Hub files are cached. Defaults to `HF_HOME / "assets"` if not provided. Can also be set with `HF_ASSETS_CACHE` environment variable. Returns: Path to the cache folder (`Path`). Example: ```py >>> from huggingface_hub import cached_assets_path >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download") PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/download') >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="extracted") PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/extracted') >>> cached_assets_path(library_name="datasets", namespace="Helsinki-NLP/tatoeba_mt") PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/Helsinki-NLP--tatoeba_mt/default') >>> cached_assets_path(library_name="datasets", assets_dir="/tmp/tmp123456") PosixPath('/tmp/tmp123456/datasets/default/default') ``` """ # Resolve assets_dir if assets_dir is None: assets_dir = HF_ASSETS_CACHE assets_dir = Path(assets_dir).expanduser().resolve() # Avoid names that could create path issues for part in (" ", "/", "\\"): library_name = library_name.replace(part, "--") namespace = namespace.replace(part, "--") subfolder = subfolder.replace(part, "--") # Path to subfolder is created path = assets_dir / library_name / namespace / subfolder try: path.mkdir(exist_ok=True, parents=True) except (FileExistsError, NotADirectoryError): raise ValueError(f"Corrupted assets folder: cannot create directory because of an existing file ({path}).") # Return return path