Repository: huggingface/huggingface_hub
Pull request #656 (2022-02-04 17:24:53)

Fix SB3 code snippet
There is some weird indentation at https://huggingface.co/ThomasSimonini/ppo-LunarLander-v2


Patch:
diff --git a/docs/hub/adding-a-model.md b/docs/hub/adding-a-model.md
index 49240cbd..c10fc246 100644
--- a/docs/hub/adding-a-model.md
+++ b/docs/hub/adding-a-model.md
@@ -46,7 +46,7 @@ If you look at the “Files and versions” tab, you’ll see that there aren’
 
 If you've used Git before, this will be very easy since Git is used to manage files in the Hub.
 
-There is only one key difference if you have large files (over 10MB). These files are usually tracked with **git-lfs** (which stands for Git Large File Storage). 
+There is only one key difference if you have large files (over 10MB, or 1MB for binary files). These files are usually tracked with **git-lfs** (which stands for Git Large File Storage). 
 
 1. Please make sure you have both **git** and **git-lfs** installed on your system.
 
diff --git a/docs/hub/model-repos.md b/docs/hub/model-repos.md
index 871c9653..068e4894 100644
--- a/docs/hub/model-repos.md
+++ b/docs/hub/model-repos.md
@@ -20,7 +20,7 @@ The model card should describe:
 ## Model card metadata
 <!-- Try not to change this header as we use the corresponding anchor link -->
 
-The model cards have a YAML section that specify metadata. These are the fields
+You can add a YAML section to the model cards (that `README.md` file) by adding three `---` to the top of your model card, including all of the metadata, and enclosing it with another group of `---` such as the example below:
 
 ```yaml
 ---
diff --git a/js/src/lib/interfaces/Libraries.ts b/js/src/lib/interfaces/Libraries.ts
index 21e3f19f..050130b3 100644
--- a/js/src/lib/interfaces/Libraries.ts
+++ b/js/src/lib/interfaces/Libraries.ts
@@ -285,10 +285,10 @@ model = fasttext.load_model(hf_hub_download("${model.id}", "model.bin"))`;
 
 const stableBaselines3 = (model: ModelData) =>
 	`from huggingface_sb3 import load_from_hub
-	checkpoint = load_from_hub(
-    		repo_id="${model.id}",
-    		filename="{MODEL FILENAME}.zip",
-	)`;
+checkpoint = load_from_hub(
+	repo_id="${model.id}",
+	filename="{MODEL FILENAME}.zip",
+)`;
 
 //#endregion
 
diff --git a/setup.py b/setup.py
index 27b78617..c63a3146 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@ extras["testing"] = [
 ]
 
 extras["quality"] = [
-    "black==21.4b0",
+    "black~=22.0",
     "isort>=5.5.4",
     "flake8>=3.8.3",
 ]
diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py
index fc11433b..52adf92e 100644
--- a/src/huggingface_hub/hf_api.py
+++ b/src/huggingface_hub/hf_api.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 import logging
 import os
-import re
 import subprocess
 import sys
 import warnings
@@ -49,10 +48,6 @@ else:
 
 USERNAME_PLACEHOLDER = "hf_user"
 
-REMOTE_FILEPATH_REGEX = re.compile(r"^\w[\w\/\-]*(\.\w+)?$")
-# ^^ No trailing slash, no backslash, no spaces, no relative parts ("." or "..")
-#    Only word characters and an optional extension
-
 
 def repo_type_and_id_from_hf_id(hf_id: str):
     """
@@ -517,6 +512,7 @@ class HfApi:
         direction: Optional[Literal[-1]] = None,
         limit: Optional[int] = None,
         full: Optional[bool] = None,
+        cardData: Optional[bool] = None,
         fetch_config: Optional[bool] = None,
         use_auth_token: Optional[Union[bool, str]] = None,
     ) -> List[ModelInfo]:
@@ -591,6 +587,9 @@ class HfApi:
             full (:obj:`bool`, `optional`):
                 Whether to fetch all model data, including the `lastModified`, the `sha`, the files and the `tags`.
                 This is set to `True` by default when using a filter.
+            cardData (:obj:`bool`, `optional`):
+                Whether to grab the metadata for the model as well. Can contain useful information such as carbon emissions,
+                metrics, and datasets trained on.
             fetch_config (:obj:`bool`, `optional`):
                 Whether to fetch the model configs as well. This is not included in `full` due to its size.
             use_auth_token (:obj:`bool` or :obj:`str`, `optional`):
@@ -635,7 +634,9 @@ class HfApi:
                 del params["full"]
         if fetch_config is not None:
             params.update({"config": fetch_config})
-        r = requests.get(path, headers=headers, params=params)
+        if cardData is not None:
+            params.update({"cardData": cardData})
+        r = requests.get(path, params=params, headers=headers)
         r.raise_for_status()
         d = r.json()
         return [ModelInfo(**x) for x in d]
@@ -708,6 +709,7 @@ class HfApi:
         sort: Union[Literal["lastModified"], str, None] = None,
         direction: Optional[Literal[-1]] = None,
         limit: Optional[int] = None,
+        cardData: Optional[bool] = None,
         full: Optional[bool] = None,
         use_auth_token: Optional[str] = None,
     ) -> List[DatasetInfo]:
@@ -779,6 +781,8 @@ class HfApi:
                 sort by ascending order.
             limit (:obj:`int`, `optional`):
                 The limit on the number of datasets fetched. Leaving this option to `None` fetches all datasets.
+            cardData (:obj:`bool`, `optional`):
+                Whether to grab the metadata for the dataset as well. Can contain useful information such as the PapersWithCode ID.
             full (:obj:`bool`, `optional`):
                 Whether to fetch all dataset data, including the `lastModified` and the `cardData`.
             use_auth_token (:obj:`bool` or :obj:`str`, `optional`):
@@ -818,7 +822,10 @@ class HfApi:
         if full is not None:
             if full:
                 params.update({"full": True})
-        r = requests.get(path, headers=headers, params=params)
+        if cardData is not None:
+            if cardData:
+                params.update({"full": True})
+        r = requests.get(path, params=params, headers=headers)
         r.raise_for_status()
         d = r.json()
         return [DatasetInfo(**x) for x in d]
@@ -1342,14 +1349,6 @@ class HfApi:
                 "If you passed a fileobj, make sure you've opened the file in binary mode."
             )
 
-        # Normalize path separators and strip leading slashes
-        if not REMOTE_FILEPATH_REGEX.match(path_in_repo):
-            raise ValueError(
-                "Invalid path_in_repo '{}', path_in_repo must match regex {}".format(
-                    path_in_repo, REMOTE_FILEPATH_REGEX.pattern
-                )
-            )
-
         if repo_type in REPO_TYPES_URL_PREFIXES:
             repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
 
@@ -1424,14 +1423,6 @@ class HfApi:
                     "`huggingface-cli login`."
                 )
 
-        # Normalize path separators and strip leading slashes
-        if not REMOTE_FILEPATH_REGEX.match(path_in_repo):
-            raise ValueError(
-                "Invalid path_in_repo '{}', path_in_repo must match regex {}".format(
-                    path_in_repo, REMOTE_FILEPATH_REGEX.pattern
-                )
-            )
-
         if repo_type in REPO_TYPES_URL_PREFIXES:
             repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
 
diff --git a/src/huggingface_hub/keras_mixin.py b/src/huggingface_hub/keras_mixin.py
index 6bea285e..a9ecb104 100644
--- a/src/huggingface_hub/keras_mixin.py
+++ b/src/huggingface_hub/keras_mixin.py
@@ -2,6 +2,7 @@ import json
 import logging
 import os
 from pathlib import Path
+from shutil import copytree
 from typing import Any, Dict, Optional, Union
 
 from huggingface_hub import ModelHubMixin
@@ -71,6 +72,7 @@ def push_to_hub_keras(
     model,
     repo_path_or_name: Optional[str] = None,
     repo_url: Optional[str] = None,
+    log_dir: Optional[str] = None,
     commit_message: Optional[str] = "Add model",
     organization: Optional[str] = None,
     private: Optional[bool] = None,
@@ -97,6 +99,9 @@ def push_to_hub_keras(
             Specify this in case you want to push to an existing repository in the hub. If unspecified, a new
             repository will be created in your namespace (unless you specify an :obj:`organization`) with
             :obj:`repo_name`.
+        log_dir (:obj:`str`, `optional`):
+            TensorBoard logging directory to be pushed. The Hub automatically hosts
+            and displays a TensorBoard instance if log files are included in the repository.
         commit_message (:obj:`str`, `optional`):
             Message to commit while pushing. Will default to :obj:`"add model"`.
         organization (:obj:`str`, `optional`):
@@ -173,7 +178,8 @@ def push_to_hub_keras(
         include_optimizer=include_optimizer,
         **model_save_kwargs,
     )
-
+    if log_dir is not None:
+        copytree(log_dir, f"{repo_path_or_name}/logs")
     # Commit and push!
     repo.git_add(auto_lfs_track=True)
     repo.git_commit(commit_message)


Test patch:
diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py
index a4bbec42..8452393f 100644
--- a/tests/test_hf_api.py
+++ b/tests/test_hf_api.py
@@ -292,20 +292,6 @@ class HfApiUploadFileTest(HfApiCommonTestWithLogin):
                 token=self._token,
             )
 
-        for (invalid_path, msg) in [
-            ("Remote\\README.md", "Has a backslash"),
-            ("/Remote/README.md", "Starts with a slash"),
-            ("Remote/../subtree/./README.md", "Has relative parts"),
-        ]:
-            with self.subTest(msg=msg):
-                with self.assertRaises(ValueError, msg="path_in_repo is invalid"):
-                    self._api.upload_file(
-                        path_or_fileobj=self.tmp_file,
-                        path_in_repo=invalid_path,
-                        repo_id=f"{USER}/{REPO_NAME}",
-                        token=self._token,
-                    )
-
     def test_upload_file_path(self):
         REPO_NAME = repo_name("path")
         self._api.create_repo(token=self._token, name=REPO_NAME)
@@ -672,6 +658,21 @@ class HfApiPublicTest(unittest.TestCase):
         self.assertGreater(len(datasets), 10)
         self.assertIsInstance(datasets[0], DatasetInfo)
 
+    @with_production_testing
+    def test_filter_datasets_with_cardData(self):
+        _api = HfApi()
+        datasets = _api.list_datasets(cardData=True)
+        self.assertGreater(
+            sum(
+                [getattr(dataset, "cardData", None) is not None for dataset in datasets]
+            ),
+            0,
+        )
+        datasets = _api.list_datasets()
+        self.assertTrue(
+            all([getattr(dataset, "cardData", None) is None for dataset in datasets])
+        )
+
     @with_production_testing
     def test_dataset_info(self):
         _api = HfApi()
@@ -780,6 +781,14 @@ class HfApiPublicTest(unittest.TestCase):
             ["pytorch" in model.tags and "tf" in model.tags for model in models]
         )
 
+    @with_production_testing
+    def test_filter_models_with_cardData(self):
+        _api = HfApi()
+        models = _api.list_models("co2_eq_emissions", cardData=True)
+        self.assertTrue([hasattr(model, "cardData") for model in models])
+        models = _api.list_models("co2_eq_emissions")
+        self.assertTrue(all([not hasattr(model, "cardData") for model in models]))
+
 
 class HfApiPrivateTest(HfApiCommonTestWithLogin):
     def setUp(self) -> None:
@@ -870,7 +879,7 @@ class HfLargefilesTest(HfApiCommonTest):
         REMOTE_URL = self._api.create_repo(
             name=self.REPO_NAME_LARGE_FILE,
             token=self._token,
-            lfsmultipartthresh=6 * 10 ** 6,
+            lfsmultipartthresh=6 * 10**6,
         )
         self.setup_local_clone(REMOTE_URL)
 
@@ -923,7 +932,7 @@ class HfLargefilesTest(HfApiCommonTest):
         REMOTE_URL = self._api.create_repo(
             name=self.REPO_NAME_LARGE_FILE,
             token=self._token,
-            lfsmultipartthresh=16 * 10 ** 6,
+            lfsmultipartthresh=16 * 10**6,
         )
         self.setup_local_clone(REMOTE_URL)
 
diff --git a/tests/test_hubmixin.py b/tests/test_hubmixin.py
index f9c1fa40..e9f673d5 100644
--- a/tests/test_hubmixin.py
+++ b/tests/test_hubmixin.py
@@ -49,7 +49,6 @@ if is_torch_available():
         def forward(self, x):
             return self.l1(x)
 
-
 else:
     DummyModel = None
 
diff --git a/tests/test_keras_integration.py b/tests/test_keras_integration.py
index d5ccbbe3..17a05044 100644
--- a/tests/test_keras_integration.py
+++ b/tests/test_keras_integration.py
@@ -58,7 +58,6 @@ if is_tf_available():
         def call(self, x):
             return self.l1(x)
 
-
 else:
     DummyModel = None
 
@@ -292,6 +291,32 @@ class HubKerasSequentialTest(HubMixingTestKeras):
 
         self._api.delete_repo(name=f"{REPO_NAME}", token=self._token)
 
+    def test_push_to_hub_tensorboard(self):
+        os.makedirs(f"{WORKING_REPO_DIR}/tb_log_dir")
+        with open(f"{WORKING_REPO_DIR}/tb_log_dir/tensorboard.txt", "w") as fp:
+            fp.write("Keras FTW")
+        REPO_NAME = repo_name("PUSH_TO_HUB")
+        model = self.model_init()
+        model.build((None, 2))
+        push_to_hub_keras(
+            model,
+            repo_path_or_name=f"{WORKING_REPO_DIR}/{REPO_NAME}",
+            log_dir=f"{WORKING_REPO_DIR}/tb_log_dir",
+            api_endpoint=ENDPOINT_STAGING,
+            use_auth_token=self._token,
+            git_user="ci",
+            git_email="ci@dummy.com",
+        )
+        model_info = HfApi(endpoint=ENDPOINT_STAGING).model_info(
+            f"{USER}/{REPO_NAME}",
+        )
+
+        self.assertTrue(
+            "logs/tensorboard.txt" in [f.rfilename for f in model_info.siblings]
+        )
+
+        self._api.delete_repo(name=f"{REPO_NAME}", token=self._token)
+
     def test_push_to_hub_model_kwargs(self):
         REPO_NAME = repo_name("PUSH_TO_HUB")
         model = self.model_init()