Repository: huggingface/huggingface_hub

Issue:
List all datasets or models using token
As of now, `list_models` and `list_datasets` don't use the token, so they only list public models. If we could pass the token here, it would be very useful to do things such as retrieving all datasets (including private) of an organization.

cc @muellerzr @LysandreJik 

https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/hf_api.py#L511


Discussion:
At one point we thought on this, now that there's a need would be a decent idea to implement that again. Can put that on my list :) 
Good issue! This would require a `use_auth_token` parameter rather than a `token` parameter that would automatically fetch the current logged-in user's token, as we could envision users wanting to use `list_{models,datasets}` without the private models.

The `use_auth_token` would behave similarly to others in the library, i.e., `None` by default, if `True` will fetch the current logged-in user's token, and if it is a string then will use that as the token.

Let me know if you want to tackle that @muellerzr otherwise I'll do it in a bit.
@LysandreJik shouldn't take too too long to implement, so I can do that quickly this morning :)

Resolving patch:
diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py
--- a/src/huggingface_hub/hf_api.py
+++ b/src/huggingface_hub/hf_api.py
@@ -518,6 +518,7 @@ def list_models(
         limit: Optional[int] = None,
         full: Optional[bool] = None,
         fetch_config: Optional[bool] = None,
+        use_auth_token: Optional[Union[bool, str]] = None,
     ) -> List[ModelInfo]:
         """
         Get the public list of all the models on huggingface.co
@@ -592,9 +593,24 @@ def list_models(
                 This is set to `True` by default when using a filter.
             fetch_config (:obj:`bool`, `optional`):
                 Whether to fetch the model configs as well. This is not included in `full` due to its size.
-
+            use_auth_token (:obj:`bool` or :obj:`str`, `optional`):
+                Whether to use the `auth_token` provided from the `huggingface_hub` cli. If not logged in,
+                a valid `auth_token` can be passed in as a string.
         """
         path = f"{self.endpoint}/api/models"
+        if use_auth_token:
+            if isinstance(use_auth_token, str):
+                if not self._is_valid_token(use_auth_token):
+                    raise ValueError("Invalid token passed!")
+                token = use_auth_token
+            else:
+                token = HfFolder.get_token()
+                if token is None:
+                    raise EnvironmentError(
+                        "You need to provide a `token` to `use_auth_token` or be logged in to Hugging Face with "
+                        "`huggingface-cli login`."
+                    )
+        headers = {"authorization": f"Bearer {token}"} if use_auth_token else None
         params = {}
         if filter is not None:
             if isinstance(filter, ModelFilter):
@@ -619,7 +635,7 @@ def list_models(
                 del params["full"]
         if fetch_config is not None:
             params.update({"config": fetch_config})
-        r = requests.get(path, params=params)
+        r = requests.get(path, headers=headers, params=params)
         r.raise_for_status()
         d = r.json()
         return [ModelInfo(**x) for x in d]
@@ -693,6 +709,7 @@ def list_datasets(
         direction: Optional[Literal[-1]] = None,
         limit: Optional[int] = None,
         full: Optional[bool] = None,
+        use_auth_token: Optional[str] = None,
     ) -> List[DatasetInfo]:
         """
         Get the public list of all the datasets on huggingface.co
@@ -764,9 +781,24 @@ def list_datasets(
                 The limit on the number of datasets fetched. Leaving this option to `None` fetches all datasets.
             full (:obj:`bool`, `optional`):
                 Whether to fetch all dataset data, including the `lastModified` and the `cardData`.
-
+            use_auth_token (:obj:`bool` or :obj:`str`, `optional`):
+                Whether to use the `auth_token` provided from the `huggingface_hub` cli. If not logged in,
+                a valid `auth_token` can be passed in as a string.
         """
         path = f"{self.endpoint}/api/datasets"
+        if use_auth_token:
+            if isinstance(use_auth_token, str):
+                if not self._is_valid_token(use_auth_token):
+                    raise ValueError("Invalid token passed!")
+                token = use_auth_token
+            else:
+                token = HfFolder.get_token()
+                if token is None:
+                    raise EnvironmentError(
+                        "You need to provide a `token` to `use_auth_token` or be logged in to Hugging Face with "
+                        "`huggingface-cli login`."
+                    )
+        headers = {"authorization": f"Bearer {token}"} if use_auth_token else None
         params = {}
         if filter is not None:
             if isinstance(filter, DatasetFilter):
@@ -786,7 +818,7 @@ def list_datasets(
         if full is not None:
             if full:
                 params.update({"full": True})
-        r = requests.get(path, params=params)
+        r = requests.get(path, headers=headers, params=params)
         r.raise_for_status()
         d = r.json()
         return [DatasetInfo(**x) for x in d]


Test patch:
diff --git a/tests/test_hf_api.py b/tests/test_hf_api.py
--- a/tests/test_hf_api.py
+++ b/tests/test_hf_api.py
@@ -801,6 +801,18 @@ def test_model_info(self):
         )
         self.assertIsInstance(model_info, ModelInfo)
 
+    @with_production_testing
+    def test_list_private_models(self):
+        orig = len(self._api.list_datasets())
+        new = len(self._api.list_datasets(use_auth_token=self._token))
+        self.assertGreater(new, orig)
+
+    @with_production_testing
+    def test_list_private_datasets(self):
+        orig = len(self._api.list_models())
+        new = len(self._api.list_models(use_auth_token=self._token))
+        self.assertGreater(new, orig)
+
 
 class HfFolderTest(unittest.TestCase):
     def test_token_workflow(self):
@@ -858,7 +870,7 @@ def test_end_to_end_thresh_6M(self):
         REMOTE_URL = self._api.create_repo(
             name=self.REPO_NAME_LARGE_FILE,
             token=self._token,
-            lfsmultipartthresh=6 * 10 ** 6,
+            lfsmultipartthresh=6 * 10**6,
         )
         self.setup_local_clone(REMOTE_URL)
 
@@ -911,7 +923,7 @@ def test_end_to_end_thresh_16M(self):
         REMOTE_URL = self._api.create_repo(
             name=self.REPO_NAME_LARGE_FILE,
             token=self._token,
-            lfsmultipartthresh=16 * 10 ** 6,
+            lfsmultipartthresh=16 * 10**6,
         )
         self.setup_local_clone(REMOTE_URL)