Repository: huggingface/huggingface_hub
Pull request #1906 (2023-12-14 10:36:47)

Send bearer auth in LFS upload
Switch from Basic auth to Bearer token in authorization header when uploading an LFS file. OAuth tokens don't work with basic auth which means uploading LFS files currently fails. See related [slack thread](https://huggingface.slack.com/archives/C02EMARJ65P/p1702485214619679) (private).
I suspect we were using Basic auth to be compatible with `git lfs` command line? In any case, the part of the code changed in the PR is not used by `huggingface-cli lfs`. 

cc @coyotte508 

HuggingFaceDocBuilderDev: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/huggingface_hub/pr_1906). All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

Patch:
diff --git a/src/huggingface_hub/lfs.py b/src/huggingface_hub/lfs.py
index a1a3ac69..709ae719 100644
--- a/src/huggingface_hub/lfs.py
+++ b/src/huggingface_hub/lfs.py
@@ -26,12 +26,17 @@ from pathlib import Path
 from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, List, Optional, Tuple, TypedDict
 from urllib.parse import unquote
 
-from requests.auth import HTTPBasicAuth
-
 from huggingface_hub.constants import ENDPOINT, HF_HUB_ENABLE_HF_TRANSFER, REPO_TYPES_URL_PREFIXES
 from huggingface_hub.utils import get_session
 
-from .utils import get_token_to_send, hf_raise_for_status, http_backoff, logging, tqdm, validate_hf_hub_args
+from .utils import (
+    build_hf_headers,
+    hf_raise_for_status,
+    http_backoff,
+    logging,
+    tqdm,
+    validate_hf_hub_args,
+)
 from .utils.sha import sha256, sha_fileobj
 
 
@@ -149,15 +154,8 @@ def post_lfs_batch_info(
     }
     if revision is not None:
         payload["ref"] = {"name": unquote(revision)}  # revision has been previously 'quoted'
-    resp = get_session().post(
-        batch_url,
-        headers=LFS_HEADERS,
-        json=payload,
-        auth=HTTPBasicAuth(
-            "access_token",
-            get_token_to_send(token or True),  # type: ignore  # Token must be provided or retrieved
-        ),
-    )
+    headers = {**LFS_HEADERS, **build_hf_headers(token=token or True)}  # Token must be provided or retrieved
+    resp = get_session().post(batch_url, headers=headers, json=payload)
     hf_raise_for_status(resp)
     batch_info = resp.json()
 
@@ -236,7 +234,7 @@ def lfs_upload(operation: "CommitOperationAdd", lfs_batch_action: Dict, token: O
         _validate_lfs_action(verify_action)
         verify_resp = get_session().post(
             verify_action["href"],
-            auth=HTTPBasicAuth(username="USER", password=get_token_to_send(token or True)),  # type: ignore
+            headers=build_hf_headers(token=token or True),
             json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size},
         )
         hf_raise_for_status(verify_resp)
diff --git a/src/huggingface_hub/utils/_errors.py b/src/huggingface_hub/utils/_errors.py
index 47afab1c..787495a1 100644
--- a/src/huggingface_hub/utils/_errors.py
+++ b/src/huggingface_hub/utils/_errors.py
@@ -1,11 +1,27 @@
+import re
 from typing import Optional
 
 from requests import HTTPError, Response
 
-from ..constants import INFERENCE_ENDPOINTS_ENDPOINT
 from ._fixes import JSONDecodeError
 
 
+REPO_API_REGEX = re.compile(
+    r"""
+        # staging or production endpoint
+        ^https://(hub-ci.)?huggingface.co
+        (
+            # on /api/repo_type/repo_id
+            /api/(models|datasets|spaces)/(.+)
+            |
+            # or /repo_id/resolve/revision/...
+            /(.+)/resolve/(.+)
+        )
+    """,
+    flags=re.VERBOSE,
+)
+
+
 class FileMetadataError(OSError):
     """Error triggered when the metadata of a file on the Hub cannot be retrieved (missing ETag or commit_hash).
 
@@ -285,25 +301,12 @@ def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None)
             )
             raise GatedRepoError(message, response) from e
 
-        elif (
-            response.status_code == 401
-            and response.request.url is not None
-            and "/api/collections" in response.request.url
-        ):
-            # Collection not found. We don't raise a custom error for this.
-            # This prevent from raising a misleading `RepositoryNotFoundError` (see below).
-            pass
-
-        elif (
+        elif error_code == "RepoNotFound" or (
             response.status_code == 401
+            and response.request is not None
             and response.request.url is not None
-            and INFERENCE_ENDPOINTS_ENDPOINT in response.request.url
+            and REPO_API_REGEX.search(response.request.url) is not None
         ):
-            # Not enough permission to list Inference Endpoints from this org. We don't raise a custom error for this.
-            # This prevent from raising a misleading `RepositoryNotFoundError` (see below).
-            pass
-
-        elif error_code == "RepoNotFound" or response.status_code == 401:
             # 401 is misleading as it is returned for:
             #    - private and gated repos if user is not authenticated
             #    - missing repos


Test patch:
diff --git a/tests/test_utils_errors.py b/tests/test_utils_errors.py
index b0362a19..65117f86 100644
--- a/tests/test_utils_errors.py
+++ b/tests/test_utils_errors.py
@@ -1,8 +1,10 @@
 import unittest
 
+import pytest
 from requests.models import PreparedRequest, Response
 
 from huggingface_hub.utils._errors import (
+    REPO_API_REGEX,
     BadRequestError,
     EntryNotFoundError,
     HfHubHTTPError,
@@ -23,17 +25,30 @@ class TestErrorUtils(unittest.TestCase):
         self.assertEqual(context.exception.response.status_code, 404)
         self.assertIn("Request ID: 123", str(context.exception))
 
-    def test_hf_raise_for_status_repo_not_found_without_error_code(self) -> None:
+    def test_hf_raise_for_status_401_repo_url(self) -> None:
         response = Response()
         response.headers = {"X-Request-Id": 123}
         response.status_code = 401
         response.request = PreparedRequest()
+        response.request.url = "https://huggingface.co/api/models/username/reponame"
         with self.assertRaisesRegex(RepositoryNotFoundError, "Repository Not Found") as context:
             hf_raise_for_status(response)
 
         self.assertEqual(context.exception.response.status_code, 401)
         self.assertIn("Request ID: 123", str(context.exception))
 
+    def test_hf_raise_for_status_401_not_repo_url(self) -> None:
+        response = Response()
+        response.headers = {"X-Request-Id": 123}
+        response.status_code = 401
+        response.request = PreparedRequest()
+        response.request.url = "https://huggingface.co/api/collections"
+        with self.assertRaises(HfHubHTTPError) as context:
+            hf_raise_for_status(response)
+
+        self.assertEqual(context.exception.response.status_code, 401)
+        self.assertIn("Request ID: 123", str(context.exception))
+
     def test_hf_raise_for_status_revision_not_found(self) -> None:
         response = Response()
         response.headers = {"X-Error-Code": "RevisionNotFound", "X-Request-Id": 123}
@@ -239,3 +254,44 @@ class TestHfHubHTTPError(unittest.TestCase):
             "this is a message\n\nError message duplicated in headers and body.",
         )
         self.assertEqual(error.server_message, "Error message duplicated in headers and body.")
+
+
+@pytest.mark.parametrize(
+    ("url", "should_match"),
+    [
+        # Listing endpoints => False
+        ("https://huggingface.co/api/models", False),
+        ("https://huggingface.co/api/datasets", False),
+        ("https://huggingface.co/api/spaces", False),
+        # Create repo endpoint => False
+        ("https://huggingface.co/api/repos/create", False),
+        # Collection endpoints => False
+        ("https://huggingface.co/api/collections", False),
+        ("https://huggingface.co/api/collections/foo/bar", False),
+        # Repo endpoints => True
+        ("https://huggingface.co/api/models/repo_id", True),
+        ("https://huggingface.co/api/datasets/repo_id", True),
+        ("https://huggingface.co/api/spaces/repo_id", True),
+        ("https://huggingface.co/api/models/username/repo_name/refs/main", True),
+        ("https://huggingface.co/api/datasets/username/repo_name/refs/main", True),
+        ("https://huggingface.co/api/spaces/username/repo_name/refs/main", True),
+        # Inference Endpoint => False
+        ("https://api.endpoints.huggingface.cloud/v2/endpoint/namespace", False),
+        # Staging Endpoint => True
+        ("https://hub-ci.huggingface.co/api/models/repo_id", True),
+        ("https://hub-ci.huggingface.co/api/datasets/repo_id", True),
+        ("https://hub-ci.huggingface.co/api/spaces/repo_id", True),
+        # /resolve Endpoint => True
+        ("https://huggingface.co/gpt2/resolve/main/README.md", True),
+        ("https://huggingface.co/datasets/google/fleurs/resolve/revision/README.md", True),
+        # Regression tests
+        ("https://huggingface.co/bert-base/resolve/main/pytorch_model.bin", True),
+        ("https://hub-ci.huggingface.co/__DUMMY_USER__/repo-1470b5/resolve/main/file.txt", True),
+    ],
+)
+def test_repo_api_regex(url: str, should_match: bool) -> None:
+    """Test the regex used to match repo API URLs."""
+    if should_match:
+        assert REPO_API_REGEX.match(url)
+    else:
+        assert REPO_API_REGEX.match(url) is None