Repository: huggingface/huggingface_hub
Pull request #2155 (2024-03-25 15:19:34)

Fix HF_ENDPOINT not handled correctly
Fix https://github.com/huggingface/huggingface_hub/issues/2152. Related to https://github.com/huggingface/huggingface_hub/pull/2120 and to this [internal slack thread](https://huggingface.slack.com/archives/C0626AL5AR4/p1711370113894589).

Uploading an LFS file fails because RepoUrl(...).endpoint was not correctly overwritten with HF_ENDPOINT. We were only overwriting `RepoUrl(...).endpoint` but not the inner value `str(RepoUrl(...))`. This PR definitely fixes this by replacing it before parsing the string value. cc @padeoe.

---

Tips from @padeoe to reproduce the issue:
1. Add this line to `/etc/hosts` or `` to deactivate traffic to huggingace.co
```
127.0.0.1 [huggingface.co](http://huggingface.co/)
```

2. Run this script
```py
import os


os.environ['HF_ENDPOINT'] = "https://hf-mirror.com"

from huggingface_hub import HfApi


api = HfApi() # optionally add your token here
repo_url = api.create_repo("tmp_test_mir", exist_ok=True)
repo_id = repo_url.repo_id
api.upload_file(repo_id=repo_id, path_or_fileobj=b"content", path_in_repo="file.txt")
api.upload_file(repo_id=repo_id, path_or_fileobj=b"content", path_in_repo="lfs.bin")
```



HuggingFaceDocBuilderDev: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/huggingface_hub/pr_2155). All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

Patch:
diff --git a/src/huggingface_hub/hf_api.py b/src/huggingface_hub/hf_api.py
index 5b00c489..85872de5 100644
--- a/src/huggingface_hub/hf_api.py
+++ b/src/huggingface_hub/hf_api.py
@@ -185,11 +185,6 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
     """
     input_hf_id = hf_id
 
-    # check if a proxy has been set => if yes, update the returned URL to use the proxy
-    if ENDPOINT not in (_HF_DEFAULT_ENDPOINT, _HF_DEFAULT_STAGING_ENDPOINT):
-        hf_id = hf_id.replace(_HF_DEFAULT_ENDPOINT, ENDPOINT)
-        hf_id = hf_id.replace(_HF_DEFAULT_STAGING_ENDPOINT, ENDPOINT)
-
     hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
     is_hf_url = hub_url in hf_id and "@" not in hf_id
 
@@ -436,6 +431,11 @@ class RepoUrl(str):
     """
 
     def __new__(cls, url: Any, endpoint: Optional[str] = None):
+        # check if a proxy has been set => if yes, update the returned URL to use the proxy
+        if ENDPOINT not in (_HF_DEFAULT_ENDPOINT, _HF_DEFAULT_STAGING_ENDPOINT):
+            url = url.replace(_HF_DEFAULT_ENDPOINT, ENDPOINT)
+            url = url.replace(_HF_DEFAULT_STAGING_ENDPOINT, ENDPOINT)
+
         return super(RepoUrl, cls).__new__(cls, url)
 
     def __init__(self, url: Any, endpoint: Optional[str] = None) -> None:
diff --git a/src/huggingface_hub/hub_mixin.py b/src/huggingface_hub/hub_mixin.py
index fa9ee5e7..eb42ef4e 100644
--- a/src/huggingface_hub/hub_mixin.py
+++ b/src/huggingface_hub/hub_mixin.py
@@ -403,7 +403,12 @@ class ModelHubMixin:
                 # Forward config to model initialization
                 model_kwargs["config"] = config
 
-            if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()):
+            # Inject config if `**kwargs` are expected
+            if is_dataclass(cls):
+                for key in cls.__dataclass_fields__:
+                    if key not in model_kwargs and key in config:
+                        model_kwargs[key] = config[key]
+            elif any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()):
                 for key, value in config.items():
                     if key not in model_kwargs:
                         model_kwargs[key] = value


Test patch:
diff --git a/tests/test_hub_mixin.py b/tests/test_hub_mixin.py
index 09dd91c5..0047bdd4 100644
--- a/tests/test_hub_mixin.py
+++ b/tests/test_hub_mixin.py
@@ -99,6 +99,28 @@ class DummyModelSavingConfig(ModelHubMixin):
         (save_directory / "config.json").write_text(json.dumps({"custom_config": "custom_config"}))
 
 
+@dataclass
+class DummyModelThatIsAlsoADataclass(ModelHubMixin):
+    foo: int
+    bar: str
+
+    @classmethod
+    def _from_pretrained(
+        cls,
+        *,
+        model_id: str,
+        revision: Optional[str],
+        cache_dir: Optional[Union[str, Path]],
+        force_download: bool,
+        proxies: Optional[Dict],
+        resume_download: bool,
+        local_files_only: bool,
+        token: Optional[Union[str, bool]],
+        **model_kwargs,
+    ):
+        return cls(**model_kwargs)
+
+
 @pytest.mark.usefixtures("fx_cache_dir")
 class HubMixinTest(unittest.TestCase):
     cache_dir: Path
@@ -328,3 +350,18 @@ class HubMixinTest(unittest.TestCase):
         # config.json IS overwritten
         with open(self.cache_dir / "config.json") as f:
             assert json.load(f) == {"a": 1, "b": 2}
+
+    def test_from_pretrained_when_cls_is_a_dataclass(self):
+        """Regression test for #2157.
+
+        When the ModelHubMixin class happens to be a dataclass, `__init__` method will accept `**kwargs` when
+        inspecting it. However, due to how dataclasses work, we cannot forward arbitrary kwargs to the `__init__`.
+        This test ensures that the `from_pretrained` method does not raise an error when the class is a dataclass.
+
+        See https://github.com/huggingface/huggingface_hub/issues/2157.
+        """
+        (self.cache_dir / "config.json").write_text('{"foo": 42, "bar": "baz", "other": "value"}')
+        model = DummyModelThatIsAlsoADataclass.from_pretrained(self.cache_dir)
+        assert model.foo == 42
+        assert model.bar == "baz"
+        assert not hasattr(model, "other")