Repository: xorbitsai/inference

Issue:
ENH: auto retry download on network errors
### Is your feature request related to a problem? Please describe
#301 describes a common problem when downloading large models.

### Describe the solution you'd like
Auto retry on network errors. Enable`resume_download` so that the previous progress won't be lost.




Discussion:

Resolving patch:
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -26,6 +26,8 @@
 
 logger = logging.getLogger(__name__)
 
+MAX_ATTEMPTS = 3
+
 
 class GgmlLLMSpecV1(BaseModel):
     model_format: Literal["ggmlv3"]
@@ -228,22 +230,51 @@ def cache_from_huggingface(
 
     if llm_spec.model_format == "pytorch":
         assert isinstance(llm_spec, PytorchLLMSpecV1)
-        huggingface_hub.snapshot_download(
-            llm_spec.model_id,
-            revision=llm_spec.model_revision,
-            local_dir=cache_dir,
-            local_dir_use_symlinks=True,
-        )
+
+        for current_attempt in range(1, MAX_ATTEMPTS + 1):
+            try:
+                huggingface_hub.snapshot_download(
+                    llm_spec.model_id,
+                    revision=llm_spec.model_revision,
+                    local_dir=cache_dir,
+                    local_dir_use_symlinks=True,
+                )
+                break
+            except huggingface_hub.utils.LocalEntryNotFoundError:
+                remaining_attempts = MAX_ATTEMPTS - current_attempt
+                logger.warning(
+                    f"Attempt {current_attempt} failed. Remaining attempts: {remaining_attempts}"
+                )
+
+        else:
+            raise RuntimeError(
+                f"Failed to download model '{llm_spec.model_name}' (size: {llm_spec.model_size}, format: {llm_spec.model_format}) after multiple retries"
+            )
+
     elif llm_spec.model_format == "ggmlv3":
         assert isinstance(llm_spec, GgmlLLMSpecV1)
         file_name = llm_spec.model_file_name_template.format(quantization=quantization)
-        huggingface_hub.hf_hub_download(
-            llm_spec.model_id,
-            revision=llm_spec.model_revision,
-            filename=file_name,
-            local_dir=cache_dir,
-            local_dir_use_symlinks=True,
-        )
+
+        for current_attempt in range(1, MAX_ATTEMPTS + 1):
+            try:
+                huggingface_hub.hf_hub_download(
+                    llm_spec.model_id,
+                    revision=llm_spec.model_revision,
+                    filename=file_name,
+                    local_dir=cache_dir,
+                    local_dir_use_symlinks=True,
+                )
+                break
+            except huggingface_hub.utils.LocalEntryNotFoundError:
+                remaining_attempts = MAX_ATTEMPTS - current_attempt
+                logger.warning(
+                    f"Attempt {current_attempt} failed. Remaining attempts: {remaining_attempts}"
+                )
+
+        else:
+            raise RuntimeError(
+                f"Failed to download model '{llm_spec.model_name}' (size: {llm_spec.model_size}, format: {llm_spec.model_format}) after multiple retries"
+            )
 
     return cache_dir
 


Test patch: