updated modeling

modeling_glap.py

@@ -19,6 +19,11 @@ from einops import rearrange
 from einops.layers.torch import Rearrange
 from transformers import PreTrainedModel
 
+try:
+    from huggingface_hub import hf_hub_download
+except ImportError:
+    hf_hub_download = None  # type: ignore[assignment,misc]
+
 from .configuration_glap import GlapConfig
 
 
@@ -815,17 +820,38 @@ class GlapModel(PreTrainedModel):
 
     def _get_tokenizer(self) -> NllbTokenizer:
         if self.tokenizer is None:
-            # Find the model directory: HuggingFace copies .py files to its cache
-            # but not .model files, so we use config._name_or_path (the original
-            # model path) to locate the tokenizer.
+            tokenizer_filename = "sentencepiece.source.256000.model"
+            tokenizer_path: Optional[Path | str] = None
+
+            # 1. Check config._name_or_path (local directory)
             model_dir = Path(self.config._name_or_path)
-            if not model_dir.is_dir():
-                model_dir = Path(__file__).parent
-            tokenizer_path = model_dir / "sentencepiece.source.256000.model"
-            if not tokenizer_path.exists():
-                tokenizer_path = (
-                    Path(__file__).parent / "sentencepiece.source.256000.model"
+            candidate = model_dir / tokenizer_filename
+            if candidate.exists():
+                tokenizer_path = candidate
+
+            # 2. Check next to this file (modules cache / local install)
+            if tokenizer_path is None:
+                candidate = Path(__file__).parent / tokenizer_filename
+                if candidate.exists():
+                    tokenizer_path = candidate
+
+            # 3. Download from HuggingFace Hub
+            if tokenizer_path is None and hf_hub_download is not None:
+                try:
+                    tokenizer_path = hf_hub_download(
+                        repo_id=self.config._name_or_path,
+                        filename=tokenizer_filename,
+                    )
+                except Exception:
+                    pass
+
+            if tokenizer_path is None:
+                raise FileNotFoundError(
+                    f"Could not find {tokenizer_filename}. "
+                    f"Searched {self.config._name_or_path} and "
+                    f"{Path(__file__).parent}."
                 )
+
             self.tokenizer = NllbTokenizer(tokenizer_path)
         return self.tokenizer