Fix tokenizer added_tokens format and numpy compatibility

app.py

@@ -29,7 +29,7 @@ def log_runtime_versions() -> None:
 
 
 def sanitize_added_tokens_file(added_tokens_file: str) -> None:
-    """Normalize malformed added_tokens.json to list format expected by tokenizers."""
+    """Normalize added_tokens.json to dict format expected by slow tokenizers."""
     if not os.path.exists(added_tokens_file):
         return
 
@@ -37,15 +37,15 @@ def sanitize_added_tokens_file(added_tokens_file: str) -> None:
         with open(added_tokens_file, "r", encoding="utf-8") as fp:
             data = json.load(fp)
     except Exception:
-        data = []
+        data = {}
 
-    if isinstance(data, list):
-        sanitized = [item for item in data if isinstance(item, str)]
-    elif isinstance(data, dict):
-        # Some repos store empty/object payloads here; tokenizer expects a list.
-        sanitized = [key for key in data.keys() if isinstance(key, str)]
+    if isinstance(data, dict):
+        sanitized = {k: v for k, v in data.items() if isinstance(k, str) and isinstance(v, int)}
+    elif isinstance(data, list):
+        # If a list was written by older workaround, convert to empty mapping.
+        sanitized = {}
     else:
-        sanitized = []
+        sanitized = {}
 
     with open(added_tokens_file, "w", encoding="utf-8") as fp:
         json.dump(sanitized, fp, ensure_ascii=True)

requirements.txt

@@ -1,6 +1,7 @@
 transformers==4.40.2
 huggingface_hub==0.36.2
 torch==2.1.2
+numpy<2
 sentencepiece>=0.1.96
 accelerate>=0.20.0
 datasets>=2.0.0