Remove added_tokens file before tokenizer init

app.py

@@ -1,16 +1,33 @@
 import gradio as gr
+import os
+from huggingface_hub import snapshot_download
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, RobertaTokenizer
 import torch
 
 # 加载 CodeT5+ 模型
 model_name = "Salesforce/codet5p-220m"
+
+
+def prepare_local_model(repo_id: str, local_dir: str = "./model_cache") -> str:
+    snapshot_download(repo_id=repo_id, local_dir=local_dir)
+
+    # Work around a transformers/tokenizers incompatibility for this repo.
+    # Its added_tokens.json is an empty dict, which can crash tokenizer init in some versions.
+    added_tokens_file = os.path.join(local_dir, "added_tokens.json")
+    if os.path.exists(added_tokens_file):
+        os.remove(added_tokens_file)
+
+    return local_dir
+
+
+local_model_dir = prepare_local_model(model_name)
 try:
-    tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, trust_remote_code=False)
+    tokenizer = AutoTokenizer.from_pretrained(local_model_dir, use_fast=False, trust_remote_code=False)
 except Exception:
     # Fallback to explicit slow tokenizer class to bypass tokenizers fast-path issues.
-    tokenizer = RobertaTokenizer.from_pretrained(model_name, trust_remote_code=False)
+    tokenizer = RobertaTokenizer.from_pretrained(local_model_dir, trust_remote_code=False)
 
-model = AutoModelForSeq2SeqLM.from_pretrained(model_name, trust_remote_code=False)
+model = AutoModelForSeq2SeqLM.from_pretrained(local_model_dir, trust_remote_code=False)
 
 def generate_code(prompt: str, max_length: int = 128) -> str:
     """代码生成/补全"""

requirements.txt

@@ -1,4 +1,5 @@
 transformers>=4.40.0
+huggingface_hub>=0.23.0
 torch>=2.0.0
 sentencepiece>=0.1.96
 accelerate>=0.20.0