File size: 384 Bytes
c766a08 bce41d0 c766a08 293bad6 c766a08 |
1 2 3 4 5 6 7 8 9 |
"""
## 默认 use_fast=True 报错
lib\site-packages\transformers\tokenization_utils_fast.py", line 504, in _batch_encode_plus
encodings = self._tokenizer.encode_batch(
pyo3_runtime.PanicException: AddedVocabulary bad split
"""
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("lmsys/fastchat-t5-3b-v1.0", trust_remote_code=True, use_fast=False) |