YiTokenizer doesn't exist
In [1]: from transformers import AutoTokenizer
In [2]: tokenizer = AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-34B")
ValueError Traceback (most recent call last)
Cell In[2], line 1
----> 1 tokenizer = AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-34B")
File /databricks/python3/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py:688, in AutoTokenizer.from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs)
686 tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
687 if tokenizer_class is None:
--> 688 raise ValueError(
689 f"Tokenizer class {tokenizer_class_candidate} does not exist or is not currently imported."
690 )
691 return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
693 # Otherwise we have to be creative.
694 # if model is an encoder decoder, the encoder tokenizer class is used by default
ValueError: Tokenizer class YiTokenizer does not exist or is not currently imported.
Setting AutoTokenizer as AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-34B", trust_remote_code=True) worked for me 👍
yes, but it does make it harder to add the model to the Huggingface LLM leaderboard benchmark
https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard