Merge branch 'main' of https://huggingface.co/IEITYuan/Yuan2-2B-Janus-hf into main

README.md

@@ -42,7 +42,7 @@ sys.path.append(
 from transformers import AutoModelForCausalLM,AutoTokenizer,LlamaTokenizer
 
 print("Creat tokenizer...")
-tokenizer = LlamaTokenizer.from_pretrained('IEITYuan/Yuan2-2B-hf', add_eos_token=False, add_bos_token=False, eos_token='<eod>')
+tokenizer = LlamaTokenizer.from_pretrained('IEITYuan/Yuan2-2B-Janus-hf', add_eos_token=False, add_bos_token=False, eos_token='<eod>')
 tokenizer.add_tokens(['<sep>', '<pad>', '<mask>', '<predict>', '<FIM_SUFFIX>', '<FIM_PREFIX>', '<FIM_MIDDLE>','<commit_before>','<commit_msg>','<commit_after>','<jupyter_start>','<jupyter_text>','<jupyter_code>','<jupyter_output>','<empty_output>'], special_tokens=True)
 
 print("Creat model...")