IEIT-Yuan commited on
Commit
df53be1
1 Parent(s): ec708fb

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +1 -1
README.md CHANGED
@@ -42,7 +42,7 @@ sys.path.append(
42
  from transformers import AutoModelForCausalLM,AutoTokenizer,LlamaTokenizer
43
 
44
  print("Creat tokenizer...")
45
- tokenizer = LlamaTokenizer.from_pretrained('IEITYuan/Yuan2-2B-hf', add_eos_token=False, add_bos_token=False, eos_token='<eod>')
46
  tokenizer.add_tokens(['<sep>', '<pad>', '<mask>', '<predict>', '<FIM_SUFFIX>', '<FIM_PREFIX>', '<FIM_MIDDLE>','<commit_before>','<commit_msg>','<commit_after>','<jupyter_start>','<jupyter_text>','<jupyter_code>','<jupyter_output>','<empty_output>'], special_tokens=True)
47
 
48
  print("Creat model...")
 
42
  from transformers import AutoModelForCausalLM,AutoTokenizer,LlamaTokenizer
43
 
44
  print("Creat tokenizer...")
45
+ tokenizer = LlamaTokenizer.from_pretrained('IEITYuan/Yuan2-51B-hf', add_eos_token=False, add_bos_token=False, eos_token='<eod>')
46
  tokenizer.add_tokens(['<sep>', '<pad>', '<mask>', '<predict>', '<FIM_SUFFIX>', '<FIM_PREFIX>', '<FIM_MIDDLE>','<commit_before>','<commit_msg>','<commit_after>','<jupyter_start>','<jupyter_text>','<jupyter_code>','<jupyter_output>','<empty_output>'], special_tokens=True)
47
 
48
  print("Creat model...")