dirkgr commited on
Commit
e3c82a0
1 Parent(s): 16fd943

Update README.md

Browse files

Trailing space breaks tokenization

Files changed (1) hide show
  1. README.md +1 -1
README.md CHANGED
@@ -75,7 +75,7 @@ Proceed as usual with HuggingFace:
75
  from transformers import AutoModelForCausalLM, AutoTokenizer
76
  olmo = AutoModelForCausalLM.from_pretrained("allenai/OLMo-7B-0424-hf")
77
  tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-7B-0424-hf")
78
- message = ["Language modeling is "]
79
  inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
80
  # optional verifying cuda
81
  # inputs = {k: v.to('cuda') for k,v in inputs.items()}
 
75
  from transformers import AutoModelForCausalLM, AutoTokenizer
76
  olmo = AutoModelForCausalLM.from_pretrained("allenai/OLMo-7B-0424-hf")
77
  tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-7B-0424-hf")
78
+ message = ["Language modeling is"]
79
  inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
80
  # optional verifying cuda
81
  # inputs = {k: v.to('cuda') for k,v in inputs.items()}