vitvit commited on
Commit
b268c72
โ€ข
1 Parent(s): d4ae5f6

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +7 -0
README.md CHANGED
@@ -18,6 +18,13 @@ from transformers import AutoModelForMaskedLM, AutoTokenizer
18
 
19
  tokenizer = AutoTokenizer.from_pretrained('HeNLP/LongHeRo')
20
  model = AutoModelForMaskedLM.from_pretrained('HeNLP/LongHeRo')
 
 
 
 
 
 
 
21
  ```
22
 
23
 
 
18
 
19
  tokenizer = AutoTokenizer.from_pretrained('HeNLP/LongHeRo')
20
  model = AutoModelForMaskedLM.from_pretrained('HeNLP/LongHeRo')
21
+
22
+ # Tokenization Example:
23
+ # Encoding and tokenizing
24
+ encoded_string = tokenizer('ืฉืœื•ื ืœื›ื•ืœื')
25
+
26
+ # Decoding
27
+ decoded_string = tokenizer.decode(encoded_string['input_ids'], skip_special_tokens=True)
28
  ```
29
 
30