Aye10032 commited on
Commit
d0fcbbf
1 Parent(s): e6a1260

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +1 -1
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "50000": {
28
  "content": "|<PAD>|",
29
  "lstrip": false,
30
  "normalized": false,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "3": {
28
  "content": "|<PAD>|",
29
  "lstrip": false,
30
  "normalized": false,