nihaal-subhash-7 commited on
Commit
1c9e7f4
1 Parent(s): 684d864

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +9 -0
tokenizer.json CHANGED
@@ -38,6 +38,15 @@
38
  "rstrip": false,
39
  "normalized": true,
40
  "special": false
 
 
 
 
 
 
 
 
 
41
  }
42
  ],
43
  "normalizer": {
 
38
  "rstrip": false,
39
  "normalized": true,
40
  "special": false
41
+ },
42
+ {
43
+ "id": 32001,
44
+ "content": "[PAD]",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
  }
51
  ],
52
  "normalizer": {