rashmi035 commited on
Commit
a480eef
1 Parent(s): 016d2ad

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -3
  2. special_tokens_map.json +0 -7
  3. vocab.json +1 -1
added_tokens.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "</s>": 82,
3
- "<s>": 81,
4
- "[PAD]": 83
5
  }
 
1
  {
2
+ "</s>": 83,
3
+ "<s>": 82
 
4
  }
special_tokens_map.json CHANGED
@@ -13,13 +13,6 @@
13
  "normalized": true,
14
  "rstrip": false,
15
  "single_word": false
16
- },
17
- {
18
- "content": "[PAD]",
19
- "lstrip": false,
20
- "normalized": true,
21
- "rstrip": false,
22
- "single_word": false
23
  }
24
  ],
25
  "bos_token": "<s>",
 
13
  "normalized": true,
14
  "rstrip": false,
15
  "single_word": false
 
 
 
 
 
 
 
16
  }
17
  ],
18
  "bos_token": "<s>",
vocab.json CHANGED
@@ -12,7 +12,7 @@
12
  "M": 10,
13
  "W": 11,
14
  "[PAD]": 81,
15
- "[UNK]": 81,
16
  "a": 12,
17
  "e": 13,
18
  "i": 14,
 
12
  "M": 10,
13
  "W": 11,
14
  "[PAD]": 81,
15
+ "[UNK]": 80,
16
  "a": 12,
17
  "e": 13,
18
  "i": 14,