ad019el commited on
Commit
16957f6
1 Parent(s): 321ba90

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +10 -11
vocab.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
- "[PAD]": 29,
3
- "[UNK]": 28,
4
  "a": 5,
5
  "b": 18,
6
- "c": 21,
7
- "d": 24,
8
  "e": 11,
9
  "f": 1,
10
  "g": 6,
@@ -15,18 +15,17 @@
15
  "l": 0,
16
  "m": 15,
17
  "n": 17,
18
- "o": 27,
19
  "p": 8,
20
- "q": 26,
21
  "r": 16,
22
- "s": 22,
23
  "t": 3,
24
  "u": 12,
25
- "v": 25,
26
- "w": 23,
27
  "x": 9,
28
  "y": 7,
29
  "z": 2,
30
- "|": 14,
31
- " ": 20
32
  }
 
1
  {
2
+ "[PAD]": 28,
3
+ "[UNK]": 27,
4
  "a": 5,
5
  "b": 18,
6
+ "c": 20,
7
+ "d": 23,
8
  "e": 11,
9
  "f": 1,
10
  "g": 6,
 
15
  "l": 0,
16
  "m": 15,
17
  "n": 17,
18
+ "o": 26,
19
  "p": 8,
20
+ "q": 25,
21
  "r": 16,
22
+ "s": 21,
23
  "t": 3,
24
  "u": 12,
25
+ "v": 24,
26
+ "w": 22,
27
  "x": 9,
28
  "y": 7,
29
  "z": 2,
30
+ "|": 14
 
31
  }