MaghribLAB commited on
Commit
95d687e
1 Parent(s): 29ce73f

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +56 -56
vocab.json CHANGED
@@ -1,60 +1,60 @@
1
  {
2
- "'": 38,
3
- "(": 7,
4
- ")": 54,
5
- "0": 16,
6
- "1": 47,
7
- "2": 42,
8
- "3": 9,
9
- "5": 4,
10
- "6": 46,
11
- "7": 44,
12
  "[PAD]": 57,
13
  "[UNK]": 56,
14
- "a": 10,
15
- "b": 37,
16
- "c": 11,
17
- "d": 13,
18
- "e": 12,
19
- "f": 2,
20
- "g": 15,
21
- "h": 53,
22
- "i": 48,
23
- "j": 3,
24
- "k": 34,
25
- "l": 27,
26
- "m": 28,
27
- "n": 51,
28
- "o": 31,
29
- "p": 41,
30
- "q": 49,
31
- "r": 33,
32
- "s": 29,
33
- "t": 25,
34
- "u": 0,
35
- "v": 45,
36
- "w": 17,
37
- "x": 26,
38
- "y": 36,
39
- "z": 18,
40
- "|": 30,
41
- "²": 14,
42
- "à": 21,
43
- "â": 5,
44
- "ç": 35,
45
- "è": 24,
46
- "é": 20,
47
- "ê": 1,
48
- "î": 19,
49
- "ô": 52,
50
- "ù": 40,
51
- "û": 50,
52
- "ď": 39,
53
- "đ": 23,
54
- "ġ": 8,
55
- "š": 32,
56
- "ŧ": 22,
57
- "ž": 6,
58
- "ԑ": 55,
59
- "ḥ": 43
60
  }
 
1
  {
2
+ "'": 41,
3
+ "(": 11,
4
+ ")": 46,
5
+ "0": 2,
6
+ "1": 38,
7
+ "2": 36,
8
+ "3": 12,
9
+ "5": 16,
10
+ "6": 4,
11
+ "7": 43,
12
  "[PAD]": 57,
13
  "[UNK]": 56,
14
+ "a": 26,
15
+ "b": 8,
16
+ "c": 54,
17
+ "d": 20,
18
+ "e": 10,
19
+ "f": 47,
20
+ "g": 48,
21
+ "h": 15,
22
+ "i": 32,
23
+ "j": 49,
24
+ "k": 1,
25
+ "l": 24,
26
+ "m": 6,
27
+ "n": 45,
28
+ "o": 19,
29
+ "p": 9,
30
+ "q": 0,
31
+ "r": 42,
32
+ "s": 44,
33
+ "t": 40,
34
+ "u": 33,
35
+ "v": 30,
36
+ "w": 29,
37
+ "x": 13,
38
+ "y": 53,
39
+ "z": 14,
40
+ "|": 50,
41
+ "²": 7,
42
+ "à": 37,
43
+ "â": 35,
44
+ "ç": 27,
45
+ "è": 21,
46
+ "é": 34,
47
+ "ê": 5,
48
+ "î": 17,
49
+ "ô": 55,
50
+ "ù": 3,
51
+ "û": 18,
52
+ "ď": 22,
53
+ "đ": 31,
54
+ "ġ": 51,
55
+ "š": 39,
56
+ "ŧ": 52,
57
+ "ž": 25,
58
+ "ԑ": 28,
59
+ "ḥ": 23
60
  }