MaghribLAB commited on
Commit
1a4872d
1 Parent(s): 1e2a327

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +55 -55
vocab.json CHANGED
@@ -1,60 +1,60 @@
1
  {
2
- "'": 52,
3
- "(": 13,
4
- ")": 0,
5
- "0": 17,
6
- "1": 5,
7
- "2": 44,
8
- "3": 51,
9
- "5": 6,
10
- "6": 30,
11
- "7": 38,
12
  "[PAD]": 57,
13
  "[UNK]": 56,
14
- "a": 16,
15
- "b": 34,
16
- "c": 4,
17
- "d": 24,
18
- "e": 15,
19
- "f": 9,
20
  "g": 26,
21
- "h": 20,
22
- "i": 19,
23
- "j": 8,
24
- "k": 3,
25
- "l": 32,
26
- "m": 27,
27
- "n": 18,
28
- "o": 48,
29
- "p": 45,
30
- "q": 14,
31
- "r": 28,
32
- "s": 12,
33
- "t": 23,
34
- "u": 25,
35
- "v": 7,
36
- "w": 29,
37
- "x": 50,
38
- "y": 43,
39
- "z": 41,
40
- "|": 53,
41
- "²": 36,
42
- "à": 54,
43
- "â": 39,
44
- "ç": 21,
45
- "è": 46,
46
- "é": 2,
47
- "ê": 35,
48
- "î": 42,
49
- "ô": 10,
50
- "ù": 47,
51
- "û": 31,
52
- "ď": 40,
53
- "đ": 55,
54
- "ġ": 1,
55
- "š": 49,
56
- "ŧ": 37,
57
- "ž": 33,
58
- "ԑ": 22,
59
- "ḥ": 11
60
  }
 
1
  {
2
+ "'": 41,
3
+ "(": 47,
4
+ ")": 51,
5
+ "0": 3,
6
+ "1": 45,
7
+ "2": 16,
8
+ "3": 24,
9
+ "5": 5,
10
+ "6": 12,
11
+ "7": 30,
12
  "[PAD]": 57,
13
  "[UNK]": 56,
14
+ "a": 31,
15
+ "b": 54,
16
+ "c": 22,
17
+ "d": 21,
18
+ "e": 32,
19
+ "f": 29,
20
  "g": 26,
21
+ "h": 9,
22
+ "i": 20,
23
+ "j": 7,
24
+ "k": 27,
25
+ "l": 35,
26
+ "m": 50,
27
+ "n": 36,
28
+ "o": 38,
29
+ "p": 11,
30
+ "q": 44,
31
+ "r": 17,
32
+ "s": 6,
33
+ "t": 8,
34
+ "u": 28,
35
+ "v": 15,
36
+ "w": 10,
37
+ "x": 43,
38
+ "y": 14,
39
+ "z": 53,
40
+ "|": 0,
41
+ "²": 52,
42
+ "à": 18,
43
+ "â": 37,
44
+ "ç": 4,
45
+ "è": 23,
46
+ "é": 49,
47
+ "ê": 13,
48
+ "î": 34,
49
+ "ô": 1,
50
+ "ù": 33,
51
+ "û": 42,
52
+ "ď": 19,
53
+ "đ": 25,
54
+ "ġ": 2,
55
+ "š": 55,
56
+ "ŧ": 40,
57
+ "ž": 39,
58
+ "ԑ": 46,
59
+ "ḥ": 48
60
  }