MaghribLAB commited on
Commit
1e2a327
1 Parent(s): 95d687e

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +55 -55
vocab.json CHANGED
@@ -1,60 +1,60 @@
1
  {
2
- "'": 41,
3
- "(": 11,
4
- ")": 46,
5
- "0": 2,
6
- "1": 38,
7
- "2": 36,
8
- "3": 12,
9
- "5": 16,
10
- "6": 4,
11
- "7": 43,
12
  "[PAD]": 57,
13
  "[UNK]": 56,
14
- "a": 26,
15
- "b": 8,
16
- "c": 54,
17
- "d": 20,
18
- "e": 10,
19
- "f": 47,
20
- "g": 48,
21
- "h": 15,
22
- "i": 32,
23
- "j": 49,
24
- "k": 1,
25
- "l": 24,
26
- "m": 6,
27
- "n": 45,
28
- "o": 19,
29
- "p": 9,
30
- "q": 0,
31
- "r": 42,
32
- "s": 44,
33
- "t": 40,
34
- "u": 33,
35
- "v": 30,
36
  "w": 29,
37
- "x": 13,
38
- "y": 53,
39
- "z": 14,
40
- "|": 50,
41
- "²": 7,
42
- "à": 37,
43
- "â": 35,
44
- "ç": 27,
45
- "è": 21,
46
- "é": 34,
47
- "ê": 5,
48
- "î": 17,
49
- "ô": 55,
50
- "ù": 3,
51
- "û": 18,
52
- "ď": 22,
53
- "đ": 31,
54
- "ġ": 51,
55
- "š": 39,
56
- "ŧ": 52,
57
- "ž": 25,
58
- "ԑ": 28,
59
- "ḥ": 23
60
  }
 
1
  {
2
+ "'": 52,
3
+ "(": 13,
4
+ ")": 0,
5
+ "0": 17,
6
+ "1": 5,
7
+ "2": 44,
8
+ "3": 51,
9
+ "5": 6,
10
+ "6": 30,
11
+ "7": 38,
12
  "[PAD]": 57,
13
  "[UNK]": 56,
14
+ "a": 16,
15
+ "b": 34,
16
+ "c": 4,
17
+ "d": 24,
18
+ "e": 15,
19
+ "f": 9,
20
+ "g": 26,
21
+ "h": 20,
22
+ "i": 19,
23
+ "j": 8,
24
+ "k": 3,
25
+ "l": 32,
26
+ "m": 27,
27
+ "n": 18,
28
+ "o": 48,
29
+ "p": 45,
30
+ "q": 14,
31
+ "r": 28,
32
+ "s": 12,
33
+ "t": 23,
34
+ "u": 25,
35
+ "v": 7,
36
  "w": 29,
37
+ "x": 50,
38
+ "y": 43,
39
+ "z": 41,
40
+ "|": 53,
41
+ "²": 36,
42
+ "à": 54,
43
+ "â": 39,
44
+ "ç": 21,
45
+ "è": 46,
46
+ "é": 2,
47
+ "ê": 35,
48
+ "î": 42,
49
+ "ô": 10,
50
+ "ù": 47,
51
+ "û": 31,
52
+ "ď": 40,
53
+ "đ": 55,
54
+ "ġ": 1,
55
+ "š": 49,
56
+ "ŧ": 37,
57
+ "ž": 33,
58
+ "ԑ": 22,
59
+ "ḥ": 11
60
  }