ad019el commited on
Commit
53b0207
1 Parent(s): 81d75b1

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +17 -18
vocab.json CHANGED
@@ -1,48 +1,47 @@
1
  {
2
- "[PAD]": 45,
3
- "[UNK]": 44,
4
  "|": 27,
5
- "ĉ": 28,
6
  "̣": 8,
7
- "ء": 37,
8
  "آ": 18,
9
  "أ": 5,
10
- "ؤ": 34,
11
  "إ": 0,
12
  "ئ": 4,
13
  "ا": 3,
14
  "ب": 2,
15
- "ة": 33,
16
  "ت": 1,
17
  "ث": 12,
18
  "ج": 9,
19
- "ح": 32,
20
  "خ": 26,
21
  "د": 25,
22
  "ذ": 20,
23
  "ر": 13,
24
  "ز": 24,
25
- "س": 30,
26
- "ش": 38,
27
  "ص": 19,
28
  "ض": 17,
29
- "ط": 41,
30
- "ظ": 29,
31
  "ع": 15,
32
  "غ": 23,
33
  "ف": 10,
34
  "ق": 16,
35
- "ك": 31,
36
  "ل": 22,
37
  "م": 11,
38
  "ن": 6,
39
- "ه": 39,
40
- "و": 35,
41
  "ى": 21,
42
- "ي": 36,
43
- "ٱ": 40,
44
  "چ": 14,
45
- "ڤ": 43,
46
  "ک": 7,
47
- "ی": 42
48
  }
 
1
  {
2
+ "[PAD]": 44,
3
+ "[UNK]": 43,
4
  "|": 27,
 
5
  "̣": 8,
6
+ "ء": 36,
7
  "آ": 18,
8
  "أ": 5,
9
+ "ؤ": 33,
10
  "إ": 0,
11
  "ئ": 4,
12
  "ا": 3,
13
  "ب": 2,
14
+ "ة": 32,
15
  "ت": 1,
16
  "ث": 12,
17
  "ج": 9,
18
+ "ح": 31,
19
  "خ": 26,
20
  "د": 25,
21
  "ذ": 20,
22
  "ر": 13,
23
  "ز": 24,
24
+ "س": 29,
25
+ "ش": 37,
26
  "ص": 19,
27
  "ض": 17,
28
+ "ط": 40,
29
+ "ظ": 28,
30
  "ع": 15,
31
  "غ": 23,
32
  "ف": 10,
33
  "ق": 16,
34
+ "ك": 30,
35
  "ل": 22,
36
  "م": 11,
37
  "ن": 6,
38
+ "ه": 38,
39
+ "و": 34,
40
  "ى": 21,
41
+ "ي": 35,
42
+ "ٱ": 39,
43
  "چ": 14,
44
+ "ڤ": 42,
45
  "ک": 7,
46
+ "ی": 41
47
  }