ALIGHASEMI931 commited on
Commit
6084a88
1 Parent(s): cf6ad8b

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +2 -2
  2. vocab.json +80 -83
added_tokens.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "</s>": 125,
3
- "<s>": 124
4
  }
 
1
  {
2
+ "</s>": 122,
3
+ "<s>": 121
4
  }
vocab.json CHANGED
@@ -3,8 +3,8 @@
3
  "&": 2,
4
  "(": 3,
5
  ")": 4,
6
- "[PAD]": 123,
7
- "[UNK]": 122,
8
  "_": 5,
9
  "a": 6,
10
  "b": 7,
@@ -42,85 +42,82 @@
42
  "ئ": 38,
43
  "ا": 39,
44
  "ب": 40,
45
- "ة": 41,
46
- "ت": 42,
47
- "ث": 43,
48
- "ج": 44,
49
- "ح": 45,
50
- "خ": 46,
51
- "د": 47,
52
- "ذ": 48,
53
- "ر": 49,
54
- "ز": 50,
55
- "س": 51,
56
- "ش": 52,
57
- "ص": 53,
58
- "ض": 54,
59
- "ط": 55,
60
- "ظ": 56,
61
- "ع": 57,
62
- "غ": 58,
63
- "ـ": 59,
64
- "ف": 60,
65
- "ق": 61,
66
- "ك": 62,
67
- "ل": 63,
68
- "م": 64,
69
- "ن": 65,
70
- "ه": 66,
71
- "و": 67,
72
- "ى": 68,
73
- "ي": 69,
74
- "ً": 70,
75
- "ٌ": 71,
76
- "َ": 72,
77
- "ُ": 73,
78
- "ِ": 74,
79
- "ّ": 75,
80
- "ْ": 76,
81
- "ٔ": 77,
82
- "٬": 78,
83
- "پ": 79,
84
- "چ": 80,
85
- "ژ": 81,
86
- "ک": 82,
87
- "گ": 83,
88
- "ۀ": 84,
89
- "ی": 85,
90
- "ے": 86,
91
- "": 87,
92
- "": 88,
93
- "": 89,
94
- "": 90,
95
- "": 91,
96
- "": 92,
97
- "": 93,
98
- "ﯿ": 94,
99
- "": 95,
100
- "": 96,
101
- "": 97,
102
- "": 98,
103
- "": 99,
104
- "": 100,
105
- "": 101,
106
- "": 102,
107
- "": 103,
108
- "": 104,
109
- "": 105,
110
- "": 106,
111
- "": 107,
112
- "": 108,
113
- "": 109,
114
- "": 110,
115
- "": 111,
116
- "": 112,
117
- "": 113,
118
- "": 114,
119
- "": 115,
120
- "": 116,
121
- "": 117,
122
- "": 118,
123
- "ﻪ": 119,
124
- "ﻭ": 120,
125
- "ﻮ": 121
126
  }
 
3
  "&": 2,
4
  "(": 3,
5
  ")": 4,
6
+ "[PAD]": 120,
7
+ "[UNK]": 119,
8
  "_": 5,
9
  "a": 6,
10
  "b": 7,
 
42
  "ئ": 38,
43
  "ا": 39,
44
  "ب": 40,
45
+ "ت": 41,
46
+ "ث": 42,
47
+ "ج": 43,
48
+ "ح": 44,
49
+ "خ": 45,
50
+ "د": 46,
51
+ "ذ": 47,
52
+ "ر": 48,
53
+ "ز": 49,
54
+ "س": 50,
55
+ "ش": 51,
56
+ "ص": 52,
57
+ "ض": 53,
58
+ "ط": 54,
59
+ "ظ": 55,
60
+ "ع": 56,
61
+ "غ": 57,
62
+ "ـ": 58,
63
+ "ف": 59,
64
+ "ق": 60,
65
+ "ل": 61,
66
+ "م": 62,
67
+ "ن": 63,
68
+ "ه": 64,
69
+ "و": 65,
70
+ "ى": 66,
71
+ "ً": 67,
72
+ "ٌ": 68,
73
+ "َ": 69,
74
+ "ُ": 70,
75
+ "ِ": 71,
76
+ "ّ": 72,
77
+ "ْ": 73,
78
+ "ٔ": 74,
79
+ "٬": 75,
80
+ "پ": 76,
81
+ "چ": 77,
82
+ "ژ": 78,
83
+ "ک": 79,
84
+ "گ": 80,
85
+ "ۀ": 81,
86
+ "ی": 82,
87
+ "ے": 83,
88
+ "": 84,
89
+ "": 85,
90
+ "": 86,
91
+ "": 87,
92
+ "": 88,
93
+ "": 89,
94
+ "": 90,
95
+ "ﯿ": 91,
96
+ "": 92,
97
+ "": 93,
98
+ "": 94,
99
+ "": 95,
100
+ "": 96,
101
+ "": 97,
102
+ "": 98,
103
+ "": 99,
104
+ "": 100,
105
+ "": 101,
106
+ "": 102,
107
+ "": 103,
108
+ "": 104,
109
+ "": 105,
110
+ "": 106,
111
+ "": 107,
112
+ "": 108,
113
+ "": 109,
114
+ "": 110,
115
+ "": 111,
116
+ "": 112,
117
+ "": 113,
118
+ "": 114,
119
+ "": 115,
120
+ "": 116,
121
+ "": 117,
122
+ "": 118
 
 
 
123
  }