Hugging Face's logo

Zaid
/
ashaar_tokenizer

Model card Files Community
Zaid commited on
Commit
cd10bb4
•
1 Parent(s): e436494

Upload tokenizer

Browse files
Files changed (5) hide show
  1. merges.txt +1 -0
  2. special_tokens_map.json +5 -0
  3. tokenizer.json +283 -0
  4. tokenizer_config.json +9 -0
  5. vocab.json +1 -0
merges.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ #version: 0.2
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "unk_token": "<|endoftext|>"
5
+ }
tokenizer.json ADDED
@@ -0,0 +1,283 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<|endoftext|>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<|vsep|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "<|bsep|>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<|pad|>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "|psep|",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 5,
53
+ "content": "<|meter_0|>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ },
60
+ {
61
+ "id": 6,
62
+ "content": "<|meter_1|>",
63
+ "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
+ },
69
+ {
70
+ "id": 7,
71
+ "content": "<|meter_2|>",
72
+ "single_word": false,
73
+ "lstrip": false,
74
+ "rstrip": false,
75
+ "normalized": false,
76
+ "special": true
77
+ },
78
+ {
79
+ "id": 8,
80
+ "content": "<|meter_3|>",
81
+ "single_word": false,
82
+ "lstrip": false,
83
+ "rstrip": false,
84
+ "normalized": false,
85
+ "special": true
86
+ },
87
+ {
88
+ "id": 9,
89
+ "content": "<|meter_4|>",
90
+ "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": false,
94
+ "special": true
95
+ },
96
+ {
97
+ "id": 10,
98
+ "content": "<|meter_5|>",
99
+ "single_word": false,
100
+ "lstrip": false,
101
+ "rstrip": false,
102
+ "normalized": false,
103
+ "special": true
104
+ },
105
+ {
106
+ "id": 11,
107
+ "content": "<|meter_6|>",
108
+ "single_word": false,
109
+ "lstrip": false,
110
+ "rstrip": false,
111
+ "normalized": false,
112
+ "special": true
113
+ },
114
+ {
115
+ "id": 12,
116
+ "content": "<|meter_7|>",
117
+ "single_word": false,
118
+ "lstrip": false,
119
+ "rstrip": false,
120
+ "normalized": false,
121
+ "special": true
122
+ },
123
+ {
124
+ "id": 13,
125
+ "content": "<|meter_8|>",
126
+ "single_word": false,
127
+ "lstrip": false,
128
+ "rstrip": false,
129
+ "normalized": false,
130
+ "special": true
131
+ },
132
+ {
133
+ "id": 14,
134
+ "content": "<|meter_9|>",
135
+ "single_word": false,
136
+ "lstrip": false,
137
+ "rstrip": false,
138
+ "normalized": false,
139
+ "special": true
140
+ },
141
+ {
142
+ "id": 15,
143
+ "content": "<|meter_10|>",
144
+ "single_word": false,
145
+ "lstrip": false,
146
+ "rstrip": false,
147
+ "normalized": false,
148
+ "special": true
149
+ },
150
+ {
151
+ "id": 16,
152
+ "content": "<|meter_11|>",
153
+ "single_word": false,
154
+ "lstrip": false,
155
+ "rstrip": false,
156
+ "normalized": false,
157
+ "special": true
158
+ },
159
+ {
160
+ "id": 17,
161
+ "content": "<|meter_12|>",
162
+ "single_word": false,
163
+ "lstrip": false,
164
+ "rstrip": false,
165
+ "normalized": false,
166
+ "special": true
167
+ },
168
+ {
169
+ "id": 18,
170
+ "content": "<|meter_13|>",
171
+ "single_word": false,
172
+ "lstrip": false,
173
+ "rstrip": false,
174
+ "normalized": false,
175
+ "special": true
176
+ }
177
+ ],
178
+ "normalizer": null,
179
+ "pre_tokenizer": {
180
+ "type": "UnicodeScripts"
181
+ },
182
+ "post_processor": null,
183
+ "decoder": null,
184
+ "model": {
185
+ "type": "BPE",
186
+ "dropout": null,
187
+ "unk_token": null,
188
+ "continuing_subword_prefix": null,
189
+ "end_of_word_suffix": null,
190
+ "fuse_unk": false,
191
+ "byte_fallback": false,
192
+ "vocab": {
193
+ "<|endoftext|>": 0,
194
+ "<|vsep|>": 1,
195
+ "<|bsep|>": 2,
196
+ "<|pad|>": 3,
197
+ "|psep|": 4,
198
+ "<|meter_0|>": 5,
199
+ "<|meter_1|>": 6,
200
+ "<|meter_2|>": 7,
201
+ "<|meter_3|>": 8,
202
+ "<|meter_4|>": 9,
203
+ "<|meter_5|>": 10,
204
+ "<|meter_6|>": 11,
205
+ "<|meter_7|>": 12,
206
+ "<|meter_8|>": 13,
207
+ "<|meter_9|>": 14,
208
+ "<|meter_10|>": 15,
209
+ "<|meter_11|>": 16,
210
+ "<|meter_12|>": 17,
211
+ "<|meter_13|>": 18,
212
+ " ": 19,
213
+ "0": 20,
214
+ "1": 21,
215
+ "2": 22,
216
+ "3": 23,
217
+ "4": 24,
218
+ "5": 25,
219
+ "6": 26,
220
+ "7": 27,
221
+ "8": 28,
222
+ "9": 29,
223
+ ":": 30,
224
+ "<": 31,
225
+ ">": 32,
226
+ "_": 33,
227
+ "b": 34,
228
+ "e": 35,
229
+ "m": 36,
230
+ "p": 37,
231
+ "r": 38,
232
+ "s": 39,
233
+ "t": 40,
234
+ "v": 41,
235
+ "|": 42,
236
+ "ء": 43,
237
+ "آ": 44,
238
+ "أ": 45,
239
+ "ؤ": 46,
240
+ "إ": 47,
241
+ "ئ": 48,
242
+ "ا": 49,
243
+ "ب": 50,
244
+ "ة": 51,
245
+ "ت": 52,
246
+ "ث": 53,
247
+ "ج": 54,
248
+ "ح": 55,
249
+ "خ": 56,
250
+ "د": 57,
251
+ "ذ": 58,
252
+ "ر": 59,
253
+ "ز": 60,
254
+ "س": 61,
255
+ "ش": 62,
256
+ "ص": 63,
257
+ "ض": 64,
258
+ "ط": 65,
259
+ "ظ": 66,
260
+ "ع": 67,
261
+ "غ": 68,
262
+ "ف": 69,
263
+ "ق": 70,
264
+ "ك": 71,
265
+ "ل": 72,
266
+ "م": 73,
267
+ "ن": 74,
268
+ "ه": 75,
269
+ "و": 76,
270
+ "ى": 77,
271
+ "ي": 78,
272
+ "ً": 79,
273
+ "ٌ": 80,
274
+ "ٍ": 81,
275
+ "َ": 82,
276
+ "ُ": 83,
277
+ "ِ": 84,
278
+ "ّ": 85,
279
+ "ْ": 86
280
+ },
281
+ "merges": []
282
+ }
283
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "eos_token": "<|endoftext|>",
6
+ "model_max_length": 1000000000000000019884624838656,
7
+ "tokenizer_class": "GPT2Tokenizer",
8
+ "unk_token": "<|endoftext|>"
9
+ }
vocab.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"<|endoftext|>":0,"<|vsep|>":1,"<|bsep|>":2,"<|pad|>":3,"|psep|":4,"<|meter_0|>":5,"<|meter_1|>":6,"<|meter_2|>":7,"<|meter_3|>":8,"<|meter_4|>":9,"<|meter_5|>":10,"<|meter_6|>":11,"<|meter_7|>":12,"<|meter_8|>":13,"<|meter_9|>":14,"<|meter_10|>":15,"<|meter_11|>":16,"<|meter_12|>":17,"<|meter_13|>":18," ":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,"<":31,">":32,"_":33,"b":34,"e":35,"m":36,"p":37,"r":38,"s":39,"t":40,"v":41,"|":42,"ء":43,"آ":44,"أ":45,"ؤ":46,"إ":47,"ئ":48,"ا":49,"ب":50,"ة":51,"ت":52,"ث":53,"ج":54,"ح":55,"خ":56,"د":57,"ذ":58,"ر":59,"ز":60,"س":61,"ش":62,"ص":63,"ض":64,"ط":65,"ظ":66,"ع":67,"غ":68,"ف":69,"ق":70,"ك":71,"ل":72,"م":73,"ن":74,"ه":75,"و":76,"ى":77,"ي":78,"ً":79,"ٌ":80,"ٍ":81,"َ":82,"ُ":83,"ِ":84,"ّ":85,"ْ":86}