nafiz09 commited on
Commit
899ded7
1 Parent(s): 004cd5e

Training done

Browse files
Files changed (2) hide show
  1. added_tokens.json +2 -22
  2. tokenizer.json +1 -181
added_tokens.json CHANGED
@@ -1,28 +1,8 @@
1
  {
2
  "</s>": 32000,
3
- "<s_cord-v2>": 32025,
4
  "কুষ্টিয়া": 32001,
5
- "খূলণা": 32022,
6
- "চট্টমেটড়ো": 32006,
7
- "চট্টো": 32007,
8
- "চট্রো": 32016,
9
- "চুয়াডাঙ্গা": 32019,
10
- "জশর": 32023,
11
- "জোশোড়": 32004,
12
- "জোষোড়": 32010,
13
- "ঝিনাইদাহ": 32020,
14
- "টাংগাইল": 32005,
15
- "টাঞাঈল": 32012,
16
- "ঢাককা": 32024,
17
- "ধাকা": 32011,
18
  "নারায়ণগঞ্জ": 32003,
19
- "নারায়নগঞ্জ": 32008,
20
- "নোয়াখালী": 32021,
21
- "নড়াইল": 32009,
22
- "বগুড়া": 32017,
23
- "মেটড়ো": 32015,
24
  "ময়মনসিংহ": 32002,
25
- "যশোর": 32018,
26
- "রাজবাড়ী": 32013,
27
- "্ডহাকা": 32014
28
  }
 
1
  {
2
  "</s>": 32000,
3
+ "<s_cord-v2>": 32005,
4
  "কুষ্টিয়া": 32001,
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "নারায়ণগঞ্জ": 32003,
 
 
 
 
 
6
  "ময়মনসিংহ": 32002,
7
+ "রাজবাড়ী": 32004
 
 
8
  }
tokenizer.json CHANGED
@@ -100,87 +100,6 @@
100
  },
101
  {
102
  "id": 32004,
103
- "content": "জোশোড়",
104
- "single_word": false,
105
- "lstrip": false,
106
- "rstrip": false,
107
- "normalized": true,
108
- "special": false
109
- },
110
- {
111
- "id": 32005,
112
- "content": "টাংগাইল",
113
- "single_word": false,
114
- "lstrip": false,
115
- "rstrip": false,
116
- "normalized": true,
117
- "special": false
118
- },
119
- {
120
- "id": 32006,
121
- "content": "চট্টমেটড়ো",
122
- "single_word": false,
123
- "lstrip": false,
124
- "rstrip": false,
125
- "normalized": true,
126
- "special": false
127
- },
128
- {
129
- "id": 32007,
130
- "content": "চট্টো",
131
- "single_word": false,
132
- "lstrip": false,
133
- "rstrip": false,
134
- "normalized": true,
135
- "special": false
136
- },
137
- {
138
- "id": 32008,
139
- "content": "নারায়নগঞ্জ",
140
- "single_word": false,
141
- "lstrip": false,
142
- "rstrip": false,
143
- "normalized": true,
144
- "special": false
145
- },
146
- {
147
- "id": 32009,
148
- "content": "নড়াইল",
149
- "single_word": false,
150
- "lstrip": false,
151
- "rstrip": false,
152
- "normalized": true,
153
- "special": false
154
- },
155
- {
156
- "id": 32010,
157
- "content": "জোষোড়",
158
- "single_word": false,
159
- "lstrip": false,
160
- "rstrip": false,
161
- "normalized": true,
162
- "special": false
163
- },
164
- {
165
- "id": 32011,
166
- "content": "ধাকা",
167
- "single_word": false,
168
- "lstrip": false,
169
- "rstrip": false,
170
- "normalized": true,
171
- "special": false
172
- },
173
- {
174
- "id": 32012,
175
- "content": "টাঞাঈল",
176
- "single_word": false,
177
- "lstrip": false,
178
- "rstrip": false,
179
- "normalized": true,
180
- "special": false
181
- },
182
- {
183
- "id": 32013,
184
  "content": "রাজবাড়ী",
185
  "single_word": false,
186
  "lstrip": false,
@@ -189,106 +108,7 @@
189
  "special": false
190
  },
191
  {
192
- "id": 32014,
193
- "content": "্ডহাকা",
194
- "single_word": false,
195
- "lstrip": false,
196
- "rstrip": false,
197
- "normalized": true,
198
- "special": false
199
- },
200
- {
201
- "id": 32015,
202
- "content": "মেটড়ো",
203
- "single_word": false,
204
- "lstrip": false,
205
- "rstrip": false,
206
- "normalized": true,
207
- "special": false
208
- },
209
- {
210
- "id": 32016,
211
- "content": "চট্রো",
212
- "single_word": false,
213
- "lstrip": false,
214
- "rstrip": false,
215
- "normalized": true,
216
- "special": false
217
- },
218
- {
219
- "id": 32017,
220
- "content": "বগুড়া",
221
- "single_word": false,
222
- "lstrip": false,
223
- "rstrip": false,
224
- "normalized": true,
225
- "special": false
226
- },
227
- {
228
- "id": 32018,
229
- "content": "যশোর",
230
- "single_word": false,
231
- "lstrip": false,
232
- "rstrip": false,
233
- "normalized": true,
234
- "special": false
235
- },
236
- {
237
- "id": 32019,
238
- "content": "চুয়াডাঙ্গা",
239
- "single_word": false,
240
- "lstrip": false,
241
- "rstrip": false,
242
- "normalized": true,
243
- "special": false
244
- },
245
- {
246
- "id": 32020,
247
- "content": "ঝিনাইদাহ",
248
- "single_word": false,
249
- "lstrip": false,
250
- "rstrip": false,
251
- "normalized": true,
252
- "special": false
253
- },
254
- {
255
- "id": 32021,
256
- "content": "নোয়াখালী",
257
- "single_word": false,
258
- "lstrip": false,
259
- "rstrip": false,
260
- "normalized": true,
261
- "special": false
262
- },
263
- {
264
- "id": 32022,
265
- "content": "খূলণা",
266
- "single_word": false,
267
- "lstrip": false,
268
- "rstrip": false,
269
- "normalized": true,
270
- "special": false
271
- },
272
- {
273
- "id": 32023,
274
- "content": "জশর",
275
- "single_word": false,
276
- "lstrip": false,
277
- "rstrip": false,
278
- "normalized": true,
279
- "special": false
280
- },
281
- {
282
- "id": 32024,
283
- "content": "ঢাককা",
284
- "single_word": false,
285
- "lstrip": false,
286
- "rstrip": false,
287
- "normalized": true,
288
- "special": false
289
- },
290
- {
291
- "id": 32025,
292
  "content": "<s_cord-v2>",
293
  "single_word": false,
294
  "lstrip": false,
 
100
  },
101
  {
102
  "id": 32004,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  "content": "রাজবাড়ী",
104
  "single_word": false,
105
  "lstrip": false,
 
108
  "special": false
109
  },
110
  {
111
+ "id": 32005,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112
  "content": "<s_cord-v2>",
113
  "single_word": false,
114
  "lstrip": false,