Rasi1610 commited on
Commit
00c66a1
1 Parent(s): 550e0ef

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +23 -21
  2. tokenizer.json +38 -20
  3. tokenizer_config.json +36 -20
added_tokens.json CHANGED
@@ -1,36 +1,38 @@
1
  {
2
- "</s_Age>": 57546,
3
  "</s_City>": 57532,
4
- "</s_Father>": 57552,
5
- "</s_Gender>": 57538,
6
- "</s_Marital Status>": 57542,
7
- "</s_Mother>": 57554,
8
- "</s_Place of birth>": 57548,
9
- "</s_Race>": 57540,
10
  "</s_State file #>": 57528,
11
- "</s_date_of_birth>": 57544,
 
12
  "</s_date_of_death>": 57534,
13
  "</s_name>": 57530,
14
  "</s_person>": 57526,
15
- "</s_person_data>": 57536,
16
- "</s_relation>": 57550,
17
- "<s_Age>": 57545,
18
  "<s_City>": 57531,
19
- "<s_Father>": 57551,
20
- "<s_Gender>": 57537,
21
- "<s_Marital Status>": 57541,
22
- "<s_Mother>": 57553,
23
- "<s_Place of birth>": 57547,
24
- "<s_Race>": 57539,
25
  "<s_State file #>": 57527,
26
- "<s_cord-v2>": 57555,
27
- "<s_date_of_birth>": 57543,
 
28
  "<s_date_of_death>": 57533,
29
  "<s_iitcdip>": 57523,
30
  "<s_name>": 57529,
31
  "<s_person>": 57525,
32
- "<s_person_data>": 57535,
33
- "<s_relation>": 57549,
34
  "<s_synthdog>": 57524,
35
  "<sep/>": 57522
36
  }
 
1
  {
2
+ "</s_Age>": 57548,
3
  "</s_City>": 57532,
4
+ "</s_Father>": 57554,
5
+ "</s_Gender>": 57540,
6
+ "</s_Marital Status>": 57544,
7
+ "</s_Mother>": 57556,
8
+ "</s_Place of birth>": 57550,
9
+ "</s_Race>": 57542,
10
  "</s_State file #>": 57528,
11
+ "</s_county>": 57536,
12
+ "</s_date_of_birth>": 57546,
13
  "</s_date_of_death>": 57534,
14
  "</s_name>": 57530,
15
  "</s_person>": 57526,
16
+ "</s_person_data>": 57538,
17
+ "</s_relation>": 57552,
18
+ "<s_Age>": 57547,
19
  "<s_City>": 57531,
20
+ "<s_Father>": 57553,
21
+ "<s_Gender>": 57539,
22
+ "<s_Marital Status>": 57543,
23
+ "<s_Mother>": 57555,
24
+ "<s_Place of birth>": 57549,
25
+ "<s_Race>": 57541,
26
  "<s_State file #>": 57527,
27
+ "<s_cord-v2>": 57557,
28
+ "<s_county>": 57535,
29
+ "<s_date_of_birth>": 57545,
30
  "<s_date_of_death>": 57533,
31
  "<s_iitcdip>": 57523,
32
  "<s_name>": 57529,
33
  "<s_person>": 57525,
34
+ "<s_person_data>": 57537,
35
+ "<s_relation>": 57551,
36
  "<s_synthdog>": 57524,
37
  "<sep/>": 57522
38
  }
tokenizer.json CHANGED
@@ -181,7 +181,7 @@
181
  },
182
  {
183
  "id": 57535,
184
- "content": "<s_person_data>",
185
  "single_word": false,
186
  "lstrip": false,
187
  "rstrip": false,
@@ -190,7 +190,7 @@
190
  },
191
  {
192
  "id": 57536,
193
- "content": "</s_person_data>",
194
  "single_word": false,
195
  "lstrip": false,
196
  "rstrip": false,
@@ -199,7 +199,7 @@
199
  },
200
  {
201
  "id": 57537,
202
- "content": "<s_Gender>",
203
  "single_word": false,
204
  "lstrip": false,
205
  "rstrip": false,
@@ -208,7 +208,7 @@
208
  },
209
  {
210
  "id": 57538,
211
- "content": "</s_Gender>",
212
  "single_word": false,
213
  "lstrip": false,
214
  "rstrip": false,
@@ -217,7 +217,7 @@
217
  },
218
  {
219
  "id": 57539,
220
- "content": "<s_Race>",
221
  "single_word": false,
222
  "lstrip": false,
223
  "rstrip": false,
@@ -226,7 +226,7 @@
226
  },
227
  {
228
  "id": 57540,
229
- "content": "</s_Race>",
230
  "single_word": false,
231
  "lstrip": false,
232
  "rstrip": false,
@@ -235,7 +235,7 @@
235
  },
236
  {
237
  "id": 57541,
238
- "content": "<s_Marital Status>",
239
  "single_word": false,
240
  "lstrip": false,
241
  "rstrip": false,
@@ -244,7 +244,7 @@
244
  },
245
  {
246
  "id": 57542,
247
- "content": "</s_Marital Status>",
248
  "single_word": false,
249
  "lstrip": false,
250
  "rstrip": false,
@@ -253,7 +253,7 @@
253
  },
254
  {
255
  "id": 57543,
256
- "content": "<s_date_of_birth>",
257
  "single_word": false,
258
  "lstrip": false,
259
  "rstrip": false,
@@ -262,7 +262,7 @@
262
  },
263
  {
264
  "id": 57544,
265
- "content": "</s_date_of_birth>",
266
  "single_word": false,
267
  "lstrip": false,
268
  "rstrip": false,
@@ -271,7 +271,7 @@
271
  },
272
  {
273
  "id": 57545,
274
- "content": "<s_Age>",
275
  "single_word": false,
276
  "lstrip": false,
277
  "rstrip": false,
@@ -280,7 +280,7 @@
280
  },
281
  {
282
  "id": 57546,
283
- "content": "</s_Age>",
284
  "single_word": false,
285
  "lstrip": false,
286
  "rstrip": false,
@@ -289,7 +289,7 @@
289
  },
290
  {
291
  "id": 57547,
292
- "content": "<s_Place of birth>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
@@ -298,7 +298,7 @@
298
  },
299
  {
300
  "id": 57548,
301
- "content": "</s_Place of birth>",
302
  "single_word": false,
303
  "lstrip": false,
304
  "rstrip": false,
@@ -307,7 +307,7 @@
307
  },
308
  {
309
  "id": 57549,
310
- "content": "<s_relation>",
311
  "single_word": false,
312
  "lstrip": false,
313
  "rstrip": false,
@@ -316,7 +316,7 @@
316
  },
317
  {
318
  "id": 57550,
319
- "content": "</s_relation>",
320
  "single_word": false,
321
  "lstrip": false,
322
  "rstrip": false,
@@ -325,7 +325,7 @@
325
  },
326
  {
327
  "id": 57551,
328
- "content": "<s_Father>",
329
  "single_word": false,
330
  "lstrip": false,
331
  "rstrip": false,
@@ -334,7 +334,7 @@
334
  },
335
  {
336
  "id": 57552,
337
- "content": "</s_Father>",
338
  "single_word": false,
339
  "lstrip": false,
340
  "rstrip": false,
@@ -343,7 +343,7 @@
343
  },
344
  {
345
  "id": 57553,
346
- "content": "<s_Mother>",
347
  "single_word": false,
348
  "lstrip": false,
349
  "rstrip": false,
@@ -352,7 +352,7 @@
352
  },
353
  {
354
  "id": 57554,
355
- "content": "</s_Mother>",
356
  "single_word": false,
357
  "lstrip": false,
358
  "rstrip": false,
@@ -361,6 +361,24 @@
361
  },
362
  {
363
  "id": 57555,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  "content": "<s_cord-v2>",
365
  "single_word": false,
366
  "lstrip": false,
 
181
  },
182
  {
183
  "id": 57535,
184
+ "content": "<s_county>",
185
  "single_word": false,
186
  "lstrip": false,
187
  "rstrip": false,
 
190
  },
191
  {
192
  "id": 57536,
193
+ "content": "</s_county>",
194
  "single_word": false,
195
  "lstrip": false,
196
  "rstrip": false,
 
199
  },
200
  {
201
  "id": 57537,
202
+ "content": "<s_person_data>",
203
  "single_word": false,
204
  "lstrip": false,
205
  "rstrip": false,
 
208
  },
209
  {
210
  "id": 57538,
211
+ "content": "</s_person_data>",
212
  "single_word": false,
213
  "lstrip": false,
214
  "rstrip": false,
 
217
  },
218
  {
219
  "id": 57539,
220
+ "content": "<s_Gender>",
221
  "single_word": false,
222
  "lstrip": false,
223
  "rstrip": false,
 
226
  },
227
  {
228
  "id": 57540,
229
+ "content": "</s_Gender>",
230
  "single_word": false,
231
  "lstrip": false,
232
  "rstrip": false,
 
235
  },
236
  {
237
  "id": 57541,
238
+ "content": "<s_Race>",
239
  "single_word": false,
240
  "lstrip": false,
241
  "rstrip": false,
 
244
  },
245
  {
246
  "id": 57542,
247
+ "content": "</s_Race>",
248
  "single_word": false,
249
  "lstrip": false,
250
  "rstrip": false,
 
253
  },
254
  {
255
  "id": 57543,
256
+ "content": "<s_Marital Status>",
257
  "single_word": false,
258
  "lstrip": false,
259
  "rstrip": false,
 
262
  },
263
  {
264
  "id": 57544,
265
+ "content": "</s_Marital Status>",
266
  "single_word": false,
267
  "lstrip": false,
268
  "rstrip": false,
 
271
  },
272
  {
273
  "id": 57545,
274
+ "content": "<s_date_of_birth>",
275
  "single_word": false,
276
  "lstrip": false,
277
  "rstrip": false,
 
280
  },
281
  {
282
  "id": 57546,
283
+ "content": "</s_date_of_birth>",
284
  "single_word": false,
285
  "lstrip": false,
286
  "rstrip": false,
 
289
  },
290
  {
291
  "id": 57547,
292
+ "content": "<s_Age>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
 
298
  },
299
  {
300
  "id": 57548,
301
+ "content": "</s_Age>",
302
  "single_word": false,
303
  "lstrip": false,
304
  "rstrip": false,
 
307
  },
308
  {
309
  "id": 57549,
310
+ "content": "<s_Place of birth>",
311
  "single_word": false,
312
  "lstrip": false,
313
  "rstrip": false,
 
316
  },
317
  {
318
  "id": 57550,
319
+ "content": "</s_Place of birth>",
320
  "single_word": false,
321
  "lstrip": false,
322
  "rstrip": false,
 
325
  },
326
  {
327
  "id": 57551,
328
+ "content": "<s_relation>",
329
  "single_word": false,
330
  "lstrip": false,
331
  "rstrip": false,
 
334
  },
335
  {
336
  "id": 57552,
337
+ "content": "</s_relation>",
338
  "single_word": false,
339
  "lstrip": false,
340
  "rstrip": false,
 
343
  },
344
  {
345
  "id": 57553,
346
+ "content": "<s_Father>",
347
  "single_word": false,
348
  "lstrip": false,
349
  "rstrip": false,
 
352
  },
353
  {
354
  "id": 57554,
355
+ "content": "</s_Father>",
356
  "single_word": false,
357
  "lstrip": false,
358
  "rstrip": false,
 
361
  },
362
  {
363
  "id": 57555,
364
+ "content": "<s_Mother>",
365
+ "single_word": false,
366
+ "lstrip": false,
367
+ "rstrip": false,
368
+ "normalized": true,
369
+ "special": false
370
+ },
371
+ {
372
+ "id": 57556,
373
+ "content": "</s_Mother>",
374
+ "single_word": false,
375
+ "lstrip": false,
376
+ "rstrip": false,
377
+ "normalized": true,
378
+ "special": false
379
+ },
380
+ {
381
+ "id": 57557,
382
  "content": "<s_cord-v2>",
383
  "single_word": false,
384
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -145,7 +145,7 @@
145
  "special": false
146
  },
147
  "57535": {
148
- "content": "<s_person_data>",
149
  "lstrip": false,
150
  "normalized": true,
151
  "rstrip": false,
@@ -153,7 +153,7 @@
153
  "special": false
154
  },
155
  "57536": {
156
- "content": "</s_person_data>",
157
  "lstrip": false,
158
  "normalized": true,
159
  "rstrip": false,
@@ -161,7 +161,7 @@
161
  "special": false
162
  },
163
  "57537": {
164
- "content": "<s_Gender>",
165
  "lstrip": false,
166
  "normalized": true,
167
  "rstrip": false,
@@ -169,7 +169,7 @@
169
  "special": false
170
  },
171
  "57538": {
172
- "content": "</s_Gender>",
173
  "lstrip": false,
174
  "normalized": true,
175
  "rstrip": false,
@@ -177,7 +177,7 @@
177
  "special": false
178
  },
179
  "57539": {
180
- "content": "<s_Race>",
181
  "lstrip": false,
182
  "normalized": true,
183
  "rstrip": false,
@@ -185,7 +185,7 @@
185
  "special": false
186
  },
187
  "57540": {
188
- "content": "</s_Race>",
189
  "lstrip": false,
190
  "normalized": true,
191
  "rstrip": false,
@@ -193,7 +193,7 @@
193
  "special": false
194
  },
195
  "57541": {
196
- "content": "<s_Marital Status>",
197
  "lstrip": false,
198
  "normalized": true,
199
  "rstrip": false,
@@ -201,7 +201,7 @@
201
  "special": false
202
  },
203
  "57542": {
204
- "content": "</s_Marital Status>",
205
  "lstrip": false,
206
  "normalized": true,
207
  "rstrip": false,
@@ -209,7 +209,7 @@
209
  "special": false
210
  },
211
  "57543": {
212
- "content": "<s_date_of_birth>",
213
  "lstrip": false,
214
  "normalized": true,
215
  "rstrip": false,
@@ -217,7 +217,7 @@
217
  "special": false
218
  },
219
  "57544": {
220
- "content": "</s_date_of_birth>",
221
  "lstrip": false,
222
  "normalized": true,
223
  "rstrip": false,
@@ -225,7 +225,7 @@
225
  "special": false
226
  },
227
  "57545": {
228
- "content": "<s_Age>",
229
  "lstrip": false,
230
  "normalized": true,
231
  "rstrip": false,
@@ -233,7 +233,7 @@
233
  "special": false
234
  },
235
  "57546": {
236
- "content": "</s_Age>",
237
  "lstrip": false,
238
  "normalized": true,
239
  "rstrip": false,
@@ -241,7 +241,7 @@
241
  "special": false
242
  },
243
  "57547": {
244
- "content": "<s_Place of birth>",
245
  "lstrip": false,
246
  "normalized": true,
247
  "rstrip": false,
@@ -249,7 +249,7 @@
249
  "special": false
250
  },
251
  "57548": {
252
- "content": "</s_Place of birth>",
253
  "lstrip": false,
254
  "normalized": true,
255
  "rstrip": false,
@@ -257,7 +257,7 @@
257
  "special": false
258
  },
259
  "57549": {
260
- "content": "<s_relation>",
261
  "lstrip": false,
262
  "normalized": true,
263
  "rstrip": false,
@@ -265,7 +265,7 @@
265
  "special": false
266
  },
267
  "57550": {
268
- "content": "</s_relation>",
269
  "lstrip": false,
270
  "normalized": true,
271
  "rstrip": false,
@@ -273,7 +273,7 @@
273
  "special": false
274
  },
275
  "57551": {
276
- "content": "<s_Father>",
277
  "lstrip": false,
278
  "normalized": true,
279
  "rstrip": false,
@@ -281,7 +281,7 @@
281
  "special": false
282
  },
283
  "57552": {
284
- "content": "</s_Father>",
285
  "lstrip": false,
286
  "normalized": true,
287
  "rstrip": false,
@@ -289,7 +289,7 @@
289
  "special": false
290
  },
291
  "57553": {
292
- "content": "<s_Mother>",
293
  "lstrip": false,
294
  "normalized": true,
295
  "rstrip": false,
@@ -297,7 +297,7 @@
297
  "special": false
298
  },
299
  "57554": {
300
- "content": "</s_Mother>",
301
  "lstrip": false,
302
  "normalized": true,
303
  "rstrip": false,
@@ -305,6 +305,22 @@
305
  "special": false
306
  },
307
  "57555": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
308
  "content": "<s_cord-v2>",
309
  "lstrip": false,
310
  "normalized": true,
 
145
  "special": false
146
  },
147
  "57535": {
148
+ "content": "<s_county>",
149
  "lstrip": false,
150
  "normalized": true,
151
  "rstrip": false,
 
153
  "special": false
154
  },
155
  "57536": {
156
+ "content": "</s_county>",
157
  "lstrip": false,
158
  "normalized": true,
159
  "rstrip": false,
 
161
  "special": false
162
  },
163
  "57537": {
164
+ "content": "<s_person_data>",
165
  "lstrip": false,
166
  "normalized": true,
167
  "rstrip": false,
 
169
  "special": false
170
  },
171
  "57538": {
172
+ "content": "</s_person_data>",
173
  "lstrip": false,
174
  "normalized": true,
175
  "rstrip": false,
 
177
  "special": false
178
  },
179
  "57539": {
180
+ "content": "<s_Gender>",
181
  "lstrip": false,
182
  "normalized": true,
183
  "rstrip": false,
 
185
  "special": false
186
  },
187
  "57540": {
188
+ "content": "</s_Gender>",
189
  "lstrip": false,
190
  "normalized": true,
191
  "rstrip": false,
 
193
  "special": false
194
  },
195
  "57541": {
196
+ "content": "<s_Race>",
197
  "lstrip": false,
198
  "normalized": true,
199
  "rstrip": false,
 
201
  "special": false
202
  },
203
  "57542": {
204
+ "content": "</s_Race>",
205
  "lstrip": false,
206
  "normalized": true,
207
  "rstrip": false,
 
209
  "special": false
210
  },
211
  "57543": {
212
+ "content": "<s_Marital Status>",
213
  "lstrip": false,
214
  "normalized": true,
215
  "rstrip": false,
 
217
  "special": false
218
  },
219
  "57544": {
220
+ "content": "</s_Marital Status>",
221
  "lstrip": false,
222
  "normalized": true,
223
  "rstrip": false,
 
225
  "special": false
226
  },
227
  "57545": {
228
+ "content": "<s_date_of_birth>",
229
  "lstrip": false,
230
  "normalized": true,
231
  "rstrip": false,
 
233
  "special": false
234
  },
235
  "57546": {
236
+ "content": "</s_date_of_birth>",
237
  "lstrip": false,
238
  "normalized": true,
239
  "rstrip": false,
 
241
  "special": false
242
  },
243
  "57547": {
244
+ "content": "<s_Age>",
245
  "lstrip": false,
246
  "normalized": true,
247
  "rstrip": false,
 
249
  "special": false
250
  },
251
  "57548": {
252
+ "content": "</s_Age>",
253
  "lstrip": false,
254
  "normalized": true,
255
  "rstrip": false,
 
257
  "special": false
258
  },
259
  "57549": {
260
+ "content": "<s_Place of birth>",
261
  "lstrip": false,
262
  "normalized": true,
263
  "rstrip": false,
 
265
  "special": false
266
  },
267
  "57550": {
268
+ "content": "</s_Place of birth>",
269
  "lstrip": false,
270
  "normalized": true,
271
  "rstrip": false,
 
273
  "special": false
274
  },
275
  "57551": {
276
+ "content": "<s_relation>",
277
  "lstrip": false,
278
  "normalized": true,
279
  "rstrip": false,
 
281
  "special": false
282
  },
283
  "57552": {
284
+ "content": "</s_relation>",
285
  "lstrip": false,
286
  "normalized": true,
287
  "rstrip": false,
 
289
  "special": false
290
  },
291
  "57553": {
292
+ "content": "<s_Father>",
293
  "lstrip": false,
294
  "normalized": true,
295
  "rstrip": false,
 
297
  "special": false
298
  },
299
  "57554": {
300
+ "content": "</s_Father>",
301
  "lstrip": false,
302
  "normalized": true,
303
  "rstrip": false,
 
305
  "special": false
306
  },
307
  "57555": {
308
+ "content": "<s_Mother>",
309
+ "lstrip": false,
310
+ "normalized": true,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": false
314
+ },
315
+ "57556": {
316
+ "content": "</s_Mother>",
317
+ "lstrip": false,
318
+ "normalized": true,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": false
322
+ },
323
+ "57557": {
324
  "content": "<s_cord-v2>",
325
  "lstrip": false,
326
  "normalized": true,