Processor at epoch 0
Browse files- added_tokens.json +1 -3
- tokenizer.json +0 -18
- tokenizer_config.json +0 -16
added_tokens.json
CHANGED
|
@@ -23,7 +23,6 @@
|
|
| 23 |
"</s_total_incl_gst>": 57566,
|
| 24 |
"</s_unit_price_ex_gst>": 57558,
|
| 25 |
"</s_uom>": 57548,
|
| 26 |
-
"<header_start>": 57573,
|
| 27 |
"<s_None>": 57567,
|
| 28 |
"<s_amount_incl_gst>": 57559,
|
| 29 |
"<s_client_address>": 57539,
|
|
@@ -50,6 +49,5 @@
|
|
| 50 |
"<s_total_incl_gst>": 57565,
|
| 51 |
"<s_unit_price_ex_gst>": 57557,
|
| 52 |
"<s_uom>": 57547,
|
| 53 |
-
"<sep/>": 57522
|
| 54 |
-
"<summary_end>": 57574
|
| 55 |
}
|
|
|
|
| 23 |
"</s_total_incl_gst>": 57566,
|
| 24 |
"</s_unit_price_ex_gst>": 57558,
|
| 25 |
"</s_uom>": 57548,
|
|
|
|
| 26 |
"<s_None>": 57567,
|
| 27 |
"<s_amount_incl_gst>": 57559,
|
| 28 |
"<s_client_address>": 57539,
|
|
|
|
| 49 |
"<s_total_incl_gst>": 57565,
|
| 50 |
"<s_unit_price_ex_gst>": 57557,
|
| 51 |
"<s_uom>": 57547,
|
| 52 |
+
"<sep/>": 57522
|
|
|
|
| 53 |
}
|
tokenizer.json
CHANGED
|
@@ -520,24 +520,6 @@
|
|
| 520 |
"rstrip": false,
|
| 521 |
"normalized": true,
|
| 522 |
"special": false
|
| 523 |
-
},
|
| 524 |
-
{
|
| 525 |
-
"id": 57573,
|
| 526 |
-
"content": "<header_start>",
|
| 527 |
-
"single_word": false,
|
| 528 |
-
"lstrip": false,
|
| 529 |
-
"rstrip": false,
|
| 530 |
-
"normalized": true,
|
| 531 |
-
"special": false
|
| 532 |
-
},
|
| 533 |
-
{
|
| 534 |
-
"id": 57574,
|
| 535 |
-
"content": "<summary_end>",
|
| 536 |
-
"single_word": false,
|
| 537 |
-
"lstrip": false,
|
| 538 |
-
"rstrip": false,
|
| 539 |
-
"normalized": true,
|
| 540 |
-
"special": false
|
| 541 |
}
|
| 542 |
],
|
| 543 |
"normalizer": {
|
|
|
|
| 520 |
"rstrip": false,
|
| 521 |
"normalized": true,
|
| 522 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 523 |
}
|
| 524 |
],
|
| 525 |
"normalizer": {
|
tokenizer_config.json
CHANGED
|
@@ -447,22 +447,6 @@
|
|
| 447 |
"rstrip": false,
|
| 448 |
"single_word": false,
|
| 449 |
"special": false
|
| 450 |
-
},
|
| 451 |
-
"57573": {
|
| 452 |
-
"content": "<header_start>",
|
| 453 |
-
"lstrip": false,
|
| 454 |
-
"normalized": true,
|
| 455 |
-
"rstrip": false,
|
| 456 |
-
"single_word": false,
|
| 457 |
-
"special": false
|
| 458 |
-
},
|
| 459 |
-
"57574": {
|
| 460 |
-
"content": "<summary_end>",
|
| 461 |
-
"lstrip": false,
|
| 462 |
-
"normalized": true,
|
| 463 |
-
"rstrip": false,
|
| 464 |
-
"single_word": false,
|
| 465 |
-
"special": false
|
| 466 |
}
|
| 467 |
},
|
| 468 |
"additional_special_tokens": [
|
|
|
|
| 447 |
"rstrip": false,
|
| 448 |
"single_word": false,
|
| 449 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
}
|
| 451 |
},
|
| 452 |
"additional_special_tokens": [
|