pandafm commited on
Commit
01c41da
1 Parent(s): cbe3c46

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -0
  2. tokenizer.json +20 -2
  3. tokenizer_config.json +18 -2
added_tokens.json CHANGED
@@ -18,6 +18,7 @@
18
  "</s_service_price>": 57537,
19
  "</s_sub>": 57547,
20
  "</s_sub_total>": 57533,
 
21
  "</s_subtotal_price>": 57535,
22
  "</s_tax_price>": 57539,
23
  "</s_total>": 57543,
@@ -47,6 +48,7 @@
47
  "<s_service_price>": 57538,
48
  "<s_sub>": 57548,
49
  "<s_sub_total>": 57534,
 
50
  "<s_subtotal_price>": 57536,
51
  "<s_synthdog>": 57524,
52
  "<s_tax_price>": 57540,
 
18
  "</s_service_price>": 57537,
19
  "</s_sub>": 57547,
20
  "</s_sub_total>": 57533,
21
+ "</s_subtotal>": 57581,
22
  "</s_subtotal_price>": 57535,
23
  "</s_tax_price>": 57539,
24
  "</s_total>": 57543,
 
48
  "<s_service_price>": 57538,
49
  "<s_sub>": 57548,
50
  "<s_sub_total>": 57534,
51
+ "<s_subtotal>": 57580,
52
  "<s_subtotal_price>": 57536,
53
  "<s_synthdog>": 57524,
54
  "<s_tax_price>": 57540,
tokenizer.json CHANGED
@@ -581,8 +581,26 @@
581
  "single_word": false,
582
  "lstrip": false,
583
  "rstrip": false,
584
- "normalized": false,
585
- "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
586
  }
587
  ],
588
  "normalizer": {
 
581
  "single_word": false,
582
  "lstrip": false,
583
  "rstrip": false,
584
+ "normalized": true,
585
+ "special": false
586
+ },
587
+ {
588
+ "id": 57580,
589
+ "content": "<s_subtotal>",
590
+ "single_word": false,
591
+ "lstrip": false,
592
+ "rstrip": false,
593
+ "normalized": true,
594
+ "special": false
595
+ },
596
+ {
597
+ "id": 57581,
598
+ "content": "</s_subtotal>",
599
+ "single_word": false,
600
+ "lstrip": false,
601
+ "rstrip": false,
602
+ "normalized": true,
603
+ "special": false
604
  }
605
  ],
606
  "normalizer": {
tokenizer_config.json CHANGED
@@ -499,10 +499,26 @@
499
  "57579": {
500
  "content": "<s_cord-v2>",
501
  "lstrip": false,
502
- "normalized": false,
503
  "rstrip": false,
504
  "single_word": false,
505
- "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
506
  }
507
  },
508
  "additional_special_tokens": [
 
499
  "57579": {
500
  "content": "<s_cord-v2>",
501
  "lstrip": false,
502
+ "normalized": true,
503
  "rstrip": false,
504
  "single_word": false,
505
+ "special": false
506
+ },
507
+ "57580": {
508
+ "content": "<s_subtotal>",
509
+ "lstrip": false,
510
+ "normalized": true,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": false
514
+ },
515
+ "57581": {
516
+ "content": "</s_subtotal>",
517
+ "lstrip": false,
518
+ "normalized": true,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": false
522
  }
523
  },
524
  "additional_special_tokens": [