Training done

added_tokens.json

@@ -32,6 +32,7 @@
   "</s_item_hscode>": 57621,
   "</s_item_quantity>": 57562,
   "</s_item_total_price>": 57566,
+  "</s_item_unit>": 57633,
   "</s_item_unit_price>": 57564,
   "</s_item_uom>": 57623,
   "</s_item_weight>": 57588,
@@ -87,6 +88,7 @@
   "<s_item_hscode>": 57620,
   "<s_item_quantity>": 57561,
   "<s_item_total_price>": 57565,
+  "<s_item_unit>": 57632,
   "<s_item_unit_price>": 57563,
   "<s_item_uom>": 57622,
   "<s_item_weight>": 57587,

tokenizer.json

@@ -1051,6 +1051,24 @@
       "rstrip": false,
       "normalized": true,
       "special": false
+    },
+    {
+      "id": 57632,
+      "content": "<s_item_unit>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57633,
+      "content": "</s_item_unit>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -919,6 +919,22 @@
       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "57632": {
+      "content": "<s_item_unit>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57633": {
+      "content": "</s_item_unit>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "additional_special_tokens": [