feat: new tokenizer and config

added_tokens.json

@@ -1,47 +1,43 @@
 {
-  "</s_Abstract>": 57566,
-  "</s_Address>": 57552,
-  "</s_Advertisement>": 57564,
-  "</s_Author>": 57550,
-  "</s_Caption>": 57536,
-  "</s_Chart>": 57562,
-  "</s_Field-Name>": 57556,
-  "</s_Footer>": 57534,
-  "</s_Formula>": 57568,
-  "</s_Header>": 57542,
-  "</s_Headline>": 57546,
-  "</s_Link>": 57540,
-  "</s_List-item>": 57532,
+  "</s_Abstract>": 57564,
+  "</s_Address>": 57538,
+  "</s_Advertisement>": 57556,
+  "</s_Author>": 57546,
+  "</s_Caption>": 57530,
+  "</s_Chart>": 57560,
+  "</s_Field-Name>": 57548,
+  "</s_Footer>": 57542,
+  "</s_Header>": 57552,
+  "</s_Headline>": 57536,
+  "</s_Link>": 57528,
+  "</s_List-item>": 57540,
   "</s_Metadata>": 57558,
-  "</s_Misc>": 57530,
-  "</s_Page number>": 57548,
-  "</s_Subheadline>": 57528,
-  "</s_Table>": 57538,
+  "</s_Misc>": 57534,
+  "</s_Page number>": 57562,
+  "</s_Subheadline>": 57532,
+  "</s_Table>": 57550,
   "</s_Text>": 57526,
-  "</s_Threading>": 57554,
-  "</s_Title>": 57560,
+  "</s_Title>": 57554,
   "</s_Value>": 57544,
-  "<s_Abstract>": 57565,
-  "<s_Address>": 57551,
-  "<s_Advertisement>": 57563,
-  "<s_Author>": 57549,
-  "<s_Caption>": 57535,
-  "<s_Chart>": 57561,
-  "<s_Field-Name>": 57555,
-  "<s_Footer>": 57533,
-  "<s_Formula>": 57567,
-  "<s_Header>": 57541,
-  "<s_Headline>": 57545,
-  "<s_Link>": 57539,
-  "<s_List-item>": 57531,
+  "<s_Abstract>": 57563,
+  "<s_Address>": 57537,
+  "<s_Advertisement>": 57555,
+  "<s_Author>": 57545,
+  "<s_Caption>": 57529,
+  "<s_Chart>": 57559,
+  "<s_Field-Name>": 57547,
+  "<s_Footer>": 57541,
+  "<s_Header>": 57551,
+  "<s_Headline>": 57535,
+  "<s_Link>": 57527,
+  "<s_List-item>": 57539,
   "<s_Metadata>": 57557,
-  "<s_Misc>": 57529,
-  "<s_Page number>": 57547,
-  "<s_Subheadline>": 57527,
-  "<s_Table>": 57537,
+  "<s_Misc>": 57533,
+  "<s_Page number>": 57561,
+  "<s_Subheadline>": 57531,
+  "<s_Table>": 57549,
   "<s_Text>": 57525,
-  "<s_Threading>": 57553,
-  "<s_Title>": 57559,
+  "<s_Title>": 57553,
   "<s_Value>": 57543,
   "<s_iitcdip>": 57523,
   "<s_synthdog>": 57524,

special_tokens_map.json

@@ -2,48 +2,44 @@
   "additional_special_tokens": [
     "<s_Text>",
     "</s_Text>",
+    "<s_Link>",
+    "</s_Link>",
+    "<s_Caption>",
+    "</s_Caption>",
     "<s_Subheadline>",
     "</s_Subheadline>",
     "<s_Misc>",
     "</s_Misc>",
+    "<s_Headline>",
+    "</s_Headline>",
+    "<s_Address>",
+    "</s_Address>",
     "<s_List-item>",
     "</s_List-item>",
     "<s_Footer>",
     "</s_Footer>",
-    "<s_Caption>",
-    "</s_Caption>",
-    "<s_Table>",
-    "</s_Table>",
-    "<s_Link>",
-    "</s_Link>",
-    "<s_Header>",
-    "</s_Header>",
     "<s_Value>",
     "</s_Value>",
-    "<s_Headline>",
-    "</s_Headline>",
-    "<s_Page number>",
-    "</s_Page number>",
     "<s_Author>",
     "</s_Author>",
-    "<s_Address>",
-    "</s_Address>",
-    "<s_Threading>",
-    "</s_Threading>",
     "<s_Field-Name>",
     "</s_Field-Name>",
-    "<s_Metadata>",
-    "</s_Metadata>",
+    "<s_Table>",
+    "</s_Table>",
+    "<s_Header>",
+    "</s_Header>",
     "<s_Title>",
     "</s_Title>",
-    "<s_Chart>",
-    "</s_Chart>",
     "<s_Advertisement>",
     "</s_Advertisement>",
+    "<s_Metadata>",
+    "</s_Metadata>",
+    "<s_Chart>",
+    "</s_Chart>",
+    "<s_Page number>",
+    "</s_Page number>",
     "<s_Abstract>",
     "</s_Abstract>",
-    "<s_Formula>",
-    "</s_Formula>",
     "<s>",
     "</s>"
   ],

tokenizer.json

@@ -109,7 +109,7 @@
     },
     {
       "id": 57527,
-      "content": "<s_Subheadline>",
+      "content": "<s_Link>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -118,7 +118,7 @@
     },
     {
       "id": 57528,
-      "content": "</s_Subheadline>",
+      "content": "</s_Link>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -127,7 +127,7 @@
     },
     {
       "id": 57529,
-      "content": "<s_Misc>",
+      "content": "<s_Caption>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -136,7 +136,7 @@
     },
     {
       "id": 57530,
-      "content": "</s_Misc>",
+      "content": "</s_Caption>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -145,7 +145,7 @@
     },
     {
       "id": 57531,
-      "content": "<s_List-item>",
+      "content": "<s_Subheadline>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -154,7 +154,7 @@
     },
     {
       "id": 57532,
-      "content": "</s_List-item>",
+      "content": "</s_Subheadline>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -163,7 +163,7 @@
     },
     {
       "id": 57533,
-      "content": "<s_Footer>",
+      "content": "<s_Misc>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -172,7 +172,7 @@
     },
     {
       "id": 57534,
-      "content": "</s_Footer>",
+      "content": "</s_Misc>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -181,7 +181,7 @@
     },
     {
       "id": 57535,
-      "content": "<s_Caption>",
+      "content": "<s_Headline>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -190,7 +190,7 @@
     },
     {
       "id": 57536,
-      "content": "</s_Caption>",
+      "content": "</s_Headline>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -199,7 +199,7 @@
     },
     {
       "id": 57537,
-      "content": "<s_Table>",
+      "content": "<s_Address>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -208,7 +208,7 @@
     },
     {
       "id": 57538,
-      "content": "</s_Table>",
+      "content": "</s_Address>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -217,7 +217,7 @@
     },
     {
       "id": 57539,
-      "content": "<s_Link>",
+      "content": "<s_List-item>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -226,7 +226,7 @@
     },
     {
       "id": 57540,
-      "content": "</s_Link>",
+      "content": "</s_List-item>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -235,7 +235,7 @@
     },
     {
       "id": 57541,
-      "content": "<s_Header>",
+      "content": "<s_Footer>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -244,7 +244,7 @@
     },
     {
       "id": 57542,
-      "content": "</s_Header>",
+      "content": "</s_Footer>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -271,7 +271,7 @@
     },
     {
       "id": 57545,
-      "content": "<s_Headline>",
+      "content": "<s_Author>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -280,7 +280,7 @@
     },
     {
       "id": 57546,
-      "content": "</s_Headline>",
+      "content": "</s_Author>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -289,7 +289,7 @@
     },
     {
       "id": 57547,
-      "content": "<s_Page number>",
+      "content": "<s_Field-Name>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -298,7 +298,7 @@
     },
     {
       "id": 57548,
-      "content": "</s_Page number>",
+      "content": "</s_Field-Name>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -307,7 +307,7 @@
     },
     {
       "id": 57549,
-      "content": "<s_Author>",
+      "content": "<s_Table>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -316,7 +316,7 @@
     },
     {
       "id": 57550,
-      "content": "</s_Author>",
+      "content": "</s_Table>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -325,7 +325,7 @@
     },
     {
       "id": 57551,
-      "content": "<s_Address>",
+      "content": "<s_Header>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -334,7 +334,7 @@
     },
     {
       "id": 57552,
-      "content": "</s_Address>",
+      "content": "</s_Header>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -343,7 +343,7 @@
     },
     {
       "id": 57553,
-      "content": "<s_Threading>",
+      "content": "<s_Title>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -352,7 +352,7 @@
     },
     {
       "id": 57554,
-      "content": "</s_Threading>",
+      "content": "</s_Title>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -361,7 +361,7 @@
     },
     {
       "id": 57555,
-      "content": "<s_Field-Name>",
+      "content": "<s_Advertisement>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -370,7 +370,7 @@
     },
     {
       "id": 57556,
-      "content": "</s_Field-Name>",
+      "content": "</s_Advertisement>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -397,7 +397,7 @@
     },
     {
       "id": 57559,
-      "content": "<s_Title>",
+      "content": "<s_Chart>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -406,7 +406,7 @@
     },
     {
       "id": 57560,
-      "content": "</s_Title>",
+      "content": "</s_Chart>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -415,7 +415,7 @@
     },
     {
       "id": 57561,
-      "content": "<s_Chart>",
+      "content": "<s_Page number>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -424,7 +424,7 @@
     },
     {
       "id": 57562,
-      "content": "</s_Chart>",
+      "content": "</s_Page number>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -433,24 +433,6 @@
     },
     {
       "id": 57563,
-      "content": "<s_Advertisement>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 57564,
-      "content": "</s_Advertisement>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 57565,
       "content": "<s_Abstract>",
       "single_word": false,
       "lstrip": false,
@@ -459,31 +441,13 @@
       "special": true
     },
     {
-      "id": 57566,
+      "id": 57564,
       "content": "</s_Abstract>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 57567,
-      "content": "<s_Formula>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 57568,
-      "content": "</s_Formula>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {