Config

added_tokens.json

@@ -1,13 +1,11 @@
 {
-  "</s_cols>": 57528,
-  "</s_rows>": 57526,
-  "</s_text>": 57530,
-  "</s_vichart>": 57532,
-  "<s_cols>": 57527,
-  "<s_iitcdip>": 57523,
-  "<s_rows>": 57525,
-  "<s_synthdog>": 57524,
-  "<s_text>": 57529,
-  "<s_vichart>": 57531,
-  "<sep/>": 57522
+  "</s_cols>": 40033,
+  "</s_rows>": 40031,
+  "</s_text>": 40035,
+  "</s_vichart>": 40037,
+  "<s_cols>": 40032,
+  "<s_rows>": 40030,
+  "<s_text>": 40034,
+  "<s_vichart>": 40036,
+  "<sep/>": 40038
 }

sentencepiece.bpe.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb9e3dce4c326195d08fc3dd0f7e2eee1da8595c847bf4c1a9c78b7a82d47e2d
-size 1296245
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -1,29 +1,7 @@
 {
-  "additional_special_tokens": [
-    "<s_iitcdip>",
-    "<s_synthdog>"
-  ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "cls_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -31,25 +9,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "sep_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -32,7 +32,7 @@
       "single_word": false,
       "special": true
     },
-    "57521": {
+    "40029": {
       "content": "<mask>",
       "lstrip": true,
       "normalized": true,
@@ -40,31 +40,7 @@
       "single_word": false,
       "special": true
     },
-    "57522": {
-      "content": "<sep/>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57523": {
-      "content": "<s_iitcdip>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "57524": {
-      "content": "<s_synthdog>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "57525": {
+    "40030": {
       "content": "<s_rows>",
       "lstrip": false,
       "normalized": true,
@@ -72,7 +48,7 @@
       "single_word": false,
       "special": false
     },
-    "57526": {
+    "40031": {
       "content": "</s_rows>",
       "lstrip": false,
       "normalized": true,
@@ -80,7 +56,7 @@
       "single_word": false,
       "special": false
     },
-    "57527": {
+    "40032": {
       "content": "<s_cols>",
       "lstrip": false,
       "normalized": true,
@@ -88,7 +64,7 @@
       "single_word": false,
       "special": false
     },
-    "57528": {
+    "40033": {
       "content": "</s_cols>",
       "lstrip": false,
       "normalized": true,
@@ -96,7 +72,7 @@
       "single_word": false,
       "special": false
     },
-    "57529": {
+    "40034": {
       "content": "<s_text>",
       "lstrip": false,
       "normalized": true,
@@ -104,7 +80,7 @@
       "single_word": false,
       "special": false
     },
-    "57530": {
+    "40035": {
       "content": "</s_text>",
       "lstrip": false,
       "normalized": true,
@@ -112,7 +88,7 @@
       "single_word": false,
       "special": false
     },
-    "57531": {
+    "40036": {
       "content": "<s_vichart>",
       "lstrip": false,
       "normalized": true,
@@ -120,19 +96,23 @@
       "single_word": false,
       "special": false
     },
-    "57532": {
+    "40037": {
       "content": "</s_vichart>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "40038": {
+      "content": "<sep/>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
-  "additional_special_tokens": [
-    "<s_iitcdip>",
-    "<s_synthdog>"
-  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",
@@ -143,6 +123,6 @@
   "processor_class": "DonutProcessor",
   "sep_token": "</s>",
   "sp_model_kwargs": {},
-  "tokenizer_class": "XLMRobertaTokenizer",
+  "tokenizer_class": "BartphoTokenizer",
   "unk_token": "<unk>"
 }