Upload tokenizer

tokenizer.json

@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 5,
+      "id": 3,
       "content": "UTT_BOUNDARY",
       "single_word": false,
       "lstrip": false,
@@ -34,13 +34,6 @@
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "\n"
-        },
-        "content": " UTT_BOUNDARY"
-      },
       {
         "type": "Strip",
         "strip_left": true,
@@ -51,58 +44,109 @@
   "pre_tokenizer": {
     "type": "Whitespace"
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
   "decoder": null,
   "model": {
     "type": "WordLevel",
     "vocab": {
       "UNK": 0,
       "PAD": 1,
-      "BOS": 2,
-      "EOS": 3,
-      "WORD_BOUNDARY": 4,
-      "UTT_BOUNDARY": 5,
-      "d": 6,
-      "e": 7,
-      "ɛ": 8,
-      "n": 9,
-      "m": 10,
-      "s": 11,
-      "t": 12,
-      "k": 13,
-      "j": 14,
-      "f": 15,
-      "ɑ": 16,
-      "ɒ": 17,
-      "ə": 18,
-      "ʋ": 19,
-      "a": 20,
-      "l": 21,
-      "h": 22,
-      "b": 23,
-      "ʁ": 24,
-      "p": 25,
-      "œ": 26,
-      "i": 27,
-      "ɡ": 28,
-      "ʌ": 29,
-      "u": 30,
-      "ʃ": 31,
-      "ɔ": 32,
-      "w": 33,
-      "ð": 34,
-      "o": 35,
-      "y": 36,
-      "ŋ": 37,
-      "aɪ": 38,
-      "œː": 39,
-      "aː": 40,
-      "d̠ʒ": 41,
-      "uː": 42,
-      "ʌː": 43,
-      "ɜ": 44,
-      "oː": 45,
-      "yː": 46
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "d": 4,
+      "e": 5,
+      "ɛ": 6,
+      "n": 7,
+      "m": 8,
+      "s": 9,
+      "t": 10,
+      "k": 11,
+      "j": 12,
+      "f": 13,
+      "ɑ": 14,
+      "ɒ": 15,
+      "ə": 16,
+      "ʋ": 17,
+      "a": 18,
+      "l": 19,
+      "h": 20,
+      "b": 21,
+      "ʁ": 22,
+      "p": 23,
+      "œ": 24,
+      "i": 25,
+      "ɡ": 26,
+      "ʌ": 27,
+      "u": 28,
+      "ʃ": 29,
+      "ɔ": 30,
+      "w": 31,
+      "ð": 32,
+      "o": 33,
+      "y": 34,
+      "ŋ": 35,
+      "aɪ": 36,
+      "œː": 37,
+      "aː": 38,
+      "d̠ʒ": 39,
+      "uː": 40,
+      "ʌː": 41,
+      "ɜ": 42,
+      "oː": 43,
+      "yː": 44
     },
     "unk_token": "UNK"
   }

tokenizer_config.json

@@ -17,7 +17,7 @@
       "single_word": false,
       "special": true
     },
-    "5": {
+    "3": {
       "content": "UTT_BOUNDARY",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"BOS":2,"EOS":3,"WORD_BOUNDARY":4,"UTT_BOUNDARY":5,"d":6,"e":7,"ɛ":8,"n":9,"m":10,"s":11,"t":12,"k":13,"j":14,"f":15,"ɑ":16,"ɒ":17,"ə":18,"ʋ":19,"a":20,"l":21,"h":22,"b":23,"ʁ":24,"p":25,"œ":26,"i":27,"ɡ":28,"ʌ":29,"u":30,"ʃ":31,"ɔ":32,"w":33,"ð":34,"o":35,"y":36,"ŋ":37,"aɪ":38,"œː":39,"aː":40,"d̠ʒ":41,"uː":42,"ʌː":43,"ɜ":44,"oː":45,"yː":46}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"d":4,"e":5,"ɛ":6,"n":7,"m":8,"s":9,"t":10,"k":11,"j":12,"f":13,"ɑ":14,"ɒ":15,"ə":16,"ʋ":17,"a":18,"l":19,"h":20,"b":21,"ʁ":22,"p":23,"œ":24,"i":25,"ɡ":26,"ʌ":27,"u":28,"ʃ":29,"ɔ":30,"w":31,"ð":32,"o":33,"y":34,"ŋ":35,"aɪ":36,"œː":37,"aː":38,"d̠ʒ":39,"uː":40,"ʌː":41,"ɜ":42,"oː":43,"yː":44}