Upload tokenizer

tokenizer.json

@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 5,
+      "id": 3,
       "content": "UTT_BOUNDARY",
       "single_word": false,
       "lstrip": false,
@@ -34,13 +34,6 @@
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": "\n"
-        },
-        "content": " UTT_BOUNDARY"
-      },
       {
         "type": "Strip",
         "strip_left": true,
@@ -51,65 +44,116 @@
   "pre_tokenizer": {
     "type": "Whitespace"
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
   "decoder": null,
   "model": {
     "type": "WordLevel",
     "vocab": {
       "UNK": 0,
       "PAD": 1,
-      "BOS": 2,
-      "EOS": 3,
-      "WORD_BOUNDARY": 4,
-      "UTT_BOUNDARY": 5,
-      "aː": 6,
-      "ɾ": 7,
-      "r": 8,
-      "ɪ": 9,
-      "ɛ": 10,
-      "d": 11,
-      "s": 12,
-      "j": 13,
-      "a": 14,
-      "b": 15,
-      "iː": 16,
-      "k": 17,
-      "ʋ": 18,
-      "ɛː": 19,
-      "θ": 20,
-      "i": 21,
-      "l": 22,
-      "n": 23,
-      "uː": 24,
-      "ð": 25,
-      "ɡ": 26,
-      "ɔ": 27,
-      "h": 28,
-      "aʊ": 29,
-      "y": 30,
-      "m": 31,
-      "f": 32,
-      "ɔː": 33,
-      "x": 34,
-      "ɟ": 35,
-      "t": 36,
-      "eɪ": 37,
-      "oʊ": 38,
-      "p": 39,
-      "ŋ": 40,
-      "ɣ": 41,
-      "yː": 42,
-      "u": 43,
-      "ɪː": 44,
-      "œ": 45,
-      "aɪ": 46,
-      "ç": 47,
-      "ə": 48,
-      "øy": 49,
-      "c": 50,
-      "ɲ": 51,
-      "œː": 52,
-      "ɔɪ": 53
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "aː": 4,
+      "ɾ": 5,
+      "r": 6,
+      "ɪ": 7,
+      "ɛ": 8,
+      "d": 9,
+      "s": 10,
+      "j": 11,
+      "a": 12,
+      "b": 13,
+      "iː": 14,
+      "k": 15,
+      "ʋ": 16,
+      "ɛː": 17,
+      "θ": 18,
+      "i": 19,
+      "l": 20,
+      "n": 21,
+      "uː": 22,
+      "ð": 23,
+      "ɡ": 24,
+      "ɔ": 25,
+      "h": 26,
+      "aʊ": 27,
+      "y": 28,
+      "m": 29,
+      "f": 30,
+      "ɔː": 31,
+      "x": 32,
+      "ɟ": 33,
+      "t": 34,
+      "eɪ": 35,
+      "oʊ": 36,
+      "p": 37,
+      "ŋ": 38,
+      "ɣ": 39,
+      "yː": 40,
+      "u": 41,
+      "ɪː": 42,
+      "œ": 43,
+      "aɪ": 44,
+      "ç": 45,
+      "ə": 46,
+      "øy": 47,
+      "c": 48,
+      "ɲ": 49,
+      "œː": 50,
+      "ɔɪ": 51
     },
     "unk_token": "UNK"
   }

tokenizer_config.json

@@ -17,7 +17,7 @@
       "single_word": false,
       "special": true
     },
-    "5": {
+    "3": {
       "content": "UTT_BOUNDARY",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"BOS":2,"EOS":3,"WORD_BOUNDARY":4,"UTT_BOUNDARY":5,"aː":6,"ɾ":7,"r":8,"ɪ":9,"ɛ":10,"d":11,"s":12,"j":13,"a":14,"b":15,"iː":16,"k":17,"ʋ":18,"ɛː":19,"θ":20,"i":21,"l":22,"n":23,"uː":24,"ð":25,"ɡ":26,"ɔ":27,"h":28,"aʊ":29,"y":30,"m":31,"f":32,"ɔː":33,"x":34,"ɟ":35,"t":36,"eɪ":37,"oʊ":38,"p":39,"ŋ":40,"ɣ":41,"yː":42,"u":43,"ɪː":44,"œ":45,"aɪ":46,"ç":47,"ə":48,"øy":49,"c":50,"ɲ":51,"œː":52,"ɔɪ":53}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"aː":4,"ɾ":5,"r":6,"ɪ":7,"ɛ":8,"d":9,"s":10,"j":11,"a":12,"b":13,"iː":14,"k":15,"ʋ":16,"ɛː":17,"θ":18,"i":19,"l":20,"n":21,"uː":22,"ð":23,"ɡ":24,"ɔ":25,"h":26,"aʊ":27,"y":28,"m":29,"f":30,"ɔː":31,"x":32,"ɟ":33,"t":34,"eɪ":35,"oʊ":36,"p":37,"ŋ":38,"ɣ":39,"yː":40,"u":41,"ɪː":42,"œ":43,"aɪ":44,"ç":45,"ə":46,"øy":47,"c":48,"ɲ":49,"œː":50,"ɔɪ":51}