add tokenizer

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "replace_word_delimiter_char": " ",
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "[UNK]",
+  "word_delimiter_token": "|"
+}

vocab.json

@@ -0,0 +1,124 @@
+{
+  "-": 1,
+  "0": 2,
+  "1": 3,
+  "2": 4,
+  "3": 5,
+  "4": 6,
+  "5": 7,
+  "6": 8,
+  "7": 9,
+  "8": 10,
+  "9": 11,
+  "[PAD]": 121,
+  "[UNK]": 120,
+  "a": 12,
+  "c": 13,
+  "f": 14,
+  "h": 15,
+  "o": 16,
+  "p": 17,
+  "u": 18,
+  "|": 0,
+  "々": 19,
+  "〇": 20,
+  "ぁ": 21,
+  "あ": 22,
+  "ぃ": 23,
+  "い": 24,
+  "ぅ": 25,
+  "う": 26,
+  "ぇ": 27,
+  "え": 28,
+  "ぉ": 29,
+  "お": 30,
+  "か": 31,
+  "が": 32,
+  "き": 33,
+  "ぎ": 34,
+  "く": 35,
+  "ぐ": 36,
+  "け": 37,
+  "げ": 38,
+  "こ": 39,
+  "ご": 40,
+  "さ": 41,
+  "ざ": 42,
+  "し": 43,
+  "じ": 44,
+  "す": 45,
+  "ず": 46,
+  "せ": 47,
+  "ぜ": 48,
+  "そ": 49,
+  "ぞ": 50,
+  "た": 51,
+  "だ": 52,
+  "ち": 53,
+  "ぢ": 54,
+  "っ": 55,
+  "つ": 56,
+  "づ": 57,
+  "て": 58,
+  "で": 59,
+  "と": 60,
+  "ど": 61,
+  "な": 62,
+  "に": 63,
+  "ぬ": 64,
+  "ね": 65,
+  "の": 66,
+  "は": 67,
+  "ば": 68,
+  "ぱ": 69,
+  "ひ": 70,
+  "び": 71,
+  "ぴ": 72,
+  "ふ": 73,
+  "ぶ": 74,
+  "ぷ": 75,
+  "へ": 76,
+  "べ": 77,
+  "ぺ": 78,
+  "ほ": 79,
+  "ぼ": 80,
+  "ぽ": 81,
+  "ま": 82,
+  "み": 83,
+  "む": 84,
+  "め": 85,
+  "も": 86,
+  "ゃ": 87,
+  "や": 88,
+  "ゅ": 89,
+  "ゆ": 90,
+  "ょ": 91,
+  "よ": 92,
+  "ら": 93,
+  "り": 94,
+  "る": 95,
+  "れ": 96,
+  "ろ": 97,
+  "わ": 98,
+  "を": 99,
+  "ん": 100,
+  "ゔ": 101,
+  "ゖ": 102,
+  "ー": 103,
+  "％": 104,
+  "０": 105,
+  "１": 106,
+  "２": 107,
+  "３": 108,
+  "４": 109,
+  "５": 110,
+  "６": 111,
+  "７": 112,
+  "８": 113,
+  "９": 114,
+  "Ｄ": 115,
+  "Ｇ": 116,
+  "Ｐ": 117,
+  "ｉ": 118,
+  "ｘ": 119
+}