add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 54, "</s>": 55}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"'": 1, "a": 2, "e": 3, "g": 4, "h": 5, "m": 6, "n": 7, "o": 8, "r": 9, "t": 10, "v": 11, "«": 12, "´": 13, "·": 14, "»": 15, "́": 16, "ΐ": 17, "ά": 18, "έ": 19, "ή": 20, "ί": 21, "α": 22, "β": 23, "γ": 24, "δ": 25, "ε": 26, "ζ": 27, "η": 28, "θ": 29, "ι": 30, "κ": 31, "λ": 32, "μ": 33, "ν": 34, "ξ": 35, "ο": 36, "π": 37, "ρ": 38, "ς": 39, "σ": 40, "τ": 41, "υ": 42, "φ": 43, "χ": 44, "ψ": 45, "ω": 46, "ϊ": 47, "ϋ": 48, "ό": 49, "ύ": 50, "ώ": 51, "|": 0, "[UNK]": 52, "[PAD]": 53}