add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "/workspace/.cache/huggingface/transformers/02c153bf53bcd0a6d3bc7f4a41f5448df74644b94a7137e6471c3cd75a57452f.a21d51735cf8667bcd610f057e88548d5d6a381401f6b4501a8bc6c1a9dc8498", "tokenizer_file": null, "name_or_path": "manandey/wav2vec2-large-xlsr-punjabi", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "/workspace/.cache/huggingface/transformers/93e627c1d485fcab8a79fcf898b021187013b29075034ece7e0e46dfa29292ec.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "Harveenchadha/vakyansh-wav2vec2-punjabi-pam-10", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"ਜ": 0, "ਖ": 1, "ਝ": 2, "ਗ਼": 3, "ੁ": 4, "ਰ": 5, "ਸ": 6, "ਢ": 7, "ਦ": 8, "ੋ": 9, "ਠ": 10, "ਣ": 11, "ਸ਼": 12, "ਊ": 13, "ੌ": 14, "ਹ": 15, "ਓ": 16, "ਆ": 17, "ਇ": 18, "ੰ": 19, "ਾ": 20, "ਘ": 21, "ਕ": 23, "ੀ": 24, "ਤ": 25, "ਐ": 26, "ਜ਼": 27, "ਡ": 28, "ਈ": 29, "ਚ": 30, "ਛ": 31, "ਖ਼": 32, "ਧ": 33, "ੜ": 34, "ੱ": 35, "ਥ": 36, "ਫ਼": 37, "ਬ": 38, "ਂ": 39, "ਟ": 40, "ਭ": 41, "ਨ": 42, "ੂ": 43, "ਮ": 44, "੍": 45, "ਪ": 46, "ਵ": 47, "ਉ": 48, "ਫ": 49, "ਏ": 50, "ਔ": 51, "ੇ": 52, "ਲ਼": 53, "਼": 54, "ਯ": 55, "ਗ": 56, "ਅ": 57, "ਿ": 58, "ੈ": 59, "ਲ": 60, "|": 22, "[UNK]": 61, "[PAD]": 62}
+{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3, "|": 4, "ਂ": 5, "ਅ": 6, "ਆ": 7, "ਇ": 8, "ਈ": 9, "ਉ": 10, "ਊ": 11, "ਏ": 12, "ਐ": 13, "ਓ": 14, "ਔ": 15, "ਕ": 16, "ਖ": 17, "ਗ": 18, "ਘ": 19, "ਚ": 20, "ਛ": 21, "ਜ": 22, "ਝ": 23, "ਟ": 24, "ਠ": 25, "ਡ": 26, "ਢ": 27, "ਣ": 28, "ਤ": 29, "ਥ": 30, "ਦ": 31, "ਧ": 32, "ਨ": 33, "ਪ": 34, "ਫ": 35, "ਬ": 36, "ਭ": 37, "ਮ": 38, "ਯ": 39, "ਰ": 40, "ਲ": 41, "ਲ਼": 42, "ਵ": 43, "ਸ਼": 44, "ਸ": 45, "ਹ": 46, "਼": 47, "ਾ": 48, "ਿ": 49, "ੀ": 50, "ੁ": 51, "ੂ": 52, "ੇ": 53, "ੈ": 54, "ੋ": 55, "ੌ": 56, "੍": 57, "ਖ਼": 58, "ਗ਼": 59, "ਜ਼": 60, "ੜ": 61, "ਫ਼": 62, "ੰ": 63, "ੱ": 64}