add tokenizer

added_tokens.json

@@ -1,5 +1,4 @@
 {
-  "</s>": 28,
-  "<pad>": 29,
-  "<s>": 27
+  "</s>": 30,
+  "<s>": 29
 }

runs/Jul21_14-56-25_Tejas-XPS22/1658429826.1307015/events.out.tfevents.1658429826.Tejas-XPS22.26960.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6667350dbbaa134b90a85b4e9365896a5ecc703e8ad15659deaec2e22faea416
+size 5385

runs/Jul21_14-56-25_Tejas-XPS22/events.out.tfevents.1658429826.Tejas-XPS22.26960.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cdb35ef0b21503c52f0b11f7ef461f580e1fc312bf01c6a1a94c6b8ae2719a2
+size 5092

special_tokens_map.json

@@ -13,17 +13,10 @@
       "normalized": true,
       "rstrip": false,
       "single_word": false
-    },
-    {
-      "content": "<pad>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false
     }
   ],
   "bos_token": "<s>",
   "eos_token": "</s>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>"
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
 }

tokenizer_config.json

@@ -3,10 +3,10 @@
   "do_lower_case": false,
   "eos_token": "</s>",
   "name_or_path": "./",
-  "pad_token": "<pad>",
+  "pad_token": "[PAD]",
   "replace_word_delimiter_char": " ",
   "special_tokens_map_file": null,
   "tokenizer_class": "Wav2Vec2CTCTokenizer",
-  "unk_token": "<unk>",
-  "word_delimiter_token": "|"
+  "unk_token": "[UNK]",
+  "word_delimiter_token": " "
 }

vocab.json

@@ -1,5 +1,7 @@
 {
   " ": 0,
+  "[PAD]": 28,
+  "[UNK]": 27,
   "a": 1,
   "b": 2,
   "c": 3,