Updated model

alphabet.json

@@ -1 +1 @@
-{"labels": ["", "R", "Z", "<s>", " ", "0", "9", "G", "F", "3", "L", "E", "Y", "Å", "A", "U", "J", "X", "T", "I", "M", "P", "N", "B", "É", "7", "</s>", "Ä", "1", "4", "C", "Ö", "⁇", "H", "V", "'", "5", "K", "W", "Q", "O", "6", "2", "8", "D", "S"], "is_bpe": false}
+{"labels": ["", "<s>", "</s>", "\u2047", " ", "T", "E", "A", "N", "R", "S", "I", "L", "D", "O", "M", "K", "G", "U", "V", "F", "H", "\u00c4", "\u00c5", "P", "\u00d6", "B", "J", "C", "Y", "X", "W", "Z", "\u00c9", "Q", "8", "2", "5", "9", "1", "6", "7", "3", "4", "0", "'"], "is_bpe": false}

get_tokens.py

@@ -5,11 +5,11 @@ processor = AutoProcessor.from_pretrained(model_id)
 
 vocab_dict = processor.tokenizer.get_vocab()
 # print(vocab_dict)
-sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}
+sorted_vocab_dict = {k: v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}
+print("the sorted vocab list is", sorted_vocab_dict)
 
 from pyctcdecode import build_ctcdecoder
 
-
 decoder = build_ctcdecoder(
     labels=list(sorted_vocab_dict.keys()),
     kenlm_model_path="5gram_correct.arpa",
@@ -21,4 +21,6 @@ processor_with_lm = Wav2Vec2ProcessorWithLM(
     feature_extractor=processor.feature_extractor,
     tokenizer=processor.tokenizer,
     decoder=decoder
-)
+)
+
+processor_with_lm.save_pretrained(".")

language_model/attrs.json

@@ -0,0 +1 @@
+{"alpha": 0.5, "beta": 1.5, "unk_score_offset": -10.0, "score_boundary": true}

language_model/unigrams.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62489a53c88421993c0cb90a248f9f662c45469db1a9106b953ceb44726215bd
+size 5139633

preprocessor_config.json

@@ -4,8 +4,7 @@
   "feature_size": 1,
   "padding_side": "right",
   "padding_value": 0,
+  "processor_class": "Wav2Vec2ProcessorWithLM",
   "return_attention_mask": true,
   "sampling_rate": 16000
 }
-
-

tokenizer_config.json

@@ -1,9 +1 @@
-{
-   "bos_token" : "<s>",
-   "do_lower_case" : true,
-   "eos_token" : "</s>",
-   "pad_token" : "<pad>",
-   "tokenizer_class" : "Wav2Vec2CTCTokenizer",
-   "unk_token" : "<unk>",
-   "word_delimiter_token" : "|"
-}
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": true, "word_delimiter_token": "|", "special_tokens_map_file": "./special_tokens_map.json", "tokenizer_file": null, "name_or_path": ".", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"}