Updated files

get_tokens.py

@@ -0,0 +1,15 @@
+from transformers import Wav2Vec2ProcessorWithLM
+import torchaudio
+
+import torch
+from datasets import load_dataset
+from transformers import AutoModelForCTC, AutoProcessor
+import torchaudio.functional as F
+
+model_id = "."
+model = AutoModelForCTC.from_pretrained(model_id)
+processor = AutoProcessor.from_pretrained(model_id)
+
+vocab_dict = processor.tokenizer.get_vocab()
+print(vocab_dict)
+sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}

lm.py

@@ -10,6 +10,7 @@ import torchaudio.functional as F
 
 model_id = "."
 
+
 sample_iter = iter(load_dataset("mozilla-foundation/common_voice_7_0", "sv-SE", split="test", streaming=True, use_auth_token=True))
 
 sample = next(sample_iter)