WIP updated lm

.gitattributes

@@ -15,3 +15,5 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.arpa filter=lfs diff=lfs merge=lfs -text
+*.txt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+kenlm/

language_model/5gram.bin → 5gram_correct.arpa

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c803936922612f71cf0abdb37763c18d24624e36bfa4abac20187cc17b88541d
-size 1981380707
+oid sha256:3aab18b31980b9b9fdf85546c8763af2a4e2220d464ab5e3fab99cf19c3158dd
+size 4394946469

build_n_gram.py

@@ -0,0 +1,9 @@
+from datasets import load_dataset
+
+target_lang="sv"  # change to your target lang
+username = "hf-test"  # change to your username
+
+dataset = load_dataset(f"{username}/{target_lang}_corpora_parliament_processed", split="train")
+
+with open("text.txt", "w") as file:
+  file.write(" ".join(dataset["text"]))

end_token.py

@@ -0,0 +1,12 @@
+with open("5gram.arpa", "r") as read_file, open("5gram_correct.arpa", "w") as write_file:
+  has_added_eos = False
+  for line in read_file:
+    if not has_added_eos and "ngram 1=" in line:
+      count=line.strip().split("=")[-1]
+      write_file.write(line.replace(f"{count}", f"{int(count)+1}"))
+    elif not has_added_eos and "<s>" in line:
+      write_file.write(line)
+      write_file.write(line.replace("<s>", "</s>"))
+      has_added_eos = True
+    else:
+      write_file.write(line)

get_tokens.py

@@ -1,15 +1,24 @@
-from transformers import Wav2Vec2ProcessorWithLM
-import torchaudio
-
-import torch
-from datasets import load_dataset
 from transformers import AutoModelForCTC, AutoProcessor
-import torchaudio.functional as F
-
 model_id = "."
 model = AutoModelForCTC.from_pretrained(model_id)
 processor = AutoProcessor.from_pretrained(model_id)
 
 vocab_dict = processor.tokenizer.get_vocab()
-print(vocab_dict)
-sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}
+# print(vocab_dict)
+sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}
+
+from pyctcdecode import build_ctcdecoder
+
+
+decoder = build_ctcdecoder(
+    labels=list(sorted_vocab_dict.keys()),
+    kenlm_model_path="5gram_correct.arpa",
+)
+
+from transformers import Wav2Vec2ProcessorWithLM
+
+processor_with_lm = Wav2Vec2ProcessorWithLM(
+    feature_extractor=processor.feature_extractor,
+    tokenizer=processor.tokenizer,
+    decoder=decoder
+)

language_model/attrs.json

@@ -1 +0,0 @@
-{"alpha": 0.5, "beta": 1.5, "unk_score_offset": -10.0, "score_boundary": true}

preprocessor_config.json

@@ -5,8 +5,7 @@
   "padding_side": "right",
   "padding_value": 0,
   "return_attention_mask": true,
-  "sampling_rate": 16000,
-	"processor_class": "Wav2Vec2ProcessorWithLM"
+  "sampling_rate": 16000
 }
 
 

text.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fba5ecc23b254f6c36c7f18e9052bb7db01c3dfe2fd5786dd105410b4b9e094f
+size 286673267