ngram.py

from transformers import AutoProcessor
from transformers import Wav2Vec2ProcessorWithLM
from huggingface_hub import Repository
from pyctcdecode import build_ctcdecoder

model_name = "cahya/wav2vec2-base-turkish-artificial-cv"
processor = AutoProcessor.from_pretrained(model_name)

vocab_dict = processor.tokenizer.get_vocab()
sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}

decoder = build_ctcdecoder(
    labels=list(sorted_vocab_dict.keys()),
    kenlm_model_path="5gram.arpa",
)

processor_with_lm = Wav2Vec2ProcessorWithLM(
    feature_extractor=processor.feature_extractor,
    tokenizer=processor.tokenizer,
    decoder=decoder
)

#repo = Repository(local_dir="wav2vec2-base-turkish", clone_from=model_name)
processor_with_lm.save_pretrained("wav2vec2-base-turkish")
#repo.push_to_hub(commit_message="Upload lm-boosted decoder")