import nltk import librosa import torch import gradio as gr from pyctcdecode import build_ctcdecoder from transformers import AutoProcessor, AutoModelForCTC nltk.download("punkt") model_name = "facebook/wav2vec2-base-960h" processor = AutoProcessor.from_pretrained(model_name) model = AutoModelForCTC.from_pretrained(model_name) def load_and_fix_data(input_file): #read the file speech, sample_rate = librosa.load(input_file) #make it 1D if len(speech.shape) > 1: speech = speech[:,0] + speech[:,1] #resampling to 16KHz if sample_rate !=16000: speech = librosa.resample(speech, sample_rate,16000) return speech def fix_transcription_casing(input_sentence): sentences = nltk.sent_tokenize(input_sentence) return (' '.join([s.replace(s[0],s[0].capitalize(),1) for s in sentences])) def predict_and_ctc_decode(input_file): speech = load_and_fix_data(input_file) input_values = processor(speech, return_tensors="pt", sampling_rate=16000).input_values logits = model(input_values).logits.cpu().detach().numpy()[0] vocab_list = list(processor.tokenizer.get_vocab().keys()) decoder = build_ctcdecoder(vocab_list) pred = decoder.decode(logits) transcribed_text = fix_transcription_casing(pred.lower()) return transcribed_text def predict_and_greedy_decode(input_file): speech = load_and_fix_data(input_file) input_values = processor(speech, return_tensors="pt", sampling_rate=16000).input_values logits = model(input_values).logits predicted_ids = torch.argmax(logits, dim=-1) pred = processor.batch_decode(predicted_ids) transcribed_text = fix_transcription_casing(pred[0].lower()) return transcribed_text def return_all_predictions(input_file, model_name): print(model_name) return predict_and_ctc_decode(input_file), predict_and_greedy_decode(input_file) gr.Interface(return_all_predictions, inputs = [gr.inputs.Audio(source="microphone", type="filepath", label="Record/ Drop audio"), gr.inputs.Dropdown(["facebook/wav2vec2-base-960h", "facebook/hubert-large-ls960-ft"], label="Model Name")], outputs = [gr.outputs.Textbox(label="Beam CTC Decoding"), gr.outputs.Textbox(label="Greedy Decoding")], title="ASR using Wav2Vec 2.0 & pyctcdecode", description = "Extending HF ASR models with pyctcdecode decoder", layout = "horizontal", examples = [["test1.wav", "facebook/wav2vec2-base-960h"], ["test2.wav", "facebook/hubert-large-ls960-ft"]], theme="huggingface").launch()