Spaces:
Runtime error
Runtime error
| # -*- coding: utf-8 -*- | |
| """app.ipynb | |
| Automatically generated by Colaboratory. | |
| Original file is located at | |
| https://colab.research.google.com/drive/143eWt9oxUTcF59OBiVybOgKXJB3QOTsK | |
| """ | |
| # Beginning of Unit 7 | |
| from transformers.models.markuplm.tokenization_markuplm import MARKUPLM_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING | |
| import torch, torchaudio | |
| from transformers import WhisperProcessor, WhisperForConditionalGeneration | |
| import sentencepiece | |
| from transformers import MarianMTModel, MarianTokenizer | |
| from datasets import load_dataset | |
| from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan | |
| from IPython.display import Audio | |
| import numpy as np | |
| target_dtype = np.int16 | |
| max_range = np.iinfo(target_dtype).max | |
| # Load Spanish Audio | |
| def transcribe(audio): | |
| model_id_asr = "openai/whisper-small" | |
| processor_asr = WhisperProcessor.from_pretrained(model_id_asr) | |
| model_asr = WhisperForConditionalGeneration.from_pretrained(model_id_asr) | |
| model_asr.config.forced_decoder_ids = None | |
| input_features = processor_asr(audio["audio"]["array"], sampling_rate=audio["audio"]["sampling_rate"], return_tensors="pt").input_features | |
| predicted_ids = model_asr.generate(input_features) | |
| # decode token ids to text | |
| transcription = processor_asr.batch_decode(predicted_ids, skip_special_tokens=True) | |
| return transcription[0] | |
| # Run inference on Spanish Audio vector | |
| def translate(text): | |
| model_id_mt = "Helsinki-NLP/opus-mt-es-fr" | |
| tokenizer_mt = MarianTokenizer.from_pretrained(model_id_mt) | |
| model_mt = MarianMTModel.from_pretrained(model_id_mt) | |
| # Tokenize the input text | |
| input_ids = tokenizer_mt.encode(text, return_tensors="pt") | |
| # Generate translation | |
| with torch.no_grad(): | |
| translated_ids = model_mt.generate(input_ids) | |
| # Decode the translated text | |
| translated_text = tokenizer_mt.decode(translated_ids[0], skip_special_tokens=True) | |
| return translated_text | |
| def synthesise(text): | |
| processor_tts = SpeechT5Processor.from_pretrained("crowbarmassage/speecht5_finetuned_voxpopuli_fr") | |
| model_tts = SpeechT5ForTextToSpeech.from_pretrained("crowbarmassage/speecht5_finetuned_voxpopuli_fr") | |
| vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") | |
| embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation") | |
| speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0) | |
| # Load your dataset from Hugging Face | |
| #embeddings_dataset = load_dataset("crowbarmassage/MAEmbed") | |
| #print(embeddings_dataset.features) | |
| #print(embeddings_dataset[0]) | |
| # Extract the embedding (assuming it's in a column named 'embedding') | |
| # Note: Adjust the index [0] if your embedding is at a different position in the dataset. | |
| #embedding_array = embeddings_dataset[0]['embedding'] | |
| # Convert the embedding to a PyTorch tensor and add a batch dimension | |
| #speaker_embeddings = torch.tensor(embedding_array).unsqueeze(0) | |
| print(speaker_embeddings) | |
| print(type(speaker_embeddings)) | |
| inputs = processor_tts(text=text, return_tensors="pt") | |
| speech = model_tts.generate_speech( | |
| inputs["input_ids"], speaker_embeddings, vocoder=vocoder | |
| ) | |
| print(speech) | |
| print(len(speech)) | |
| print(torch.norm(speech)) | |
| return speech | |
| def speech_to_speech_translation(audio_filepath): | |
| # Load the audio file | |
| waveform, sampling_rate = torchaudio.load(audio_filepath) | |
| if sampling_rate != 16000: | |
| resampler = torchaudio.transforms.Resample(orig_freq=sampling_rate, new_freq=16000) | |
| waveform = resampler(waveform) | |
| sampling_rate = 16000 | |
| # Convert the waveform to a numpy array and construct the expected dictionary format | |
| audio_dict = { | |
| "audio": { | |
| "array": waveform.numpy(), | |
| "sampling_rate": sampling_rate | |
| } | |
| } | |
| transcribed_text = transcribe(audio_dict) | |
| translated_text = translate(transcribed_text) | |
| synthesised_speech = synthesise(translated_text) | |
| #print(transcribed_text) | |
| #print(translated_text) | |
| #print(synthesised_speech) | |
| #print(torch.min(synthesised_speech), torch.max(synthesised_speech)) | |
| synthesised_speech = (synthesised_speech * 32767).numpy().astype(np.int16) | |
| #print(synthesised_speech) | |
| #print(np.min(synthesised_speech), np.max(synthesised_speech)) | |
| return 16000, synthesised_speech | |
| import gradio as gr | |
| demo = gr.Blocks() | |
| mic_translate = gr.Interface( | |
| fn=speech_to_speech_translation, | |
| inputs=gr.Audio(source="microphone", type="filepath"), | |
| outputs=gr.Audio(label="Generated Speech", type="numpy"), | |
| ) | |
| file_translate = gr.Interface( | |
| fn=speech_to_speech_translation, | |
| inputs=gr.Audio(source="upload", type="filepath"), | |
| outputs=gr.Audio(label="Generated Speech", type="numpy"), | |
| ) | |
| with demo: | |
| gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"]) | |
| demo.launch(debug=True, share=False) |