Code Added

app.py

@@ -0,0 +1,122 @@
+import gradio as gr
+import torchaudio
+import torch
+import torch.nn.functional as F
+from speechbrain.inference.speaker import EncoderClassifier
+from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
+import noisereduce as nr
+import librosa
+
+# Load the classifier model
+classifier = EncoderClassifier.from_hparams(source="speechbrain/spkrec-xvect-voxceleb", savedir="pretrained_models/spkrec-xvect-voxceleb")
+
+
+def f2embed(wav_file, classifier, size_embed):
+
+    signal, fs = stereo_to_mono(wav_file)
+    if signal is None:
+        return None
+    # print(fs, "FS")
+    if fs != 16000:
+        signal, fs = resample_to_16000(signal, fs)
+        if signal is None:
+            return None
+    assert fs == 16000, fs
+    with torch.no_grad():
+        embeddings = classifier.encode_batch(signal)
+        embeddings = F.normalize(embeddings, dim=2)
+        embeddings = embeddings.squeeze().cpu().numpy()
+    assert embeddings.shape[0] == size_embed, embeddings.shape[0]
+    return embeddings
+
+def stereo_to_mono(wav_file):
+    try:
+        signal, fs = torchaudio.load(wav_file)
+        signal_np = signal.numpy()
+        if signal_np.shape[0] == 2:  # If stereo
+            signal_mono = librosa.to_mono(signal_np)
+            signal_mono = torch.from_numpy(signal_mono).unsqueeze(0)
+        else:
+            signal_mono = signal  # Already mono
+        print(f"Converted to mono: {signal_mono.shape}")
+        return signal_mono, fs
+    except Exception as e:
+        print(f"Error in stereo_to_mono: {e}")
+        return None, None
+
+def resample_to_16000(signal, original_sr):
+    try:
+        signal_np = signal.numpy().flatten()
+        signal_resampled = librosa.resample(signal_np, orig_sr=original_sr, target_sr=16000)
+        signal_resampled = torch.from_numpy(signal_resampled).unsqueeze(0)
+        print(f"Resampled to 16000 Hz: {signal_resampled.shape}")
+        return signal_resampled, 16000
+    except Exception as e:
+        print(f"Error in resample_to_16000: {e}")
+        return None, None
+
+def reduce_noise(speech, noise_reduction_amount=0.5):
+    try:
+        denoised_speech = nr.reduce_noise(y=speech, sr=16000)
+        return denoised_speech
+    except Exception as e:
+        print(f"Error in reduce_noise: {e}")
+        return speech
+
+
+
+def process_audio(wav_file, text):
+    try:
+        # Extract speaker embeddings
+        speaker_embeddings = f2embed(wav_file, classifier, 512)
+        if speaker_embeddings is None:
+            return None, "Error in speaker embedding extraction"
+
+        embeddings = torch.tensor(speaker_embeddings).unsqueeze(0)
+        
+        # Load and process the speech file
+        signal, fs = torchaudio.load(wav_file)
+        signal_np = signal.numpy().flatten()
+        print(f"Loaded signal: {signal_np.shape}, Sample rate: {fs}")
+        
+        # Convert text to speech using the speaker embeddings
+        processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
+        model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
+        inputs = processor(text=text, return_tensors="pt")
+        inputs.update({"speaker_embeddings": embeddings})
+        vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+        speech = model.generate_speech(inputs["input_ids"], speaker_embeddings=inputs["speaker_embeddings"],vocoder=vocoder)
+        print(f"Generated speech, shape: {speech.shape}")
+
+        # Reduce noise
+        speech_denoised = reduce_noise(speech)
+        print(f"Reduced noise, signal shape: {speech_denoised.shape}")
+        return speech_denoised, 16000
+    except Exception as e:
+        print(f"Error in process_audio: {e}")
+        return None, "Error in audio processing"
+
+# Gradio interface
+def gradio_interface(wav_file, text):
+    try:
+        processed_audio, rate = process_audio(wav_file, text)
+        if processed_audio is None:
+            return "Error occurred during processing"
+        return (rate, processed_audio)
+    except Exception as e:
+        print(f"Error in gradio_interface: {e}")
+        return "Error occurred during processing"
+
+# Create Gradio interface
+gr_interface = gr.Interface(
+    fn=gradio_interface,
+    inputs=[gr.Audio(type="filepath"), gr.Textbox(lines=2, placeholder="Enter text here...")],
+    outputs=gr.Audio(type="numpy"),
+    title="Text-to-Speech with Speaker Embeddings",
+    description="Upload a speaker audio file and enter text to convert the text to speech using the speaker's voice.",
+)
+
+gr_interface.launch()
+
+
+# process_audio("/content/Network Chunck.mp3","Hello this network chunk")

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+torchaudio
+transformers
+noisereduce
+librosa
+speechbrain