File size: 2,361 Bytes
627c134 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
from bark.generation import load_codec_model, generate_text_semantic, grab_best_device
from bark import SAMPLE_RATE
from encodec.utils import convert_audio
from bark.hubert.hubert_manager import HuBERTManager
from bark.hubert.pre_kmeans_hubert import CustomHubert
from bark.hubert.customtokenizer import CustomTokenizer
from bark.api import semantic_to_waveform
from scipy.io.wavfile import write as write_wav
from util.helper import create_filename
from util.settings import Settings
import torchaudio
import torch
import os
import gradio
def swap_voice_from_audio(swap_audio_filename, selected_speaker, tokenizer_lang, seed, batchcount, progress=gradio.Progress(track_tqdm=True)):
use_gpu = not os.environ.get("BARK_FORCE_CPU", False)
progress(0, desc="Loading Codec")
# From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer
hubert_manager = HuBERTManager()
hubert_manager.make_sure_hubert_installed()
hubert_manager.make_sure_tokenizer_installed(tokenizer_lang=tokenizer_lang)
# From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer
# Load HuBERT for semantic tokens
# Load the HuBERT model
device = grab_best_device(use_gpu)
hubert_model = CustomHubert(checkpoint_path='./models/hubert/hubert.pt').to(device)
model = load_codec_model(use_gpu=use_gpu)
# Load the CustomTokenizer model
tokenizer = CustomTokenizer.load_from_checkpoint(f'./models/hubert/{tokenizer_lang}_tokenizer.pth').to(device) # Automatically uses the right layers
progress(0.25, desc="Converting WAV")
# Load and pre-process the audio waveform
wav, sr = torchaudio.load(swap_audio_filename)
if wav.shape[0] == 2: # Stereo to mono if needed
wav = wav.mean(0, keepdim=True)
wav = convert_audio(wav, sr, model.sample_rate, model.channels)
wav = wav.to(device)
semantic_vectors = hubert_model.forward(wav, input_sample_hz=model.sample_rate)
semantic_tokens = tokenizer.get_token(semantic_vectors)
audio = semantic_to_waveform(
semantic_tokens,
history_prompt=selected_speaker,
temp=0.7,
silent=False,
output_full=False)
settings = Settings('config.yaml')
result = create_filename(settings.output_folder_path, None, "swapvoice",".wav")
write_wav(result, SAMPLE_RATE, audio)
return result
|