File size: 2,639 Bytes
79a08d6 411cc33 79a08d6 7e1c39f 79a08d6 7e1c39f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
from bark.generation import load_codec_model, generate_text_semantic, grab_best_device
from encodec.utils import convert_audio
from bark.hubert.hubert_manager import HuBERTManager
from bark.hubert.pre_kmeans_hubert import CustomHubert
from bark.hubert.customtokenizer import CustomTokenizer
import torchaudio
import torch
import os
import gradio
def clone_voice(audio_filepath, dest_filename, progress=gradio.Progress(track_tqdm=True)):
# if len(text) < 1:
# raise gradio.Error('No transcription text entered!')
use_gpu = False # not os.environ.get("BARK_FORCE_CPU", False)
progress(0, desc="Loading Codec")
model = load_codec_model(use_gpu=use_gpu)
# From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer
hubert_manager = HuBERTManager()
hubert_manager.make_sure_hubert_installed()
hubert_manager.make_sure_tokenizer_installed()
# From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer
# Load HuBERT for semantic tokens
# Load the HuBERT model
device = grab_best_device(use_gpu)
hubert_model = CustomHubert(checkpoint_path='./models/hubert/hubert.pt').to(device)
# Load the CustomTokenizer model
tokenizer = CustomTokenizer.load_from_checkpoint('./models/hubert/en_tokenizer.pth').to(device) # change to the correct path
progress(0.25, desc="Converting WAV")
# Load and pre-process the audio waveform
wav, sr = torchaudio.load(audio_filepath)
if wav.shape[0] == 2: # Stereo to mono if needed
wav = wav.mean(0, keepdim=True)
wav = convert_audio(wav, sr, model.sample_rate, model.channels)
wav = wav.to(device)
progress(0.5, desc="Extracting codes")
semantic_vectors = hubert_model.forward(wav, input_sample_hz=model.sample_rate)
semantic_tokens = tokenizer.get_token(semantic_vectors)
# Extract discrete codes from EnCodec
with torch.no_grad():
encoded_frames = model.encode(wav.unsqueeze(0))
codes = torch.cat([encoded[0] for encoded in encoded_frames], dim=-1).squeeze() # [n_q, T]
# get seconds of audio
# seconds = wav.shape[-1] / model.sample_rate
# generate semantic tokens
# semantic_tokens = generate_text_semantic(text, max_gen_duration_s=seconds, top_k=50, top_p=.95, temp=0.7)
# move codes to cpu
codes = codes.cpu().numpy()
# move semantic tokens to cpu
semantic_tokens = semantic_tokens.cpu().numpy()
import numpy as np
output_path = dest_filename + '.npz'
np.savez(output_path, fine_prompt=codes, coarse_prompt=codes[:2, :], semantic_prompt=semantic_tokens)
return ["Finished", output_path] |