import argparse
import os
from pathlib import Path
import logging
import re_matching
logging.getLogger("numba").setLevel(logging.WARNING)
logging.getLogger("markdown_it").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("matplotlib").setLevel(logging.WARNING)
logging.basicConfig(
level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
)
logger = logging.getLogger(__name__)
import librosa
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import Dataset
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm
from clap_wrapper import get_clap_audio_feature, get_clap_text_feature
from tools.sentence import extrac, is_japanese, is_chinese, seconds_to_ass_time, extract_text_from_file, remove_annotations,extract_and_convert
import re
import gradio as gr
import utils
from config import config
import torch
import commons
from text import cleaned_text_to_sequence, get_bert
from text.cleaner import clean_text
import utils
from scipy.io.wavfile import write
from models import SynthesizerTrn
from text.symbols import symbols
import sys
import shutil
net_g = None
device = (
"cuda:0"
if torch.cuda.is_available()
else (
"mps"
if sys.platform == "darwin" and torch.backends.mps.is_available()
else "cpu"
)
)
BandList = {
"PoppinParty":["香澄","有咲","たえ","りみ","沙綾"],
"Afterglow":["蘭","モカ","ひまり","巴","つぐみ"],
"HelloHappyWorld":["こころ","美咲","薫","花音","はぐみ"],
"PastelPalettes":["彩","日菜","千聖","イヴ","麻弥"],
"Roselia":["友希那","紗夜","リサ","燐子","あこ"],
"RaiseASuilen":["レイヤ","ロック","ますき","チュチュ","パレオ"],
"Morfonica":["ましろ","瑠唯","つくし","七深","透子"],
"MyGo":["燈","愛音","そよ","立希","楽奈"],
"AveMujica":["祥子","睦","海鈴","にゃむ","初華"],
"圣翔音乐学园":["華戀","光","香子","雙葉","真晝","純那","克洛迪娜","真矢","奈奈"],
"凛明馆女子学校":["珠緒","壘","文","悠悠子","一愛"],
"弗隆提亚艺术学校":["艾露","艾露露","菈樂菲","司","靜羽"],
"西克菲尔特音乐学院":["晶","未知留","八千代","栞","美帆"]
}
def get_net_g(model_path: str, device: str, hps):
# 当前版本模型 net_g
net_g = SynthesizerTrn(
len(symbols),
hps.data.filter_length // 2 + 1,
hps.train.segment_size // hps.data.hop_length,
n_speakers=hps.data.n_speakers,
**hps.model,
).to(device)
_ = net_g.eval()
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
return net_g
def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
style_text = None if style_text == "" else style_text
# 在此处实现当前版本的get_text
norm_text, phone, tone, word2ph = clean_text(text, language_str)
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
if hps.data.add_blank:
phone = commons.intersperse(phone, 0)
tone = commons.intersperse(tone, 0)
language = commons.intersperse(language, 0)
for i in range(len(word2ph)):
word2ph[i] = word2ph[i] * 2
word2ph[0] += 1
bert = get_bert(norm_text, word2ph, language_str, device, style_text, style_weight)
del word2ph
assert bert.shape[-1] == len(
phone
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
phone = torch.LongTensor(phone)
tone = torch.LongTensor(tone)
language = torch.LongTensor(language)
return bert, phone, tone, language
def infer(
text,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
emotion,
reference_audio=None,
skip_start=False,
skip_end=False,
style_text=None,
style_weight=0.7,
):
language = "JP"
if isinstance(reference_audio, np.ndarray):
emo = get_clap_audio_feature(reference_audio, device)
else:
emo = get_clap_text_feature(emotion, device)
emo = torch.squeeze(emo, dim=1)
bert, phones, tones, lang_ids = get_text(
text,
language,
hps,
device,
style_text=style_text,
style_weight=style_weight,
)
if skip_start:
phones = phones[3:]
tones = tones[3:]
lang_ids = lang_ids[3:]
bert = bert[:, 3:]
if skip_end:
phones = phones[:-2]
tones = tones[:-2]
lang_ids = lang_ids[:-2]
bert = bert[:, :-2]
with torch.no_grad():
x_tst = phones.to(device).unsqueeze(0)
tones = tones.to(device).unsqueeze(0)
lang_ids = lang_ids.to(device).unsqueeze(0)
bert = bert.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
emo = emo.to(device).unsqueeze(0)
del phones
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
audio = (
net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
emo,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
del (
x_tst,
tones,
lang_ids,
bert,
x_tst_lengths,
speakers,
emo,
) # , emo
if torch.cuda.is_available():
torch.cuda.empty_cache()
return (hps.data.sampling_rate,gr.processing_utils.convert_to_16_bit_wav(audio))
'''srt格式
def generate_audio_and_srt_for_group(group, outputPath, group_index, sampling_rate, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime):
audio_fin = []
ass_entries = []
start_time = 0
#speaker = random.choice(cara_list)
ass_header = """[Script Info]
; 我没意见
Title: Audiobook
ScriptType: v4.00+
WrapStyle: 0
PlayResX: 640
PlayResY: 360
ScaledBorderAndShadow: yes
[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
Style: Default,Arial,20,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,1,2,10,10,10,1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
"""
for sentence in group:
try:
print(sentence)
FakeSpeaker = sentence.split("|")[0]
print(FakeSpeaker)
SpeakersList = re.split('\n', spealerList)
if FakeSpeaker in list(hps.data.spk2id.keys()):
speaker = FakeSpeaker
for i in SpeakersList:
if FakeSpeaker == i.split("|")[1]:
speaker = i.split("|")[0]
if sentence != '\n':
audio = infer_simple((remove_annotations(sentence.split("|")[-1]).replace(" ","")+"。").replace("，。","。").replace("。。","。"), sdp_ratio, noise_scale, noise_scale_w, length_scale,speaker)
silence_frames = int(silenceTime * 44010) if is_chinese(sentence) else int(silenceTime * 44010)
silence_data = np.zeros((silence_frames,), dtype=audio.dtype)
audio_fin.append(audio)
audio_fin.append(silence_data)
duration = len(audio) / sampling_rate
print(duration)
end_time = start_time + duration + silenceTime
ass_entries.append("Dialogue: 0,{},{},".format(seconds_to_ass_time(start_time), seconds_to_ass_time(end_time)) + "Default,,0,0,0,,{}".format(sentence.replace("|","：")))
start_time = end_time
except:
pass
wav_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.wav')
ass_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.ass')
write(wav_filename, sampling_rate, np.concatenate(audio_fin))
with open(ass_filename, 'w', encoding='utf-8') as f:
f.write(ass_header + '\n'.join(ass_entries))
return (hps.data.sampling_rate, np.concatenate(audio_fin))
'''
def format_srt_timestamp(seconds):
ms = int((seconds - int(seconds)) * 1000)
seconds = int(seconds)
hours = seconds // 3600
minutes = (seconds % 3600) // 60
seconds = seconds % 60
return f"{hours:02}:{minutes:02}:{seconds:02},{ms:03}"
def clean_sentence(sentence):
return sentence.replace('\n', '').replace('\r', '').replace(' ', '')
def generate_audio_and_srt_for_group(group, outputPath, group_index, sampling_rate, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale, spealerList, silenceTime):
audio_fin = []
srt_entries = []
start_time = 0
for i, sentence in enumerate(group):
try:
FakeSpeaker = sentence.split("|")[0]
SpeakersList = re.split('\n', spealerList)
if FakeSpeaker in list(hps.data.spk2id.keys()):
speaker = FakeSpeaker
for s in SpeakersList:
if FakeSpeaker == s.split("|")[1]:
speaker = s.split("|")[0]
if len(sentence)>2 and (sentence != '\n' or sentence != '\r' or sentence != '' or sentence != ' ' or sentence != '\r\n'):
clean_msg = clean_sentence(sentence.split("|")[-1])
audio = infer_simple((remove_annotations(clean_msg) + "。").replace("，。", "。").replace("。。", "。"), sdp_ratio, noise_scale, noise_scale_w, length_scale, speaker)
silence_frames = int(silenceTime * 44100) if is_chinese(sentence) else int(silenceTime * 44100)
silence_data = np.zeros((silence_frames,), dtype=audio.dtype)
audio_fin.append(audio)
audio_fin.append(silence_data)
duration = len(audio) / sampling_rate
end_time = start_time + duration + silenceTime
srt_entries.append(f"{i+1}\n{format_srt_timestamp(start_time)} --> {format_srt_timestamp(end_time)}\n{clean_msg.replace('|', '：')}\n\n")
start_time = end_time
except:
pass
wav_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.wav')
srt_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.srt')
write(wav_filename, sampling_rate, np.concatenate(audio_fin))
with open(srt_filename, 'w', encoding='utf-8') as f:
f.writelines(srt_entries)
return (hps.data.sampling_rate, np.concatenate(audio_fin))
def infer_simple(
text,
sdp_ratio,
noise_scale,
noise_scale_w,
length_scale,
sid,
emotion = '',
reference_audio=None,
skip_start=False,
skip_end=False,
style_text=None,
style_weight=0.7,
):
language = "JP"
if isinstance(reference_audio, np.ndarray):
emo = get_clap_audio_feature(reference_audio, device)
else:
emo = get_clap_text_feature(emotion, device)
emo = torch.squeeze(emo, dim=1)
bert, phones, tones, lang_ids = get_text(
text,
language,
hps,
device,
style_text=style_text,
style_weight=style_weight,
)
if skip_start:
phones = phones[3:]
tones = tones[3:]
lang_ids = lang_ids[3:]
bert = bert[:, 3:]
if skip_end:
phones = phones[:-2]
tones = tones[:-2]
lang_ids = lang_ids[:-2]
bert = bert[:, :-2]
with torch.no_grad():
x_tst = phones.to(device).unsqueeze(0)
tones = tones.to(device).unsqueeze(0)
lang_ids = lang_ids.to(device).unsqueeze(0)
bert = bert.to(device).unsqueeze(0)
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
emo = emo.to(device).unsqueeze(0)
del phones
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
audio = (
net_g.infer(
x_tst,
x_tst_lengths,
speakers,
tones,
lang_ids,
bert,
emo,
sdp_ratio=sdp_ratio,
noise_scale=noise_scale,
noise_scale_w=noise_scale_w,
length_scale=length_scale,
)[0][0, 0]
.data.cpu()
.float()
.numpy()
)
del (
x_tst,
tones,
lang_ids,
bert,
x_tst_lengths,
speakers,
emo,
) # , emo
if torch.cuda.is_available():
torch.cuda.empty_cache()
return audio
def audiobook(inputFile, groupsize, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime,filepath,raw_text):
directory_path = filepath if torch.cuda.is_available() else "books"
if os.path.exists(directory_path):
shutil.rmtree(directory_path)
os.makedirs(directory_path)
if inputFile:
text = extract_text_from_file(inputFile.name)
else:
text = raw_text
sentences = extrac(extract_and_convert(text))
GROUP_SIZE = groupsize
for i in range(0, len(sentences), GROUP_SIZE):
group = sentences[i:i+GROUP_SIZE]
if spealerList == "":
spealerList = "无"
result = generate_audio_and_srt_for_group(group,directory_path, i//GROUP_SIZE + 1, 44100, speaker, sdp_ratio, noise_scale, noise_scale_w, length_scale,spealerList,silenceTime)
if not torch.cuda.is_available():
return result
return result
def loadmodel(model):
_ = net_g.eval()
_ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
return "success"
if __name__ == "__main__":
modelPaths = []
for dirpath, dirnames, filenames in os.walk('Data/BangDream/models/'):
for filename in filenames:
modelPaths.append(os.path.join(dirpath, filename))
hps = utils.get_hparams_from_file('Data/BangDream//config.json')
net_g = get_net_g(
model_path=modelPaths[-1], device=device, hps=hps
)
speaker_ids = hps.data.spk2id
speakers = list(speaker_ids.keys())
with gr.Blocks() as app:
for band in BandList:
with gr.TabItem(band):
for name in BandList[band]:
with gr.TabItem(name):
with gr.Row():
with gr.Column():
with gr.Row():
gr.Markdown(
''
f'
'
'