Spaces:

Mahiruoshi
/

MyGO_VIts-bert

Running

App Files Files Community

MyGO_VIts-bert / app.py

Mahiruoshi

Update app.py

cb3cd10 verified 4 months ago

raw history blame contribute delete

No virus

25.9 kB

	import argparse
	import os
	from pathlib import Path

	import logging
	import re_matching

	logging.getLogger("numba").setLevel(logging.WARNING)
	logging.getLogger("markdown_it").setLevel(logging.WARNING)
	logging.getLogger("urllib3").setLevel(logging.WARNING)
	logging.getLogger("matplotlib").setLevel(logging.WARNING)

	logging.basicConfig(
	level=logging.INFO, format="\| %(name)s \| %(levelname)s \| %(message)s"
	)

	logger = logging.getLogger(__name__)
	import shutil
	from scipy.io.wavfile import write
	import librosa
	import numpy as np
	import torch
	import torch.nn as nn
	from torch.utils.data import Dataset
	from torch.utils.data import DataLoader, Dataset
	from tqdm import tqdm

	import gradio as gr

	import utils
	from config import config

	import torch
	import commons
	from text import cleaned_text_to_sequence, get_bert

	from tools.sentence import extrac, is_japanese, is_chinese, seconds_to_ass_time, extract_text_from_file, remove_annotations,extract_and_convert

	from text.cleaner import clean_text
	import utils

	from tools.translate import translate

	from models import SynthesizerTrn
	from text.symbols import symbols
	import sys
	import re

	import random
	import hashlib

	from fugashi import Tagger
	import jaconv
	import unidic
	import subprocess

	import requests

	from ebooklib import epub
	import PyPDF2
	from PyPDF2 import PdfReader
	from bs4 import BeautifulSoup
	import jieba
	import romajitable

	webBase = {
	'pyopenjtalk-V2.3-Katakana': 'https://mahiruoshi-mygo-vits-bert.hf.space/',
	'fugashi-V2.3-Katakana': 'https://mahiruoshi-mygo-vits-bert.hf.space/',
	}

	languages = [ "Auto", "ZH", "JP"]
	modelPaths = []
	modes = ['pyopenjtalk-V2.3']
	if torch.cuda.is_available():
	modes = ['pyopenjtalk-V2.3','fugashi-V2.3']
	sentence_modes = ['sentence','paragraph']

	net_g = None

	device = (
	"cuda:0"
	if torch.cuda.is_available()
	else (
	"mps"
	if sys.platform == "darwin" and torch.backends.mps.is_available()
	else "cpu"
	)
	)

	#device = "cpu"
	BandList = {
	"PoppinParty":["香澄","有咲","たえ","りみ","沙綾"],
	"Afterglow":["蘭","モカ","ひまり","巴","つぐみ"],
	"HelloHappyWorld":["こころ","美咲","薫","花音","はぐみ"],
	"PastelPalettes":["彩","日菜","千聖","イヴ","麻弥"],
	"Roselia":["友希那","紗夜","リサ","燐子","あこ"],
	"RaiseASuilen":["レイヤ","ロック","ますき","チュチュ","パレオ"],
	"Morfonica":["ましろ","瑠唯","つくし","七深","透子"],
	"MyGo":["燈","愛音","そよ","立希","楽奈"],
	"AveMujica":["祥子","睦","海鈴","にゃむ","初華"],
	"圣翔音乐学园":["華戀","光","香子","雙葉","真晝","純那","克洛迪娜","真矢","奈奈"],
	"凛明馆女子学校":["珠緒","壘","文","悠悠子","一愛"],
	"弗隆提亚艺术学校":["艾露","艾露露","菈樂菲","司","靜羽"],
	"西克菲尔特音乐学院":["晶","未知留","八千代","栞","美帆"]
	}

	# 推理工具
	def download_unidic():
	try:
	Tagger()
	print("Tagger launch successfully.")
	except Exception as e:
	print("UNIDIC dictionary not found, downloading...")
	subprocess.run([sys.executable, "-m", "unidic", "download"])
	print("Download completed.")

	def kanji_to_hiragana(text):
	global tagger
	output = ""

	# 更新正则表达式以更准确地区分文本和标点符号
	segments = re.findall(r'[一-龥ぁ-んァ-ン\w]+\|[^\一-龥ぁ-んァ-ン\w\s]', text, re.UNICODE)

	for segment in segments:
	if re.match(r'[一-龥ぁ-んァ-ン\w]+', segment):
	# 如果是单词或汉字，转换为平假名
	for word in tagger(segment):
	kana = word.feature.kana or word.surface
	hiragana = jaconv.kata2hira(kana) # 将片假名转换为平假名
	output += hiragana
	else:
	# 如果是标点符号，保持不变
	output += segment

	return output

	def get_net_g(model_path: str, device: str, hps):
	net_g = SynthesizerTrn(
	len(symbols),
	hps.data.filter_length // 2 + 1,
	hps.train.segment_size // hps.data.hop_length,
	n_speakers=hps.data.n_speakers,
	**hps.model,
	).to(device)
	_ = net_g.eval()
	_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
	return net_g

	def get_text(text, language_str, hps, device, style_text=None, style_weight=0.7):
	style_text = None if style_text == "" else style_text
	norm_text, phone, tone, word2ph = clean_text(text, language_str)
	phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)

	if hps.data.add_blank:
	phone = commons.intersperse(phone, 0)
	tone = commons.intersperse(tone, 0)
	language = commons.intersperse(language, 0)
	for i in range(len(word2ph)):
	word2ph[i] = word2ph[i] * 2
	word2ph[0] += 1
	bert_ori = get_bert(
	norm_text, word2ph, language_str, device, style_text, style_weight
	)
	del word2ph
	assert bert_ori.shape[-1] == len(phone), phone

	if language_str == "ZH":
	bert = bert_ori
	ja_bert = torch.randn(1024, len(phone))
	en_bert = torch.randn(1024, len(phone))
	elif language_str == "JP":
	bert = torch.randn(1024, len(phone))
	ja_bert = bert_ori
	en_bert = torch.randn(1024, len(phone))
	elif language_str == "EN":
	bert = torch.randn(1024, len(phone))
	ja_bert = torch.randn(1024, len(phone))
	en_bert = bert_ori
	else:
	raise ValueError("language_str should be ZH, JP or EN")

	assert bert.shape[-1] == len(
	phone
	), f"Bert seq len {bert.shape[-1]} != {len(phone)}"

	phone = torch.LongTensor(phone)
	tone = torch.LongTensor(tone)
	language = torch.LongTensor(language)
	return bert, ja_bert, en_bert, phone, tone, language

	def infer(
	text,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	sid,
	style_text=None,
	style_weight=0.7,
	language = "Auto",
	mode = 'pyopenjtalk-V2.3',
	skip_start=False,
	skip_end=False,
	):
	if style_text == None:
	style_text = ""
	style_weight=0,
	if mode == 'fugashi-V2.3':
	text = kanji_to_hiragana(text) if is_japanese(text) else text
	if language == "JP":
	text = translate(text,"jp")
	if language == "ZH":
	text = translate(text,"zh")
	if language == "Auto":
	language= 'JP' if is_japanese(text) else 'ZH'
	#print(f'{text}:{sdp_ratio}:{noise_scale}:{noise_scale_w}:{length_scale}:{length_scale}:{sid}:{language}:{mode}:{skip_start}:{skip_end}')
	bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
	text,
	language,
	hps,
	device,
	style_text=style_text,
	style_weight=style_weight,
	)
	if skip_start:
	phones = phones[3:]
	tones = tones[3:]
	lang_ids = lang_ids[3:]
	bert = bert[:, 3:]
	ja_bert = ja_bert[:, 3:]
	en_bert = en_bert[:, 3:]
	if skip_end:
	phones = phones[:-2]
	tones = tones[:-2]
	lang_ids = lang_ids[:-2]
	bert = bert[:, :-2]
	ja_bert = ja_bert[:, :-2]
	en_bert = en_bert[:, :-2]
	with torch.no_grad():
	x_tst = phones.to(device).unsqueeze(0)
	tones = tones.to(device).unsqueeze(0)
	lang_ids = lang_ids.to(device).unsqueeze(0)
	bert = bert.to(device).unsqueeze(0)
	ja_bert = ja_bert.to(device).unsqueeze(0)
	en_bert = en_bert.to(device).unsqueeze(0)
	x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
	# emo = emo.to(device).unsqueeze(0)
	del phones
	speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
	audio = (
	net_g.infer(
	x_tst,
	x_tst_lengths,
	speakers,
	tones,
	lang_ids,
	bert,
	ja_bert,
	en_bert,
	sdp_ratio=sdp_ratio,
	noise_scale=noise_scale,
	noise_scale_w=noise_scale_w,
	length_scale=length_scale,
	)[0][0, 0]
	.data.cpu()
	.float()
	.numpy()
	)
	del (
	x_tst,
	tones,
	lang_ids,
	bert,
	x_tst_lengths,
	speakers,
	ja_bert,
	en_bert,
	) # , emo
	if torch.cuda.is_available():
	torch.cuda.empty_cache()
	print("Success.")
	return audio

	def loadmodel(model):
	_ = net_g.eval()
	_ = utils.load_checkpoint(model, net_g, None, skip_optimizer=True)
	return "success"

	def generate_audio_and_srt_for_group(
	group,
	outputPath,
	group_index,
	sampling_rate,
	speaker,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	speakerList,
	silenceTime,
	language,
	mode,
	skip_start,
	skip_end,
	style_text,
	style_weight,
	):
	audio_fin = []
	ass_entries = []
	start_time = 0
	#speaker = random.choice(cara_list)
	ass_header = """[Script Info]
	; 我没意见
	Title: Audiobook
	ScriptType: v4.00+
	WrapStyle: 0
	PlayResX: 640
	PlayResY: 360
	ScaledBorderAndShadow: yes
	[V4+ Styles]
	Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
	Style: Default,Arial,20,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,1,2,10,10,10,1
	[Events]
	Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
	"""

	for sentence in group:
	try:
	if len(sentence) > 1:
	FakeSpeaker = sentence.split("\|")[0]
	print(FakeSpeaker)
	SpeakersList = re.split('\n', speakerList)
	if FakeSpeaker in list(hps.data.spk2id.keys()):
	speaker = FakeSpeaker
	for i in SpeakersList:
	if FakeSpeaker == i.split("\|")[1]:
	speaker = i.split("\|")[0]
	if sentence != '\n':
	text = (remove_annotations(sentence.split("\|")[-1]).replace(" ","")+"。").replace("，。","。")
	if mode == 'pyopenjtalk-V2.3' or mode == 'fugashi-V2.3':
	#print(f'{text}:{sdp_ratio}:{noise_scale}:{noise_scale_w}:{length_scale}:{length_scale}:{speaker}:{language}:{mode}:{skip_start}:{skip_end}')
	audio = infer(
	text,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	speaker,
	style_text,
	style_weight,
	language,
	mode,
	skip_start,
	skip_end,
	)
	silence_frames = int(silenceTime * 44010) if is_chinese(sentence) else int(silenceTime * 44010)
	silence_data = np.zeros((silence_frames,), dtype=audio.dtype)
	audio_fin.append(audio)
	audio_fin.append(silence_data)
	duration = len(audio) / sampling_rate
	print(duration)
	end_time = start_time + duration + silenceTime
	ass_entries.append("Dialogue: 0,{},{},".format(seconds_to_ass_time(start_time), seconds_to_ass_time(end_time)) + "Default,,0,0,0,,{}".format(sentence.replace("\|","：")))
	start_time = end_time
	except:
	pass
	wav_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.wav')
	ass_filename = os.path.join(outputPath, f'audiobook_part_{group_index}.ass')
	write(wav_filename, sampling_rate, gr.processing_utils.convert_to_16_bit_wav(np.concatenate(audio_fin)))

	with open(ass_filename, 'w', encoding='utf-8') as f:
	f.write(ass_header + '\n'.join(ass_entries))
	return (hps.data.sampling_rate, gr.processing_utils.convert_to_16_bit_wav(np.concatenate(audio_fin)))

	def generate_audio(
	inputFile,
	groupSize,
	filepath,
	silenceTime,
	speakerList,
	text,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	sid,
	style_text=None,
	style_weight=0.7,
	language = "Auto",
	mode = 'pyopenjtalk-V2.3',
	sentence_mode = 'sentence',
	skip_start=False,
	skip_end=False,
	):
	if inputFile:
	text = extract_text_from_file(inputFile.name)
	sentence_mode = 'paragraph'
	if mode == 'pyopenjtalk-V2.3' or mode == 'fugashi-V2.3':
	if sentence_mode == 'sentence':
	audio = infer(
	text,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	sid,
	style_text,
	style_weight,
	language,
	mode,
	skip_start,
	skip_end,
	)
	return (hps.data.sampling_rate,gr.processing_utils.convert_to_16_bit_wav(audio))
	if sentence_mode == 'paragraph':
	GROUP_SIZE = groupSize
	directory_path = filepath if torch.cuda.is_available() else "books"
	if os.path.exists(directory_path):
	shutil.rmtree(directory_path)
	os.makedirs(directory_path)
	if language == 'Auto':
	sentences = extrac(extract_and_convert(text))
	else:
	sentences = extrac(text)
	for i in range(0, len(sentences), GROUP_SIZE):
	group = sentences[i:i+GROUP_SIZE]
	if speakerList == "":
	speakerList = "无"
	result = generate_audio_and_srt_for_group(
	group,
	directory_path,
	i//GROUP_SIZE + 1,
	44100,
	sid,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	speakerList,
	silenceTime,
	language,
	mode,
	skip_start,
	skip_end,
	style_text,
	style_weight,
	)
	if not torch.cuda.is_available():
	return result
	return result
	#url = f'{webBase[mode]}?text={text}&speaker={sid}&sdp_ratio={sdp_ratio}&noise_scale={noise_scale}&noise_scale_w={noise_scale_w}&length_scale={length_scale}&language={language}&skip_start={skip_start}&skip_end={skip_end}'
	#print(url)
	#res = requests.get(url)
	#改用post
	res = requests.post(webBase[mode], json = {
	"groupSize": groupSize,
	"filepath": filepath,
	"silenceTime": silenceTime,
	"speakerList": speakerList,
	"text": text,
	"speaker": sid,
	"sdp_ratio": sdp_ratio,
	"noise_scale": noise_scale,
	"noise_scale_w": noise_scale_w,
	"length_scale": length_scale,
	"language": language,
	"skip_start": skip_start,
	"skip_end": skip_end,
	"mode": mode,
	"sentence_mode": sentence_mode,
	"style_text": style_text,
	"style_weight": style_weight
	})
	audio = res.content
	with open('output.wav', 'wb') as code:
	code.write(audio)
	file_path = "output.wav"
	return file_path

	if __name__ == "__main__":
	if torch.cuda.is_available():
	download_unidic()
	tagger = Tagger()
	for dirpath, dirnames, filenames in os.walk('Data/BangDream/models/'):
	for filename in filenames:
	modelPaths.append(os.path.join(dirpath, filename))
	hps = utils.get_hparams_from_file('Data/BangDream/config.json')
	net_g = get_net_g(
	model_path=modelPaths[-1], device=device, hps=hps
	)
	speaker_ids = hps.data.spk2id
	speakers = list(speaker_ids.keys())
	with gr.Blocks() as app:
	gr.Markdown(value="""
	[日语特化版(推荐)](https://huggingface.co/spaces/Mahiruoshi/BangStarlight),国内可用连接: https://mahiruoshi-BangStarlight.hf.space/\n
	[假名标注版](https://huggingface.co/spaces/Mahiruoshi/MyGO_VIts-bert)，国内可用连接: https://mahiruoshi-MyGO-VIts-bert.hf.space/\n
	该界面的真实链接(国内可用): https://mahiruoshi-bangdream-bert-vits2.hf.space/\n
	([Bert-Vits2](https://github.com/Stardust-minus/Bert-VITS2) V2.3)少歌邦邦全员在线语音合成\n
	[好玩的](http://love.soyorin.top/)\n
	API: https://mahiruoshi-bert-vits2-api.hf.space/ \n
	调用方式: https://mahiruoshi-bert-vits2-api.hf.space/?text={{speakText}}&speaker=chosen_speaker\n
	推荐搭配[Legado开源阅读](https://github.com/gedoor/legado)或[聊天bot](https://github.com/Paraworks/BangDreamAi)使用\n
	二创请标注作者：B站@Mahiroshi: https://space.bilibili.com/19874615\n
	训练数据集归属：BangDream及少歌手游,提取自BestDori,[数据集获取流程](https://nijigaku.top/2023/09/29/Bestbushiroad%E8%AE%A1%E5%88%92-vits-%E9%9F%B3%E9%A2%91%E6%8A%93%E5%8F%96%E5%8F%8A%E6%95%B0%E6%8D%AE%E9%9B%86%E5%AF%B9%E9%BD%90/)\n
	BangDream数据集下载[链接](https://huggingface.co/spaces/Mahiruoshi/BangDream-Bert-VITS2/blob/main/%E7%88%AC%E8%99%AB/SortPathUrl.txt)\n
	！！！注意：huggingface容器仅用作展示，建议在右上角更多选项中克隆本项目或Docker运行app.py/server.py,环境参考requirements.txt\n""")
	for band in BandList:
	with gr.TabItem(band):
	for name in BandList[band]:
	with gr.TabItem(name):
	with gr.Row():
	with gr.Column():
	with gr.Row():
	gr.Markdown(
	'<div align="center">'
	f'<img style="width:auto;height:400px;" src="https://mahiruoshi-bangdream-bert-vits2.hf.space/file/image/{name}.png">'
	'</div>'
	)
	with gr.Accordion(label="参数设定", open=False):
	sdp_ratio = gr.Slider(
	minimum=0, maximum=1, value=0.5, step=0.01, label="SDP/DP混合比"
	)
	noise_scale = gr.Slider(
	minimum=0.1, maximum=2, value=0.6, step=0.01, label="Noise:感情调节"
	)
	noise_scale_w = gr.Slider(
	minimum=0.1, maximum=2, value=0.667, step=0.01, label="Noise_W:音素长度"
	)
	skip_start = gr.Checkbox(label="skip_start")
	skip_end = gr.Checkbox(label="skip_end")
	speaker = gr.Dropdown(
	choices=speakers, value=name, label="说话人"
	)
	length_scale = gr.Slider(
	minimum=0.1, maximum=2, value=1, step=0.01, label="语速调节"
	)
	language = gr.Dropdown(
	choices=languages, value="Auto", label="语言选择,若不选自动则会将输入语言翻译为日语或中文"
	)
	mode = gr.Dropdown(
	choices=modes, value="pyopenjtalk-V2.3", label="TTS模式，合成少歌角色需要切换成 pyopenjtalk-V2.3-Katakana "
	)
	sentence_mode = gr.Dropdown(
	choices=sentence_modes, value="paragraph", label="文本合成模式"
	)
	with gr.Accordion(label="扩展选项", open=False):
	inputFile = gr.UploadButton(label="txt文件输入")
	speakerList = gr.TextArea(
	label="角色对应表,如果你记不住角色名可以这样,左边是你想要在每一句话合成中用到的speaker(见角色清单)右边是你上传文本时分隔符左边设置的说话人:{ChoseSpeakerFromConfigList}\|{SeakerInUploadText}",
	value = "ましろ\|天音\n七深\|七深\n透子\|透子\nつくし\|筑紫\n瑠唯\|瑠唯\nそよ\|素世\n祥子\|祥子",
	)
	groupSize = gr.Slider(
	minimum=10, maximum=1000 if torch.cuda.is_available() else 50,value = 50, step=1, label="单个音频文件包含的最大句子数"
	)
	filepath = gr.TextArea(
	label="本地合成时的音频存储文件夹(会清空文件夹,别把C盘删了)",
	value = "D:/audiobook/book1",
	)
	silenceTime = gr.Slider(
	minimum=0, maximum=1, value=0.5, step=0.01, label="句子的间隔"
	)
	modelstrs = gr.Dropdown(label = "模型", choices = modelPaths, value = modelPaths[0], type = "value")
	btnMod = gr.Button("载入模型")
	statusa = gr.TextArea(label = "模型加载状态")
	btnMod.click(loadmodel, inputs=[modelstrs], outputs = [statusa])
	with gr.Column():
	text = gr.TextArea(
	label="文本输入,可用'\|'分割说话人和文本,注意换行",
	info="输入纯日语或者中文",
	value=f"{name}\|你是职业歌手吗\n天音\|我觉得我是",
	placeholder=f"私は{name}です、あの子はだれ？ "
	)
	style_text = gr.Textbox(
	label="情感辅助文本",
	info="语言保持跟主文本一致,文本可以参考训练集:https://huggingface.co/spaces/Mahiruoshi/BangDream-Bert-VITS2/blob/main/filelists/Mygo.list)",
	placeholder="使用辅助文本的语意来辅助生成对话（语言保持与主文本相同）\n\n"
	"注意：不要使用指令式文本（如：开心），要使用带有强烈情感的文本（如：我好快乐！！！）"
	)
	style_weight = gr.Slider(
	minimum=0,
	maximum=1,
	value=0.7,
	step=0.1,
	label="Weight",
	info="主文本和辅助文本的bert混合比率，0表示仅主文本，1表示仅辅助文本",
	)
	btn = gr.Button("点击生成", variant="primary")
	audio_output = gr.Audio(label="Output Audio")
	btntran = gr.Button("快速中翻日")
	translateResult = gr.TextArea(label="使用百度翻译",placeholder="从这里复制翻译后的文本")
	btntran.click(translate, inputs=[text], outputs = [translateResult])
	btn.click(
	generate_audio,
	inputs=[
	inputFile,
	groupSize,
	filepath,
	silenceTime,
	speakerList,
	text,
	sdp_ratio,
	noise_scale,
	noise_scale_w,
	length_scale,
	speaker,
	style_text,
	style_weight,
	language,
	mode,
	sentence_mode,
	skip_start,
	skip_end
	],
	outputs=[audio_output],
	)
	print("推理页面已开启!")
	app.launch()