lj1995 commited on
Commit
1c75048
1 Parent(s): 2619423

Upload 67 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.py +38 -0
  2. go-web.bat +1 -0
  3. go.bat +1 -0
  4. hubert_base.pt +3 -0
  5. infer-web.py +193 -0
  6. infer.py +48 -0
  7. infer_pack/__pycache__/attentions.cpython-39.pyc +0 -0
  8. infer_pack/__pycache__/commons.cpython-39.pyc +0 -0
  9. infer_pack/__pycache__/models.cpython-39.pyc +0 -0
  10. infer_pack/__pycache__/modules.cpython-39.pyc +0 -0
  11. infer_pack/__pycache__/transforms.cpython-39.pyc +0 -0
  12. infer_pack/attentions.py +417 -0
  13. infer_pack/commons.py +164 -0
  14. infer_pack/models.py +664 -0
  15. infer_pack/modules.py +522 -0
  16. infer_pack/transforms.py +193 -0
  17. infer_uvr5.py +108 -0
  18. slicer.py +151 -0
  19. trainset_preprocess_pipeline.py +63 -0
  20. uvr5_pack/__pycache__/utils.cpython-39.pyc +0 -0
  21. uvr5_pack/lib_v5/__pycache__/layers_123821KB.cpython-39.pyc +0 -0
  22. uvr5_pack/lib_v5/__pycache__/model_param_init.cpython-39.pyc +0 -0
  23. uvr5_pack/lib_v5/__pycache__/nets_61968KB.cpython-39.pyc +0 -0
  24. uvr5_pack/lib_v5/__pycache__/spec_utils.cpython-39.pyc +0 -0
  25. uvr5_pack/lib_v5/dataset.py +170 -0
  26. uvr5_pack/lib_v5/layers.py +116 -0
  27. uvr5_pack/lib_v5/layers_123812KB .py +116 -0
  28. uvr5_pack/lib_v5/layers_123821KB.py +116 -0
  29. uvr5_pack/lib_v5/layers_33966KB.py +122 -0
  30. uvr5_pack/lib_v5/layers_537227KB.py +122 -0
  31. uvr5_pack/lib_v5/layers_537238KB.py +122 -0
  32. uvr5_pack/lib_v5/model_param_init.py +60 -0
  33. uvr5_pack/lib_v5/modelparams/1band_sr16000_hl512.json +19 -0
  34. uvr5_pack/lib_v5/modelparams/1band_sr32000_hl512.json +19 -0
  35. uvr5_pack/lib_v5/modelparams/1band_sr33075_hl384.json +19 -0
  36. uvr5_pack/lib_v5/modelparams/1band_sr44100_hl1024.json +19 -0
  37. uvr5_pack/lib_v5/modelparams/1band_sr44100_hl256.json +19 -0
  38. uvr5_pack/lib_v5/modelparams/1band_sr44100_hl512.json +19 -0
  39. uvr5_pack/lib_v5/modelparams/1band_sr44100_hl512_cut.json +19 -0
  40. uvr5_pack/lib_v5/modelparams/2band_32000.json +30 -0
  41. uvr5_pack/lib_v5/modelparams/2band_44100_lofi.json +30 -0
  42. uvr5_pack/lib_v5/modelparams/2band_48000.json +30 -0
  43. uvr5_pack/lib_v5/modelparams/3band_44100.json +42 -0
  44. uvr5_pack/lib_v5/modelparams/3band_44100_mid.json +43 -0
  45. uvr5_pack/lib_v5/modelparams/3band_44100_msb2.json +43 -0
  46. uvr5_pack/lib_v5/modelparams/4band_44100.json +54 -0
  47. uvr5_pack/lib_v5/modelparams/4band_44100_mid.json +55 -0
  48. uvr5_pack/lib_v5/modelparams/4band_44100_msb.json +55 -0
  49. uvr5_pack/lib_v5/modelparams/4band_44100_msb2.json +55 -0
  50. uvr5_pack/lib_v5/modelparams/4band_44100_reverse.json +55 -0
config.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ############离线VC参数
2
+ inp_root=r"白鹭霜华长条"#对输入目录下所有音频进行转换,别放非音频文件
3
+ opt_root=r"opt"#输出目录
4
+ f0_up_key=0#升降调,整数,男转女12,女转男-12
5
+ person=r"weights\洛天依v3.pt"#目前只有洛天依v3
6
+ ############硬件参数
7
+ device = "cuda:0"#填写cuda:x或cpu,x指代第几张卡,只支持N卡加速
8
+ is_half=True#9-10-20-30-40系显卡无脑True,不影响质量,>=20显卡开启有加速
9
+ n_cpu=0#默认0用上所有线程,写数字限制CPU资源使用
10
+ ############下头别动
11
+ import torch
12
+ if(torch.cuda.is_available()==False):
13
+ print("没有发现支持的N卡,使用CPU进行推理")
14
+ device="cpu"
15
+ is_half=False
16
+ if(device!="cpu"):
17
+ gpu_name=torch.cuda.get_device_name(int(device.split(":")[-1]))
18
+ if("16"in gpu_name):
19
+ print("16系显卡强制单精度")
20
+ is_half=False
21
+ from multiprocessing import cpu_count
22
+ if(n_cpu==0):n_cpu=cpu_count()
23
+ if(is_half==True):
24
+ #6G显存配置
25
+ x_pad = 3
26
+ x_query = 10
27
+ x_center = 60
28
+ x_max = 65
29
+ else:
30
+ #5G显存配置
31
+ x_pad = 1
32
+ # x_query = 6
33
+ # x_center = 30
34
+ # x_max = 32
35
+ #6G显存配置
36
+ x_query = 6
37
+ x_center = 38
38
+ x_max = 41
go-web.bat ADDED
@@ -0,0 +1 @@
 
1
+ runtime\python.exe infer-web.py
go.bat ADDED
@@ -0,0 +1 @@
 
1
+ runtime\python.exe infer.py
hubert_base.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f54b40fd2802423a5643779c4861af1e9ee9c1564dc9d32f54f20b5ffba7db96
3
+ size 189507909
infer-web.py ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch, pdb, os,traceback,sys,warnings,shutil
2
+ now_dir=os.getcwd()
3
+ sys.path.append(now_dir)
4
+ tmp=os.path.join(now_dir,"TEMP")
5
+ shutil.rmtree(tmp,ignore_errors=True)
6
+ os.makedirs(tmp,exist_ok=True)
7
+ os.environ["TEMP"]=tmp
8
+ warnings.filterwarnings("ignore")
9
+ torch.manual_seed(114514)
10
+ from infer_pack.models import SynthesizerTrnMs256NSF as SynthesizerTrn256
11
+ from scipy.io import wavfile
12
+ from fairseq import checkpoint_utils
13
+ import gradio as gr
14
+ import librosa
15
+ import logging
16
+ from vc_infer_pipeline import VC
17
+ import soundfile as sf
18
+ from config import is_half,device,is_half
19
+ from infer_uvr5 import _audio_pre_
20
+ logging.getLogger('numba').setLevel(logging.WARNING)
21
+
22
+ models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(["hubert_base.pt"],suffix="",)
23
+ hubert_model = models[0]
24
+ hubert_model = hubert_model.to(device)
25
+ if(is_half):hubert_model = hubert_model.half()
26
+ else:hubert_model = hubert_model.float()
27
+ hubert_model.eval()
28
+
29
+
30
+ weight_root="weights"
31
+ weight_uvr5_root="uvr5_weights"
32
+ names=[]
33
+ for name in os.listdir(weight_root):names.append(name.replace(".pt",""))
34
+ uvr5_names=[]
35
+ for name in os.listdir(weight_uvr5_root):uvr5_names.append(name.replace(".pth",""))
36
+
37
+ def get_vc(sid):
38
+ person = "%s/%s.pt" % (weight_root, sid)
39
+ cpt = torch.load(person, map_location="cpu")
40
+ dv = cpt["dv"]
41
+ tgt_sr = cpt["config"][-1]
42
+ net_g = SynthesizerTrn256(*cpt["config"], is_half=is_half)
43
+ net_g.load_state_dict(cpt["weight"], strict=True)
44
+ net_g.eval().to(device)
45
+ if (is_half):net_g = net_g.half()
46
+ else:net_g = net_g.float()
47
+ vc = VC(tgt_sr, device, is_half)
48
+ return dv,tgt_sr,net_g,vc
49
+
50
+ def vc_single(sid,input_audio,f0_up_key,f0_file):
51
+ if input_audio is None:return "You need to upload an audio", None
52
+ f0_up_key = int(f0_up_key)
53
+ try:
54
+ if(type(input_audio)==str):
55
+ print("processing %s" % input_audio)
56
+ audio, sampling_rate = sf.read(input_audio)
57
+ else:
58
+ sampling_rate, audio = input_audio
59
+ audio = audio.astype("float32") / 32768
60
+ if(type(sid)==str):dv, tgt_sr, net_g, vc=get_vc(sid)
61
+ else:dv,tgt_sr,net_g,vc=sid
62
+ if len(audio.shape) > 1:
63
+ audio = librosa.to_mono(audio.transpose(1, 0))
64
+ if sampling_rate != 16000:
65
+ audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
66
+ times = [0, 0, 0]
67
+ audio_opt=vc.pipeline(hubert_model,net_g,dv,audio,times,f0_up_key,f0_file=f0_file)
68
+ print(times)
69
+ return "Success", (tgt_sr, audio_opt)
70
+ except:
71
+ info=traceback.format_exc()
72
+ print(info)
73
+ return info,(None,None)
74
+ finally:
75
+ print("clean_empty_cache")
76
+ del net_g,dv,vc
77
+ torch.cuda.empty_cache()
78
+
79
+ def vc_multi(sid,dir_path,opt_root,paths,f0_up_key):
80
+ try:
81
+ dir_path=dir_path.strip(" ")#防止小白拷路径头尾带了空格
82
+ opt_root=opt_root.strip(" ")
83
+ os.makedirs(opt_root, exist_ok=True)
84
+ dv, tgt_sr, net_g, vc = get_vc(sid)
85
+ try:
86
+ if(dir_path!=""):paths=[os.path.join(dir_path,name)for name in os.listdir(dir_path)]
87
+ else:paths=[path.name for path in paths]
88
+ except:
89
+ traceback.print_exc()
90
+ paths = [path.name for path in paths]
91
+ infos=[]
92
+ for path in paths:
93
+ info,opt=vc_single([dv,tgt_sr,net_g,vc],path,f0_up_key,f0_file=None)
94
+ if(info=="Success"):
95
+ try:
96
+ tgt_sr,audio_opt=opt
97
+ wavfile.write("%s/%s" % (opt_root, os.path.basename(path)), tgt_sr, audio_opt)
98
+ except:
99
+ info=traceback.format_exc()
100
+ infos.append("%s->%s"%(os.path.basename(path),info))
101
+ return "\n".join(infos)
102
+ except:
103
+ return traceback.format_exc()
104
+ finally:
105
+ print("clean_empty_cache")
106
+ del net_g,dv,vc
107
+ torch.cuda.empty_cache()
108
+
109
+ def uvr(model_name,inp_root,save_root_vocal,save_root_ins):
110
+ infos = []
111
+ try:
112
+ inp_root = inp_root.strip(" ")# 防止小白拷路径头尾带了空格
113
+ save_root_vocal = save_root_vocal.strip(" ")
114
+ save_root_ins = save_root_ins.strip(" ")
115
+ pre_fun = _audio_pre_(model_path=os.path.join(weight_uvr5_root,model_name+".pth"), device=device, is_half=is_half)
116
+ for name in os.listdir(inp_root):
117
+ inp_path=os.path.join(inp_root,name)
118
+ try:
119
+ pre_fun._path_audio_(inp_path , save_root_ins,save_root_vocal)
120
+ infos.append("%s->Success"%(os.path.basename(inp_path)))
121
+ except:
122
+ infos.append("%s->%s" % (os.path.basename(inp_path),traceback.format_exc()))
123
+ except:
124
+ infos.append(traceback.format_exc())
125
+ finally:
126
+ try:
127
+ del pre_fun.model
128
+ del pre_fun
129
+ except:
130
+ traceback.print_exc()
131
+ print("clean_empty_cache")
132
+ torch.cuda.empty_cache()
133
+ return "\n".join(infos)
134
+
135
+ with gr.Blocks() as app:
136
+ with gr.Tabs():
137
+ with gr.TabItem("推理"):
138
+ with gr.Group():
139
+ gr.Markdown(value="""
140
+ 使用软件者、传播软件导出的声音者自负全责。如不认可该条款,则不能使用/引用软件包内所有代码和文件。<br>
141
+ 目前仅开放白菜音色,后续将扩展为本地训练推理工具,用户可训练自己的音色进行社区共享。<br>
142
+ 男转女推荐+12key,女转男推荐-12key,如果音域爆炸导致音色失真也可以自己调整到合适音域
143
+ """)
144
+ with gr.Row():
145
+ with gr.Column():
146
+ sid0 = gr.Dropdown(label="音色", choices=names)
147
+ vc_transform0 = gr.Number(label="变调(整数,半音数量,升八度12降八度-12)", value=12)
148
+ f0_file = gr.File(label="F0曲线文件,可选,一行一个音高,代替默认F0及升降调")
149
+ input_audio0 = gr.Audio(label="上传音频")
150
+ but0=gr.Button("转换", variant="primary")
151
+ with gr.Column():
152
+ vc_output1 = gr.Textbox(label="输出信息")
153
+ vc_output2 = gr.Audio(label="输出音频")
154
+ but0.click(vc_single, [sid0, input_audio0, vc_transform0,f0_file], [vc_output1, vc_output2])
155
+ with gr.Group():
156
+ gr.Markdown(value="""
157
+ 批量转换,上传多个音频文件,在指定文件夹(默认opt)下输出转换的音频。<br>
158
+ 合格的文件夹路径格式举例:E:\codes\py39\\vits_vc_gpu\白鹭霜华测试样例(去文件管理器地址栏拷就行了)
159
+ """)
160
+ with gr.Row():
161
+ with gr.Column():
162
+ sid1 = gr.Dropdown(label="音色", choices=names)
163
+ vc_transform1 = gr.Number(label="变调(整数,半音数量,升八度12降八度-12)", value=12)
164
+ opt_input = gr.Textbox(label="指定输出文件夹",value="opt")
165
+ with gr.Column():
166
+ dir_input = gr.Textbox(label="输入待处理音频文件夹路径")
167
+ inputs = gr.File(file_count="multiple", label="也可批量输入音频文件,二选一,优先读文件夹")
168
+ but1=gr.Button("转换", variant="primary")
169
+ vc_output3 = gr.Textbox(label="输出信息")
170
+ but1.click(vc_multi, [sid1, dir_input,opt_input,inputs, vc_transform1], [vc_output3])
171
+
172
+ with gr.TabItem("数据处理"):
173
+ with gr.Group():
174
+ gr.Markdown(value="""
175
+ 人声伴奏分离批量处理,使用UVR5模型。<br>
176
+ 不带和声用HP2,带和声且提取的人声不需要和声用HP5<br>
177
+ 合格的文件夹路径格式举例:E:\codes\py39\\vits_vc_gpu\白鹭霜华测试样例(去文件管理器地址栏拷就行了)
178
+ """)
179
+ with gr.Row():
180
+ with gr.Column():
181
+ dir_wav_input = gr.Textbox(label="输入待处理音频文件夹路径")
182
+ wav_inputs = gr.File(file_count="multiple", label="也可批量输入音频文件,二选一,优先读文件夹")
183
+ with gr.Column():
184
+ model_choose = gr.Dropdown(label="模型", choices=uvr5_names)
185
+ opt_vocal_root = gr.Textbox(label="指定输出人声文件夹",value="opt")
186
+ opt_ins_root = gr.Textbox(label="指定输出乐器文件夹",value="opt")
187
+ but2=gr.Button("转换", variant="primary")
188
+ vc_output4 = gr.Textbox(label="输出信息")
189
+ but2.click(uvr, [model_choose, dir_wav_input,opt_vocal_root,opt_ins_root], [vc_output4])
190
+ with gr.TabItem("训练-待开放"):pass
191
+
192
+ # app.launch(server_name="0.0.0.0",server_port=7860)
193
+ app.launch(server_name="127.0.0.1",server_port=7860)
infer.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch, pdb, os,sys,librosa,warnings,traceback
2
+ warnings.filterwarnings("ignore")
3
+ torch.manual_seed(114514)
4
+ sys.path.append(os.getcwd())
5
+ from config import inp_root,opt_root,f0_up_key,person,is_half,device
6
+ os.makedirs(opt_root,exist_ok=True)
7
+ import soundfile as sf
8
+ from infer_pack.models import SynthesizerTrnMs256NSF as SynthesizerTrn256
9
+ from scipy.io import wavfile
10
+ from fairseq import checkpoint_utils
11
+ import scipy.signal as signal
12
+ from vc_infer_pipeline import VC
13
+
14
+ models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(["hubert_base.pt"],suffix="",)
15
+ model = models[0]
16
+ model = model.to(device)
17
+ if(is_half):model = model.half()
18
+ else:model = model.float()
19
+ model.eval()
20
+
21
+ cpt=torch.load(person,map_location="cpu")
22
+ dv=cpt["dv"]
23
+ tgt_sr=cpt["config"][-1]
24
+ net_g = SynthesizerTrn256(*cpt["config"],is_half=is_half)
25
+ net_g.load_state_dict(cpt["weight"],strict=True)
26
+ net_g.eval().to(device)
27
+ if(is_half):net_g = net_g.half()
28
+ else:net_g = net_g.float()
29
+
30
+ vc=VC(tgt_sr,device,is_half)
31
+
32
+ for name in os.listdir(inp_root):
33
+ try:
34
+ wav_path="%s\%s"%(inp_root,name)
35
+ print("processing %s"%wav_path)
36
+ audio, sampling_rate = sf.read(wav_path)
37
+ if len(audio.shape) > 1:
38
+ audio = librosa.to_mono(audio.transpose(1, 0))
39
+ if sampling_rate != vc.sr:
40
+ audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=vc.sr)
41
+
42
+ times = [0, 0, 0]
43
+ audio_opt=vc.pipeline(model,net_g,dv,audio,times,f0_up_key,f0_file=None)
44
+ wavfile.write("%s/%s"%(opt_root,name), tgt_sr, audio_opt)
45
+ except:
46
+ traceback.print_exc()
47
+
48
+ print(times)
infer_pack/__pycache__/attentions.cpython-39.pyc ADDED
Binary file (9.92 kB). View file
infer_pack/__pycache__/commons.cpython-39.pyc ADDED
Binary file (5.88 kB). View file
infer_pack/__pycache__/models.cpython-39.pyc ADDED
Binary file (16.7 kB). View file
infer_pack/__pycache__/modules.cpython-39.pyc ADDED
Binary file (11.9 kB). View file
infer_pack/__pycache__/transforms.cpython-39.pyc ADDED
Binary file (3.92 kB). View file
infer_pack/attentions.py ADDED
@@ -0,0 +1,417 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import math
3
+ import numpy as np
4
+ import torch
5
+ from torch import nn
6
+ from torch.nn import functional as F
7
+
8
+ from infer_pack import commons
9
+ from infer_pack import modules
10
+ from infer_pack.modules import LayerNorm
11
+
12
+
13
+ class Encoder(nn.Module):
14
+ def __init__(
15
+ self,
16
+ hidden_channels,
17
+ filter_channels,
18
+ n_heads,
19
+ n_layers,
20
+ kernel_size=1,
21
+ p_dropout=0.0,
22
+ window_size=10,
23
+ **kwargs
24
+ ):
25
+ super().__init__()
26
+ self.hidden_channels = hidden_channels
27
+ self.filter_channels = filter_channels
28
+ self.n_heads = n_heads
29
+ self.n_layers = n_layers
30
+ self.kernel_size = kernel_size
31
+ self.p_dropout = p_dropout
32
+ self.window_size = window_size
33
+
34
+ self.drop = nn.Dropout(p_dropout)
35
+ self.attn_layers = nn.ModuleList()
36
+ self.norm_layers_1 = nn.ModuleList()
37
+ self.ffn_layers = nn.ModuleList()
38
+ self.norm_layers_2 = nn.ModuleList()
39
+ for i in range(self.n_layers):
40
+ self.attn_layers.append(
41
+ MultiHeadAttention(
42
+ hidden_channels,
43
+ hidden_channels,
44
+ n_heads,
45
+ p_dropout=p_dropout,
46
+ window_size=window_size,
47
+ )
48
+ )
49
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
50
+ self.ffn_layers.append(
51
+ FFN(
52
+ hidden_channels,
53
+ hidden_channels,
54
+ filter_channels,
55
+ kernel_size,
56
+ p_dropout=p_dropout,
57
+ )
58
+ )
59
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
60
+
61
+ def forward(self, x, x_mask):
62
+ attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
63
+ x = x * x_mask
64
+ for i in range(self.n_layers):
65
+ y = self.attn_layers[i](x, x, attn_mask)
66
+ y = self.drop(y)
67
+ x = self.norm_layers_1[i](x + y)
68
+
69
+ y = self.ffn_layers[i](x, x_mask)
70
+ y = self.drop(y)
71
+ x = self.norm_layers_2[i](x + y)
72
+ x = x * x_mask
73
+ return x
74
+
75
+
76
+ class Decoder(nn.Module):
77
+ def __init__(
78
+ self,
79
+ hidden_channels,
80
+ filter_channels,
81
+ n_heads,
82
+ n_layers,
83
+ kernel_size=1,
84
+ p_dropout=0.0,
85
+ proximal_bias=False,
86
+ proximal_init=True,
87
+ **kwargs
88
+ ):
89
+ super().__init__()
90
+ self.hidden_channels = hidden_channels
91
+ self.filter_channels = filter_channels
92
+ self.n_heads = n_heads
93
+ self.n_layers = n_layers
94
+ self.kernel_size = kernel_size
95
+ self.p_dropout = p_dropout
96
+ self.proximal_bias = proximal_bias
97
+ self.proximal_init = proximal_init
98
+
99
+ self.drop = nn.Dropout(p_dropout)
100
+ self.self_attn_layers = nn.ModuleList()
101
+ self.norm_layers_0 = nn.ModuleList()
102
+ self.encdec_attn_layers = nn.ModuleList()
103
+ self.norm_layers_1 = nn.ModuleList()
104
+ self.ffn_layers = nn.ModuleList()
105
+ self.norm_layers_2 = nn.ModuleList()
106
+ for i in range(self.n_layers):
107
+ self.self_attn_layers.append(
108
+ MultiHeadAttention(
109
+ hidden_channels,
110
+ hidden_channels,
111
+ n_heads,
112
+ p_dropout=p_dropout,
113
+ proximal_bias=proximal_bias,
114
+ proximal_init=proximal_init,
115
+ )
116
+ )
117
+ self.norm_layers_0.append(LayerNorm(hidden_channels))
118
+ self.encdec_attn_layers.append(
119
+ MultiHeadAttention(
120
+ hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
121
+ )
122
+ )
123
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
124
+ self.ffn_layers.append(
125
+ FFN(
126
+ hidden_channels,
127
+ hidden_channels,
128
+ filter_channels,
129
+ kernel_size,
130
+ p_dropout=p_dropout,
131
+ causal=True,
132
+ )
133
+ )
134
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
135
+
136
+ def forward(self, x, x_mask, h, h_mask):
137
+ """
138
+ x: decoder input
139
+ h: encoder output
140
+ """
141
+ self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
142
+ device=x.device, dtype=x.dtype
143
+ )
144
+ encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
145
+ x = x * x_mask
146
+ for i in range(self.n_layers):
147
+ y = self.self_attn_layers[i](x, x, self_attn_mask)
148
+ y = self.drop(y)
149
+ x = self.norm_layers_0[i](x + y)
150
+
151
+ y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
152
+ y = self.drop(y)
153
+ x = self.norm_layers_1[i](x + y)
154
+
155
+ y = self.ffn_layers[i](x, x_mask)
156
+ y = self.drop(y)
157
+ x = self.norm_layers_2[i](x + y)
158
+ x = x * x_mask
159
+ return x
160
+
161
+
162
+ class MultiHeadAttention(nn.Module):
163
+ def __init__(
164
+ self,
165
+ channels,
166
+ out_channels,
167
+ n_heads,
168
+ p_dropout=0.0,
169
+ window_size=None,
170
+ heads_share=True,
171
+ block_length=None,
172
+ proximal_bias=False,
173
+ proximal_init=False,
174
+ ):
175
+ super().__init__()
176
+ assert channels % n_heads == 0
177
+
178
+ self.channels = channels
179
+ self.out_channels = out_channels
180
+ self.n_heads = n_heads
181
+ self.p_dropout = p_dropout
182
+ self.window_size = window_size
183
+ self.heads_share = heads_share
184
+ self.block_length = block_length
185
+ self.proximal_bias = proximal_bias
186
+ self.proximal_init = proximal_init
187
+ self.attn = None
188
+
189
+ self.k_channels = channels // n_heads
190
+ self.conv_q = nn.Conv1d(channels, channels, 1)
191
+ self.conv_k = nn.Conv1d(channels, channels, 1)
192
+ self.conv_v = nn.Conv1d(channels, channels, 1)
193
+ self.conv_o = nn.Conv1d(channels, out_channels, 1)
194
+ self.drop = nn.Dropout(p_dropout)
195
+
196
+ if window_size is not None:
197
+ n_heads_rel = 1 if heads_share else n_heads
198
+ rel_stddev = self.k_channels**-0.5
199
+ self.emb_rel_k = nn.Parameter(
200
+ torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
201
+ * rel_stddev
202
+ )
203
+ self.emb_rel_v = nn.Parameter(
204
+ torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
205
+ * rel_stddev
206
+ )
207
+
208
+ nn.init.xavier_uniform_(self.conv_q.weight)
209
+ nn.init.xavier_uniform_(self.conv_k.weight)
210
+ nn.init.xavier_uniform_(self.conv_v.weight)
211
+ if proximal_init:
212
+ with torch.no_grad():
213
+ self.conv_k.weight.copy_(self.conv_q.weight)
214
+ self.conv_k.bias.copy_(self.conv_q.bias)
215
+
216
+ def forward(self, x, c, attn_mask=None):
217
+ q = self.conv_q(x)
218
+ k = self.conv_k(c)
219
+ v = self.conv_v(c)
220
+
221
+ x, self.attn = self.attention(q, k, v, mask=attn_mask)
222
+
223
+ x = self.conv_o(x)
224
+ return x
225
+
226
+ def attention(self, query, key, value, mask=None):
227
+ # reshape [b, d, t] -> [b, n_h, t, d_k]
228
+ b, d, t_s, t_t = (*key.size(), query.size(2))
229
+ query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
230
+ key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
231
+ value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
232
+
233
+ scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
234
+ if self.window_size is not None:
235
+ assert (
236
+ t_s == t_t
237
+ ), "Relative attention is only available for self-attention."
238
+ key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
239
+ rel_logits = self._matmul_with_relative_keys(
240
+ query / math.sqrt(self.k_channels), key_relative_embeddings
241
+ )
242
+ scores_local = self._relative_position_to_absolute_position(rel_logits)
243
+ scores = scores + scores_local
244
+ if self.proximal_bias:
245
+ assert t_s == t_t, "Proximal bias is only available for self-attention."
246
+ scores = scores + self._attention_bias_proximal(t_s).to(
247
+ device=scores.device, dtype=scores.dtype
248
+ )
249
+ if mask is not None:
250
+ scores = scores.masked_fill(mask == 0, -1e4)
251
+ if self.block_length is not None:
252
+ assert (
253
+ t_s == t_t
254
+ ), "Local attention is only available for self-attention."
255
+ block_mask = (
256
+ torch.ones_like(scores)
257
+ .triu(-self.block_length)
258
+ .tril(self.block_length)
259
+ )
260
+ scores = scores.masked_fill(block_mask == 0, -1e4)
261
+ p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
262
+ p_attn = self.drop(p_attn)
263
+ output = torch.matmul(p_attn, value)
264
+ if self.window_size is not None:
265
+ relative_weights = self._absolute_position_to_relative_position(p_attn)
266
+ value_relative_embeddings = self._get_relative_embeddings(
267
+ self.emb_rel_v, t_s
268
+ )
269
+ output = output + self._matmul_with_relative_values(
270
+ relative_weights, value_relative_embeddings
271
+ )
272
+ output = (
273
+ output.transpose(2, 3).contiguous().view(b, d, t_t)
274
+ ) # [b, n_h, t_t, d_k] -> [b, d, t_t]
275
+ return output, p_attn
276
+
277
+ def _matmul_with_relative_values(self, x, y):
278
+ """
279
+ x: [b, h, l, m]
280
+ y: [h or 1, m, d]
281
+ ret: [b, h, l, d]
282
+ """
283
+ ret = torch.matmul(x, y.unsqueeze(0))
284
+ return ret
285
+
286
+ def _matmul_with_relative_keys(self, x, y):
287
+ """
288
+ x: [b, h, l, d]
289
+ y: [h or 1, m, d]
290
+ ret: [b, h, l, m]
291
+ """
292
+ ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
293
+ return ret
294
+
295
+ def _get_relative_embeddings(self, relative_embeddings, length):
296
+ max_relative_position = 2 * self.window_size + 1
297
+ # Pad first before slice to avoid using cond ops.
298
+ pad_length = max(length - (self.window_size + 1), 0)
299
+ slice_start_position = max((self.window_size + 1) - length, 0)
300
+ slice_end_position = slice_start_position + 2 * length - 1
301
+ if pad_length > 0:
302
+ padded_relative_embeddings = F.pad(
303
+ relative_embeddings,
304
+ commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
305
+ )
306
+ else:
307
+ padded_relative_embeddings = relative_embeddings
308
+ used_relative_embeddings = padded_relative_embeddings[
309
+ :, slice_start_position:slice_end_position
310
+ ]
311
+ return used_relative_embeddings
312
+
313
+ def _relative_position_to_absolute_position(self, x):
314
+ """
315
+ x: [b, h, l, 2*l-1]
316
+ ret: [b, h, l, l]
317
+ """
318
+ batch, heads, length, _ = x.size()
319
+ # Concat columns of pad to shift from relative to absolute indexing.
320
+ x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
321
+
322
+ # Concat extra elements so to add up to shape (len+1, 2*len-1).
323
+ x_flat = x.view([batch, heads, length * 2 * length])
324
+ x_flat = F.pad(
325
+ x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
326
+ )
327
+
328
+ # Reshape and slice out the padded elements.
329
+ x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
330
+ :, :, :length, length - 1 :
331
+ ]
332
+ return x_final
333
+
334
+ def _absolute_position_to_relative_position(self, x):
335
+ """
336
+ x: [b, h, l, l]
337
+ ret: [b, h, l, 2*l-1]
338
+ """
339
+ batch, heads, length, _ = x.size()
340
+ # padd along column
341
+ x = F.pad(
342
+ x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
343
+ )
344
+ x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
345
+ # add 0's in the beginning that will skew the elements after reshape
346
+ x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
347
+ x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
348
+ return x_final
349
+
350
+ def _attention_bias_proximal(self, length):
351
+ """Bias for self-attention to encourage attention to close positions.
352
+ Args:
353
+ length: an integer scalar.
354
+ Returns:
355
+ a Tensor with shape [1, 1, length, length]
356
+ """
357
+ r = torch.arange(length, dtype=torch.float32)
358
+ diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
359
+ return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
360
+
361
+
362
+ class FFN(nn.Module):
363
+ def __init__(
364
+ self,
365
+ in_channels,
366
+ out_channels,
367
+ filter_channels,
368
+ kernel_size,
369
+ p_dropout=0.0,
370
+ activation=None,
371
+ causal=False,
372
+ ):
373
+ super().__init__()
374
+ self.in_channels = in_channels
375
+ self.out_channels = out_channels
376
+ self.filter_channels = filter_channels
377
+ self.kernel_size = kernel_size
378
+ self.p_dropout = p_dropout
379
+ self.activation = activation
380
+ self.causal = causal
381
+
382
+ if causal:
383
+ self.padding = self._causal_padding
384
+ else:
385
+ self.padding = self._same_padding
386
+
387
+ self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
388
+ self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
389
+ self.drop = nn.Dropout(p_dropout)
390
+
391
+ def forward(self, x, x_mask):
392
+ x = self.conv_1(self.padding(x * x_mask))
393
+ if self.activation == "gelu":
394
+ x = x * torch.sigmoid(1.702 * x)
395
+ else:
396
+ x = torch.relu(x)
397
+ x = self.drop(x)
398
+ x = self.conv_2(self.padding(x * x_mask))
399
+ return x * x_mask
400
+
401
+ def _causal_padding(self, x):
402
+ if self.kernel_size == 1:
403
+ return x
404
+ pad_l = self.kernel_size - 1
405
+ pad_r = 0
406
+ padding = [[0, 0], [0, 0], [pad_l, pad_r]]
407
+ x = F.pad(x, commons.convert_pad_shape(padding))
408
+ return x
409
+
410
+ def _same_padding(self, x):
411
+ if self.kernel_size == 1:
412
+ return x
413
+ pad_l = (self.kernel_size - 1) // 2
414
+ pad_r = self.kernel_size // 2
415
+ padding = [[0, 0], [0, 0], [pad_l, pad_r]]
416
+ x = F.pad(x, commons.convert_pad_shape(padding))
417
+ return x
infer_pack/commons.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import numpy as np
3
+ import torch
4
+ from torch import nn
5
+ from torch.nn import functional as F
6
+
7
+
8
+ def init_weights(m, mean=0.0, std=0.01):
9
+ classname = m.__class__.__name__
10
+ if classname.find("Conv") != -1:
11
+ m.weight.data.normal_(mean, std)
12
+
13
+
14
+ def get_padding(kernel_size, dilation=1):
15
+ return int((kernel_size * dilation - dilation) / 2)
16
+
17
+
18
+ def convert_pad_shape(pad_shape):
19
+ l = pad_shape[::-1]
20
+ pad_shape = [item for sublist in l for item in sublist]
21
+ return pad_shape
22
+
23
+
24
+ def kl_divergence(m_p, logs_p, m_q, logs_q):
25
+ """KL(P||Q)"""
26
+ kl = (logs_q - logs_p) - 0.5
27
+ kl += (
28
+ 0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
29
+ )
30
+ return kl
31
+
32
+
33
+ def rand_gumbel(shape):
34
+ """Sample from the Gumbel distribution, protect from overflows."""
35
+ uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
36
+ return -torch.log(-torch.log(uniform_samples))
37
+
38
+
39
+ def rand_gumbel_like(x):
40
+ g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
41
+ return g
42
+
43
+
44
+ def slice_segments(x, ids_str, segment_size=4):
45
+ ret = torch.zeros_like(x[:, :, :segment_size])
46
+ for i in range(x.size(0)):
47
+ idx_str = ids_str[i]
48
+ idx_end = idx_str + segment_size
49
+ ret[i] = x[i, :, idx_str:idx_end]
50
+ return ret
51
+ def slice_segments2(x, ids_str, segment_size=4):
52
+ ret = torch.zeros_like(x[:, :segment_size])
53
+ for i in range(x.size(0)):
54
+ idx_str = ids_str[i]
55
+ idx_end = idx_str + segment_size
56
+ ret[i] = x[i, idx_str:idx_end]
57
+ return ret
58
+
59
+
60
+ def rand_slice_segments(x, x_lengths=None, segment_size=4):
61
+ b, d, t = x.size()
62
+ if x_lengths is None:
63
+ x_lengths = t
64
+ ids_str_max = x_lengths - segment_size + 1
65
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
66
+ ret = slice_segments(x, ids_str, segment_size)
67
+ return ret, ids_str
68
+
69
+
70
+ def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
71
+ position = torch.arange(length, dtype=torch.float)
72
+ num_timescales = channels // 2
73
+ log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
74
+ num_timescales - 1
75
+ )
76
+ inv_timescales = min_timescale * torch.exp(
77
+ torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
78
+ )
79
+ scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
80
+ signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
81
+ signal = F.pad(signal, [0, 0, 0, channels % 2])
82
+ signal = signal.view(1, channels, length)
83
+ return signal
84
+
85
+
86
+ def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
87
+ b, channels, length = x.size()
88
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
89
+ return x + signal.to(dtype=x.dtype, device=x.device)
90
+
91
+
92
+ def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
93
+ b, channels, length = x.size()
94
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
95
+ return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
96
+
97
+
98
+ def subsequent_mask(length):
99
+ mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
100
+ return mask
101
+
102
+
103
+ @torch.jit.script
104
+ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
105
+ n_channels_int = n_channels[0]
106
+ in_act = input_a + input_b
107
+ t_act = torch.tanh(in_act[:, :n_channels_int, :])
108
+ s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
109
+ acts = t_act * s_act
110
+ return acts
111
+
112
+
113
+ def convert_pad_shape(pad_shape):
114
+ l = pad_shape[::-1]
115
+ pad_shape = [item for sublist in l for item in sublist]
116
+ return pad_shape
117
+
118
+
119
+ def shift_1d(x):
120
+ x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
121
+ return x
122
+
123
+
124
+ def sequence_mask(length, max_length=None):
125
+ if max_length is None:
126
+ max_length = length.max()
127
+ x = torch.arange(max_length, dtype=length.dtype, device=length.device)
128
+ return x.unsqueeze(0) < length.unsqueeze(1)
129
+
130
+
131
+ def generate_path(duration, mask):
132
+ """
133
+ duration: [b, 1, t_x]
134
+ mask: [b, 1, t_y, t_x]
135
+ """
136
+ device = duration.device
137
+
138
+ b, _, t_y, t_x = mask.shape
139
+ cum_duration = torch.cumsum(duration, -1)
140
+
141
+ cum_duration_flat = cum_duration.view(b * t_x)
142
+ path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
143
+ path = path.view(b, t_x, t_y)
144
+ path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
145
+ path = path.unsqueeze(1).transpose(2, 3) * mask
146
+ return path
147
+
148
+
149
+ def clip_grad_value_(parameters, clip_value, norm_type=2):
150
+ if isinstance(parameters, torch.Tensor):
151
+ parameters = [parameters]
152
+ parameters = list(filter(lambda p: p.grad is not None, parameters))
153
+ norm_type = float(norm_type)
154
+ if clip_value is not None:
155
+ clip_value = float(clip_value)
156
+
157
+ total_norm = 0
158
+ for p in parameters:
159
+ param_norm = p.grad.data.norm(norm_type)
160
+ total_norm += param_norm.item() ** norm_type
161
+ if clip_value is not None:
162
+ p.grad.data.clamp_(min=-clip_value, max=clip_value)
163
+ total_norm = total_norm ** (1.0 / norm_type)
164
+ return total_norm
infer_pack/models.py ADDED
@@ -0,0 +1,664 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math,pdb,os
2
+ from time import time as ttime
3
+ import torch
4
+ from torch import nn
5
+ from torch.nn import functional as F
6
+ from infer_pack import modules
7
+ from infer_pack import attentions
8
+ from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
9
+ from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
10
+ from infer_pack.commons import init_weights
11
+ import numpy as np
12
+ from infer_pack import commons
13
+ class TextEncoder256(nn.Module):
14
+ def __init__(
15
+ self, out_channels, hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, f0=True ):
16
+ super().__init__()
17
+ self.out_channels = out_channels
18
+ self.hidden_channels = hidden_channels
19
+ self.filter_channels = filter_channels
20
+ self.n_heads = n_heads
21
+ self.n_layers = n_layers
22
+ self.kernel_size = kernel_size
23
+ self.p_dropout = p_dropout
24
+ self.emb_phone = nn.Linear(256, hidden_channels)
25
+ self.lrelu=nn.LeakyReLU(0.1,inplace=True)
26
+ if(f0==True):
27
+ self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256
28
+ self.encoder = attentions.Encoder(
29
+ hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout
30
+ )
31
+ self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
32
+
33
+ def forward(self, phone, pitch, lengths):
34
+ if(pitch==None):
35
+ x = self.emb_phone(phone)
36
+ else:
37
+ x = self.emb_phone(phone) + self.emb_pitch(pitch)
38
+ x = x * math.sqrt(self.hidden_channels) # [b, t, h]
39
+ x=self.lrelu(x)
40
+ x = torch.transpose(x, 1, -1) # [b, h, t]
41
+ x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(
42
+ x.dtype
43
+ )
44
+ x = self.encoder(x * x_mask, x_mask)
45
+ stats = self.proj(x) * x_mask
46
+
47
+ m, logs = torch.split(stats, self.out_channels, dim=1)
48
+ return m, logs, x_mask
49
+ class TextEncoder256km(nn.Module):
50
+ def __init__(
51
+ self, out_channels, hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, f0=True ):
52
+ super().__init__()
53
+ self.out_channels = out_channels
54
+ self.hidden_channels = hidden_channels
55
+ self.filter_channels = filter_channels
56
+ self.n_heads = n_heads
57
+ self.n_layers = n_layers
58
+ self.kernel_size = kernel_size
59
+ self.p_dropout = p_dropout
60
+ # self.emb_phone = nn.Linear(256, hidden_channels)
61
+ self.emb_phone = nn.Embedding(500, hidden_channels)
62
+ self.lrelu=nn.LeakyReLU(0.1,inplace=True)
63
+ if(f0==True):
64
+ self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256
65
+ self.encoder = attentions.Encoder(
66
+ hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout
67
+ )
68
+ self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
69
+
70
+ def forward(self, phone, pitch, lengths):
71
+ if(pitch==None):
72
+ x = self.emb_phone(phone)
73
+ else:
74
+ x = self.emb_phone(phone) + self.emb_pitch(pitch)
75
+ x = x * math.sqrt(self.hidden_channels) # [b, t, h]
76
+ x=self.lrelu(x)
77
+ x = torch.transpose(x, 1, -1) # [b, h, t]
78
+ x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(
79
+ x.dtype
80
+ )
81
+ x = self.encoder(x * x_mask, x_mask)
82
+ stats = self.proj(x) * x_mask
83
+
84
+ m, logs = torch.split(stats, self.out_channels, dim=1)
85
+ return m, logs, x_mask
86
+ class ResidualCouplingBlock(nn.Module):
87
+ def __init__(
88
+ self,
89
+ channels,
90
+ hidden_channels,
91
+ kernel_size,
92
+ dilation_rate,
93
+ n_layers,
94
+ n_flows=4,
95
+ gin_channels=0,
96
+ ):
97
+ super().__init__()
98
+ self.channels = channels
99
+ self.hidden_channels = hidden_channels
100
+ self.kernel_size = kernel_size
101
+ self.dilation_rate = dilation_rate
102
+ self.n_layers = n_layers
103
+ self.n_flows = n_flows
104
+ self.gin_channels = gin_channels
105
+
106
+ self.flows = nn.ModuleList()
107
+ for i in range(n_flows):
108
+ self.flows.append(
109
+ modules.ResidualCouplingLayer(
110
+ channels,
111
+ hidden_channels,
112
+ kernel_size,
113
+ dilation_rate,
114
+ n_layers,
115
+ gin_channels=gin_channels,
116
+ mean_only=True,
117
+ )
118
+ )
119
+ self.flows.append(modules.Flip())
120
+
121
+ def forward(self, x, x_mask, g=None, reverse=False):
122
+ if not reverse:
123
+ for flow in self.flows:
124
+ x, _ = flow(x, x_mask, g=g, reverse=reverse)
125
+ else:
126
+ for flow in reversed(self.flows):
127
+ x = flow(x, x_mask, g=g, reverse=reverse)
128
+ return x
129
+
130
+ def remove_weight_norm(self):
131
+ for i in range(self.n_flows):
132
+ self.flows[i * 2].remove_weight_norm()
133
+ class PosteriorEncoder(nn.Module):
134
+ def __init__(
135
+ self,
136
+ in_channels,
137
+ out_channels,
138
+ hidden_channels,
139
+ kernel_size,
140
+ dilation_rate,
141
+ n_layers,
142
+ gin_channels=0,
143
+ ):
144
+ super().__init__()
145
+ self.in_channels = in_channels
146
+ self.out_channels = out_channels
147
+ self.hidden_channels = hidden_channels
148
+ self.kernel_size = kernel_size
149
+ self.dilation_rate = dilation_rate
150
+ self.n_layers = n_layers
151
+ self.gin_channels = gin_channels
152
+
153
+ self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
154
+ self.enc = modules.WN(
155
+ hidden_channels,
156
+ kernel_size,
157
+ dilation_rate,
158
+ n_layers,
159
+ gin_channels=gin_channels,
160
+ )
161
+ self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
162
+
163
+ def forward(self, x, x_lengths, g=None):
164
+ x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
165
+ x.dtype
166
+ )
167
+ x = self.pre(x) * x_mask
168
+ x = self.enc(x, x_mask, g=g)
169
+ stats = self.proj(x) * x_mask
170
+ m, logs = torch.split(stats, self.out_channels, dim=1)
171
+ z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
172
+ return z, m, logs, x_mask
173
+
174
+ def remove_weight_norm(self):
175
+ self.enc.remove_weight_norm()
176
+ class Generator(torch.nn.Module):
177
+ def __init__(
178
+ self,
179
+ initial_channel,
180
+ resblock,
181
+ resblock_kernel_sizes,
182
+ resblock_dilation_sizes,
183
+ upsample_rates,
184
+ upsample_initial_channel,
185
+ upsample_kernel_sizes,
186
+ gin_channels=0,
187
+ ):
188
+ super(Generator, self).__init__()
189
+ self.num_kernels = len(resblock_kernel_sizes)
190
+ self.num_upsamples = len(upsample_rates)
191
+ self.conv_pre = Conv1d(
192
+ initial_channel, upsample_initial_channel, 7, 1, padding=3
193
+ )
194
+ resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
195
+
196
+ self.ups = nn.ModuleList()
197
+ for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
198
+ self.ups.append(
199
+ weight_norm(
200
+ ConvTranspose1d(
201
+ upsample_initial_channel // (2**i),
202
+ upsample_initial_channel // (2 ** (i + 1)),
203
+ k,
204
+ u,
205
+ padding=(k - u) // 2,
206
+ )
207
+ )
208
+ )
209
+
210
+ self.resblocks = nn.ModuleList()
211
+ for i in range(len(self.ups)):
212
+ ch = upsample_initial_channel // (2 ** (i + 1))
213
+ for j, (k, d) in enumerate(
214
+ zip(resblock_kernel_sizes, resblock_dilation_sizes)
215
+ ):
216
+ self.resblocks.append(resblock(ch, k, d))
217
+
218
+ self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
219
+ self.ups.apply(init_weights)
220
+
221
+ if gin_channels != 0:
222
+ self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
223
+
224
+ def forward(self, x, g=None):
225
+ x = self.conv_pre(x)
226
+ if g is not None:
227
+ x = x + self.cond(g)
228
+
229
+ for i in range(self.num_upsamples):
230
+ x = F.leaky_relu(x, modules.LRELU_SLOPE)
231
+ x = self.ups[i](x)
232
+ xs = None
233
+ for j in range(self.num_kernels):
234
+ if xs is None:
235
+ xs = self.resblocks[i * self.num_kernels + j](x)
236
+ else:
237
+ xs += self.resblocks[i * self.num_kernels + j](x)
238
+ x = xs / self.num_kernels
239
+ x = F.leaky_relu(x)
240
+ x = self.conv_post(x)
241
+ x = torch.tanh(x)
242
+
243
+ return x
244
+
245
+ def remove_weight_norm(self):
246
+ for l in self.ups:
247
+ remove_weight_norm(l)
248
+ for l in self.resblocks:
249
+ l.remove_weight_norm()
250
+ class SineGen(torch.nn.Module):
251
+ """ Definition of sine generator
252
+ SineGen(samp_rate, harmonic_num = 0,
253
+ sine_amp = 0.1, noise_std = 0.003,
254
+ voiced_threshold = 0,
255
+ flag_for_pulse=False)
256
+ samp_rate: sampling rate in Hz
257
+ harmonic_num: number of harmonic overtones (default 0)
258
+ sine_amp: amplitude of sine-wavefrom (default 0.1)
259
+ noise_std: std of Gaussian noise (default 0.003)
260
+ voiced_thoreshold: F0 threshold for U/V classification (default 0)
261
+ flag_for_pulse: this SinGen is used inside PulseGen (default False)
262
+ Note: when flag_for_pulse is True, the first time step of a voiced
263
+ segment is always sin(np.pi) or cos(0)
264
+ """
265
+
266
+ def __init__(self, samp_rate, harmonic_num=0,
267
+ sine_amp=0.1, noise_std=0.003,
268
+ voiced_threshold=0,
269
+ flag_for_pulse=False):
270
+ super(SineGen, self).__init__()
271
+ self.sine_amp = sine_amp
272
+ self.noise_std = noise_std
273
+ self.harmonic_num = harmonic_num
274
+ self.dim = self.harmonic_num + 1
275
+ self.sampling_rate = samp_rate
276
+ self.voiced_threshold = voiced_threshold
277
+
278
+ def _f02uv(self, f0):
279
+ # generate uv signal
280
+ uv = torch.ones_like(f0)
281