yl12053 commited on
Commit
1d329dd
1 Parent(s): 28d412f
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. LICENSE +28 -0
  3. README.md +4 -4
  4. README_zh_CN.md +496 -0
  5. app.py +1066 -0
  6. auto_slicer.py +106 -0
  7. cluster/__init__.py +29 -0
  8. cluster/__pycache__/__init__.cpython-38.pyc +0 -0
  9. cluster/__pycache__/kmeans.cpython-38.pyc +0 -0
  10. cluster/kmeans.py +201 -0
  11. cluster/train_cluster.py +84 -0
  12. compress_model.py +69 -0
  13. configs/.ipynb_checkpoints/config-checkpoint.json +96 -0
  14. configs/config.json +100 -0
  15. configs/diffusion.yaml +49 -0
  16. configs_template/config_template.json +72 -0
  17. configs_template/diffusion_template.yaml +49 -0
  18. data_utils.py +185 -0
  19. diffusion/__init__.py +0 -0
  20. diffusion/__pycache__/__init__.cpython-38.pyc +0 -0
  21. diffusion/__pycache__/diffusion.cpython-38.pyc +0 -0
  22. diffusion/__pycache__/unit2mel.cpython-38.pyc +0 -0
  23. diffusion/__pycache__/vocoder.cpython-38.pyc +0 -0
  24. diffusion/__pycache__/wavenet.cpython-38.pyc +0 -0
  25. diffusion/data_loaders.py +288 -0
  26. diffusion/diffusion.py +317 -0
  27. diffusion/diffusion_onnx.py +612 -0
  28. diffusion/dpm_solver_pytorch.py +1201 -0
  29. diffusion/how to export onnx.md +4 -0
  30. diffusion/infer_gt_mel.py +74 -0
  31. diffusion/logger/__init__.py +0 -0
  32. diffusion/logger/__pycache__/__init__.cpython-38.pyc +0 -0
  33. diffusion/logger/__pycache__/utils.cpython-38.pyc +0 -0
  34. diffusion/logger/saver.py +150 -0
  35. diffusion/logger/utils.py +126 -0
  36. diffusion/onnx_export.py +226 -0
  37. diffusion/solver.py +195 -0
  38. diffusion/unit2mel.py +147 -0
  39. diffusion/vocoder.py +94 -0
  40. diffusion/wavenet.py +108 -0
  41. filelists/test.txt +4 -0
  42. filelists/train.txt +924 -0
  43. filelists/val.txt +2 -0
  44. flask_api.py +60 -0
  45. flask_api_full_song.py +55 -0
  46. inference/__init__.py +0 -0
  47. inference/__pycache__/__init__.cpython-38.pyc +0 -0
  48. inference/__pycache__/infer_tool.cpython-38.pyc +0 -0
  49. inference/__pycache__/slicer.cpython-38.pyc +0 -0
  50. inference/infer_tool.py +529 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ pretrain/nsf_hifigan/model filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2023, SVC Develop Team
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
README.md CHANGED
@@ -1,13 +1,13 @@
1
  ---
2
  title: So Vits 4.1 Mejiro Mcqueen
3
- emoji: 📊
4
- colorFrom: yellow
5
- colorTo: yellow
6
  sdk: gradio
7
  sdk_version: 3.38.0
8
  app_file: app.py
 
9
  pinned: false
10
- license: bsd-3-clause
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: So Vits 4.1 Mejiro Mcqueen
3
+ emoji: 🐨
4
+ colorFrom: pink
5
+ colorTo: red
6
  sdk: gradio
7
  sdk_version: 3.38.0
8
  app_file: app.py
9
+ python_version: 3.8.10
10
  pinned: false
 
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
README_zh_CN.md ADDED
@@ -0,0 +1,496 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 马上要高考了,SvcDevelopTeam在此助各位考生高考旗开得胜,超常发挥。
2
+
3
+ # SoftVC VITS Singing Voice Conversion
4
+
5
+ [**English**](./README.md) | [**中文简体**](./README_zh_CN.md)
6
+
7
+ #### ✨ 带有F0曲线编辑器,角色混合时间轴编辑器的推理端 (Onnx模型的用途) : [MoeVoiceStudio(即将到来)](https://github.com/NaruseMioShirakana/MoeVoiceStudio)
8
+
9
+ #### ✨ 改善了交互的一个分支推荐:[34j/so-vits-svc-fork](https://github.com/34j/so-vits-svc-fork)
10
+
11
+ #### ✨ 支持实时转换的一个客户端:[w-okada/voice-changer](https://github.com/w-okada/voice-changer)
12
+
13
+ **本项目与Vits有着根本上的不同。Vits是TTS,本项目是SVC。本项目无法实现TTS,Vits也无法实现SVC,这两个项目的模型是完全不通用的。**
14
+
15
+ ## 重要通知
16
+
17
+ 这个项目是为了让开发者最喜欢的动画角色唱歌而开发的,任何涉及真人的东西都与开发者的意图背道而驰。
18
+
19
+ ## 声明
20
+
21
+ 本项目为开源、离线的项目,SvcDevelopTeam的所有成员与本项目的所有开发者以及维护者(以下简称贡献者)对本项目没有控制力。本项目的贡献者从未向任何组织或个人提供包括但不限于数据集提取、数据集加工、算力支持、训练支持、推理等一切形式的帮助;本项目的贡献者不知晓也无法知晓使用者使用该项目的用途。故一切基于本项目训练的AI模型和合成的音频都与本项目贡献者无关。一切由此造成的问题由使用者自行承担。
22
+
23
+ 此项目完全离线运行,不能收集任何用户信息或获取用户输入数据。因此,这个项目的贡献者不知道所有的用户输入和模型,因此不负责任何用户输入。
24
+
25
+ 本项目只是一个框架项目,本身并没有语音合成的功能,所有的功能都需要用户自己训练模型。同时,这个项目没有任何模型,任何二次分发的项目都与这个项目的贡献者无关。
26
+
27
+ ## 📏 使用规约
28
+
29
+ # Warning:请自行解决数据集授权问题,禁止使用非授权数据集进行训练!任何由于使用非授权数据集进行训练造成的问题,需自行承担全部责任和后果!与仓库、仓库维护者、svc develop team 无关!
30
+
31
+ 1. 本项目是基于学术交流目的建立,仅供交流与学习使用,并非为生产环境准备。
32
+ 2. 任何发布到视频平台的基于 sovits 制作的视频,都必须要在简介明确指明用于变声器转换的输入源歌声、音频,例如:使用他人发布的视频 / 音频,通过分离的人声作为输入源进行转换的,必须要给出明确的原视频、音乐链接;若使用是自己的人声,或是使用其他歌声合成引擎合成的声音作为输入源进行转换的,也必须在简介加以说明。
33
+ 3. 由输入源造成的侵权问题需自行承担全部责任和一切后果。使用其他商用歌声合成软件作为输入源时,请确保遵守该软件的使用条例,注意,许多歌声合成引擎使用条例中明确指明不可用于输入源进行转换!
34
+ 4. 禁止使用该项目从事违法行为与宗教、政治等活动,该项目维护者坚决抵制上述行为,不同意此条则禁止使用该项目。
35
+ 5. 继续使用视为已同意本仓库 README 所述相关条例,本仓库 README 已进行劝导义务,不对后续可能存在问题负责。
36
+ 6. 如果将此项目用于任何其他企划,请提前联系并告知本仓库作者,十分感谢。
37
+
38
+ ## 📝 模型简介
39
+
40
+ 歌声音色转换模型,通过SoftVC内容编码器提取源音频语音特征,与F0同时输入VITS替换原本的文本输入达到歌声转换的效果。同时,更换声码器为 [NSF HiFiGAN](https://github.com/openvpi/DiffSinger/tree/refactor/modules/nsf_hifigan)解决断音问题。
41
+
42
+ ### 🆕 4.1-Stable 版本更新内容
43
+
44
+ + 特征输入更换为 [Content Vec](https://github.com/auspicious3000/contentvec) 的第12层Transformer输出,并兼容4.0分支
45
+ + 更新浅层扩散,可以使用浅层扩散模型提升音质
46
+ + 增加whisper语音编码器的支持
47
+ + 增加静态/动态声线融合
48
+ + 增加响度嵌入
49
+ + 增加特征检索,来自于[RVC](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI)
50
+
51
+ ### 🆕 关于兼容4.0模型的问题
52
+
53
+ + 可通过修改4.0模型的config.json对4.0的模型进行支持,需要在config.json的model字段中添加speech_encoder字段,具体见下
54
+
55
+ ```
56
+ "model": {
57
+ .........
58
+ "ssl_dim": 256,
59
+ "n_speakers": 200,
60
+ "speech_encoder":"vec256l9"
61
+ }
62
+ ```
63
+
64
+ ### 🆕 关于浅扩散
65
+ ![Diagram](shadowdiffusion.png)
66
+
67
+ ## 💬 关于 Python 版本问题
68
+
69
+ 在进行测试后,我们认为`Python 3.8.9`能够稳定地运行该项目
70
+
71
+ ## 📥 预先下载的模型文件
72
+
73
+ #### **必须项**
74
+
75
+ **以下编码器需要选择一个使用**
76
+
77
+ ##### **1. 若使用contentvec作为声音编码器(推荐)**
78
+
79
+ `vec768l12`与`vec256l9` 需要该编码器
80
+
81
+ + contentvec :[checkpoint_best_legacy_500.pt](https://ibm.box.com/s/z1wgl1stco8ffooyatzdwsqn2psd9lrr)
82
+ + 放在`pretrain`目录下
83
+
84
+ 或者下载下面的ContentVec,大小只有199MB,但效果相同:
85
+ + contentvec :[hubert_base.pt](https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt)
86
+ + 将文件名改为`checkpoint_best_legacy_500.pt`后,放在`pretrain`目录下
87
+
88
+ ```shell
89
+ # contentvec
90
+ wget -P pretrain/ http://obs.cstcloud.cn/share/obs/sankagenkeshi/checkpoint_best_legacy_500.pt
91
+ # 也可手动下载放在pretrain目录
92
+ ```
93
+
94
+ ##### **2. 若使用hubertsoft作为声音编码器**
95
+ + soft vc hubert:[hubert-soft-0d54a1f4.pt](https://github.com/bshall/hubert/releases/download/v0.1/hubert-soft-0d54a1f4.pt)
96
+ + 放在`pretrain`目录下
97
+
98
+ ##### **3. 若使用Whisper-ppg作为声音编码器**
99
+ + 下载模型 [medium.pt](https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt), 该模型适配`whisper-ppg`
100
+ + 下载模型 [large-v2.pt](https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt), 该模型适配`whisper-ppg-large`
101
+ + 放在`pretrain`目录下
102
+
103
+ ##### **4. 若使用cnhubertlarge作为声音编码器**
104
+ + 下载模型 [chinese-hubert-large-fairseq-ckpt.pt](https://huggingface.co/TencentGameMate/chinese-hubert-large/resolve/main/chinese-hubert-large-fairseq-ckpt.pt)
105
+ + 放在`pretrain`目录下
106
+
107
+ ##### **5. 若使用dphubert作为声音编码器**
108
+ + 下载模型 [DPHuBERT-sp0.75.pth](https://huggingface.co/pyf98/DPHuBERT/resolve/main/DPHuBERT-sp0.75.pth)
109
+ + 放在`pretrain`目录下
110
+
111
+ ##### **6. 若使用WavLM作为声音编码器**
112
+ + 下载模型 [WavLM-Base+.pt](https://valle.blob.core.windows.net/share/wavlm/WavLM-Base+.pt?sv=2020-08-04&st=2023-03-01T07%3A51%3A05Z&se=2033-03-02T07%3A51%3A00Z&sr=c&sp=rl&sig=QJXmSJG9DbMKf48UDIU1MfzIro8HQOf3sqlNXiflY1I%3D), 该模型适配`wavlmbase+`
113
+ + 放在`pretrain`目录下
114
+
115
+ ##### **7. 若使用OnnxHubert/ContentVec作为声音编码器**
116
+ + 下载模型 [MoeSS-SUBModel](https://huggingface.co/NaruseMioShirakana/MoeSS-SUBModel/tree/main)
117
+ + 放在`pretrain`目录下
118
+
119
+ #### **编码器列表**
120
+ - "vec768l12"
121
+ - "vec256l9"
122
+ - "vec256l9-onnx"
123
+ - "vec256l12-onnx"
124
+ - "vec768l9-onnx"
125
+ - "vec768l12-onnx"
126
+ - "hubertsoft-onnx"
127
+ - "hubertsoft"
128
+ - "whisper-ppg"
129
+ - "cnhubertlarge"
130
+ - "dphubert"
131
+ - "whisper-ppg-large"
132
+ - "wavlmbase+"
133
+
134
+ #### **可选项(强烈建议使用)**
135
+
136
+ + 预训练底模文件: `G_0.pth` `D_0.pth`
137
+ + 放在`logs/44k`目录下
138
+
139
+ + 扩散模型预训练底模文件: `model_0.pt `
140
+ + 放在`logs/44k/diffusion`目录下
141
+
142
+ 从svc-develop-team(待定)或任何其他地方获取Sovits底模
143
+
144
+ 扩散模型引用了[Diffusion-SVC](https://github.com/CNChTu/Diffusion-SVC)的Diffusion Model,底模与[Diffusion-SVC](https://github.com/CNChTu/Diffusion-SVC)的扩散模型底模通用,可以去[Diffusion-SVC](https://github.com/CNChTu/Diffusion-SVC)获取扩散模型的底模
145
+
146
+ 虽然底模一般不会引起什么版权问题,但还是请注意一下,比如事先询问作者,又或者作者在模型描述中明确写明了可行的用途
147
+
148
+ #### **可选项(根据情况选择)**
149
+
150
+ 如果使用`NSF-HIFIGAN增强器`或`浅层扩散`的话,需要下载预训练的NSF-HIFIGAN模型,如果不需要可以不下载
151
+
152
+ + 预训练的NSF-HIFIGAN声码器 :[nsf_hifigan_20221211.zip](https://github.com/openvpi/vocoders/releases/download/nsf-hifigan-v1/nsf_hifigan_20221211.zip)
153
+ + 解压后,将四个文件放在`pretrain/nsf_hifigan`目录下
154
+
155
+ ```shell
156
+ # nsf_hifigan
157
+ wget -P pretrain/ https://github.com/openvpi/vocoders/releases/download/nsf-hifigan-v1/nsf_hifigan_20221211.zip
158
+ unzip -od pretrain/nsf_hifigan pretrain/nsf_hifigan_20221211.zip
159
+ # 也可手动下载放在pretrain/nsf_hifigan目录
160
+ # 地址:https://github.com/openvpi/vocoders/releases/tag/nsf-hifigan-v1
161
+ ```
162
+
163
+ ## 📊 数据集准备
164
+
165
+ 仅需要以以下文件结构将数据集放入dataset_raw目录即可
166
+
167
+ ```
168
+ dataset_raw
169
+ ├───speaker0
170
+ │ ├───xxx1-xxx1.wav
171
+ │ ├───...
172
+ │ └───Lxx-0xx8.wav
173
+ └───speaker1
174
+ ├───xx2-0xxx2.wav
175
+ ├───...
176
+ └───xxx7-xxx007.wav
177
+ ```
178
+
179
+ 可以自定义说话人名称
180
+
181
+ ```
182
+ dataset_raw
183
+ └───suijiSUI
184
+ ├───1.wav
185
+ ├───...
186
+ └───25788785-20221210-200143-856_01_(Vocals)_0_0.wav
187
+ ```
188
+
189
+ ## 🛠️ 数据预处理
190
+
191
+ ### 0. 音频切片
192
+
193
+ 将音频切片至`5s - 15s`, 稍微长点也无伤大雅,实在太长可能会导致训练中途甚至预处理就爆显存
194
+
195
+ 可以使用[audio-slicer-GUI](https://github.com/flutydeer/audio-slicer)、[audio-slicer-CLI](https://github.com/openvpi/audio-slicer)
196
+
197
+ 一般情况下只需调整其中的`Minimum Interval`,普通陈述素材通常保持默认即可,歌唱素材可以调整至`100`甚至`50`
198
+
199
+ 切完之后手动删除过长过短的音频
200
+
201
+ **如果你使用Whisper-ppg声音编码器进行训练,所有的切片长���必须小于30s**
202
+
203
+ ### 1. 重采样至44100Hz单声道
204
+
205
+ ```shell
206
+ python resample.py
207
+ ```
208
+
209
+ #### 注意
210
+
211
+ 虽然本项目拥有重采样、转换单声道与响度匹配的脚本resample.py,但是默认的响度匹配是匹配到0db。这可能会造成音质的受损。而python的响度匹配包pyloudnorm无法对电平进行压限,这会导致爆音。所以建议可以考虑使用专业声音处理软件如`adobe audition`等软件做响度匹配处理。若已经使用其他软件做响度匹配,可以在运行上述命令时添加`--skip_loudnorm`跳过响度匹配步骤。如:
212
+
213
+ ```shell
214
+ python resample.py --skip_loudnorm
215
+ ```
216
+
217
+ ### 2. 自动划分训练集、验证集,以及自动生成配置文件
218
+
219
+ ```shell
220
+ python preprocess_flist_config.py --speech_encoder vec768l12
221
+ ```
222
+
223
+ speech_encoder拥有以下选择
224
+
225
+ ```
226
+ vec768l12
227
+ vec256l9
228
+ hubertsoft
229
+ whisper-ppg
230
+ whisper-ppg-large
231
+ cnhubertlarge
232
+ dphubert
233
+ wavlmbase+
234
+ ```
235
+
236
+ 如果省略speech_encoder参数,默认值为vec768l12
237
+
238
+ **使用响度嵌入**
239
+
240
+ 若使用响度嵌入,需要增加`--vol_aug`参数,比如:
241
+
242
+ ```shell
243
+ python preprocess_flist_config.py --speech_encoder vec768l12 --vol_aug
244
+ ```
245
+
246
+ 使用后训练出的模型将匹配到输入源响度,否则为训练集响度。
247
+
248
+ #### 此时可以在生成的config.json与diffusion.yaml修改部分参数
249
+
250
+ * `keep_ckpts`:训练时保留最后几个模型,`0`为保留所有,默认只保留最后`3`个
251
+
252
+ * `all_in_mem`,`cache_all_data`:加载所有数据集到内存中,某些平台的硬盘IO过于低下、同时内存容量 **远大于** 数据集体积时可以启用
253
+
254
+ * `batch_size`:单次训练加载到GPU的数据量,调整到低于显存容量的大小即可
255
+
256
+ * `vocoder_name` : 选择一种声码器,默认为`nsf-hifigan`.
257
+
258
+ ##### **声码器列表**
259
+
260
+ ```
261
+ nsf-hifigan
262
+ nsf-snake-hifigan
263
+ ```
264
+
265
+ ### 3. 生成hubert与f0
266
+
267
+ ```shell
268
+ python preprocess_hubert_f0.py --f0_predictor dio
269
+ ```
270
+
271
+ f0_predictor拥有四个选择
272
+
273
+ ```
274
+ crepe
275
+ dio
276
+ pm
277
+ harvest
278
+ ```
279
+
280
+ 如果训练集过于嘈杂,请使用crepe处理f0
281
+
282
+ 如果省略f0_predictor参数,默认值为dio
283
+
284
+ 尚若需要浅扩散功能(可选),需要增加--use_diff参数,比如
285
+
286
+ ```shell
287
+ python preprocess_hubert_f0.py --f0_predictor dio --use_diff
288
+ ```
289
+
290
+ 执行完以上步骤后 dataset 目录便是预处理完成的数据,可以删除 dataset_raw 文件夹了
291
+
292
+ ## 🏋️‍♀️ 训练
293
+
294
+ ### 扩散模型(可选)
295
+
296
+ 尚若需要浅扩散功能,需要训练扩散模型,扩散模型训练方法为:
297
+
298
+ ```shell
299
+ python train_diff.py -c configs/diffusion.yaml
300
+ ```
301
+
302
+ ### 主模型训练
303
+
304
+ ```shell
305
+ python train.py -c configs/config.json -m 44k
306
+ ```
307
+
308
+ 模型训练结束后,模型文件保存在`logs/44k`目录下,扩散模型在`logs/44k/diffusion`下
309
+
310
+ ## 🤖 推理
311
+
312
+ 使用 [inference_main.py](inference_main.py)
313
+
314
+ ```shell
315
+ # 例
316
+ python inference_main.py -m "logs/44k/G_30400.pth" -c "configs/config.json" -n "君の知らない物語-src.wav" -t 0 -s "nen"
317
+ ```
318
+
319
+ 必填项部分:
320
+ + `-m` | `--model_path`:模型路径
321
+ + `-c` | `--config_path`:配置文件路径
322
+ + `-n` | `--clean_names`:wav 文件名列表,放在 raw 文件夹下
323
+ + `-t` | `--trans`:音高调整,支持正负(半音)
324
+ + `-s` | `--spk_list`:合成目标说话人名称
325
+ + `-cl` | `--clip`:音频强制切片,默认0为自动切片,单位为秒/s
326
+
327
+ 可选项部分:部分具体见下一节
328
+ + `-lg` | `--linear_gradient`:两段音频切片的交叉淡入长度,如果强制切片后出现人声不连贯可调整该数值,如果连贯建议采用默认值0,单位为秒
329
+ + `-f0p` | `--f0_predictor`:选择F0预测器,可选择crepe,pm,dio,harvest,默认为pm(注意:crepe为原F0使用均值滤波器)
330
+ + `-a` | `--auto_predict_f0`:语音转换自动预测音高,转换歌声时不要打开这个会严重跑调
331
+ + `-cm` | `--cluster_model_path`:聚类模型或特征检索索引路径,如果没有训练聚类或特征检索则随便填
332
+ + `-cr` | `--cluster_infer_ratio`:聚类方案或特征检索占比,范围0-1,若没有训练聚类模型或特征检索则默认0即可
333
+ + `-eh` | `--enhance`:是否使用NSF_HIFIGAN增强器,该选项对部分训练集少的模型有一定的音质增强效果,但是对训练好的模型有反面效果,默认关闭
334
+ + `-shd` | `--shallow_diffusion`:是否使用浅层扩散,使用后可解决一部分电音问题,默认关闭,该选项打开时,NSF_HIFIGAN增强器将会被禁止
335
+ + `-usm` | `--use_spk_mix`:是否使用角色融合/动态声线融合
336
+ + `-lea` | `--loudness_envelope_adjustment`:输入源响度包络替换输出响度包络融合比例,越靠近1越使用输出响度包络
337
+ + `-fr` | `--feature_retrieval`:是否使用特征检索,如果使用聚类模型将被禁用,且cm与cr参数将会变成特征检索的索引路径与混合比例
338
+
339
+ 浅扩散设置:
340
+ + `-dm` | `--diffusion_model_path`:扩散模型路径
341
+ + `-dc` | `--diffusion_config_path`:扩散模型配置文��路径
342
+ + `-ks` | `--k_step`:扩散步数,越大越接近扩散模型的结果,默认100
343
+ + `-od` | `--only_diffusion`:纯扩散模式,该模式不会加载sovits模型,以扩散模型推理
344
+ + `-se` | `--second_encoding`:二次编码,浅扩散前会对原始音频进行二次编码,玄学选项,有时候效果好,有时候效果差
345
+
346
+ ### 注意!
347
+
348
+ 如果使用`whisper-ppg` 声音编码器进行推理,需要将`--clip`设置为25,`-lg`设置为1。否则将无法正常推理。
349
+
350
+ ## 🤔 可选项
351
+
352
+ 如果前面的效果已经满意,或者没看明白下面在讲啥,那后面的内容都可以忽略,不影响模型使用(这些可选项影响比较小,可能在某些特定数据上有点效果,但大部分情况似乎都感知不太明显)
353
+
354
+ ### 自动f0预测
355
+
356
+ 4.0模型训练过程会训练一个f0预测器,对于语音转换可以开启自动音高预测,如果效果不好也可以使用手动的,但转换歌声时请不要启用此功能!!!会严重跑调!!
357
+ + 在inference_main中设置auto_predict_f0为true即可
358
+
359
+ ### 聚类音色泄漏控制
360
+
361
+ 介绍:聚类方案可以减小音色泄漏,使得模型训练出来更像目标的音色(但其实不是特别明显),但是单纯的聚类方案会降低模型的咬字(会口齿不清)(这个很明显),本模型采用了融合的方式,可以线性控制聚类方案与非聚类方案的占比,也就是可以手动在"像目标音色" 和 "咬字清晰" 之间调整比例,找到合适的折中点
362
+
363
+ 使用聚类前面的已有步骤不用进行任何的变动,只需要额外训练一个聚类模型,虽然效果比较有限,但训练成本也比较低
364
+
365
+ + 训练过程:
366
+ + 使用cpu性能较好的机器训练,据我的经验在腾讯云6核cpu训练每个speaker需要约4分钟即可完成训练
367
+ + 执行`python cluster/train_cluster.py`,模型的输出会在`logs/44k/kmeans_10000.pt`
368
+ + 聚类模型目前可以使用gpu进行训练,执行`python cluster/train_cluster.py --gpu`
369
+ + 推理过程:
370
+ + `inference_main.py`中指定`cluster_model_path`
371
+ + `inference_main.py`中指定`cluster_infer_ratio`,`0`为完全不使用聚类,`1`为只使用聚类,通常设置`0.5`即可
372
+
373
+ ### 特征检索
374
+
375
+ 介绍:跟聚类方案一样可以减小音色泄漏,咬字比聚类稍好,但会降低推理速度,采用了融合的方式,可以线性控制特征检索与非特征检索的占比,
376
+
377
+ + 训练过程:
378
+ 首先需要在生成hubert与f0后执行:
379
+
380
+ ```shell
381
+ python train_index.py -c configs/config.json
382
+ ```
383
+
384
+ 模型的输出会在`logs/44k/feature_and_index.pkl`
385
+
386
+ + 推理过程:
387
+ + 需要首先制定`--feature_retrieval`,此时聚类方案会自动切换到特征检索方案
388
+ + `inference_main.py`中指定`cluster_model_path` 为模型输出文件
389
+ + `inference_main.py`中指定`cluster_infer_ratio`,`0`为完全不使用特征检索,`1`为只使用特征检索,通常设置`0.5`即可
390
+
391
+ ### [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/svc-develop-team/so-vits-svc/blob/4.1-Stable/sovits4_for_colab.ipynb) [sovits4_for_colab.ipynb](https://colab.research.google.com/github/svc-develop-team/so-vits-svc/blob/4.1-Stable/sovits4_for_colab.ipynb)
392
+
393
+ ## 🗜️ 模型压缩
394
+
395
+ 生成的模型含有继续训练所需的信息。如果确认不再训练,可以移除模型中此部分信息,得到约 1/3 大小的最终模型。
396
+
397
+ 使用 [compress_model.py](compress_model.py)
398
+
399
+ ```shell
400
+ # 例
401
+ python compress_model.py -c="configs/config.json" -i="logs/44k/G_30400.pth" -o="logs/44k/release.pth"
402
+ ```
403
+
404
+ ## 👨‍🔧 声线混合
405
+
406
+ ### 静态声线混合
407
+
408
+ **参考`webUI.py`文件中,小工具/实验室特性的静态声线融合。**
409
+
410
+ 介绍:该功能可以将多个声音模型合成为一个声音模型(多个模型参数的凸组合或线性组合),从而制造出现实中不存在的声线
411
+ **注意:**
412
+
413
+ 1. 该功能仅支持单说话人的模型
414
+ 2. 如果强行使用多说话人模型,需要保证多个模型的说话人数量相同,这样可以混合同一个SpaekerID下的声音
415
+ 3. 保证所有待混合模型的config.json中的model字段是相同的
416
+ 4. 输出的混合模型可以使用待合成模型的任意一个config.json,但聚类模型将不能使用
417
+ 5. 批量上传模型的时候最好把模型放到一个文件夹选中后一起上传
418
+ 6. 混合比例调整建议大小在0-100之间,也可以调为其他数字,但在线性组合模式下会出现未知的效果
419
+ 7. 混合完毕后,文件将会保存在项目根目录中,文件名为output.pth
420
+ 8. 凸组合模式会将混合比例执行Softmax使混合比例相加为1,而线性组合模式不会
421
+
422
+ ### 动态声线混合
423
+
424
+ **参考`spkmix.py`文件中关于动态声线混合的介绍**
425
+
426
+ 角色混合轨道 编写规则:
427
+
428
+ 角色ID : \[\[起始时间1, 终止时间1, 起始数值1, 起始数值1], [起始时间2, 终止时间2, 起始数值2, 起始数值2]]
429
+
430
+ 起始时间和前一个的终止时间必须相同,第一个起始时间必须为0,最后一个终止时间必须为1 (时间的范围为0-1)
431
+
432
+ 全部角色必须填写,不使用的角色填\[\[0., 1., 0., 0.]]即可
433
+
434
+ 融合数值可以随便填,在指定的时间段内从起始数值线性变化为终止数值,内部会自动确保线性组合为1(凸组合条件),可以放心使用
435
+
436
+ 推理的时候使用`--use_spk_mix`参数即可启用动态声线混合
437
+
438
+ ## 📤 Onnx导出
439
+
440
+ 使用 [onnx_export.py](onnx_export.py)
441
+
442
+ + 新建文件夹:`checkpoints` 并打开
443
+ + 在`checkpoints`文件夹中新建一个文件夹作为项目文件夹,文件夹名为你的项目名称,比如`aziplayer`
444
+ + 将你的模型更名为`model.pth`,配置文件更名为`config.json`,并放置到刚才创建的`aziplayer`文件夹下
445
+ + 将 [onnx_export.py](onnx_export.py) 中`path = "NyaruTaffy"` 的 `"NyaruTaffy"` 修改为你的项目名称,`path = "aziplayer" (onnx_export_speaker_mix,为支持角色混合的onnx导出)`
446
+ + 运行 [onnx_export.py](onnx_export.py)
447
+ + 等待执行完毕,在你的项目文件夹下会生成一个`model.onnx`,即为导出的模型
448
+
449
+ 注意:Hubert Onnx模型请使用MoeSS提供的模型,目前无法自行导出(fairseq中Hubert有不少onnx不支持的算子和涉及到常量的东西,在导出时会报错或者导出的模型输入输出shape和结果都有问题)
450
+
451
+ ## ☀️ 旧贡献者
452
+
453
+ 因为某些原因原作者进行了删库处理,本仓库重建之初由于组织成员疏忽直接重新上传了所有文件导致以前的contributors全部木大,现在在README里重新添加一个旧贡献者列表
454
+
455
+ *某些成员已根据其个人意愿不将其列出*
456
+
457
+ <table>
458
+ <tr>
459
+ <td align="center"><a href="https://github.com/MistEO"><img src="https://avatars.githubusercontent.com/u/18511905?v=4" width="100px;" alt=""/><br /><sub><b>MistEO</b></sub></a><br /></td>
460
+ <td align="center"><a href="https://github.com/XiaoMiku01"><img src="https://avatars.githubusercontent.com/u/54094119?v=4" width="100px;" alt=""/><br /><sub><b>XiaoMiku01</b></sub></a><br /></td>
461
+ <td align="center"><a href="https://github.com/ForsakenRei"><img src="https://avatars.githubusercontent.com/u/23041178?v=4" width="100px;" alt=""/><br /><sub><b>しぐれ</b></sub></a><br /></td>
462
+ <td align="center"><a href="https://github.com/TomoGaSukunai"><img src="https://avatars.githubusercontent.com/u/25863522?v=4" width="100px;" alt=""/><br /><sub><b>TomoGaSukunai</b></sub></a><br /></td>
463
+ <td align="center"><a href="https://github.com/Plachtaa"><img src="https://avatars.githubusercontent.com/u/112609742?v=4" width="100px;" alt=""/><br /><sub><b>Plachtaa</b></sub></a><br /></td>
464
+ <td align="center"><a href="https://github.com/zdxiaoda"><img src="https://avatars.githubusercontent.com/u/45501959?v=4" width="100px;" alt=""/><br /><sub><b>zd小达</b></sub></a><br /></td>
465
+ <td align="center"><a href="https://github.com/Archivoice"><img src="https://avatars.githubusercontent.com/u/107520869?v=4" width="100px;" alt=""/><br /><sub><b>凍聲響世</b></sub></a><br /></td>
466
+ </tr>
467
+ </table>
468
+
469
+ ## 📚 一些法律条例参考
470
+
471
+ #### 任何国家,地区,组织和个人使用此项目必须遵守以下法律
472
+
473
+ #### 《民法典》
474
+
475
+ ##### 第一千零一十九条
476
+
477
+ 任何组织或者个人不得以丑化、污损,或者利用信息技术手段伪造等方式侵害他人的肖像权。未经肖像权人同意,不得制作、使用、公开肖像权人的肖像,但是法律另有规定的除外。未经肖像权人同意,肖像作品权利人不得以发表、复制、发行、出租、展览等方式使用或者公开肖像权人的肖像。对自然人声音的保护,参照适用肖像权保护的有关规定。
478
+
479
+ ##### 第一千零二十四条
480
+
481
+ 【名誉权】民事主体享有名誉权。任何组织或者个人不得以侮辱、诽谤等方式侵害他人的名誉权。
482
+
483
+ ##### 第一千零二十七条
484
+
485
+ 【作品侵害名誉权】行为人发表的文学、艺术作品以真人真事或者特定人为描述对象,含有侮辱、诽谤内容,侵害他人名誉权的,受害人有权依法请求该行为人承担民事责任。行为人发表的文学、艺术作品不以特定人为描述对象,仅其中的情节与该特定人的情况相似的,不承担民事责任。
486
+
487
+ #### 《[中华人民共和国宪法](http://www.gov.cn/guoqing/2018-03/22/content_5276318.htm)》
488
+
489
+ #### 《[中华人民共和国刑法](http://gongbao.court.gov.cn/Details/f8e30d0689b23f57bfc782d21035c3.html?sw=中华人民共和国刑法)》
490
+
491
+ #### 《[中华人民共和国民法典](http://gongbao.court.gov.cn/Details/51eb6750b8361f79be8f90d09bc202.html)》
492
+
493
+ ## 💪 感谢所有的贡献者
494
+ <a href="https://github.com/svc-develop-team/so-vits-svc/graphs/contributors" target="_blank">
495
+ <img src="https://contrib.rocks/image?repo=svc-develop-team/so-vits-svc" />
496
+ </a>
app.py ADDED
@@ -0,0 +1,1066 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import multiprocessing
2
+ import os
3
+ import re
4
+ import torch
5
+ import glob
6
+ import gradio as gr
7
+ import librosa
8
+ import numpy as np
9
+ import soundfile as sf
10
+ from inference.infer_tool import Svc
11
+ import logging
12
+ import json
13
+ import yaml
14
+ import time
15
+ import subprocess
16
+ import shutil
17
+ import utils
18
+ import datetime
19
+ import traceback
20
+ from utils import mix_model
21
+ from onnxexport.model_onnx import SynthesizerTrn
22
+ from itertools import chain
23
+ from compress_model import removeOptimizer
24
+ from auto_slicer import AutoSlicer
25
+
26
+ logging.getLogger('numba').setLevel(logging.WARNING)
27
+ logging.getLogger('markdown_it').setLevel(logging.WARNING)
28
+ logging.getLogger('urllib3').setLevel(logging.WARNING)
29
+ logging.getLogger('matplotlib').setLevel(logging.WARNING)
30
+
31
+ workdir = "logs/44k"
32
+ diff_workdir = "logs/44k/diffusion"
33
+ config_dir = "configs/"
34
+ raw_path = "dataset_raw"
35
+ raw_wavs_path = "raw"
36
+ models_backup_path = 'models_backup'
37
+ root_dir = "checkpoints"
38
+ debug = False
39
+ sovits_params = {}
40
+ diff_params = {}
41
+
42
+ loaded = None
43
+
44
+ def debug_change():
45
+ global debug
46
+ debug = debug_button.value
47
+
48
+ def get_default_settings():
49
+ global sovits_params, diff_params
50
+ yaml_path = "settings.yaml"
51
+ with open(yaml_path, 'r') as f:
52
+ default_settings = yaml.safe_load(f)
53
+ sovits_params = default_settings['sovits_params']
54
+ diff_params = default_settings['diff_params']
55
+ return sovits_params, diff_params
56
+
57
+ def save_default_settings(log_interval,eval_interval,keep_ckpts,batch_size,learning_rate,fp16_run,all_in_mem,num_workers,cache_all_data,cache_device,amp_dtype,diff_batch_size,diff_lr,diff_interval_log,diff_interval_val,diff_force_save):
58
+ yaml_path = "settings.yaml"
59
+ with open(yaml_path, 'r') as f:
60
+ default_settings = yaml.safe_load(f)
61
+ default_settings['sovits_params']['log_interval'] = int(log_interval)
62
+ default_settings['sovits_params']['eval_interval'] = int(eval_interval)
63
+ default_settings['sovits_params']['keep_ckpts'] = int(keep_ckpts)
64
+ default_settings['sovits_params']['batch_size'] = int(batch_size)
65
+ default_settings['sovits_params']['learning_rate'] = float(learning_rate)
66
+ default_settings['sovits_params']['fp16_run'] = fp16_run
67
+ default_settings['sovits_params']['all_in_mem'] = all_in_mem
68
+ default_settings['diff_params']['num_workers'] = int(num_workers)
69
+ default_settings['diff_params']['cache_all_data'] = cache_all_data
70
+ default_settings['diff_params']['cache_device'] = str(cache_device)
71
+ default_settings['diff_params']['amp_dtype'] = str(amp_dtype)
72
+ default_settings['diff_params']['diff_batch_size'] = int(diff_batch_size)
73
+ default_settings['diff_params']['diff_lr'] = float(diff_lr)
74
+ default_settings['diff_params']['diff_interval_log'] = int(diff_interval_log)
75
+ default_settings['diff_params']['diff_interval_val'] = int(diff_interval_val)
76
+ default_settings['diff_params']['diff_force_save'] = int(diff_force_save)
77
+ with open(yaml_path, 'w') as y:
78
+ yaml.safe_dump(default_settings, y, default_flow_style=False, sort_keys=False)
79
+ return "成功保存默认配置"
80
+
81
+ def get_model_info(choice_ckpt):
82
+ pthfile = os.path.join(workdir, choice_ckpt)
83
+ net = torch.load(pthfile, map_location=torch.device('cpu')) #cpu load
84
+ spk_emb = net["model"].get("emb_g.weight")
85
+ if spk_emb is None:
86
+ return "所选模型缺少emb_g.weight,你可能选择了一个底模"
87
+ _dim, _layer = spk_emb.size()
88
+ model_type = {
89
+ 768: "Vec768-Layer12",
90
+ 256: "Vec256-Layer9 / HubertSoft",
91
+ 1024: "Whisper-PPG"
92
+ }
93
+ return model_type.get(_layer, "不受支持的模型")
94
+
95
+ def load_json_encoder(config_choice):
96
+ config_file = os.path.join(config_dir + config_choice)
97
+ with open(config_file, 'r') as f:
98
+ config = json.load(f)
99
+ try:
100
+ config_encoder = str(config["model"]["speech_encoder"])
101
+ return config_encoder
102
+ except Exception as e:
103
+ if "speech_encoder" in str(e):
104
+ return "你的配置文件似乎是未作兼容的旧版,请根据文档指示对你的配置文件进行修改"
105
+ else:
106
+ return f"出错了: {e}"
107
+
108
+ def load_model_func(ckpt_name,cluster_name,config_name,enhance,diff_model_name,diff_config_name,only_diffusion,encoder,using_device):
109
+ global model
110
+ config_path = os.path.join(config_dir, config_name)
111
+ diff_config_path = os.path.join(config_dir, diff_config_name) if diff_config_name != "no_diff_config" else "configs/diffusion.yaml"
112
+ with open(config_path, 'r') as f:
113
+ config = json.load(f)
114
+ spk_dict = config["spk"]
115
+ spk_name = config.get('spk', None)
116
+ spk_choice = next(iter(spk_name)) if spk_name else "未检测到音色"
117
+ ckpt_path = os.path.join(workdir, ckpt_name)
118
+ _, _suffix = os.path.splitext(cluster_name)
119
+ fr = True if _suffix == ".pkl" else False #如果是pkl后缀就启用特征检索
120
+ cluster_path = os.path.join(workdir, cluster_name)
121
+ diff_model_path = os.path.join(diff_workdir, diff_model_name)
122
+ shallow_diffusion = True if diff_model_name != "no_diff" else False
123
+ use_spk_mix = False
124
+ device = None if using_device == "Auto" else using_device
125
+ model = Svc(ckpt_path,
126
+ config_path,
127
+ device,
128
+ cluster_path,
129
+ enhance,
130
+ diff_model_path,
131
+ diff_config_path,
132
+ shallow_diffusion,
133
+ only_diffusion,
134
+ use_spk_mix,
135
+ fr)
136
+ spk_list = list(spk_dict.keys())
137
+ clip = 25 if encoder == "Whisper-PPG" else 0 #Whisper必须强制切片25秒
138
+ device_name = torch.cuda.get_device_properties(model.dev).name if "cuda" in str(model.dev) else str(model.dev)
139
+ index_or_kmeans = "特征索引" if fr is True else "聚类模型"
140
+ clu_load = "未加载" if cluster_name == "no_clu" else cluster_name
141
+ diff_load = "未加载" if diff_model_name == "no_diff" else diff_model_name
142
+ output_msg = f"模型被成功加载到了{device_name}上\n{index_or_kmeans}:{clu_load}\n扩散模型:{diff_load}"
143
+ return output_msg, gr.Dropdown.update(choices=spk_list, value=spk_choice), clip
144
+
145
+ def Newload_model_func(ckpt_name,cluster_name,config_name2,enhance2,diff_model_name2,diff_config_name2,only_diffusion2,encoder2,using_device2):
146
+ global model, loaded
147
+ config_name = config_name2.value
148
+ enhance = enhance2.value
149
+ diff_model_name = diff_model_name2.value
150
+ diff_config_name = (diff_config_name2).value
151
+ only_diffusion = (only_diffusion2).value
152
+ encoder = (encoder2).value
153
+ using_device = (using_device2).value
154
+ config_path = os.path.join(config_dir, config_name)
155
+ diff_config_path = os.path.join(config_dir, diff_config_name) if diff_config_name != "no_diff_config" else "configs/diffusion.yaml"
156
+ with open(config_path, 'r') as f:
157
+ config = json.load(f)
158
+ spk_dict = config["spk"]
159
+ spk_name = config.get('spk', None)
160
+ spk_choice = next(iter(spk_name)) if spk_name else "未检测到音色"
161
+ ckpt_path = os.path.join(workdir, ckpt_name)
162
+ _, _suffix = os.path.splitext(cluster_name)
163
+ fr = True if _suffix == ".pkl" else False #如果是pkl后缀就启用特征检索
164
+ cluster_path = os.path.join(workdir, cluster_name)
165
+ diff_model_path = os.path.join(diff_workdir, diff_model_name)
166
+ shallow_diffusion = True if diff_model_name != "no_diff" else False
167
+ use_spk_mix = False
168
+ device = None if using_device == "Auto" else using_device
169
+ model = Svc(ckpt_path,
170
+ config_path,
171
+ device,
172
+ cluster_path,
173
+ enhance,
174
+ diff_model_path,
175
+ diff_config_path,
176
+ shallow_diffusion,
177
+ only_diffusion,
178
+ use_spk_mix,
179
+ fr)
180
+ spk_list = list(spk_dict.keys())
181
+ clip = 25 if encoder == "Whisper-PPG" else 0 #Whisper必须强制切片25秒
182
+ device_name = torch.cuda.get_device_properties(model.dev).name if "cuda" in str(model.dev) else str(model.dev)
183
+ index_or_kmeans = "特征索引" if fr is True else "聚类模型"
184
+ clu_load = "未加载" if cluster_name == "no_clu" else cluster_name
185
+ diff_load = "未加载" if diff_model_name == "no_diff" else diff_model_name
186
+ loaded = cluster_name
187
+ #output_msg = f"模型被成功加载到了{device_name}上\n{index_or_kmeans}:{clu_load}\n扩散模型:{diff_load}"
188
+ #return output_msg, gr.Dropdown.update(choices=spk_list, value=spk_choice), clip
189
+
190
+ def get_file_options(directory, extension):
191
+ return [file for file in os.listdir(directory) if file.endswith(extension)]
192
+
193
+ def load_options():
194
+ ckpt_list = [file for file in get_file_options(workdir, ".pth") if not file.startswith("D_")]
195
+ config_list = get_file_options(config_dir, ".json")
196
+ cluster_list = ["no_clu"] + get_file_options(workdir, ".pt") + get_file_options(workdir, ".pkl") # 聚类和特征检索模型
197
+ diff_list = ["no_diff"] + get_file_options(diff_workdir, ".pt")
198
+ diff_config_list = get_file_options(config_dir, ".yaml")
199
+ return ckpt_list, config_list, cluster_list, diff_list, diff_config_list
200
+
201
+ def refresh_options():
202
+ ckpt_list, config_list, cluster_list, diff_list, diff_config_list = load_options()
203
+ return (
204
+ choice_ckpt.update(choices=ckpt_list),
205
+ config_choice.update(choices=config_list),
206
+ cluster_choice.update(choices=cluster_list),
207
+ diff_choice.update(choices=diff_list),
208
+ diff_config_choice.update(choices=diff_config_list)
209
+ )
210
+
211
+ def vc_infer(sid, input_audio, input_audio_path, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment):
212
+ if np.issubdtype(input_audio.dtype, np.integer):
213
+ input_audio = (input_audio / np.iinfo(input_audio.dtype).max).astype(np.float32)
214
+ if len(input_audio.shape) > 1:
215
+ input_audio = librosa.to_mono(input_audio.transpose(1, 0))
216
+ _audio = model.slice_inference(
217
+ input_audio_path,
218
+ sid,
219
+ vc_transform,
220
+ slice_db,
221
+ cluster_ratio,
222
+ auto_f0,
223
+ noise_scale,
224
+ pad_seconds,
225
+ cl_num,
226
+ lg_num,
227
+ lgr_num,
228
+ f0_predictor,
229
+ enhancer_adaptive_key,
230
+ cr_threshold,
231
+ k_step,
232
+ use_spk_mix,
233
+ second_encoding,
234
+ loudness_envelope_adjustment
235
+ )
236
+ model.clear_empty()
237
+ timestamp = str(int(time.time()))
238
+ if not os.path.exists("results"):
239
+ os.makedirs("results")
240
+ output_file_name = os.path.splitext(os.path.basename(input_audio_path))[0] + "_" + sid + "_" + timestamp + ".wav"
241
+ output_file_path = os.path.join("results", output_file_name)
242
+ sf.write(output_file_path, _audio, model.target_sample, format="wav")
243
+ return output_file_path
244
+
245
+ def vc_fn(sid, input_audio, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment):
246
+ global model
247
+ try:
248
+ if input_audio is None:
249
+ return "You need to upload an audio", None
250
+ if model is None:
251
+ return "You need to upload an model", None
252
+ sampling_rate, audio = input_audio
253
+ temp_path = "temp.wav"
254
+ sf.write(temp_path, audio, sampling_rate, format="wav")
255
+ output_file_path = vc_infer(sid, audio, temp_path, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment)
256
+ os.remove(temp_path)
257
+ return "Success", output_file_path
258
+ except Exception as e:
259
+ if debug: traceback.print_exc()
260
+ raise gr.Error(e)
261
+
262
+ def vc_batch_fn(sid, input_audio_files, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment):
263
+ global model
264
+ try:
265
+ if input_audio_files is None or len(input_audio_files) == 0:
266
+ return "You need to upload at least one audio file"
267
+ if model is None:
268
+ return "You need to upload a model"
269
+ for file_obj in input_audio_files:
270
+ input_audio_path = file_obj.name
271
+ audio, sampling_rate = sf.read(input_audio_path)
272
+ vc_infer(sid, audio, input_audio_path, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment)
273
+ return "批量推理完成,音频已经被保存到results文件夹"
274
+ except Exception as e:
275
+ if debug: traceback.print_exc()
276
+ raise gr.Error(e)
277
+
278
+ def tts_fn(_text, _speaker, sid, vc_transform, auto_f0,cluster_ratio, slice_db, noise_scale,pad_seconds,cl_num,lg_num,lgr_num,f0_predictor,enhancer_adaptive_key,cr_threshold, k_step,use_spk_mix,second_encoding,loudness_envelope_adjustment):
279
+ global model
280
+ try:
281
+ subprocess.run([r"python", "tts.py", _text, _speaker])
282
+ sr = 44100
283
+ y, sr = librosa.load("tts.wav")
284
+ resampled_y = librosa.resample(y, orig_sr=sr, target_sr=sr)
285
+ sf.write("tts.wav", resampled_y, sr, subtype = "PCM_16")
286
+ input_audio = "tts.wav"
287
+ audio, sampling_rate = sf.read(input_audio)
288
+ if model is None:
289
+ return "You need to upload a model", None
290
+ output_file_path = vc_infer(sid, audio, input_audio, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment)
291
+ return "Success", output_file_path
292
+ except Exception as e:
293
+ if debug: traceback.print_exc()
294
+ raise gr.Error(e)
295
+
296
+ def load_raw_dirs():
297
+ illegal_files = []
298
+ #检查文件名
299
+ allowed_pattern = re.compile(r'^[a-zA-Z0-9_@#$%^&()_+\-=\s\.]*$')
300
+ for root, dirs, files in os.walk(raw_path):
301
+ if root != raw_path: # 只处理子文件夹内的文件
302
+ for file in files:
303
+ file_name, _ = os.path.splitext(file)
304
+ if not allowed_pattern.match(file_name):
305
+ illegal_files.append(file)
306
+ if len(illegal_files)!=0:
307
+ return f"数据集文件名只能包含数字、字母、下划线,以下文件不符合要求,请改名后再试:{illegal_files}"
308
+ #检查有没有小可爱不用wav文件当数据集
309
+ for root, dirs, files in os.walk(raw_path):
310
+ if root != raw_path: # 只处理子文件夹内的文件
311
+ for file in files:
312
+ if not file.lower().endswith('.wav'):
313
+ illegal_files.append(file)
314
+ if len(illegal_files)!=0:
315
+ return f"以下文件为非wav格式文件,请删除后再试:{illegal_files}"
316
+ spk_dirs = []
317
+ with os.scandir(raw_path) as entries:
318
+ for entry in entries:
319
+ if entry.is_dir():
320
+ spk_dirs.append(entry.name)
321
+ if len(spk_dirs) != 0:
322
+ return raw_dirs_list.update(value=spk_dirs)
323
+ else:
324
+ return raw_dirs_list.update(value="未找到数据集,请检查dataset_raw文件夹")
325
+
326
+ def dataset_preprocess(encoder, f0_predictor, use_diff, vol_aug, skip_loudnorm, num_processes):
327
+ diff_arg = "--use_diff" if use_diff else ""
328
+ vol_aug_arg = "--vol_aug" if vol_aug else ""
329
+ skip_loudnorm_arg = "--skip_loudnorm" if skip_loudnorm else ""
330
+ preprocess_commands = [
331
+ r"python resample.py %s" % (skip_loudnorm_arg),
332
+ r"python preprocess_flist_config.py --speech_encoder %s %s" % (encoder, vol_aug_arg),
333
+ r"python preprocess_hubert_f0.py --num_processes %s --f0_predictor %s %s" % (num_processes ,f0_predictor, diff_arg)
334
+ ]
335
+ accumulated_output = ""
336
+ #清空dataset
337
+ dataset = os.listdir("dataset/44k")
338
+ if len(dataset) != 0:
339
+ for dir in dataset:
340
+ dataset_dir = "dataset/44k/" + str(dir)
341
+ if os.path.isdir(dataset_dir):
342
+ shutil.rmtree(dataset_dir)
343
+ accumulated_output += f"Deleting previous dataset: {dir}\n"
344
+ for command in preprocess_commands:
345
+ try:
346
+ result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, text=True)
347
+ accumulated_output += f"Command: {command}, Using Encoder: {encoder}, Using f0 Predictor: {f0_predictor}\n"
348
+ yield accumulated_output, None
349
+ progress_line = None
350
+ for line in result.stdout:
351
+ if r"it/s" in line or r"s/it" in line: #防止进度条刷屏
352
+ progress_line = line
353
+ else:
354
+ accumulated_output += line
355
+ if progress_line is None:
356
+ yield accumulated_output, None
357
+ else:
358
+ yield accumulated_output + progress_line, None
359
+ result.communicate()
360
+ except subprocess.CalledProcessError as e:
361
+ result = e.output
362
+ accumulated_output += f"Error: {result}\n"
363
+ yield accumulated_output, None
364
+ if progress_line is not None:
365
+ accumulated_output += progress_line
366
+ accumulated_output += '-' * 50 + '\n'
367
+ yield accumulated_output, None
368
+ config_path = "configs/config.json"
369
+ with open(config_path, 'r') as f:
370
+ config = json.load(f)
371
+ spk_name = config.get('spk', None)
372
+ yield accumulated_output, gr.Textbox.update(value=spk_name)
373
+
374
+ def regenerate_config(encoder, vol_aug):
375
+ vol_aug_arg = "--vol_aug" if vol_aug else ""
376
+ cmd = r"python preprocess_flist_config.py --speech_encoder %s %s" % (encoder, vol_aug_arg)
377
+ output = ""
378
+ try:
379
+ result = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, text=True)
380
+ for line in result.stdout:
381
+ output += line
382
+ output += "Regenerate config file successfully."
383
+ except subprocess.CalledProcessError as e:
384
+ result = e.output
385
+ output += f"Error: {result}\n"
386
+ return output
387
+
388
+ def clear_output():
389
+ return gr.Textbox.update(value="Cleared!>_<")
390
+
391
+ def read_config(config_path):
392
+ with open(config_path, 'r') as config_file:
393
+ config_data = json.load(config_file)
394
+ return config_data
395
+
396
+ def config_fn(log_interval, eval_interval, keep_ckpts, batch_size, lr, fp16_run, all_in_mem, diff_num_workers, diff_cache_all_data, diff_batch_size, diff_lr, diff_interval_log, diff_interval_val, diff_cache_device, diff_amp_dtype, diff_force_save):
397
+ config_origin = "configs/config.json"
398
+ diff_config = "configs/diffusion.yaml"
399
+ config_data = read_config(config_origin)
400
+ config_data['train']['log_interval'] = int(log_interval)
401
+ config_data['train']['eval_interval'] = int(eval_interval)
402
+ config_data['train']['keep_ckpts'] = int(keep_ckpts)
403
+ config_data['train']['batch_size'] = int(batch_size)
404
+ config_data['train']['learning_rate'] = float(lr)
405
+ config_data['train']['fp16_run'] = fp16_run
406
+ config_data['train']['all_in_mem'] = all_in_mem
407
+ with open(config_origin, 'w') as config_file:
408
+ json.dump(config_data, config_file, indent=4)
409
+ with open(diff_config, 'r') as diff_yaml:
410
+ diff_config_data = yaml.safe_load(diff_yaml)
411
+ diff_config_data['train']['num_workers'] = int(diff_num_workers)
412
+ diff_config_data['train']['cache_all_data'] = diff_cache_all_data
413
+ diff_config_data['train']['batch_size'] = int(diff_batch_size)
414
+ diff_config_data['train']['lr'] = float(diff_lr)
415
+ diff_config_data['train']['interval_log'] = int(diff_interval_log)
416
+ diff_config_data['train']['interval_val'] = int(diff_interval_val)
417
+ diff_config_data['train']['cache_device'] = str(diff_cache_device)
418
+ diff_config_data['train']['amp_dtype'] = str(diff_amp_dtype)
419
+ diff_config_data['train']['interval_force_save'] = int(diff_force_save)
420
+ with open(diff_config, 'w') as diff_yaml:
421
+ yaml.safe_dump(diff_config_data, diff_yaml, default_flow_style=False, sort_keys=False)
422
+ return "配置文件写入完成"
423
+
424
+ def check_dataset(dataset_path):
425
+ if not os.listdir(dataset_path):
426
+ return "数据集不存在,请检查dataset文件夹"
427
+ no_npy_pt_files = True
428
+ for root, dirs, files in os.walk(dataset_path):
429
+ for file in files:
430
+ if file.endswith('.npy') or file.endswith('.pt'):
431
+ no_npy_pt_files = False
432
+ break
433
+ if no_npy_pt_files:
434
+ return "数据集中未检测到f0和hubert文件,可能是预处理未完成"
435
+ return None
436
+
437
+ def training(gpu_selection, encoder):
438
+ config_data = read_config("configs/config.json")
439
+ vol_emb = config_data["model"]["vol_embedding"]
440
+ dataset_warn = check_dataset("dataset/44k")
441
+ if dataset_warn is not None:
442
+ return dataset_warn
443
+ encoder_models = { #编码器好多,要塞不下了
444
+ "vec256l9": ("D_0.pth", "G_0.pth", "pre_trained_model"),
445
+ "vec768l12": ("D_0.pth", "G_0.pth", "pre_trained_model/768l12/vol_emb" if vol_emb else "pre_trained_model/768l12"),
446
+ "hubertsoft": ("D_0.pth", "G_0.pth", "pre_trained_model/hubertsoft"),
447
+ "whisper-ppg": ("D_0.pth", "G_0.pth", "pre_trained_model/whisper-ppg"),
448
+ "cnhubertlarge": ("D_0.pth", "G_0.pth", "pre_trained_model/cnhubertlarge"),
449
+ "dphubert": ("D_0.pth", "G_0.pth", "pre_trained_model/dphubert"),
450
+ "whisper-ppg-large": ("D_0.pth", "G_0.pth", "pre_trained_model/whisper-ppg-large")
451
+ }
452
+ if encoder not in encoder_models:
453
+ return "未知编码器"
454
+ d_0_file, g_0_file, encoder_model_path = encoder_models[encoder]
455
+ d_0_path = os.path.join(encoder_model_path, d_0_file)
456
+ g_0_path = os.path.join(encoder_model_path, g_0_file)
457
+ timestamp = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M')
458
+ new_backup_folder = os.path.join(models_backup_path, str(timestamp))
459
+ if os.listdir(workdir) != ['diffusion']:
460
+ os.makedirs(new_backup_folder, exist_ok=True)
461
+ for file in os.listdir(workdir):
462
+ if file != "diffusion":
463
+ shutil.move(os.path.join(workdir, file), os.path.join(new_backup_folder, file))
464
+ shutil.copy(d_0_path, os.path.join(workdir, "D_0.pth"))
465
+ shutil.copy(g_0_path, os.path.join(workdir, "G_0.pth"))
466
+ cmd = r"set CUDA_VISIBLE_DEVICES=%s && python train.py -c configs/config.json -m 44k" % (gpu_selection)
467
+ subprocess.Popen(["cmd", "/c", "start", "cmd", "/k", cmd])
468
+ return "已经在新的终端窗口开始训练,请监看终端窗口的训练日志。在终端中按Ctrl+C可暂停训练。"
469
+
470
+ def continue_training(gpu_selection, encoder):
471
+ dataset_warn = check_dataset("dataset/44k")
472
+ if dataset_warn is not None:
473
+ return dataset_warn
474
+ if encoder == "":
475
+ return "请先选择预处理对应的编码器"
476
+ all_files = os.listdir(workdir)
477
+ model_files = [f for f in all_files if f.startswith('G_') and f.endswith('.pth')]
478
+ if len(model_files) == 0:
479
+ return "你还没有已开始的训练"
480
+ cmd = r"set CUDA_VISIBLE_DEVICES=%s && python train.py -c configs/config.json -m 44k" % (gpu_selection)
481
+ subprocess.Popen(["cmd", "/c", "start", "cmd", "/k", cmd])
482
+ return "已经在新的终端窗口开始训练,请监看终端窗口的训练日志。在终端中按Ctrl+C可暂停训练。"
483
+
484
+ def kmeans_training(kmeans_gpu):
485
+ if not os.listdir(r"dataset/44k"):
486
+ return "数据集不存在,请检查dataset文件夹"
487
+ cmd = r"python cluster/train_cluster.py --gpu" if kmeans_gpu else r"python cluster/train_cluster.py"
488
+ subprocess.Popen(["cmd", "/c", "start", "cmd", "/k", cmd])
489
+ return "已经在新的终端窗口开始训练,训练聚类模型不会输出日志,CPU训练一般需要5-10分钟左右"
490
+
491
+ def index_training():
492
+ if not os.listdir(r"dataset/44k"):
493
+ return "数据集不存在,请检查dataset文件夹"
494
+ cmd = r"python train_index.py -c configs/config.json"
495
+ subprocess.Popen(["cmd", "/c", "start", "cmd", "/k", cmd])
496
+ return "已经在新的终端窗口开始训练"
497
+
498
+ def diff_training(encoder):
499
+ if not os.listdir(r"dataset/44k"):
500
+ return "数据集不存在,请检查dataset文件夹"
501
+ pre_trained_model_768l12 = "pre_trained_model/diffusion/768l12/model_0.pt"
502
+ pre_trained_model_hubertsoft = "pre_trained_model/diffusion/hubertsoft/model_0.pt"
503
+ timestamp = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M')
504
+ new_backup_folder = os.path.join(models_backup_path, "diffusion", str(timestamp))
505
+ if len(os.listdir(diff_workdir)) != 0:
506
+ os.makedirs(new_backup_folder, exist_ok=True)
507
+ for file in os.listdir(diff_workdir):
508
+ shutil.move(os.path.join(diff_workdir, file), os.path.join(new_backup_folder, file))
509
+ if encoder == "vec256l9" or encoder == "whisper-ppg":
510
+ return "你所选的编码器暂时不支持训练扩散模型"
511
+ elif encoder == "vec768l12":
512
+ shutil.copy(pre_trained_model_768l12, os.path.join(diff_workdir, "model_0.pt"))
513
+ elif encoder == "hubertsoft":
514
+ shutil.copy(pre_trained_model_hubertsoft, os.path.join(diff_workdir, "model_0.pt"))
515
+ else:
516
+ return "请先选择编码器"
517
+ subprocess.Popen(["cmd", "/c", "start", "cmd", "/k", r"python train_diff.py -c configs/diffusion.yaml"])
518
+ return "已经在新的终端窗口开始训练,请监看终端窗口的训练日志。在终端中按Ctrl+C可暂停训练。"
519
+
520
+ def diff_continue_training(encoder):
521
+ if not os.listdir(r"dataset/44k"):
522
+ return "数据集不存在,请检查dataset文件夹"
523
+ if encoder == "":
524
+ return "请先选择预处理对应的编码器"
525
+ all_files = os.listdir(diff_workdir)
526
+ model_files = [f for f in all_files if f.endswith('.pt')]
527
+ if len(model_files) == 0:
528
+ return "你还没有已开始的训练"
529
+ subprocess.Popen(["cmd", "/c", "start", "cmd", "/k", r"python train_diff.py -c configs/diffusion.yaml"])
530
+ return "已经在新的终端窗口开始训练,请监看终端窗口的训练日志。在终端中按Ctrl+C可暂停训练。"
531
+
532
+ def upload_mix_append_file(files,sfiles):
533
+ try:
534
+ if(sfiles == None):
535
+ file_paths = [file.name for file in files]
536
+ else:
537
+ file_paths = [file.name for file in chain(files,sfiles)]
538
+ p = {file:100 for file in file_paths}
539
+ return file_paths,mix_model_output1.update(value=json.dumps(p,indent=2))
540
+ except Exception as e:
541
+ if debug: traceback.print_exc()
542
+ raise gr.Error(e)
543
+
544
+ def mix_submit_click(js,mode):
545
+ try:
546
+ assert js.lstrip()!=""
547
+ modes = {"凸组合":0, "线性组合":1}
548
+ mode = modes[mode]
549
+ data = json.loads(js)
550
+ data = list(data.items())
551
+ model_path,mix_rate = zip(*data)
552
+ path = mix_model(model_path,mix_rate,mode)
553
+ return f"成功,文件被保存在了{path}"
554
+ except Exception as e:
555
+ if debug: traceback.print_exc()
556
+ raise gr.Error(e)
557
+
558
+ def updata_mix_info(files):
559
+ try:
560
+ if files == None : return mix_model_output1.update(value="")
561
+ p = {file.name:100 for file in files}
562
+ return mix_model_output1.update(value=json.dumps(p,indent=2))
563
+ except Exception as e:
564
+ if debug: traceback.print_exc()
565
+ raise gr.Error(e)
566
+
567
+ def pth_identify():
568
+ if not os.path.exists(root_dir):
569
+ return f"未找到{root_dir}文件夹,请先创建一个{root_dir}文件夹并按第一步流程操作"
570
+ model_dirs = [d for d in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, d))]
571
+ if not model_dirs:
572
+ return f"未在{root_dir}文件夹中找到模型文件夹,请确保每个模型和配置文件都被放置在单独的文件夹中"
573
+ valid_model_dirs = []
574
+ for path in model_dirs:
575
+ pth_files = glob.glob(f"{root_dir}/{path}/*.pth")
576
+ json_files = glob.glob(f"{root_dir}/{path}/*.json")
577
+ if len(pth_files) != 1 or len(json_files) != 1:
578
+ return f"错误: 在{root_dir}/{path}中找到了{len(pth_files)}个.pth文件和{len(json_files)}个.json文件。应当确保每个文件夹内有且只有一个.pth文件和.json文件"
579
+ valid_model_dirs.append(path)
580
+
581
+ return f"成功识别了{len(valid_model_dirs)}个模型:{valid_model_dirs}"
582
+
583
+ def onnx_export():
584
+ model_dirs = [d for d in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, d))]
585
+ try:
586
+ for path in model_dirs:
587
+ pth_files = glob.glob(f"{root_dir}/{path}/*.pth")
588
+ json_files = glob.glob(f"{root_dir}/{path}/*.json")
589
+ model_file = pth_files[0]
590
+ json_file = json_files[0]
591
+ with open(json_file, 'r') as config_file:
592
+ config_data = json.load(config_file)
593
+ channels = config_data["model"]["gin_channels"]
594
+ if str(channels) == "256":
595
+ para1 = 1
596
+ if str(channels) == "768":
597
+ para1 = 192
598
+ device = torch.device("cpu")
599
+ hps = utils.get_hparams_from_file(json_file)
600
+ SVCVITS = SynthesizerTrn(
601
+ hps.data.filter_length // 2 + 1,
602
+ hps.train.segment_size // hps.data.hop_length,
603
+ **hps.model)
604
+ _ = utils.load_checkpoint(model_file, SVCVITS, None)
605
+ _ = SVCVITS.eval().to(device)
606
+ for i in SVCVITS.parameters():
607
+ i.requires_grad = False
608
+ n_frame = 10
609
+ test_hidden_unit = torch.rand(para1, n_frame, channels)
610
+ test_pitch = torch.rand(1, n_frame)
611
+ test_mel2ph = torch.arange(0, n_frame, dtype=torch.int64)[None] # torch.LongTensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]).unsqueeze(0)
612
+ test_uv = torch.ones(1, n_frame, dtype=torch.float32)
613
+ test_noise = torch.randn(1, 192, n_frame)
614
+ test_sid = torch.LongTensor([0])
615
+ input_names = ["c", "f0", "mel2ph", "uv", "noise", "sid"]
616
+ output_names = ["audio", ]
617
+ onnx_file = os.path.splitext(model_file)[0] + ".onnx"
618
+ torch.onnx.export(SVCVITS,
619
+ (
620
+ test_hidden_unit.to(device),
621
+ test_pitch.to(device),
622
+ test_mel2ph.to(device),
623
+ test_uv.to(device),
624
+ test_noise.to(device),
625
+ test_sid.to(device)
626
+ ),
627
+ onnx_file,
628
+ dynamic_axes={
629
+ "c": [0, 1],
630
+ "f0": [1],
631
+ "mel2ph": [1],
632
+ "uv": [1],
633
+ "noise": [2],
634
+ },
635
+ do_constant_folding=False,
636
+ opset_version=16,
637
+ verbose=False,
638
+ input_names=input_names,
639
+ output_names=output_names)
640
+ return "转换成功,模型被保存在了checkpoints下的对应目录"
641
+ except Exception as e:
642
+ if debug: traceback.print_exc()
643
+ return "转换错误:"+str(e)
644
+
645
+ def load_raw_audio(audio_path):
646
+ if not os.path.isdir(audio_path):
647
+ return "请输入正确的目录", None
648
+ files = os.listdir(audio_path)
649
+ wav_files = [file for file in files if file.lower().endswith('.wav')]
650
+ if not wav_files:
651
+ return "未在目录中找到.wav音频文件", None
652
+ return "成功加载", wav_files
653
+
654
+ def slicer_fn(input_dir, output_dir, process_method, max_sec, min_sec):
655
+ if output_dir == "":
656
+ return "请先选择输出的文件夹"
657
+ slicer = AutoSlicer()
658
+ if not os.path.exists(output_dir):
659
+ os.makedirs(output_dir)
660
+ for filename in os.listdir(input_dir):
661
+ if filename.lower().endswith(".wav"):
662
+ slicer.auto_slice(filename, input_dir, output_dir, max_sec)
663
+ if process_method == "丢弃":
664
+ for filename in os.listdir(output_dir):
665
+ if filename.endswith(".wav"):
666
+ filepath = os.path.join(output_dir, filename)
667
+ audio, sr = librosa.load(filepath, sr=None, mono=False)
668
+ if librosa.get_duration(y=audio, sr=sr) < min_sec:
669
+ os.remove(filepath)
670
+ elif process_method == "将过短音频整合为长音频":
671
+ slicer.merge_short(output_dir, max_sec, min_sec)
672
+ file_count, max_duration, min_duration, orig_duration, final_duration = slicer.slice_count(input_dir, output_dir)
673
+ hrs = int(final_duration / 3600)
674
+ mins = int((final_duration % 3600) / 60)
675
+ sec = format(float(final_duration % 60), '.2f')
676
+ rate = format(100 * (final_duration / orig_duration), '.2f')
677
+ return f"成功将音频切分为{file_count}条片段,其中最长{max_duration}秒,最短{min_duration}秒,切片后的音频总时长{hrs:02d}小时{mins:02d}分{sec}秒,为原始音频时长的{rate}%"
678
+
679
+ def model_compression(_model):
680
+ if _model == "":
681
+ return "请先选择要压缩的模型"
682
+ else:
683
+ model_path = os.path.join(workdir, _model)
684
+ filename, extension = os.path.splitext(_model)
685
+ output_model_name = f"{filename}_compressed{extension}"
686
+ output_path = os.path.join(workdir, output_model_name)
687
+ removeOptimizer(model_path, output_path)
688
+ return f"模型已成功被保存在了{output_path}"
689
+
690
+ # read ckpt list
691
+ ckpt_list, config_list, cluster_list, diff_list, diff_config_list = load_options()
692
+
693
+ #read GPU info
694
+ ngpu=torch.cuda.device_count()
695
+ gpu_infos=[]
696
+ if(torch.cuda.is_available()==False or ngpu==0):if_gpu_ok=False
697
+ else:
698
+ if_gpu_ok = False
699
+ for i in range(ngpu):
700
+ gpu_name=torch.cuda.get_device_name(i)
701
+ if("MX"in gpu_name):continue
702
+ if("10"in gpu_name or "16"in gpu_name or "20"in gpu_name or "30"in gpu_name or "40"in gpu_name or "A50"in gpu_name.upper() or "70"in gpu_name or "80"in gpu_name or "90"in gpu_name or "M4"in gpu_name or"P4"in gpu_name or "T4"in gpu_name or "TITAN"in gpu_name.upper()):#A10#A100#V100#A40#P40#M40#K80
703
+ if_gpu_ok=True#至少有一张能用的N卡
704
+ gpu_infos.append("%s\t%s"%(i,gpu_name))
705
+ gpu_info="\n".join(gpu_infos)if if_gpu_ok==True and len(gpu_infos)>0 else "很遗憾您这没有能用的显卡来支持您训练"
706
+ gpus="-".join([i[0]for i in gpu_infos])
707
+
708
+ #read default params
709
+ sovits_params, diff_params = get_default_settings()
710
+
711
+ app = gr.Blocks()
712
+
713
+ def Newget_model_info(choice_ckpt2):
714
+ choice_ckpt = str(choice_ckpt2)
715
+ pthfile = os.path.join(workdir, choice_ckpt)
716
+ net = torch.load(pthfile, map_location=torch.device('cpu')) #cpu load
717
+ spk_emb = net["model"].get("emb_g.weight")
718
+ if spk_emb is None:
719
+ return "所选模型缺��emb_g.weight,你可能选择了一个底模"
720
+ _dim, _layer = spk_emb.size()
721
+ model_type = {
722
+ 768: "Vec768-Layer12",
723
+ 256: "Vec256-Layer9 / HubertSoft",
724
+ 1024: "Whisper-PPG"
725
+ }
726
+ return gr.Textbox(visible=False, value=model_type.get(_layer, "不受支持的模型"))
727
+
728
+ with app:
729
+ gr.Markdown(value="""
730
+ ### So-VITS-SVC 4.1-Stable
731
+
732
+ 修改自原项目及bilibili@麦哲云
733
+
734
+ 仅供个人娱乐和非商业用途,禁止用于血腥、暴力、性相关、政治相关内容
735
+
736
+ weiui来自:bilibili@羽毛布団,交流③群:416656175
737
+
738
+ 镜像作者:bilibili@kiss丿冷鸟鸟,交流群:829974025
739
+
740
+ """)
741
+ with gr.Tabs():
742
+ with gr.TabItem("目白麦昆 (Mejiro Mcqueen)"):
743
+ #with gr.Row():
744
+ # choice_ckpt = gr.Dropdown(label="模型选择", choices=ckpt_list, value="no_model")
745
+ # model_branch = gr.Textbox(label="模型编码器", placeholder="请先选择模型", interactive=False)
746
+ #choice_ckpt = gr.Dropdown(value="G_90400.pth", visible=False)
747
+ #with gr.Row():
748
+ # config_choice = gr.Dropdown(label="配置文件", choices=config_list, value="no_config")
749
+ # config_info = gr.Textbox(label="配置文件编码器", placeholder="请选择配置文件")
750
+ config_choice = gr.Dropdown(value="config.json", visible=False)
751
+ #gr.Markdown(value="""**请检查模型和配置文件的编码器是否匹配**""")
752
+ #with gr.Row():
753
+ # diff_choice = gr.Dropdown(label="(可选)选择扩散模型", choices=diff_list, value="no_diff", interactive=True)
754
+ # diff_config_choice = gr.Dropdown(label="扩散模型配置文件", choices=diff_config_list, value="no_diff_config", interactive=True)
755
+ diff_choice = gr.Dropdown(value="no_diff", visible=False)
756
+ diff_config_choice = gr.Dropdown(value="no_diff_config", visible=False)
757
+ with gr.Row():
758
+ cluster_choice = gr.Dropdown(label="(可选)选择聚类模型/特征检索模型", choices=cluster_list, value="no_clu")
759
+ with gr.Row():
760
+ enhance = gr.Checkbox(label="是否使用NSF_HIFIGAN增强,该选项对部分训练集少的模型有一定的音质增强效果,但是对训练好的模型有反面效果,默认关闭", value=False)
761
+ #only_diffusion = gr.Checkbox(label="是否使用全扩散推理,开启后将不使用So-VITS模型,仅使用扩散模型进行完整扩散推理,默认关闭", value=False)
762
+ only_diffusion = gr.Checkbox(value=False, visible=False)
763
+ #using_device = gr.Dropdown(label="推理设备,默认为自动选择", choices=["Auto","cuda","cpu"], value="Auto")
764
+ using_device = gr.Dropdown(value='Auto', visible=False)
765
+ #refresh = gr.Button("刷新选项")
766
+ #loadckpt = gr.Button("加载模型", variant="primary")
767
+ #with gr.Row():
768
+ # model_message = gr.Textbox(label="Output Message")
769
+ # sid = gr.Dropdown(label="So-VITS说话人", value="speaker0")
770
+ sid = gr.Dropdown(value="1001", visible=False)
771
+
772
+ #choice_ckpt.change(get_model_info, [choice_ckpt], [model_branch])
773
+ model_branch = Newget_model_info("G_90400.pth")
774
+ #config_choice.change(load_json_encoder, [config_choice], [config_info])
775
+ #refresh.click(refresh_options,[],[choice_ckpt,config_choice,cluster_choice,diff_choice,diff_config_choice])
776
+
777
+ gr.Markdown(value="""
778
+ 请稍等片刻,模型加载大约需要10秒。后续操作不需要重新加载模型
779
+ """)
780
+ with gr.Tabs():
781
+ with gr.TabItem("单个音频上传"):
782
+ vc_input3 = gr.Audio(label="单个音频上传")
783
+ with gr.TabItem("批量音频上传"):
784
+ vc_batch_files = gr.Files(label="批量音频上传", file_types=["audio"], file_count="multiple")
785
+ with gr.TabItem("文字转语音(实验性)"):
786
+ gr.Markdown("""
787
+ 文字转语音(TTS)说明:使用edge_tts服务生成音频,并转换为So-VITS模型音色。可以在输入文字中使用标点符号简单控制情绪
788
+ zh-CN-XiaoyiNeural:中文女声
789
+ zh-CN-YunxiNeural: 中文男声
790
+ ja-JP-NanamiNeural:日文女声
791
+ ja-JP-KeitaNeural:日文男声
792
+ zh-CN-liaoning-XiaobeiNeural:东北话女声
793
+ zh-CN-shaanxi-XiaoniNeural: 陕西话女声
794
+ zh-HK-HiuMaanNeural: 粤语女声
795
+ zh-HK-WanLungNeural: 粤语男声
796
+ """)
797
+ with gr.Row():
798
+ text_input = gr.Textbox(label = "在此输入需要转译的文字(建议打开自动f0预测��",)
799
+ tts_spk = gr.Dropdown(label = "选择原始音频音色(来自微软TTS)", choices=["zh-CN-XiaoyiNeural", "zh-CN-YunxiNeural", "zh-CN-liaoning-XiaobeiNeural", "zh-CN-shaanxi-XiaoniNeural", "zh-HK-HiuMaanNeural", "zh-HK-WanLungNeural", "ja-JP-NanamiNeural", "ja-JP-KeitaNeural"], value = "zh-CN-XiaoyiNeural")
800
+ #with gr.Row():
801
+ # tts_rate = gr.Slider(label = "TTS语音变速(倍速)", minimum = 0, maximum = 3, value = 1)
802
+ # tts_volume = gr.Slider(label = "TTS语音音量(相对值)", minimum = 0, maximum = 1.5, value = 1)
803
+
804
+ with gr.Row():
805
+ auto_f0 = gr.Checkbox(label="自动f0预测,配合聚类模型f0预测效果更好,会导致变调功能失效(仅限转换语音,歌声不要勾选此项会跑调)", value=False)
806
+ f0_predictor = gr.Radio(label="f0预测器选择(如遇哑音可以更换f0预测器解决,crepe为原F0使用均值滤波器)", choices=["pm","crepe","harvest","dio"], value="pm")
807
+ cr_threshold = gr.Number(label="F0过滤阈值,只有使用crepe时有效. 数值范围从0-1. 降低该值可减少跑调概率,但会增加哑音", value=0.05)
808
+ with gr.Row():
809
+ vc_transform = gr.Number(label="变调(整数,可以正负,半音数量,升高八度就是12)", value=0)
810
+ cluster_ratio = gr.Number(label="聚类模型/特征检索混合比例,0-1之间,默认为0不启用聚类或特征检索,能提升音色相似度,但会导致咬字下降", value=0)
811
+ k_step = gr.Slider(label="浅扩散步数,只有使用了扩散模型才有效,步数越大越接近扩散模型的结果", value=100, minimum = 1, maximum = 1000)
812
+ with gr.Row():
813
+ enhancer_adaptive_key = gr.Number(label="使NSF-HIFIGAN增强器适应更高的音域(单位为半音数)|默认为0", value=0,interactive=True)
814
+ slice_db = gr.Number(label="切片阈值", value=-50)
815
+ cl_num = gr.Number(label="音频自动切片,0为按默认方式切片,单位为秒/s,爆显存可以设置此处强制切片", value=0)
816
+ with gr.Accordion("高级设置(一般不需要动)", open=False):
817
+ noise_scale = gr.Number(label="noise_scale 建议不要动,会影响音质,玄学参数", value=0.4)
818
+ pad_seconds = gr.Number(label="推理音频pad秒数,由于未知原因开头结尾会有异响,pad一小段静音段后就不会出现", value=0.5)
819
+ lg_num = gr.Number(label="两端音频切片的交叉淡入长度,如果自动切片后出现人声不连贯可调整该数值,如果连贯建议采用默认值0,注意,该设置会影响推理速度,单位为秒/s", value=1)
820
+ lgr_num = gr.Number(label="自动音频切片后,需要舍弃每段切片的头尾。该参数设置交叉长度保留的比例,范围0-1,左开右闭", value=0.75,interactive=True)
821
+ second_encoding = gr.Checkbox(label = "二次编码,浅扩散前会对原始音频进行二次编码,玄学选项,效果时好时差,默认关闭", value=False)
822
+ loudness_envelope_adjustment = gr.Number(label="输入源响度包络替换输出响度包络融合比例,越靠近1越使用输出响度包络", value = 0)
823
+ use_spk_mix = gr.Checkbox(label="动态声线融合,暂时没做完", value=False, interactive=False)
824
+ with gr.Row():
825
+ vc_submit = gr.Button("音频转换", variant="primary")
826
+ vc_batch_submit = gr.Button("批量转换", variant="primary")
827
+ vc_tts_submit = gr.Button("文本转语音", variant="primary")
828
+ vc_output1 = gr.Textbox(label="Output Message")
829
+ vc_output2 = gr.Audio(label="Output Audio")
830
+
831
+ def Newvc_fn(sid, input_audio, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment, clus2):
832
+ global model, loaded
833
+ if loaded != clus2:
834
+ Newload_model_func("G_90400.pth",clus2,config_choice,enhance,diff_choice,diff_config_choice,only_diffusion,model_branch,using_device)
835
+ loaded = clus2
836
+ try:
837
+ if input_audio is None:
838
+ return "You need to upload an audio", None
839
+ if model is None:
840
+ return "You need to upload an model", None
841
+ sampling_rate, audio = input_audio
842
+ temp_path = "temp.wav"
843
+ sf.write(temp_path, audio, sampling_rate, format="wav")
844
+ output_file_path = vc_infer(sid, audio, temp_path, vc_transform, auto_f0, cluster_ratio, slice_db, noise_scale, pad_seconds, cl_num, lg_num, lgr_num, f0_predictor, enhancer_adaptive_key, cr_threshold, k_step, use_spk_mix, second_encoding, loudness_envelope_adjustment)
845
+ os.remove(temp_path)
846
+ return "Success", output_file_path
847
+ except Exception as e:
848
+ if debug: traceback.print_exc()
849
+ raise gr.Error(e)
850
+
851
+ #loadckpt.click(load_model_func,[choice_ckpt,cluster_choice,config_choice,enhance,diff_choice,diff_config_choice,only_diffusion,model_branch,using_device],[model_message, sid, cl_num])
852
+ vc_submit.click(Newvc_fn, [sid, vc_input3, vc_transform,auto_f0,cluster_ratio, slice_db, noise_scale,pad_seconds,cl_num,lg_num,lgr_num,f0_predictor,enhancer_adaptive_key,cr_threshold,k_step,use_spk_mix,second_encoding,loudness_envelope_adjustment,cluster_choice], [vc_output1, vc_output2])
853
+ vc_batch_submit.click(vc_batch_fn, [sid, vc_batch_files, vc_transform,auto_f0,cluster_ratio, slice_db, noise_scale,pad_seconds,cl_num,lg_num,lgr_num,f0_predictor,enhancer_adaptive_key,cr_threshold,k_step,use_spk_mix,second_encoding,loudness_envelope_adjustment], [vc_output1])
854
+ vc_tts_submit.click(tts_fn, [text_input, tts_spk, sid, vc_transform,auto_f0,cluster_ratio, slice_db, noise_scale,pad_seconds,cl_num,lg_num,lgr_num,f0_predictor,enhancer_adaptive_key,cr_threshold,k_step,use_spk_mix,second_encoding,loudness_envelope_adjustment], [vc_output1, vc_output2])
855
+ '''
856
+ with gr.TabItem("训练"):
857
+ gr.Markdown(value="""请将数据集文件夹放置在dataset_raw文件夹下,确认放置正确后点击下方获取数据集名称""")
858
+ raw_dirs_list=gr.Textbox(label="Raw dataset directory(s):")
859
+ get_raw_dirs=gr.Button("识别数据集", variant="primary")
860
+ gr.Markdown(value="""确认数据集正确识别后请选择训练使用的特征编码器和f0预测器,**如果要训练扩散模型,请选择Vec768l12或hubertsoft,并确保So-VITS和扩散模型使用同一个编码器**""")
861
+ with gr.Row():
862
+ gr.Markdown(value="""**vec256l9**: ContentVec(256Layer9),旧版本叫v1,So-VITS-SVC 4.0的基础版本,**暂不支持扩散模型**
863
+ **vec768l12**: 特征输入更换为ContentVec的第12层Transformer输出,模型理论上会更加还原训练集音色
864
+ **hubertsoft**: So-VITS-SVC 3.0使用的编码器,咬字更为准确,但可能存在多说话人音色泄露问题
865
+ **whisper-ppg**: 来自OpenAI,咬字最为准确,但和Hubertsoft一样存在多说话人音色泄露,且显存占用和训练时间有明显增加。**暂不支持扩散模型**
866
+ """)
867
+ gr.Markdown(value="""**crepe**: 抗噪能力最强,但预处理速度慢(不过如果你的显卡很强的话速度会很快)
868
+ **pm**: 预处理速度快,但抗噪能力较弱
869
+ **dio**: 先前版本预处理默认使用的f0预测器
870
+ **harvest**: 有一定抗噪能力,预处理显存占用友好,速度比较慢
871
+ """)
872
+ with gr.Row():
873
+ branch_selection = gr.Radio(label="选择训练使用的编码器", choices=["vec256l9","vec768l12","hubertsoft","whisper-ppg"], value="vec768l12", interactive=True)
874
+ f0_predictor_selection = gr.Radio(label="选择训练使用的f0预测器", choices=["crepe","pm","dio","harvest"], value="crepe", interactive=True)
875
+ use_diff = gr.Checkbox(label="是否使用浅扩散模型,如要训练浅扩散模型请勾选此项", value=True)
876
+ vol_aug=gr.Checkbox(label="是否启用响度嵌入和音量增强,启用后可以根据输入源控制输出响度,但对数据集质量的要求更高。**仅支持vec768l12编码器**", value=False)
877
+ with gr.Row():
878
+ skip_loudnorm = gr.Checkbox(label="是否跳过响度匹配,如果你已经用音频处理软件做过响度匹配,请勾选此处")
879
+ num_processes = gr.Slider(label="预处理使用的CPU线程数,可以大幅加快预处理速度,但线程数过大容易爆显存,建议12G显存设置为2", minimum=1, maximum=multiprocessing.cpu_count(), value=1, step=1)
880
+ with gr.Row():
881
+ raw_preprocess=gr.Button("数据预处理", variant="primary")
882
+ regenerate_config_btn=gr.Button("重新生成配置文件", variant="primary")
883
+ preprocess_output=gr.Textbox(label="预处理输出信息,完成后请检查一下是否有报错信息,如无则可以进行下一步", max_lines=999)
884
+ clear_preprocess_output=gr.Button("清空输出信息")
885
+ with gr.Group():
886
+ gr.Markdown(value="""填写训练设置和超参数""")
887
+ with gr.Row():
888
+ gr.Textbox(label="当前使用显卡信息", value=gpu_info)
889
+ gpu_selection=gr.Textbox(label="多卡用户请指定希望训练使用的显卡ID(0,1,2...)", value=gpus, interactive=True)
890
+ with gr.Row():
891
+ log_interval=gr.Textbox(label="每隔多少步(steps)生成一次评估日志", value=sovits_params['log_interval'])
892
+ eval_interval=gr.Textbox(label="每隔多少步(steps)验证并保存一次模型", value=sovits_params['eval_interval'])
893
+ keep_ckpts=gr.Textbox(label="仅保留最新的X个模型,超出该数字的旧模型会被删除。设置为0则永不删除", value=sovits_params['keep_ckpts'])
894
+ with gr.Row():
895
+ batch_size=gr.Textbox(label="批量大小,每步取多少条数据进行训练,大batch有助于训练但显著增加显存占用。6G显存建议设定为4", value=sovits_params['batch_size'])
896
+ lr=gr.Textbox(label="学习率,一般不用动,批量大小较大时可以适当增大学习率,但强烈不建议超过0.0002,有炸炉风险", value=sovits_params['learning_rate'])
897
+ fp16_run=gr.Checkbox(label="是否使用fp16混合精度训练,fp16训练可能降低显存占用和训练时间,但对模型质量的影响尚未查证", value=sovits_params['fp16_run'])
898
+ all_in_mem=gr.Checkbox(label="是否加载所有数据集到内存中,硬盘IO过于低下、同时内存容量远大于数据集体积时可以启用,能显著加快训练速度", value=sovits_params['all_in_mem'])
899
+ with gr.Row():
900
+ gr.Markdown("请检查右侧的说话人列表是否和你要训练的目标说话人一致,确认无误后点击写入配置文件,然后就可以开始训练了")
901
+ speakers=gr.Textbox(label="说话人列表")
902
+ with gr.Accordion(label = "扩散模型配置(训练扩散模型需要写入此处)", open=True):
903
+ with gr.Row():
904
+ diff_num_workers = gr.Number(label="num_workers, 如果你的电脑配置较高,可以将这里设置为0加快训练速度", value=diff_params['num_workers'])
905
+ diff_cache_all_data = gr.Checkbox(label="是否缓存数据,启用后可以加快训练速度,关闭后可以节省显存或内存,但会减慢训练速度", value=diff_params['cache_all_data'])
906
+ diff_cache_device = gr.Radio(label="若启用缓存数据,使用显存(cuda)还是内存(cpu)缓存,如果显卡显存充足,选择cuda以加快训练速度", choices=["cuda","cpu"], value=diff_params['cache_device'])
907
+ diff_amp_dtype = gr.Radio(label="训练数据类型,fp16可能会有更快的训练速度,前提是你的显卡支持", choices=["fp32","fp16"], value=diff_params['amp_dtype'])
908
+ with gr.Row():
909
+ diff_batch_size = gr.Number(label="批量大小(batch_size),根据显卡显存设置,小显存适当降低该项,6G显存可以设定为48,但该数值不要超过数据集总数量的1/4", value=diff_params['diff_batch_size'])
910
+ diff_lr = gr.Number(label="学习率(一般不需要动)", value=diff_params['diff_lr'])
911
+ diff_interval_log = gr.Number(label="每隔多少步(steps)生成一次评估日志", value = diff_params['diff_interval_log'])
912
+ diff_interval_val = gr.Number(label="每隔多少步(steps)验证并保存一次模型,如果你的批量大小较大,可以适当减少这里的数字,但不建议设置为1000以下", value=diff_params['diff_interval_val'])
913
+ diff_force_save = gr.Number(label="每隔多少步强制保留模型,只有该步数的倍数保存的模型会被保留,其余会被删除。设置为与验证步数相同的值则每个模型都会被保留", value=diff_params['diff_force_save'])
914
+ with gr.Row():
915
+ save_params=gr.Button("将当前设置保存为默认设置", variant="primary")
916
+ write_config=gr.Button("写入配置文件", variant="primary")
917
+ write_config_output=gr.Textbox(label="输出信息")
918
+
919
+ gr.Markdown(value="""**点击从头开始训练**将会自动将已有的训练进度保存到models_backup文件夹,并自动装载预训练模型。
920
+ **继续上一次的训练进度**将从上一个保存模型的进度继续训练。继续训练进度无需重新预处理和写入配置文件。
921
+ 关于扩散、聚类和特征检索的详细说明请看[此处](https://www.yuque.com/umoubuton/ueupp5/kmui02dszo5zrqkz)。
922
+ """)
923
+ with gr.Row():
924
+ with gr.Column():
925
+ start_training=gr.Button("从头开始训练", variant="primary")
926
+ training_output=gr.Textbox(label="训练输出信息")
927
+ with gr.Column():
928
+ continue_training_btn=gr.Button("继续上一次的训练进度", variant="primary")
929
+ continue_training_output=gr.Textbox(label="训练输出信息")
930
+ with gr.Row():
931
+ with gr.Column():
932
+ diff_training_btn=gr.Button("从头训练扩散模型", variant="primary")
933
+ diff_training_output=gr.Textbox(label="训练输出信息")
934
+ with gr.Column():
935
+ diff_continue_training_btn=gr.Button("继续训练扩散模型", variant="primary")
936
+ diff_continue_training_output=gr.Textbox(label="训练输出信息")
937
+ with gr.Accordion(label = "聚类、特征检索训练", open=False):
938
+ with gr.Row():
939
+ with gr.Column():
940
+ kmeans_button=gr.Button("训练聚类模型", variant="primary")
941
+ kmeans_gpu = gr.Checkbox(label="使用GPU训练", value=True)
942
+ kmeans_output=gr.Textbox(label="训练输出信息")
943
+ with gr.Column():
944
+ index_button=gr.Button("训练特征检索模型", variant="primary")
945
+ index_output=gr.Textbox(label="训练输出信息")
946
+ '''
947
+ with gr.TabItem("小工具/实验室特性"):
948
+ gr.Markdown(value="""
949
+ ### So-vits-svc 4.1 小工具/实验室特性
950
+ 提供了一些有趣或实用的小工具,可以自行探索
951
+ """)
952
+ with gr.Tabs():
953
+ with gr.TabItem("静态声线融合"):
954
+ gr.Markdown(value="""
955
+ <font size=2> 介绍:该功能可以将多个声音模型合成为一个声音模型(多个模型参数的凸组合或线性组合),从而制造出现实中不存在的声线
956
+ 注意:
957
+ 1.该功能仅支持单说话人的模型
958
+ 2.如果强行使用多说话人模型,需要保证多个模型的说话人数量相同,这样可以混合同一个SpaekerID下的声音
959
+ 3.保证所有待混合模型的config.json中的model字段是相同的
960
+ 4.输出的混合模型可以使用待合成模型的任意一个config.json,但聚类模型将不能使用
961
+ 5.批量上传模型的时候最好把模型放到一个文件夹选中后一起上传
962
+ 6.混合比例调整建议大小在0-100之间,也可以调为其他数字,但在线性组合模式下会出现未知的效果
963
+ 7.混合完毕后,文件将会保存在项目根目录中,文件名为output.pth
964
+ 8.凸组合模式会将混合比例执行Softmax使混合比例相加为1,而线性组合模式不会
965
+ </font>
966
+ """)
967
+ mix_model_path = gr.Files(label="选择需要混合模型文件")
968
+ mix_model_upload_button = gr.UploadButton("选择/追加需要混合模型文件", file_count="multiple")
969
+ mix_model_output1 = gr.Textbox(
970
+ label="混合比例调整,单位/%",
971
+ interactive = True
972
+ )
973
+ mix_mode = gr.Radio(choices=["凸组合", "线性组合"], label="融合模式",value="凸组合",interactive = True)
974
+ mix_submit = gr.Button("声线融合启动", variant="primary")
975
+ mix_model_output2 = gr.Textbox(
976
+ label="Output Message"
977
+ )
978
+ with gr.TabItem("onnx转换"):
979
+ gr.Markdown(value="""
980
+ 提供了将.pth模型(批量)转换为.onnx模型的功能
981
+ 源项目本身自带转换的功能,但不支持批量,操作也不够简单,这个工具可以支持在WebUI中以可视化的操作方式批量转换.onnx模型
982
+ 有人可能会问,转.onnx模型有什么作用呢?相信我,如果你问出了这个问题,说明这个工具你应该用不上
983
+
984
+ ### Step 1:
985
+ 在整合包根目录下新建一个"checkpoints"文件夹,将pth模型和对应的json配置文件按目录分别放置到checkpoints文件夹下
986
+ 看起来应该像这样:
987
+ checkpoints
988
+ ├───xxxx
989
+ │ ├───xxxx.pth
990
+ │ └───xxxx.json
991
+ ├───xxxx
992
+ │ ├───xxxx.pth
993
+ │ └───xxxx.json
994
+ └───……
995
+ """)
996
+ pth_dir_msg = gr.Textbox(label="识别待转换模型", placeholder="请将模型和配置文件按上述说明放置在正确位置")
997
+ pth_dir_identify_btn = gr.Button("识别", variant="primary")
998
+ gr.Markdown(value="""
999
+ ### Step 2:
1000
+ 识别正确后点击下方开始转换,转换一个模型可能需要一分钟甚至更久
1001
+ """)
1002
+ pth2onnx_btn = gr.Button("开始转换", variant="primary")
1003
+ pth2onnx_msg = gr.Textbox(label="输出信息")
1004
+
1005
+ with gr.TabItem("智能音频切片"):
1006
+ gr.Markdown(value="""
1007
+ 该工具可以实现对音频的切片,无需调整参数即可完成符合要求的数据集制作。
1008
+ 数据集要求的音频切片约在2-15秒内,用传统的Slicer-GUI切片工具需要精准调参和二次切片才能符合要求,该工具省去了上述繁琐的操作,只要上传原始音频即可一键制作数据集。
1009
+ """)
1010
+ with gr.Row():
1011
+ raw_audio_path = gr.Textbox(label="原始音频文件夹", placeholder="包含所有待切片音频的文件夹,示例: D:\干声\speakers")
1012
+ load_raw_audio_btn = gr.Button("加载原始音频", variant = "primary")
1013
+ load_raw_audio_output = gr.Textbox(label = "输出信息")
1014
+ raw_audio_dataset = gr.Textbox(label = "音频列表", value = "")
1015
+ slicer_output_dir = gr.Textbox(label = "输出目录", placeholder = "选择输出目录")
1016
+ with gr.Row():
1017
+ process_method = gr.Radio(label = "对过短音频的处理方式", choices = ["丢弃","将过短音频整合为长音频"], value = "丢弃")
1018
+ max_sec = gr.Number(label = "切片的最长秒数", value = 15)
1019
+ min_sec = gr.Number(label = "切片的最短秒数", value = 2)
1020
+ slicer_btn = gr.Button("开始切片", variant = "primary")
1021
+ slicer_output_msg = gr.Textbox(label = "输出信息")
1022
+
1023
+ mix_model_path.change(updata_mix_info,[mix_model_path],[mix_model_output1])
1024
+ mix_model_upload_button.upload(upload_mix_append_file, [mix_model_upload_button,mix_model_path], [mix_model_path,mix_model_output1])
1025
+ mix_submit.click(mix_submit_click, [mix_model_output1,mix_mode], [mix_model_output2])
1026
+ pth_dir_identify_btn.click(pth_identify, [], [pth_dir_msg])
1027
+ pth2onnx_btn.click(onnx_export, [], [pth2onnx_msg])
1028
+ load_raw_audio_btn.click(load_raw_audio, [raw_audio_path], [load_raw_audio_output, raw_audio_dataset])
1029
+ slicer_btn.click(slicer_fn, [raw_audio_path, slicer_output_dir, process_method, max_sec, min_sec], [slicer_output_msg])
1030
+
1031
+ with gr.TabItem("模型压缩工具"):
1032
+ gr.Markdown(value="""
1033
+ 该工具可以实现对模型的体积压缩,在**不影响模型推理功能**的情况下,将原本约600M的So-VITS模型压缩至约200M, 大大减少了硬盘的压力。
1034
+ **注意:压缩后的模型将无法继续训练,请在确认封炉后再压缩。**
1035
+ 将模型文件放置在logs/44k下,然后选择需要压缩的模型
1036
+ """)
1037
+ model_to_compress = gr.Dropdown(label="模型选择", choices=ckpt_list, value="")
1038
+ compress_model_btn = gr.Button("压缩模型", variant="primary")
1039
+ compress_model_output = gr.Textbox(label="输出信息", value="")
1040
+
1041
+ compress_model_btn.click(model_compression, [model_to_compress], [compress_model_output])
1042
+ """
1043
+ get_raw_dirs.click(load_raw_dirs,[],[raw_dirs_list])
1044
+ raw_preprocess.click(dataset_preprocess,[branch_selection, f0_predictor_selection, use_diff, vol_aug, skip_loudnorm, num_processes],[preprocess_output, speakers])
1045
+ regenerate_config_btn.click(regenerate_config,[branch_selection, vol_aug],[preprocess_output])
1046
+ clear_preprocess_output.click(clear_output,[],[preprocess_output])
1047
+ save_params.click(save_default_settings, [log_interval,eval_interval,keep_ckpts,batch_size,lr,fp16_run,all_in_mem,diff_num_workers,diff_cache_all_data,diff_cache_device,diff_amp_dtype,diff_batch_size,diff_lr,diff_interval_log,diff_interval_val,diff_force_save], [write_config_output])
1048
+ write_config.click(config_fn,[log_interval, eval_interval, keep_ckpts, batch_size, lr, fp16_run, all_in_mem, diff_num_workers, diff_cache_all_data, diff_batch_size, diff_lr, diff_interval_log, diff_interval_val, diff_cache_device, diff_amp_dtype, diff_force_save],[write_config_output])
1049
+ start_training.click(training,[gpu_selection, branch_selection],[training_output])
1050
+ diff_training_btn.click(diff_training,[branch_selection],[diff_training_output])
1051
+ continue_training_btn.click(continue_training,[gpu_selection, branch_selection],[continue_training_output])
1052
+ diff_continue_training_btn.click(diff_continue_training,[branch_selection],[diff_continue_training_output])
1053
+ kmeans_button.click(kmeans_training,[kmeans_gpu],[kmeans_output])
1054
+ index_button.click(index_training, [], [index_output])
1055
+ """
1056
+ with gr.Tabs():
1057
+ with gr.Row(variant="panel"):
1058
+ with gr.Column():
1059
+ gr.Markdown(value="""
1060
+ <font size=2> WebUI设置</font>
1061
+ """)
1062
+ debug_button = gr.Checkbox(label="Debug模式,反馈BUG需要打开,打开后控制台可以显示具体错误提示", value=debug)
1063
+
1064
+ debug_button.change(debug_change,[],[])
1065
+
1066
+ app.queue(concurrency_count=1022, max_size=2044).launch()
auto_slicer.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import numpy as np
3
+ import librosa
4
+ import soundfile as sf
5
+ from modules.slicer2 import Slicer
6
+
7
+ class AutoSlicer:
8
+ def __init__(self):
9
+ self.slicer_params = {
10
+ "threshold": -40,
11
+ "min_length": 5000,
12
+ "min_interval": 300,
13
+ "hop_size": 10,
14
+ "max_sil_kept": 500,
15
+ }
16
+ self.original_min_interval = self.slicer_params["min_interval"]
17
+
18
+ def auto_slice(self, filename, input_dir, output_dir, max_sec):
19
+ audio, sr = librosa.load(os.path.join(input_dir, filename), sr=None, mono=False)
20
+ slicer = Slicer(sr=sr, **self.slicer_params)
21
+ chunks = slicer.slice(audio)
22
+ files_to_delete = []
23
+ for i, chunk in enumerate(chunks):
24
+ if len(chunk.shape) > 1:
25
+ chunk = chunk.T
26
+ output_filename = f"{os.path.splitext(filename)[0]}_{i}"
27
+ output_filename = "".join(c for c in output_filename if c.isascii() or c == "_") + ".wav"
28
+ output_filepath = os.path.join(output_dir, output_filename)
29
+ sf.write(output_filepath, chunk, sr)
30
+ #Check and re-slice audio that more than max_sec.
31
+ while True:
32
+ new_audio, sr = librosa.load(output_filepath, sr=None, mono=False)
33
+ if librosa.get_duration(y=new_audio, sr=sr) <= max_sec:
34
+ break
35
+ self.slicer_params["min_interval"] = self.slicer_params["min_interval"] // 2
36
+ if self.slicer_params["min_interval"] >= self.slicer_params["hop_size"]:
37
+ new_chunks = Slicer(sr=sr, **self.slicer_params).slice(new_audio)
38
+ for j, new_chunk in enumerate(new_chunks):
39
+ if len(new_chunk.shape) > 1:
40
+ new_chunk = new_chunk.T
41
+ new_output_filename = f"{os.path.splitext(output_filename)[0]}_{j}.wav"
42
+ sf.write(os.path.join(output_dir, new_output_filename), new_chunk, sr)
43
+ files_to_delete.append(output_filepath)
44
+ else:
45
+ break
46
+ self.slicer_params["min_interval"] = self.original_min_interval
47
+ for file_path in files_to_delete:
48
+ if os.path.exists(file_path):
49
+ os.remove(file_path)
50
+
51
+ def merge_short(self, output_dir, max_sec, min_sec):
52
+ short_files = []
53
+ for filename in os.listdir(output_dir):
54
+ filepath = os.path.join(output_dir, filename)
55
+ if filename.endswith(".wav"):
56
+ audio, sr = librosa.load(filepath, sr=None, mono=False)
57
+ duration = librosa.get_duration(y=audio, sr=sr)
58
+ if duration < min_sec:
59
+ short_files.append((filepath, audio, duration))
60
+ short_files.sort(key=lambda x: x[2], reverse=True)
61
+ merged_audio = []
62
+ current_duration = 0
63
+ for filepath, audio, duration in short_files:
64
+ if current_duration + duration <= max_sec:
65
+ merged_audio.append(audio)
66
+ current_duration += duration
67
+ os.remove(filepath)
68
+ else:
69
+ if merged_audio:
70
+ output_audio = np.concatenate(merged_audio, axis=-1)
71
+ if len(output_audio.shape) > 1:
72
+ output_audio = output_audio.T
73
+ output_filename = f"merged_{len(os.listdir(output_dir))}.wav"
74
+ sf.write(os.path.join(output_dir, output_filename), output_audio, sr)
75
+ merged_audio = [audio]
76
+ current_duration = duration
77
+ os.remove(filepath)
78
+ if merged_audio and current_duration >= min_sec:
79
+ output_audio = np.concatenate(merged_audio, axis=-1)
80
+ if len(output_audio.shape) > 1:
81
+ output_audio = output_audio.T
82
+ output_filename = f"merged_{len(os.listdir(output_dir))}.wav"
83
+ sf.write(os.path.join(output_dir, output_filename), output_audio, sr)
84
+
85
+ def slice_count(self, input_dir, output_dir):
86
+ orig_duration = final_duration = 0
87
+ for file in os.listdir(input_dir):
88
+ if file.endswith(".wav"):
89
+ _audio, _sr = librosa.load(os.path.join(input_dir, file), sr=None, mono=False)
90
+ orig_duration += librosa.get_duration(y=_audio, sr=_sr)
91
+ wav_files = [file for file in os.listdir(output_dir) if file.endswith(".wav")]
92
+ num_files = len(wav_files)
93
+ max_duration = -1
94
+ min_duration = float("inf")
95
+ for file in wav_files:
96
+ file_path = os.path.join(output_dir, file)
97
+ audio, sr = librosa.load(file_path, sr=None, mono=False)
98
+ duration = librosa.get_duration(y=audio, sr=sr)
99
+ final_duration += float(duration)
100
+ if duration > max_duration:
101
+ max_duration = float(duration)
102
+ if duration < min_duration:
103
+ min_duration = float(duration)
104
+ return num_files, max_duration, min_duration, orig_duration, final_duration
105
+
106
+
cluster/__init__.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+ from sklearn.cluster import KMeans
4
+
5
+ def get_cluster_model(ckpt_path):
6
+ checkpoint = torch.load(ckpt_path)
7
+ kmeans_dict = {}
8
+ for spk, ckpt in checkpoint.items():
9
+ km = KMeans(ckpt["n_features_in_"])
10
+ km.__dict__["n_features_in_"] = ckpt["n_features_in_"]
11
+ km.__dict__["_n_threads"] = ckpt["_n_threads"]
12
+ km.__dict__["cluster_centers_"] = ckpt["cluster_centers_"]
13
+ kmeans_dict[spk] = km
14
+ return kmeans_dict
15
+
16
+ def get_cluster_result(model, x, speaker):
17
+ """
18
+ x: np.array [t, 256]
19
+ return cluster class result
20
+ """
21
+ return model[speaker].predict(x)
22
+
23
+ def get_cluster_center_result(model, x,speaker):
24
+ """x: np.array [t, 256]"""
25
+ predict = model[speaker].predict(x)
26
+ return model[speaker].cluster_centers_[predict]
27
+
28
+ def get_center(model, x,speaker):
29
+ return model[speaker].cluster_centers_[x]
cluster/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (1.08 kB). View file
 
cluster/__pycache__/kmeans.cpython-38.pyc ADDED
Binary file (6.96 kB). View file
 
cluster/kmeans.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math,pdb
2
+ import torch,pynvml
3
+ from torch.nn.functional import normalize
4
+ from time import time
5
+ import numpy as np
6
+ # device=torch.device("cuda:0")
7
+ def _kpp(data: torch.Tensor, k: int, sample_size: int = -1):
8
+ """ Picks k points in the data based on the kmeans++ method.
9
+
10
+ Parameters
11
+ ----------
12
+ data : torch.Tensor
13
+ Expect a rank 1 or 2 array. Rank 1 is assumed to describe 1-D
14
+ data, rank 2 multidimensional data, in which case one
15
+ row is one observation.
16
+ k : int
17
+ Number of samples to generate.
18
+ sample_size : int
19
+ sample data to avoid memory overflow during calculation
20
+
21
+ Returns
22
+ -------
23
+ init : ndarray
24
+ A 'k' by 'N' containing the initial centroids.
25
+
26
+ References
27
+ ----------
28
+ .. [1] D. Arthur and S. Vassilvitskii, "k-means++: the advantages of
29
+ careful seeding", Proceedings of the Eighteenth Annual ACM-SIAM Symposium
30
+ on Discrete Algorithms, 2007.
31
+ .. [2] scipy/cluster/vq.py: _kpp
32
+ """
33
+ batch_size=data.shape[0]
34
+ if batch_size>sample_size:
35
+ data = data[torch.randint(0, batch_size,[sample_size], device=data.device)]
36
+ dims = data.shape[1] if len(data.shape) > 1 else 1
37
+ init = torch.zeros((k, dims)).to(data.device)
38
+ r = torch.distributions.uniform.Uniform(0, 1)
39
+ for i in range(k):
40
+ if i == 0:
41
+ init[i, :] = data[torch.randint(data.shape[0], [1])]
42
+ else:
43
+ D2 = torch.cdist(init[:i, :][None, :], data[None, :], p=2)[0].amin(dim=0)
44
+ probs = D2 / torch.sum(D2)
45
+ cumprobs = torch.cumsum(probs, dim=0)
46
+ init[i, :] = data[torch.searchsorted(cumprobs, r.sample([1]).to(data.device))]
47
+ return init
48
+ class KMeansGPU:
49
+ '''
50
+ Kmeans clustering algorithm implemented with PyTorch
51
+
52
+ Parameters:
53
+ n_clusters: int,
54
+ Number of clusters
55
+
56
+ max_iter: int, default: 100
57
+ Maximum number of iterations
58
+
59
+ tol: float, default: 0.0001
60
+ Tolerance
61
+
62
+ verbose: int, default: 0
63
+ Verbosity
64
+
65
+ mode: {'euclidean', 'cosine'}, default: 'euclidean'
66
+ Type of distance measure
67
+
68
+ init_method: {'random', 'point', '++'}
69
+ Type of initialization
70
+
71
+ minibatch: {None, int}, default: None
72
+ Batch size of MinibatchKmeans algorithm
73
+ if None perform full KMeans algorithm
74
+
75
+ Attributes:
76
+ centroids: torch.Tensor, shape: [n_clusters, n_features]
77
+ cluster centroids
78
+ '''
79
+ def __init__(self, n_clusters, max_iter=200, tol=1e-4, verbose=0, mode="euclidean",device=torch.device("cuda:0")):
80
+ self.n_clusters = n_clusters
81
+ self.max_iter = max_iter
82
+ self.tol = tol
83
+ self.verbose = verbose
84
+ self.mode = mode
85
+ self.device=device
86
+ pynvml.nvmlInit()
87
+ gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(device.index)
88
+ info = pynvml.nvmlDeviceGetMemoryInfo(gpu_handle)
89
+ self.minibatch=int(33e6/self.n_clusters*info.free/ 1024 / 1024 / 1024)
90
+ print("free_mem/GB:",info.free/ 1024 / 1024 / 1024,"minibatch:",self.minibatch)
91
+
92
+ @staticmethod
93
+ def cos_sim(a, b):
94
+ """
95
+ Compute cosine similarity of 2 sets of vectors
96
+
97
+ Parameters:
98
+ a: torch.Tensor, shape: [m, n_features]
99
+
100
+ b: torch.Tensor, shape: [n, n_features]
101
+ """
102
+ return normalize(a, dim=-1) @ normalize(b, dim=-1).transpose(-2, -1)
103
+
104
+ @staticmethod
105
+ def euc_sim(a, b):
106
+ """
107
+ Compute euclidean similarity of 2 sets of vectors
108
+ Parameters:
109
+ a: torch.Tensor, shape: [m, n_features]
110
+ b: torch.Tensor, shape: [n, n_features]
111
+ """
112
+ return 2 * a @ b.transpose(-2, -1) -(a**2).sum(dim=1)[..., :, None] - (b**2).sum(dim=1)[..., None, :]
113
+
114
+ def max_sim(self, a, b):
115
+ """
116
+ Compute maximum similarity (or minimum distance) of each vector
117
+ in a with all of the vectors in b
118
+ Parameters:
119
+ a: torch.Tensor, shape: [m, n_features]
120
+ b: torch.Tensor, shape: [n, n_features]
121
+ """
122
+ if self.mode == 'cosine':
123
+ sim_func = self.cos_sim
124
+ elif self.mode == 'euclidean':
125
+ sim_func = self.euc_sim
126
+ sim = sim_func(a, b)
127
+ max_sim_v, max_sim_i = sim.max(dim=-1)
128
+ return max_sim_v, max_sim_i
129
+
130
+ def fit_predict(self, X):
131
+ """
132
+ Combination of fit() and predict() methods.
133
+ This is faster than calling fit() and predict() seperately.
134
+ Parameters:
135
+ X: torch.Tensor, shape: [n_samples, n_features]
136
+ centroids: {torch.Tensor, None}, default: None
137
+ if given, centroids will be initialized with given tensor
138
+ if None, centroids will be randomly chosen from X
139
+ Return:
140
+ labels: torch.Tensor, shape: [n_samples]
141
+
142
+ mini_=33kk/k*remain
143
+ mini=min(mini_,fea_shape)
144
+ offset=log2(k/1000)*1.5
145
+ kpp_all=min(mini_*10/offset,fea_shape)
146
+ kpp_sample=min(mini_/12/offset,fea_shape)
147
+ """
148
+ assert isinstance(X, torch.Tensor), "input must be torch.Tensor"
149
+ assert X.dtype in [torch.half, torch.float, torch.double], "input must be floating point"
150
+ assert X.ndim == 2, "input must be a 2d tensor with shape: [n_samples, n_features] "
151
+ # print("verbose:%s"%self.verbose)
152
+
153
+ offset = np.power(1.5,np.log(self.n_clusters / 1000))/np.log(2)
154
+ with torch.no_grad():
155
+ batch_size= X.shape[0]
156
+ # print(self.minibatch, int(self.minibatch * 10 / offset), batch_size)
157
+ start_time = time()
158
+ if (self.minibatch*10//offset< batch_size):
159
+ x = X[torch.randint(0, batch_size,[int(self.minibatch*10/offset)])].to(self.device)
160
+ else:
161
+ x = X.to(self.device)
162
+ # print(x.device)
163
+ self.centroids = _kpp(x, self.n_clusters, min(int(self.minibatch/12/offset),batch_size))
164
+ del x
165
+ torch.cuda.empty_cache()
166
+ # self.centroids = self.centroids.to(self.device)
167
+ num_points_in_clusters = torch.ones(self.n_clusters, device=self.device, dtype=X.dtype)#全1
168
+ closest = None#[3098036]#int64
169
+ if(self.minibatch>=batch_size//2 and self.minibatch<batch_size):
170
+ X = X[torch.randint(0, batch_size,[self.minibatch])].to(self.device)
171
+ elif(self.minibatch>=batch_size):
172
+ X=X.to(self.device)
173
+ for i in range(self.max_iter):
174
+ iter_time = time()
175
+ if self.minibatch<batch_size//2:#可用minibatch数太小,每次都得从内存倒腾到显存
176
+ x = X[torch.randint(0, batch_size, [self.minibatch])].to(self.device)
177
+ else:#否则直接全部缓存
178
+ x = X
179
+
180
+ closest = self.max_sim(a=x, b=self.centroids)[1].to(torch.int16)#[3098036]#int64#0~999
181
+ matched_clusters, counts = closest.unique(return_counts=True)#int64#1k
182
+ expanded_closest = closest[None].expand(self.n_clusters, -1)#[1000, 3098036]#int16#0~999
183
+ mask = (expanded_closest==torch.arange(self.n_clusters, device=self.device)[:, None]).to(X.dtype)#==后者是int64*1000
184
+ c_grad = mask @ x / mask.sum(-1)[..., :, None]
185
+ c_grad[c_grad!=c_grad] = 0 # remove NaNs
186
+ error = (c_grad - self.centroids).pow(2).sum()
187
+ if self.minibatch is not None:
188
+ lr = 1/num_points_in_clusters[:,None] * 0.9 + 0.1
189
+ else:
190
+ lr = 1
191
+ matched_clusters=matched_clusters.long()
192
+ num_points_in_clusters[matched_clusters] += counts#IndexError: tensors used as indices must be long, byte or bool tensors
193
+ self.centroids = self.centroids * (1-lr) + c_grad * lr
194
+ if self.verbose >= 2:
195
+ print('iter:', i, 'error:', error.item(), 'time spent:', round(time()-iter_time, 4))
196
+ if error <= self.tol:
197
+ break
198
+
199
+ if self.verbose >= 1:
200
+ print(f'used {i+1} iterations ({round(time()-start_time, 4)}s) to cluster {batch_size} items into {self.n_clusters} clusters')
201
+ return closest
cluster/train_cluster.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time,pdb
2
+ import tqdm
3
+ from time import time as ttime
4
+ import os
5
+ from pathlib import Path
6
+ import logging
7
+ import argparse
8
+ from kmeans import KMeansGPU
9
+ import torch
10
+ import numpy as np
11
+ from sklearn.cluster import KMeans,MiniBatchKMeans
12
+
13
+ logging.basicConfig(level=logging.INFO)
14
+ logger = logging.getLogger(__name__)
15
+ from time import time as ttime
16
+ import pynvml,torch
17
+
18
+ def train_cluster(in_dir, n_clusters, use_minibatch=True, verbose=False,use_gpu=False):#gpu_minibatch真拉,虽然库支持但是也不考虑
19
+ logger.info(f"Loading features from {in_dir}")
20
+ features = []
21
+ nums = 0
22
+ for path in tqdm.tqdm(in_dir.glob("*.soft.pt")):
23
+ # for name in os.listdir(in_dir):
24
+ # path="%s/%s"%(in_dir,name)
25
+ features.append(torch.load(path,map_location="cpu").squeeze(0).numpy().T)
26
+ # print(features[-1].shape)
27
+ features = np.concatenate(features, axis=0)
28
+ print(nums, features.nbytes/ 1024**2, "MB , shape:",features.shape, features.dtype)
29
+ features = features.astype(np.float32)
30
+ logger.info(f"Clustering features of shape: {features.shape}")
31
+ t = time.time()
32
+ if(use_gpu==False):
33
+ if use_minibatch:
34
+ kmeans = MiniBatchKMeans(n_clusters=n_clusters,verbose=verbose, batch_size=4096, max_iter=80).fit(features)
35
+ else:
36
+ kmeans = KMeans(n_clusters=n_clusters,verbose=verbose).fit(features)
37
+ else:
38
+ kmeans = KMeansGPU(n_clusters=n_clusters, mode='euclidean', verbose=2 if verbose else 0,max_iter=500,tol=1e-2)#
39
+ features=torch.from_numpy(features)#.to(device)
40
+ labels = kmeans.fit_predict(features)#
41
+
42
+ print(time.time()-t, "s")
43
+
44
+ x = {
45
+ "n_features_in_": kmeans.n_features_in_ if use_gpu==False else features.shape[1],
46
+ "_n_threads": kmeans._n_threads if use_gpu==False else 4,
47
+ "cluster_centers_": kmeans.cluster_centers_ if use_gpu==False else kmeans.centroids.cpu().numpy(),
48
+ }
49
+ print("end")
50
+
51
+ return x
52
+
53
+ if __name__ == "__main__":
54
+ parser = argparse.ArgumentParser()
55
+ parser.add_argument('--dataset', type=Path, default="./dataset/44k",
56
+ help='path of training data directory')
57
+ parser.add_argument('--output', type=Path, default="logs/44k",
58
+ help='path of model output directory')
59
+ parser.add_argument('--gpu',action='store_true', default=False ,
60
+ help='to use GPU')
61
+
62
+
63
+ args = parser.parse_args()
64
+
65
+ checkpoint_dir = args.output
66
+ dataset = args.dataset
67
+ use_gpu = args.gpu
68
+ n_clusters = 10000
69
+
70
+ ckpt = {}
71
+ for spk in os.listdir(dataset):
72
+ if os.path.isdir(dataset/spk):
73
+ print(f"train kmeans for {spk}...")
74
+ in_dir = dataset/spk
75
+ x = train_cluster(in_dir, n_clusters,use_minibatch=False,verbose=False,use_gpu=use_gpu)
76
+ ckpt[spk] = x
77
+
78
+ checkpoint_path = checkpoint_dir / f"kmeans_{n_clusters}.pt"
79
+ checkpoint_path.parent.mkdir(exist_ok=True, parents=True)
80
+ torch.save(
81
+ ckpt,
82
+ checkpoint_path,
83
+ )
84
+
compress_model.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import OrderedDict
2
+
3
+ import torch
4
+
5
+ import utils
6
+ from models import SynthesizerTrn
7
+
8
+
9
+ def copyStateDict(state_dict):
10
+ if list(state_dict.keys())[0].startswith('module'):
11
+ start_idx = 1
12
+ else:
13
+ start_idx = 0
14
+ new_state_dict = OrderedDict()
15
+ for k, v in state_dict.items():
16
+ name = ','.join(k.split('.')[start_idx:])
17
+ new_state_dict[name] = v
18
+ return new_state_dict
19
+
20
+
21
+ def removeOptimizer(config: str, input_model: str, output_model: str):
22
+ hps = utils.get_hparams_from_file(config)
23
+
24
+ net_g = SynthesizerTrn(hps.data.filter_length // 2 + 1,
25
+ hps.train.segment_size // hps.data.hop_length,
26
+ **hps.model)
27
+
28
+ optim_g = torch.optim.AdamW(net_g.parameters(),
29
+ hps.train.learning_rate,
30
+ betas=hps.train.betas,
31
+ eps=hps.train.eps)
32
+
33
+ state_dict_g = torch.load(input_model, map_location="cpu")
34
+ new_dict_g = copyStateDict(state_dict_g)
35
+ keys = []
36
+ for k, v in new_dict_g['model'].items():
37
+ keys.append(k)
38
+
39
+ new_dict_g = {k: new_dict_g['model'][k] for k in keys}
40
+
41
+ torch.save(
42
+ {
43
+ 'model': new_dict_g,
44
+ 'iteration': 0,
45
+ 'optimizer': optim_g.state_dict(),
46
+ 'learning_rate': 0.0001
47
+ }, output_model)
48
+
49
+
50
+ if __name__ == "__main__":
51
+ import argparse
52
+ parser = argparse.ArgumentParser()
53
+ parser.add_argument("-c",
54
+ "--config",
55
+ type=str,
56
+ default='configs/config.json')
57
+ parser.add_argument("-i", "--input", type=str)
58
+ parser.add_argument("-o", "--output", type=str, default=None)
59
+
60
+ args = parser.parse_args()
61
+
62
+ output = args.output
63
+
64
+ if output is None:
65
+ import os.path
66
+ filename, ext = os.path.splitext(args.input)
67
+ output = filename + "_release" + ext
68
+
69
+ removeOptimizer(args.config, args.input, output)
configs/.ipynb_checkpoints/config-checkpoint.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 200,
4
+ "eval_interval": 200,
5
+ "seed": 1234,
6
+ "epochs": 10000,
7
+ "learning_rate": 0.0001,
8
+ "betas": [
9
+ 0.8,
10
+ 0.99
11
+ ],
12
+ "eps": 1e-09,
13
+ "batch_size": 6,
14
+ "fp16_run": false,
15
+ "lr_decay": 0.999875,
16
+ "segment_size": 10240,
17
+ "init_lr_ratio": 1,
18
+ "warmup_epochs": 0,
19
+ "c_mel": 45,
20
+ "c_kl": 1.0,
21
+ "use_sr": true,
22
+ "max_speclen": 512,
23
+ "port": "8001",
24
+ "keep_ckpts": 20,
25
+ "all_in_mem": true
26
+ },
27
+ "data": {
28
+ "training_files": "filelists/train.txt",
29
+ "validation_files": "filelists/val.txt",
30
+ "max_wav_value": 32768.0,
31
+ "sampling_rate": 44100,
32
+ "filter_length": 2048,
33
+ "hop_length": 512,
34
+ "win_length": 2048,
35
+ "n_mel_channels": 80,
36
+ "mel_fmin": 0.0,
37
+ "mel_fmax": 22050
38
+ },
39
+ "model": {
40
+ "inter_channels": 192,
41
+ "hidden_channels": 192,
42
+ "filter_channels": 768,
43
+ "n_heads": 2,
44
+ "n_layers": 6,
45
+ "kernel_size": 3,
46
+ "p_dropout": 0.1,
47
+ "resblock": "1",
48
+ "resblock_kernel_sizes": [
49
+ 3,
50
+ 7,
51
+ 11
52
+ ],
53
+ "resblock_dilation_sizes": [
54
+ [
55
+ 1,
56
+ 3,
57
+ 5
58
+ ],
59
+ [
60
+ 1,
61
+ 3,
62
+ 5
63
+ ],
64
+ [
65
+ 1,
66
+ 3,
67
+ 5
68
+ ]
69
+ ],
70
+ "upsample_rates": [
71
+ 8,
72
+ 8,
73
+ 2,
74
+ 2,
75
+ 2
76
+ ],
77
+ "upsample_initial_channel": 512,
78
+ "upsample_kernel_sizes": [
79
+ 16,
80
+ 16,
81
+ 4,
82
+ 4,
83
+ 4
84
+ ],
85
+ "n_layers_q": 3,
86
+ "use_spectral_norm": false,
87
+ "gin_channels": 768,
88
+ "ssl_dim": 768,
89
+ "n_speakers": 1,
90
+ "speech_encoder": "vec768l12",
91
+ "speaker_embedding": false
92
+ },
93
+ "spk": {
94
+ "renge": 0
95
+ }
96
+ }
configs/config.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 200,
4
+ "eval_interval": 800,
5
+ "seed": 1234,
6
+ "epochs": 10000,
7
+ "learning_rate": 0.0001,
8
+ "betas": [
9
+ 0.8,
10
+ 0.99
11
+ ],
12
+ "eps": 1e-09,
13
+ "batch_size": 6,
14
+ "fp16_run": false,
15
+ "lr_decay": 0.999875,
16
+ "segment_size": 10240,
17
+ "init_lr_ratio": 1,
18
+ "warmup_epochs": 0,
19
+ "c_mel": 45,
20
+ "c_kl": 1.0,
21
+ "use_sr": true,
22
+ "max_speclen": 512,
23
+ "port": "8001",
24
+ "keep_ckpts": 3,
25
+ "all_in_mem": false,
26
+ "vol_aug": true
27
+ },
28
+ "data": {
29
+ "training_files": "filelists/train.txt",
30
+ "validation_files": "filelists/val.txt",
31
+ "max_wav_value": 32768.0,
32
+ "sampling_rate": 44100,
33
+ "filter_length": 2048,
34
+ "hop_length": 512,
35
+ "win_length": 2048,
36
+ "n_mel_channels": 80,
37
+ "mel_fmin": 0.0,
38
+ "mel_fmax": 22050,
39
+ "unit_interpolate_mode": "nearest"
40
+ },
41
+ "model": {
42
+ "inter_channels": 192,
43
+ "hidden_channels": 192,
44
+ "filter_channels": 768,
45
+ "n_heads": 2,
46
+ "n_layers": 6,
47
+ "kernel_size": 3,
48
+ "p_dropout": 0.1,
49
+ "resblock": "1",
50
+ "resblock_kernel_sizes": [
51
+ 3,
52
+ 7,
53
+ 11
54
+ ],
55
+ "resblock_dilation_sizes": [
56
+ [
57
+ 1,
58
+ 3,
59
+ 5
60
+ ],
61
+ [
62
+ 1,
63
+ 3,
64
+ 5
65
+ ],
66
+ [
67
+ 1,
68
+ 3,
69
+ 5
70
+ ]
71
+ ],
72
+ "upsample_rates": [
73
+ 8,
74
+ 8,
75
+ 2,
76
+ 2,
77
+ 2
78
+ ],
79
+ "upsample_initial_channel": 512,
80
+ "upsample_kernel_sizes": [
81
+ 16,
82
+ 16,
83
+ 4,
84
+ 4,
85
+ 4
86
+ ],
87
+ "n_layers_q": 3,
88
+ "use_spectral_norm": false,
89
+ "gin_channels": 768,
90
+ "ssl_dim": 768,
91
+ "n_speakers": 1,
92
+ "vocoder_name": "nsf-hifigan",
93
+ "speech_encoder": "vec768l12",
94
+ "speaker_embedding": false,
95
+ "vol_embedding": true
96
+ },
97
+ "spk": {
98
+ "1013": 0
99
+ }
100
+ }
configs/diffusion.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data:
2
+ block_size: 512
3
+ cnhubertsoft_gate: 10
4
+ duration: 2
5
+ encoder: vec768l12
6
+ encoder_hop_size: 320
7
+ encoder_out_channels: 768
8
+ encoder_sample_rate: 16000
9
+ extensions:
10
+ - wav
11
+ sampling_rate: 44100
12
+ training_files: filelists/train.txt
13
+ unit_interpolate_mode: nearest
14
+ validation_files: filelists/val.txt
15
+ device: cuda
16
+ env:
17
+ expdir: logs/44k/diffusion
18
+ gpu_id: 0
19
+ infer:
20
+ method: dpm-solver
21
+ speedup: 10
22
+ model:
23
+ n_chans: 512
24
+ n_hidden: 256
25
+ n_layers: 20
26
+ n_spk: 1
27
+ type: Diffusion
28
+ use_pitch_aug: true
29
+ spk:
30
+ '1013': 0
31
+ train:
32
+ amp_dtype: fp32
33
+ batch_size: 48
34
+ cache_all_data: true
35
+ cache_device: cpu
36
+ cache_fp16: true
37
+ decay_step: 100000
38
+ epochs: 100000
39
+ gamma: 0.5
40
+ interval_force_save: 10000
41
+ interval_log: 10
42
+ interval_val: 2000
43
+ lr: 0.0002
44
+ num_workers: 2
45
+ save_opt: false
46
+ weight_decay: 0
47
+ vocoder:
48
+ ckpt: pretrain/nsf_hifigan/model
49
+ type: nsf-hifigan
configs_template/config_template.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 200,
4
+ "eval_interval": 800,
5
+ "seed": 1234,
6
+ "epochs": 10000,
7
+ "learning_rate": 0.0001,
8
+ "betas": [
9
+ 0.8,
10
+ 0.99
11
+ ],
12
+ "eps": 1e-09,
13
+ "batch_size": 6,
14
+ "fp16_run": false,
15
+ "lr_decay": 0.999875,
16
+ "segment_size": 10240,
17
+ "init_lr_ratio": 1,
18
+ "warmup_epochs": 0,
19
+ "c_mel": 45,
20
+ "c_kl": 1.0,
21
+ "use_sr": true,
22
+ "max_speclen": 512,
23
+ "port": "8001",
24
+ "keep_ckpts": 3,
25
+ "all_in_mem": false,
26
+ "vol_aug":false
27
+ },
28
+ "data": {
29
+ "training_files": "filelists/train.txt",
30
+ "validation_files": "filelists/val.txt",
31
+ "max_wav_value": 32768.0,
32
+ "sampling_rate": 44100,
33
+ "filter_length": 2048,
34
+ "hop_length": 512,
35
+ "win_length": 2048,
36
+ "n_mel_channels": 80,
37
+ "mel_fmin": 0.0,
38
+ "mel_fmax": 22050,
39
+ "unit_interpolate_mode":"nearest"
40
+ },
41
+ "model": {
42
+ "inter_channels": 192,
43
+ "hidden_channels": 192,
44
+ "filter_channels": 768,
45
+ "n_heads": 2,
46
+ "n_layers": 6,
47
+ "kernel_size": 3,
48
+ "p_dropout": 0.1,
49
+ "resblock": "1",
50
+ "resblock_kernel_sizes": [3,7,11],
51
+ "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
52
+ "upsample_rates": [ 8, 8, 2, 2, 2],
53
+ "upsample_initial_channel": 512,
54
+ "upsample_kernel_sizes": [16,16, 4, 4, 4],
55
+ "n_layers_q": 3,
56
+ "use_spectral_norm": false,
57
+ "gin_channels": 768,
58
+ "ssl_dim": 768,
59
+ "n_speakers": 200,
60
+ "vocoder_name":"nsf-hifigan",
61
+ "speech_encoder":"vec768l12",
62
+ "speaker_embedding":false,
63
+ "vol_embedding":false
64
+ },
65
+ "spk": {
66
+ "nyaru": 0,
67
+ "huiyu": 1,
68
+ "nen": 2,
69
+ "paimon": 3,
70
+ "yunhao": 4
71
+ }
72
+ }
configs_template/diffusion_template.yaml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ data:
2
+ sampling_rate: 44100
3
+ block_size: 512 # Equal to hop_length
4
+ duration: 2 # Audio duration during training, must be less than the duration of the shortest audio clip
5
+ encoder: 'vec768l12' # 'hubertsoft', 'vec256l9', 'vec768l12'
6
+ cnhubertsoft_gate: 10
7
+ encoder_sample_rate: 16000
8
+ encoder_hop_size: 320
9
+ encoder_out_channels: 768 # 256 if using 'hubertsoft'
10
+ training_files: "filelists/train.txt"
11
+ validation_files: "filelists/val.txt"
12
+ extensions: # List of extension included in the data collection
13
+ - wav
14
+ unit_interpolate_mode: "nearest"
15
+ model:
16
+ type: 'Diffusion'
17
+ n_layers: 20
18
+ n_chans: 512
19
+ n_hidden: 256
20
+ use_pitch_aug: true
21
+ n_spk: 1 # max number of different speakers
22
+ device: cuda
23
+ vocoder:
24
+ type: 'nsf-hifigan'
25
+ ckpt: 'pretrain/nsf_hifigan/model'
26
+ infer:
27
+ speedup: 10
28
+ method: 'dpm-solver' # 'pndm' or 'dpm-solver'
29
+ env:
30
+ expdir: logs/44k/diffusion
31
+ gpu_id: 0
32
+ train:
33
+ num_workers: 2 # If your cpu and gpu are both very strong, set to 0 may be faster!
34
+ amp_dtype: fp32 # fp32, fp16 or bf16 (fp16 or bf16 may be faster if it is supported by your gpu)
35
+ batch_size: 48
36
+ cache_all_data: true # Save Internal-Memory or Graphics-Memory if it is false, but may be slow
37
+ cache_device: 'cpu' # Set to 'cuda' to cache the data into the Graphics-Memory, fastest speed for strong gpu
38
+ cache_fp16: true
39
+ epochs: 100000
40
+ interval_log: 10
41
+ interval_val: 2000
42
+ interval_force_save: 10000
43
+ lr: 0.0002
44
+ decay_step: 100000
45
+ gamma: 0.5
46
+ weight_decay: 0
47
+ save_opt: false
48
+ spk:
49
+ 'nyaru': 0
data_utils.py ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ import os
3
+ import random
4
+ import numpy as np
5
+ import torch
6
+ import torch.utils.data
7
+
8
+ import modules.commons as commons
9
+ import utils
10
+ from modules.mel_processing import spectrogram_torch, spec_to_mel_torch, spectrogram_torch
11
+ from utils import load_wav_to_torch, load_filepaths_and_text
12
+
13
+ # import h5py
14
+
15
+
16
+ """Multi speaker version"""
17
+
18
+
19
+ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
20
+ """
21
+ 1) loads audio, speaker_id, text pairs
22
+ 2) normalizes text and converts them to sequences of integers
23
+ 3) computes spectrograms from audio files.
24
+ """
25
+
26
+ def __init__(self, audiopaths, hparams, all_in_mem: bool = False, vol_aug: bool = True):
27
+ self.audiopaths = load_filepaths_and_text(audiopaths)
28
+ self.hparams = hparams
29
+ self.max_wav_value = hparams.data.max_wav_value
30
+ self.sampling_rate = hparams.data.sampling_rate
31
+ self.filter_length = hparams.data.filter_length
32
+ self.hop_length = hparams.data.hop_length
33
+ self.win_length = hparams.data.win_length
34
+ self.unit_interpolate_mode = hparams.data.unit_interpolate_mode
35
+ self.sampling_rate = hparams.data.sampling_rate
36
+ self.use_sr = hparams.train.use_sr
37
+ self.spec_len = hparams.train.max_speclen
38
+ self.spk_map = hparams.spk
39
+ self.vol_emb = hparams.model.vol_embedding
40
+ self.vol_aug = hparams.train.vol_aug and vol_aug
41
+ random.seed(1234)
42
+ random.shuffle(self.audiopaths)
43
+
44
+ self.all_in_mem = all_in_mem
45
+ if self.all_in_mem:
46
+ self.cache = [self.get_audio(p[0]) for p in self.audiopaths]
47
+
48
+ def get_audio(self, filename):
49
+ filename = filename.replace("\\", "/")
50
+ audio, sampling_rate = load_wav_to_torch(filename)
51
+ if sampling_rate != self.sampling_rate:
52
+ raise ValueError("{} SR doesn't match target {} SR".format(
53
+ sampling_rate, self.sampling_rate))
54
+ audio_norm = audio / self.max_wav_value
55
+ audio_norm = audio_norm.unsqueeze(0)
56
+ spec_filename = filename.replace(".wav", ".spec.pt")
57
+
58
+ # Ideally, all data generated after Mar 25 should have .spec.pt
59
+ if os.path.exists(spec_filename):
60
+ spec = torch.load(spec_filename)
61
+ else:
62
+ spec = spectrogram_torch(audio_norm, self.filter_length,
63
+ self.sampling_rate, self.hop_length, self.win_length,
64
+ center=False)
65
+ spec = torch.squeeze(spec, 0)
66
+ torch.save(spec, spec_filename)
67
+
68
+ spk = filename.split("/")[-2]
69
+ spk = torch.LongTensor([self.spk_map[spk]])
70
+
71
+ f0, uv = np.load(filename + ".f0.npy",allow_pickle=True)
72
+
73
+ f0 = torch.FloatTensor(np.array(f0,dtype=float))
74
+ uv = torch.FloatTensor(np.array(uv,dtype=float))
75
+
76
+ c = torch.load(filename+ ".soft.pt")
77
+ c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[0], mode=self.unit_interpolate_mode)
78
+ if self.vol_emb:
79
+ volume_path = filename + ".vol.npy"
80
+ volume = np.load(volume_path)
81
+ volume = torch.from_numpy(volume).float()
82
+ else:
83
+ volume = None
84
+
85
+ lmin = min(c.size(-1), spec.size(-1))
86
+ assert abs(c.size(-1) - spec.size(-1)) < 3, (c.size(-1), spec.size(-1), f0.shape, filename)
87
+ assert abs(audio_norm.shape[1]-lmin * self.hop_length) < 3 * self.hop_length
88
+ spec, c, f0, uv = spec[:, :lmin], c[:, :lmin], f0[:lmin], uv[:lmin]
89
+ audio_norm = audio_norm[:, :lmin * self.hop_length]
90
+ if volume!= None:
91
+ volume = volume[:lmin]
92
+ return c, f0, spec, audio_norm, spk, uv, volume
93
+
94
+ def random_slice(self, c, f0, spec, audio_norm, spk, uv, volume):
95
+ # if spec.shape[1] < 30:
96
+ # print("skip too short audio:", filename)
97
+ # return None
98
+
99
+ if random.choice([True, False]) and self.vol_aug and volume!=None:
100
+ max_amp = float(torch.max(torch.abs(audio_norm))) + 1e-5
101
+ max_shift = min(1, np.log10(1/max_amp))
102
+ log10_vol_shift = random.uniform(-1, max_shift)
103
+ audio_norm = audio_norm * (10 ** log10_vol_shift)
104
+ volume = volume * (10 ** log10_vol_shift)
105
+ spec = spectrogram_torch(audio_norm,
106
+ self.hparams.data.filter_length,
107
+ self.hparams.data.sampling_rate,
108
+ self.hparams.data.hop_length,
109
+ self.hparams.data.win_length,
110
+ center=False)[0]
111
+
112
+ if spec.shape[1] > 800:
113
+ start = random.randint(0, spec.shape[1]-800)
114
+ end = start + 790
115
+ spec, c, f0, uv = spec[:, start:end], c[:, start:end], f0[start:end], uv[start:end]
116
+ audio_norm = audio_norm[:, start * self.hop_length : end * self.hop_length]
117
+ if volume !=None:
118
+ volume = volume[start:end]
119
+ return c, f0, spec, audio_norm, spk, uv,volume
120
+
121
+ def __getitem__(self, index):
122
+ if self.all_in_mem:
123
+ return self.random_slice(*self.cache[index])
124
+ else:
125
+ return self.random_slice(*self.get_audio(self.audiopaths[index][0]))
126
+
127
+ def __len__(self):
128
+ return len(self.audiopaths)
129
+
130
+
131
+ class TextAudioCollate:
132
+
133
+ def __call__(self, batch):
134
+ batch = [b for b in batch if b is not None]
135
+
136
+ input_lengths, ids_sorted_decreasing = torch.sort(
137
+ torch.LongTensor([x[0].shape[1] for x in batch]),
138
+ dim=0, descending=True)
139
+
140
+ max_c_len = max([x[0].size(1) for x in batch])
141
+ max_wav_len = max([x[3].size(1) for x in batch])
142
+
143
+ lengths = torch.LongTensor(len(batch))
144
+
145
+ c_padded = torch.FloatTensor(len(batch), batch[0][0].shape[0], max_c_len)
146
+ f0_padded = torch.FloatTensor(len(batch), max_c_len)
147
+ spec_padded = torch.FloatTensor(len(batch), batch[0][2].shape[0], max_c_len)
148
+ wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
149
+ spkids = torch.LongTensor(len(batch), 1)
150
+ uv_padded = torch.FloatTensor(len(batch), max_c_len)
151
+ volume_padded = torch.FloatTensor(len(batch), max_c_len)
152
+
153
+ c_padded.zero_()
154
+ spec_padded.zero_()
155
+ f0_padded.zero_()
156
+ wav_padded.zero_()
157
+ uv_padded.zero_()
158
+ volume_padded.zero_()
159
+
160
+ for i in range(len(ids_sorted_decreasing)):
161
+ row = batch[ids_sorted_decreasing[i]]
162
+
163
+ c = row[0]
164
+ c_padded[i, :, :c.size(1)] = c
165
+ lengths[i] = c.size(1)
166
+
167
+ f0 = row[1]
168
+ f0_padded[i, :f0.size(0)] = f0
169
+
170
+ spec = row[2]
171
+ spec_padded[i, :, :spec.size(1)] = spec
172
+
173
+ wav = row[3]
174
+ wav_padded[i, :, :wav.size(1)] = wav
175
+
176
+ spkids[i, 0] = row[4]
177
+
178
+ uv = row[5]
179
+ uv_padded[i, :uv.size(0)] = uv
180
+ volume = row[6]
181
+ if volume != None:
182
+ volume_padded[i, :volume.size(0)] = volume
183
+ else :
184
+ volume_padded = None
185
+ return c_padded, f0_padded, spec_padded, wav_padded, spkids, lengths, uv_padded, volume_padded
diffusion/__init__.py ADDED
File without changes
diffusion/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (130 Bytes). View file
 
diffusion/__pycache__/diffusion.cpython-38.pyc ADDED
Binary file (10.1 kB). View file
 
diffusion/__pycache__/unit2mel.cpython-38.pyc ADDED
Binary file (4.43 kB). View file
 
diffusion/__pycache__/vocoder.cpython-38.pyc ADDED
Binary file (3.5 kB). View file
 
diffusion/__pycache__/wavenet.cpython-38.pyc ADDED
Binary file (3.84 kB). View file
 
diffusion/data_loaders.py ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import random
3
+ import re
4
+ import numpy as np
5
+ import librosa
6
+ import torch
7
+ import random
8
+ from utils import repeat_expand_2d
9
+ from tqdm import tqdm
10
+ from torch.utils.data import Dataset
11
+
12
+ def traverse_dir(
13
+ root_dir,
14
+ extensions,
15
+ amount=None,
16
+ str_include=None,
17
+ str_exclude=None,
18
+ is_pure=False,
19
+ is_sort=False,
20
+ is_ext=True):
21
+
22
+ file_list = []
23
+ cnt = 0
24
+ for root, _, files in os.walk(root_dir):
25
+ for file in files:
26
+ if any([file.endswith(f".{ext}") for ext in extensions]):
27
+ # path
28
+ mix_path = os.path.join(root, file)
29
+ pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
30
+
31
+ # amount
32
+ if (amount is not None) and (cnt == amount):
33
+ if is_sort:
34
+ file_list.sort()
35
+ return file_list
36
+
37
+ # check string
38
+ if (str_include is not None) and (str_include not in pure_path):
39
+ continue
40
+ if (str_exclude is not None) and (str_exclude in pure_path):
41
+ continue
42
+
43
+ if not is_ext:
44
+ ext = pure_path.split('.')[-1]
45
+ pure_path = pure_path[:-(len(ext)+1)]
46
+ file_list.append(pure_path)
47
+ cnt += 1
48
+ if is_sort:
49
+ file_list.sort()
50
+ return file_list
51
+
52
+
53
+ def get_data_loaders(args, whole_audio=False):
54
+ data_train = AudioDataset(
55
+ filelists = args.data.training_files,
56
+ waveform_sec=args.data.duration,
57
+ hop_size=args.data.block_size,
58
+ sample_rate=args.data.sampling_rate,
59
+ load_all_data=args.train.cache_all_data,
60
+ whole_audio=whole_audio,
61
+ extensions=args.data.extensions,
62
+ n_spk=args.model.n_spk,
63
+ spk=args.spk,
64
+ device=args.train.cache_device,
65
+ fp16=args.train.cache_fp16,
66
+ unit_interpolate_mode = args.data.unit_interpolate_mode,
67
+ use_aug=True)
68
+ loader_train = torch.utils.data.DataLoader(
69
+ data_train ,
70
+ batch_size=args.train.batch_size if not whole_audio else 1,
71
+ shuffle=True,
72
+ num_workers=args.train.num_workers if args.train.cache_device=='cpu' else 0,
73
+ persistent_workers=(args.train.num_workers > 0) if args.train.cache_device=='cpu' else False,
74
+ pin_memory=True if args.train.cache_device=='cpu' else False
75
+ )
76
+ data_valid = AudioDataset(
77
+ filelists = args.data.validation_files,
78
+ waveform_sec=args.data.duration,
79
+ hop_size=args.data.block_size,
80
+ sample_rate=args.data.sampling_rate,
81
+ load_all_data=args.train.cache_all_data,
82
+ whole_audio=True,
83
+ spk=args.spk,
84
+ extensions=args.data.extensions,
85
+ unit_interpolate_mode = args.data.unit_interpolate_mode,
86
+ n_spk=args.model.n_spk)
87
+ loader_valid = torch.utils.data.DataLoader(
88
+ data_valid,
89
+ batch_size=1,
90
+ shuffle=False,
91
+ num_workers=0,
92
+ pin_memory=True
93
+ )
94
+ return loader_train, loader_valid
95
+
96
+
97
+ class AudioDataset(Dataset):
98
+ def __init__(
99
+ self,
100
+ filelists,
101
+ waveform_sec,
102
+ hop_size,
103
+ sample_rate,
104
+ spk,
105
+ load_all_data=True,
106
+ whole_audio=False,
107
+ extensions=['wav'],
108
+ n_spk=1,
109
+ device='cpu',
110
+ fp16=False,
111
+ use_aug=False,
112
+ unit_interpolate_mode = 'left'
113
+ ):
114
+ super().__init__()
115
+
116
+ self.waveform_sec = waveform_sec
117
+ self.sample_rate = sample_rate
118
+ self.hop_size = hop_size
119
+ self.filelists = filelists
120
+ self.whole_audio = whole_audio
121
+ self.use_aug = use_aug
122
+ self.data_buffer={}
123
+ self.pitch_aug_dict = {}
124
+ self.unit_interpolate_mode = unit_interpolate_mode
125
+ # np.load(os.path.join(self.path_root, 'pitch_aug_dict.npy'), allow_pickle=True).item()
126
+ if load_all_data:
127
+ print('Load all the data filelists:', filelists)
128
+ else:
129
+ print('Load the f0, volume data filelists:', filelists)
130
+ with open(filelists,"r") as f:
131
+ self.paths = f.read().splitlines()
132
+ for name_ext in tqdm(self.paths, total=len(self.paths)):
133
+ name = os.path.splitext(name_ext)[0]
134
+ path_audio = name_ext
135
+ duration = librosa.get_duration(filename = path_audio, sr = self.sample_rate)
136
+
137
+ path_f0 = name_ext + ".f0.npy"
138
+ f0,_ = np.load(path_f0,allow_pickle=True)
139
+ f0 = torch.from_numpy(np.array(f0,dtype=float)).float().unsqueeze(-1).to(device)
140
+
141
+ path_volume = name_ext + ".vol.npy"
142
+ volume = np.load(path_volume)
143
+ volume = torch.from_numpy(volume).float().unsqueeze(-1).to(device)
144
+
145
+ path_augvol = name_ext + ".aug_vol.npy"
146
+ aug_vol = np.load(path_augvol)
147
+ aug_vol = torch.from_numpy(aug_vol).float().unsqueeze(-1).to(device)
148
+
149
+ if n_spk is not None and n_spk > 1:
150
+ spk_name = name_ext.split("/")[-2]
151
+ spk_id = spk[spk_name] if spk_name in spk else 0
152
+ if spk_id < 0 or spk_id >= n_spk:
153
+ raise ValueError(' [x] Muiti-speaker traing error : spk_id must be a positive integer from 0 to n_spk-1 ')
154
+ else:
155
+ spk_id = 0
156
+ spk_id = torch.LongTensor(np.array([spk_id])).to(device)
157
+
158
+ if load_all_data:
159
+ '''
160
+ audio, sr = librosa.load(path_audio, sr=self.sample_rate)
161
+ if len(audio.shape) > 1:
162
+ audio = librosa.to_mono(audio)
163
+ audio = torch.from_numpy(audio).to(device)
164
+ '''
165
+ path_mel = name_ext + ".mel.npy"
166
+ mel = np.load(path_mel)
167
+ mel = torch.from_numpy(mel).to(device)
168
+
169
+ path_augmel = name_ext + ".aug_mel.npy"
170
+ aug_mel,keyshift = np.load(path_augmel, allow_pickle=True)
171
+ aug_mel = np.array(aug_mel,dtype=float)
172
+ aug_mel = torch.from_numpy(aug_mel).to(device)
173
+ self.pitch_aug_dict[name_ext] = keyshift
174
+
175
+ path_units = name_ext + ".soft.pt"
176
+ units = torch.load(path_units).to(device)
177
+ units = units[0]
178
+ units = repeat_expand_2d(units,f0.size(0),unit_interpolate_mode).transpose(0,1)
179
+
180
+ if fp16:
181
+ mel = mel.half()
182
+ aug_mel = aug_mel.half()
183
+ units = units.half()
184
+
185
+ self.data_buffer[name_ext] = {
186
+ 'duration': duration,
187
+ 'mel': mel,
188
+ 'aug_mel': aug_mel,
189
+ 'units': units,
190
+ 'f0': f0,
191
+ 'volume': volume,
192
+ 'aug_vol': aug_vol,
193
+ 'spk_id': spk_id
194
+ }
195
+ else:
196
+ path_augmel = name_ext + ".aug_mel.npy"
197
+ aug_mel,keyshift = np.load(path_augmel, allow_pickle=True)
198
+ self.pitch_aug_dict[name_ext] = keyshift
199
+ self.data_buffer[name_ext] = {
200
+ 'duration': duration,
201
+ 'f0': f0,
202
+ 'volume': volume,
203
+ 'aug_vol': aug_vol,
204
+ 'spk_id': spk_id
205
+ }
206
+
207
+
208
+ def __getitem__(self, file_idx):
209
+ name_ext = self.paths[file_idx]
210
+ data_buffer = self.data_buffer[name_ext]
211
+ # check duration. if too short, then skip
212
+ if data_buffer['duration'] < (self.waveform_sec + 0.1):
213
+ return self.__getitem__( (file_idx + 1) % len(self.paths))
214
+
215
+ # get item
216
+ return self.get_data(name_ext, data_buffer)
217
+
218
+ def get_data(self, name_ext, data_buffer):
219
+ name = os.path.splitext(name_ext)[0]
220
+ frame_resolution = self.hop_size / self.sample_rate
221
+ duration = data_buffer['duration']
222
+ waveform_sec = duration if self.whole_audio else self.waveform_sec
223
+
224
+ # load audio
225
+ idx_from = 0 if self.whole_audio else random.uniform(0, duration - waveform_sec - 0.1)
226
+ start_frame = int(idx_from / frame_resolution)
227
+ units_frame_len = int(waveform_sec / frame_resolution)
228
+ aug_flag = random.choice([True, False]) and self.use_aug
229
+ '''
230
+ audio = data_buffer.get('audio')
231
+ if audio is None:
232
+ path_audio = os.path.join(self.path_root, 'audio', name) + '.wav'
233
+ audio, sr = librosa.load(
234
+ path_audio,
235
+ sr = self.sample_rate,
236
+ offset = start_frame * frame_resolution,
237
+ duration = waveform_sec)
238
+ if len(audio.shape) > 1:
239
+ audio = librosa.to_mono(audio)
240
+ # clip audio into N seconds
241
+ audio = audio[ : audio.shape[-1] // self.hop_size * self.hop_size]
242
+ audio = torch.from_numpy(audio).float()
243
+ else:
244
+ audio = audio[start_frame * self.hop_size : (start_frame + units_frame_len) * self.hop_size]
245
+ '''
246
+ # load mel
247
+ mel_key = 'aug_mel' if aug_flag else 'mel'
248
+ mel = data_buffer.get(mel_key)
249
+ if mel is None:
250
+ mel = name_ext + ".mel.npy"
251
+ mel = np.load(mel)
252
+ mel = mel[start_frame : start_frame + units_frame_len]
253
+ mel = torch.from_numpy(mel).float()
254
+ else:
255
+ mel = mel[start_frame : start_frame + units_frame_len]
256
+
257
+ # load f0
258
+ f0 = data_buffer.get('f0')
259
+ aug_shift = 0
260
+ if aug_flag:
261
+ aug_shift = self.pitch_aug_dict[name_ext]
262
+ f0_frames = 2 ** (aug_shift / 12) * f0[start_frame : start_frame + units_frame_len]
263
+
264
+ # load units
265
+ units = data_buffer.get('units')
266
+ if units is None:
267
+ path_units = name_ext + ".soft.pt"
268
+ units = torch.load(path_units)
269
+ units = units[0]
270
+ units = repeat_expand_2d(units,f0.size(0),self.unit_interpolate_mode).transpose(0,1)
271
+
272
+ units = units[start_frame : start_frame + units_frame_len]
273
+
274
+ # load volume
275
+ vol_key = 'aug_vol' if aug_flag else 'volume'
276
+ volume = data_buffer.get(vol_key)
277
+ volume_frames = volume[start_frame : start_frame + units_frame_len]
278
+
279
+ # load spk_id
280
+ spk_id = data_buffer.get('spk_id')
281
+
282
+ # load shift
283
+ aug_shift = torch.from_numpy(np.array([[aug_shift]])).float()
284
+
285
+ return dict(mel=mel, f0=f0_frames, volume=volume_frames, units=units, spk_id=spk_id, aug_shift=aug_shift, name=name, name_ext=name_ext)
286
+
287
+ def __len__(self):
288
+ return len(self.paths)
diffusion/diffusion.py ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import deque
2
+ from functools import partial
3
+ from inspect import isfunction
4
+ import torch.nn.functional as F
5
+ import librosa.sequence
6
+ import numpy as np
7
+ import torch
8
+ from torch import nn
9
+ from tqdm import tqdm
10
+
11
+
12
+ def exists(x):
13
+ return x is not None
14
+
15
+
16
+ def default(val, d):
17
+ if exists(val):
18
+ return val
19
+ return d() if isfunction(d) else d
20
+
21
+
22
+ def extract(a, t, x_shape):
23
+ b, *_ = t.shape
24
+ out = a.gather(-1, t)
25
+ return out.reshape(b, *((1,) * (len(x_shape) - 1)))
26
+
27
+
28
+ def noise_like(shape, device, repeat=False):
29
+ repeat_noise = lambda: torch.randn((1, *shape[1:]), device=device).repeat(shape[0], *((1,) * (len(shape) - 1)))
30
+ noise = lambda: torch.randn(shape, device=device)
31
+ return repeat_noise() if repeat else noise()
32
+
33
+
34
+ def linear_beta_schedule(timesteps, max_beta=0.02):
35
+ """
36
+ linear schedule
37
+ """
38
+ betas = np.linspace(1e-4, max_beta, timesteps)
39
+ return betas
40
+
41
+
42
+ def cosine_beta_schedule(timesteps, s=0.008):
43
+ """
44
+ cosine schedule
45
+ as proposed in https://openreview.net/forum?id=-NEXDKk8gZ
46
+ """
47
+ steps = timesteps + 1
48
+ x = np.linspace(0, steps, steps)
49
+ alphas_cumprod = np.cos(((x / steps) + s) / (1 + s) * np.pi * 0.5) ** 2
50
+ alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
51
+ betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
52
+ return np.clip(betas, a_min=0, a_max=0.999)
53
+
54
+
55
+ beta_schedule = {
56
+ "cosine": cosine_beta_schedule,
57
+ "linear": linear_beta_schedule,
58
+ }
59
+
60
+
61
+ class GaussianDiffusion(nn.Module):
62
+ def __init__(self,
63
+ denoise_fn,
64
+ out_dims=128,
65
+ timesteps=1000,
66
+ k_step=1000,
67
+ max_beta=0.02,
68
+ spec_min=-12,
69
+ spec_max=2):
70
+ super().__init__()
71
+ self.denoise_fn = denoise_fn
72
+ self.out_dims = out_dims
73
+ betas = beta_schedule['linear'](timesteps, max_beta=max_beta)
74
+
75
+ alphas = 1. - betas
76
+ alphas_cumprod = np.cumprod(alphas, axis=0)
77
+ alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1])
78
+
79
+ timesteps, = betas.shape
80
+ self.num_timesteps = int(timesteps)
81
+ self.k_step = k_step
82
+
83
+ self.noise_list = deque(maxlen=4)
84
+
85
+ to_torch = partial(torch.tensor, dtype=torch.float32)
86
+
87
+ self.register_buffer('betas', to_torch(betas))
88
+ self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod))
89
+ self.register_buffer('alphas_cumprod_prev', to_torch(alphas_cumprod_prev))
90
+
91
+ # calculations for diffusion q(x_t | x_{t-1}) and others
92
+ self.register_buffer('sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod)))
93
+ self.register_buffer('sqrt_one_minus_alphas_cumprod', to_torch(np.sqrt(1. - alphas_cumprod)))
94
+ self.register_buffer('log_one_minus_alphas_cumprod', to_torch(np.log(1. - alphas_cumprod)))
95
+ self.register_buffer('sqrt_recip_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod)))
96
+ self.register_buffer('sqrt_recipm1_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod - 1)))
97
+
98
+ # calculations for posterior q(x_{t-1} | x_t, x_0)
99
+ posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod)
100
+ # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t)
101
+ self.register_buffer('posterior_variance', to_torch(posterior_variance))
102
+ # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
103
+ self.register_buffer('posterior_log_variance_clipped', to_torch(np.log(np.maximum(posterior_variance, 1e-20))))
104
+ self.register_buffer('posterior_mean_coef1', to_torch(
105
+ betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)))
106
+ self.register_buffer('posterior_mean_coef2', to_torch(
107
+ (1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod)))
108
+
109
+ self.register_buffer('spec_min', torch.FloatTensor([spec_min])[None, None, :out_dims])
110
+ self.register_buffer('spec_max', torch.FloatTensor([spec_max])[None, None, :out_dims])
111
+
112
+ def q_mean_variance(self, x_start, t):
113
+ mean = extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
114
+ variance = extract(1. - self.alphas_cumprod, t, x_start.shape)
115
+ log_variance = extract(self.log_one_minus_alphas_cumprod, t, x_start.shape)
116
+ return mean, variance, log_variance
117
+
118
+ def predict_start_from_noise(self, x_t, t, noise):
119
+ return (
120
+ extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t -
121
+ extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise
122
+ )
123
+
124
+ def q_posterior(self, x_start, x_t, t):
125
+ posterior_mean = (
126
+ extract(self.posterior_mean_coef1, t, x_t.shape) * x_start +
127
+ extract(self.posterior_mean_coef2, t, x_t.shape) * x_t
128
+ )
129
+ posterior_variance = extract(self.posterior_variance, t, x_t.shape)
130
+ posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape)
131
+ return posterior_mean, posterior_variance, posterior_log_variance_clipped
132
+
133
+ def p_mean_variance(self, x, t, cond):
134
+ noise_pred = self.denoise_fn(x, t, cond=cond)
135
+ x_recon = self.predict_start_from_noise(x, t=t, noise=noise_pred)
136
+
137
+ x_recon.clamp_(-1., 1.)
138
+
139
+ model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start=x_recon, x_t=x, t=t)
140
+ return model_mean, posterior_variance, posterior_log_variance
141
+
142
+ @torch.no_grad()
143
+ def p_sample(self, x, t, cond, clip_denoised=True, repeat_noise=False):
144
+ b, *_, device = *x.shape, x.device
145
+ model_mean, _, model_log_variance = self.p_mean_variance(x=x, t=t, cond=cond)
146
+ noise = noise_like(x.shape, device, repeat_noise)
147
+ # no noise when t == 0
148
+ nonzero_mask = (1 - (t == 0).float()).reshape(b, *((1,) * (len(x.shape) - 1)))
149
+ return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise
150
+
151
+ @torch.no_grad()
152
+ def p_sample_plms(self, x, t, interval, cond, clip_denoised=True, repeat_noise=False):
153
+ """
154
+ Use the PLMS method from
155
+ [Pseudo Numerical Methods for Diffusion Models on Manifolds](https://arxiv.org/abs/2202.09778).
156
+ """
157
+
158
+ def get_x_pred(x, noise_t, t):
159
+ a_t = extract(self.alphas_cumprod, t, x.shape)
160
+ a_prev = extract(self.alphas_cumprod, torch.max(t - interval, torch.zeros_like(t)), x.shape)
161
+ a_t_sq, a_prev_sq = a_t.sqrt(), a_prev.sqrt()
162
+
163
+ x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x - 1 / (
164
+ a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
165
+ x_pred = x + x_delta
166
+
167
+ return x_pred
168
+
169
+ noise_list = self.noise_list
170
+ noise_pred = self.denoise_fn(x, t, cond=cond)
171
+
172
+ if len(noise_list) == 0:
173
+ x_pred = get_x_pred(x, noise_pred, t)
174
+ noise_pred_prev = self.denoise_fn(x_pred, max(t - interval, 0), cond=cond)
175
+ noise_pred_prime = (noise_pred + noise_pred_prev) / 2
176
+ elif len(noise_list) == 1:
177
+ noise_pred_prime = (3 * noise_pred - noise_list[-1]) / 2
178
+ elif len(noise_list) == 2:
179
+ noise_pred_prime = (23 * noise_pred - 16 * noise_list[-1] + 5 * noise_list[-2]) / 12
180
+ else:
181
+ noise_pred_prime = (55 * noise_pred - 59 * noise_list[-1] + 37 * noise_list[-2] - 9 * noise_list[-3]) / 24
182
+
183
+ x_prev = get_x_pred(x, noise_pred_prime, t)
184
+ noise_list.append(noise_pred)
185
+
186
+ return x_prev
187
+
188
+ def q_sample(self, x_start, t, noise=None):
189
+ noise = default(noise, lambda: torch.randn_like(x_start))
190
+ return (
191
+ extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
192
+ extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise
193
+ )
194
+
195
+ def p_losses(self, x_start, t, cond, noise=None, loss_type='l2'):
196
+ noise = default(noise, lambda: torch.randn_like(x_start))
197
+
198
+ x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise)
199
+ x_recon = self.denoise_fn(x_noisy, t, cond)
200
+
201
+ if loss_type == 'l1':
202
+ loss = (noise - x_recon).abs().mean()
203
+ elif loss_type == 'l2':
204
+ loss = F.mse_loss(noise, x_recon)
205
+ else:
206
+ raise NotImplementedError()
207
+
208
+ return loss
209
+
210
+ def forward(self,
211
+ condition,
212
+ gt_spec=None,
213
+ infer=True,
214
+ infer_speedup=10,
215
+ method='dpm-solver',
216
+ k_step=300,
217
+ use_tqdm=True):
218
+ """
219
+ conditioning diffusion, use fastspeech2 encoder output as the condition
220
+ """
221
+ cond = condition.transpose(1, 2)
222
+ b, device = condition.shape[0], condition.device
223
+
224
+ if not infer:
225
+ spec = self.norm_spec(gt_spec)
226
+ t = torch.randint(0, self.k_step, (b,), device=device).long()
227
+ norm_spec = spec.transpose(1, 2)[:, None, :, :] # [B, 1, M, T]
228
+ return self.p_losses(norm_spec, t, cond=cond)
229
+ else:
230
+ shape = (cond.shape[0], 1, self.out_dims, cond.shape[2])
231
+
232
+ if gt_spec is None:
233
+ t = self.k_step
234
+ x = torch.randn(shape, device=device)
235
+ else:
236
+ t = k_step
237
+ norm_spec = self.norm_spec(gt_spec)
238
+ norm_spec = norm_spec.transpose(1, 2)[:, None, :, :]
239
+ x = self.q_sample(x_start=norm_spec, t=torch.tensor([t - 1], device=device).long())
240
+
241
+ if method is not None and infer_speedup > 1:
242
+ if method == 'dpm-solver':
243
+ from .dpm_solver_pytorch import NoiseScheduleVP, model_wrapper, DPM_Solver
244
+ # 1. Define the noise schedule.
245
+ noise_schedule = NoiseScheduleVP(schedule='discrete', betas=self.betas[:t])
246
+
247
+ # 2. Convert your discrete-time `model` to the continuous-time
248
+ # noise prediction model. Here is an example for a diffusion model
249
+ # `model` with the noise prediction type ("noise") .
250
+ def my_wrapper(fn):
251
+ def wrapped(x, t, **kwargs):
252
+ ret = fn(x, t, **kwargs)
253
+ if use_tqdm:
254
+ self.bar.update(1)
255
+ return ret
256
+
257
+ return wrapped
258
+
259
+ model_fn = model_wrapper(
260
+ my_wrapper(self.denoise_fn),
261
+ noise_schedule,
262
+ model_type="noise", # or "x_start" or "v" or "score"
263
+ model_kwargs={"cond": cond}
264
+ )
265
+
266
+ # 3. Define dpm-solver and sample by singlestep DPM-Solver.
267
+ # (We recommend singlestep DPM-Solver for unconditional sampling)
268
+ # You can adjust the `steps` to balance the computation
269
+ # costs and the sample quality.
270
+ dpm_solver = DPM_Solver(model_fn, noise_schedule)
271
+
272
+ steps = t // infer_speedup
273
+ if use_tqdm:
274
+ self.bar = tqdm(desc="sample time step", total=steps)
275
+ x = dpm_solver.sample(
276
+ x,
277
+ steps=steps,
278
+ order=3,
279
+ skip_type="time_uniform",
280
+ method="singlestep",
281
+ )
282
+ if use_tqdm:
283
+ self.bar.close()
284
+ elif method == 'pndm':
285
+ self.noise_list = deque(maxlen=4)
286
+ if use_tqdm:
287
+ for i in tqdm(
288
+ reversed(range(0, t, infer_speedup)), desc='sample time step',
289
+ total=t // infer_speedup,
290
+ ):
291
+ x = self.p_sample_plms(
292
+ x, torch.full((b,), i, device=device, dtype=torch.long),
293
+ infer_speedup, cond=cond
294
+ )
295
+ else:
296
+ for i in reversed(range(0, t, infer_speedup)):
297
+ x = self.p_sample_plms(
298
+ x, torch.full((b,), i, device=device, dtype=torch.long),
299
+ infer_speedup, cond=cond
300
+ )
301
+ else:
302
+ raise NotImplementedError(method)
303
+ else:
304
+ if use_tqdm:
305
+ for i in tqdm(reversed(range(0, t)), desc='sample time step', total=t):
306
+ x = self.p_sample(x, torch.full((b,), i, device=device, dtype=torch.long), cond)
307
+ else:
308
+ for i in reversed(range(0, t)):
309
+ x = self.p_sample(x, torch.full((b,), i, device=device, dtype=torch.long), cond)
310
+ x = x.squeeze(1).transpose(1, 2) # [B, T, M]
311
+ return self.denorm_spec(x)
312
+
313
+ def norm_spec(self, x):
314
+ return (x - self.spec_min) / (self.spec_max - self.spec_min) * 2 - 1
315
+
316
+ def denorm_spec(self, x):
317
+ return (x + 1) / 2 * (self.spec_max - self.spec_min) + self.spec_min
diffusion/diffusion_onnx.py ADDED
@@ -0,0 +1,612 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import deque
2
+ from functools import partial
3
+ from inspect import isfunction
4
+ import torch.nn.functional as F
5
+ import librosa.sequence
6
+ import numpy as np
7
+ from torch.nn import Conv1d
8
+ from torch.nn import Mish
9
+ import torch
10
+ from torch import nn
11
+ from tqdm import tqdm
12
+ import math
13
+
14
+
15
+ def exists(x):
16
+ return x is not None
17
+
18
+
19
+ def default(val, d):
20
+ if exists(val):
21
+ return val
22
+ return d() if isfunction(d) else d
23
+
24
+
25
+ def extract(a, t):
26
+ return a[t].reshape((1, 1, 1, 1))
27
+
28
+
29
+ def noise_like(shape, device, repeat=False):
30
+ repeat_noise = lambda: torch.randn((1, *shape[1:]), device=device).repeat(shape[0], *((1,) * (len(shape) - 1)))
31
+ noise = lambda: torch.randn(shape, device=device)
32
+ return repeat_noise() if repeat else noise()
33
+
34
+
35
+ def linear_beta_schedule(timesteps, max_beta=0.02):
36
+ """
37
+ linear schedule
38
+ """
39
+ betas = np.linspace(1e-4, max_beta, timesteps)
40
+ return betas
41
+
42
+
43
+ def cosine_beta_schedule(timesteps, s=0.008):
44
+ """
45
+ cosine schedule
46
+ as proposed in https://openreview.net/forum?id=-NEXDKk8gZ
47
+ """
48
+ steps = timesteps + 1
49
+ x = np.linspace(0, steps, steps)
50
+ alphas_cumprod = np.cos(((x / steps) + s) / (1 + s) * np.pi * 0.5) ** 2
51
+ alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
52
+ betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
53
+ return np.clip(betas, a_min=0, a_max=0.999)
54
+
55
+
56
+ beta_schedule = {
57
+ "cosine": cosine_beta_schedule,
58
+ "linear": linear_beta_schedule,
59
+ }
60
+
61
+
62
+ def extract_1(a, t):
63
+ return a[t].reshape((1, 1, 1, 1))
64
+
65
+
66
+ def predict_stage0(noise_pred, noise_pred_prev):
67
+ return (noise_pred + noise_pred_prev) / 2
68
+
69
+
70
+ def predict_stage1(noise_pred, noise_list):
71
+ return (noise_pred * 3
72
+ - noise_list[-1]) / 2
73
+
74
+
75
+ def predict_stage2(noise_pred, noise_list):
76
+ return (noise_pred * 23
77
+ - noise_list[-1] * 16
78
+ + noise_list[-2] * 5) / 12
79
+
80
+
81
+ def predict_stage3(noise_pred, noise_list):
82
+ return (noise_pred * 55
83
+ - noise_list[-1] * 59
84
+ + noise_list[-2] * 37
85
+ - noise_list[-3] * 9) / 24
86
+
87
+
88
+ class SinusoidalPosEmb(nn.Module):
89
+ def __init__(self, dim):
90
+ super().__init__()
91
+ self.dim = dim
92
+ self.half_dim = dim // 2
93
+ self.emb = 9.21034037 / (self.half_dim - 1)
94
+ self.emb = torch.exp(torch.arange(self.half_dim) * torch.tensor(-self.emb)).unsqueeze(0)
95
+ self.emb = self.emb.cpu()
96
+
97
+ def forward(self, x):
98
+ emb = self.emb * x
99
+ emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
100
+ return emb
101
+
102
+
103
+ class ResidualBlock(nn.Module):
104
+ def __init__(self, encoder_hidden, residual_channels, dilation):
105
+ super().__init__()
106
+ self.residual_channels = residual_channels
107
+ self.dilated_conv = Conv1d(residual_channels, 2 * residual_channels, 3, padding=dilation, dilation=dilation)
108
+ self.diffusion_projection = nn.Linear(residual_channels, residual_channels)
109
+ self.conditioner_projection = Conv1d(encoder_hidden, 2 * residual_channels, 1)
110
+ self.output_projection = Conv1d(residual_channels, 2 * residual_channels, 1)
111
+
112
+ def forward(self, x, conditioner, diffusion_step):
113
+ diffusion_step = self.diffusion_projection(diffusion_step).unsqueeze(-1)
114
+ conditioner = self.conditioner_projection(conditioner)
115
+ y = x + diffusion_step
116
+ y = self.dilated_conv(y) + conditioner
117
+
118
+ gate, filter_1 = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
119
+
120
+ y = torch.sigmoid(gate) * torch.tanh(filter_1)
121
+ y = self.output_projection(y)
122
+
123
+ residual, skip = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
124
+
125
+ return (x + residual) / 1.41421356, skip
126
+
127
+
128
+ class DiffNet(nn.Module):
129
+ def __init__(self, in_dims, n_layers, n_chans, n_hidden):
130
+ super().__init__()
131
+ self.encoder_hidden = n_hidden
132
+ self.residual_layers = n_layers
133
+ self.residual_channels = n_chans
134
+ self.input_projection = Conv1d(in_dims, self.residual_channels, 1)
135
+ self.diffusion_embedding = SinusoidalPosEmb(self.residual_channels)
136
+ dim = self.residual_channels
137
+ self.mlp = nn.Sequential(
138
+ nn.Linear(dim, dim * 4),
139
+ Mish(),
140
+ nn.Linear(dim * 4, dim)
141
+ )
142
+ self.residual_layers = nn.ModuleList([
143
+ ResidualBlock(self.encoder_hidden, self.residual_channels, 1)
144
+ for i in range(self.residual_layers)
145
+ ])
146
+ self.skip_projection = Conv1d(self.residual_channels, self.residual_channels, 1)
147
+ self.output_projection = Conv1d(self.residual_channels, in_dims, 1)
148
+ nn.init.zeros_(self.output_projection.weight)
149
+
150
+ def forward(self, spec, diffusion_step, cond):
151
+ x = spec.squeeze(0)
152
+ x = self.input_projection(x) # x [B, residual_channel, T]
153
+ x = F.relu(x)
154
+ # skip = torch.randn_like(x)
155
+ diffusion_step = diffusion_step.float()
156
+ diffusion_step = self.diffusion_embedding(diffusion_step)
157
+ diffusion_step = self.mlp(diffusion_step)
158
+
159
+ x, skip = self.residual_layers[0](x, cond, diffusion_step)
160
+ # noinspection PyTypeChecker
161
+ for layer in self.residual_layers[1:]:
162
+ x, skip_connection = layer.forward(x, cond, diffusion_step)
163
+ skip = skip + skip_connection
164
+ x = skip / math.sqrt(len(self.residual_layers))
165
+ x = self.skip_projection(x)
166
+ x = F.relu(x)
167
+ x = self.output_projection(x) # [B, 80, T]
168
+ return x.unsqueeze(1)
169
+
170
+
171
+ class AfterDiffusion(nn.Module):
172
+ def __init__(self, spec_max, spec_min, v_type='a'):
173
+ super().__init__()
174
+ self.spec_max = spec_max
175
+ self.spec_min = spec_min
176
+ self.type = v_type
177
+
178
+ def forward(self, x):
179
+ x = x.squeeze(1).permute(0, 2, 1)
180
+ mel_out = (x + 1) / 2 * (self.spec_max - self.spec_min) + self.spec_min
181
+ if self.type == 'nsf-hifigan-log10':
182
+ mel_out = mel_out * 0.434294
183
+ return mel_out.transpose(2, 1)
184
+
185
+
186
+ class Pred(nn.Module):
187
+ def __init__(self, alphas_cumprod):
188
+ super().__init__()
189
+ self.alphas_cumprod = alphas_cumprod
190
+
191
+ def forward(self, x_1, noise_t, t_1, t_prev):
192
+ a_t = extract(self.alphas_cumprod, t_1).cpu()
193
+ a_prev = extract(self.alphas_cumprod, t_prev).cpu()
194
+ a_t_sq, a_prev_sq = a_t.sqrt().cpu(), a_prev.sqrt().cpu()
195
+ x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x_1 - 1 / (
196
+ a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
197
+ x_pred = x_1 + x_delta.cpu()
198
+
199
+ return x_pred
200
+
201
+
202
+ class GaussianDiffusion(nn.Module):
203
+ def __init__(self,
204
+ out_dims=128,
205
+ n_layers=20,
206
+ n_chans=384,
207
+ n_hidden=256,
208
+ timesteps=1000,
209
+ k_step=1000,
210
+ max_beta=0.02,
211
+ spec_min=-12,
212
+ spec_max=2):
213
+ super().__init__()
214
+ self.denoise_fn = DiffNet(out_dims, n_layers, n_chans, n_hidden)
215
+ self.out_dims = out_dims
216
+ self.mel_bins = out_dims
217
+ self.n_hidden = n_hidden
218
+ betas = beta_schedule['linear'](timesteps, max_beta=max_beta)
219
+
220
+ alphas = 1. - betas
221
+ alphas_cumprod = np.cumprod(alphas, axis=0)
222
+ alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1])
223
+ timesteps, = betas.shape
224
+ self.num_timesteps = int(timesteps)
225
+ self.k_step = k_step
226
+
227
+ self.noise_list = deque(maxlen=4)
228
+
229
+ to_torch = partial(torch.tensor, dtype=torch.float32)
230
+
231
+ self.register_buffer('betas', to_torch(betas))
232
+ self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod))
233
+ self.register_buffer('alphas_cumprod_prev', to_torch(alphas_cumprod_prev))
234
+
235
+ # calculations for diffusion q(x_t | x_{t-1}) and others
236
+ self.register_buffer('sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod)))
237
+ self.register_buffer('sqrt_one_minus_alphas_cumprod', to_torch(np.sqrt(1. - alphas_cumprod)))
238
+ self.register_buffer('log_one_minus_alphas_cumprod', to_torch(np.log(1. - alphas_cumprod)))
239
+ self.register_buffer('sqrt_recip_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod)))
240
+ self.register_buffer('sqrt_recipm1_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod - 1)))
241
+
242
+ # calculations for posterior q(x_{t-1} | x_t, x_0)
243
+ posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod)
244
+ # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t)
245
+ self.register_buffer('posterior_variance', to_torch(posterior_variance))
246
+ # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
247
+ self.register_buffer('posterior_log_variance_clipped', to_torch(np.log(np.maximum(posterior_variance, 1e-20))))
248
+ self.register_buffer('posterior_mean_coef1', to_torch(
249
+ betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)))
250
+ self.register_buffer('posterior_mean_coef2', to_torch(
251
+ (1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod)))
252
+
253
+ self.register_buffer('spec_min', torch.FloatTensor([spec_min])[None, None, :out_dims])
254
+ self.register_buffer('spec_max', torch.FloatTensor([spec_max])[None, None, :out_dims])
255
+ self.ad = AfterDiffusion(self.spec_max, self.spec_min)
256
+ self.xp = Pred(self.alphas_cumprod)
257
+
258
+ def q_mean_variance(self, x_start, t):
259
+ mean = extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
260
+ variance = extract(1. - self.alphas_cumprod, t, x_start.shape)
261
+ log_variance = extract(self.log_one_minus_alphas_cumprod, t, x_start.shape)
262
+ return mean, variance, log_variance
263
+
264
+ def predict_start_from_noise(self, x_t, t, noise):
265
+ return (
266
+ extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t -
267
+ extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise
268
+ )
269
+
270
+ def q_posterior(self, x_start, x_t, t):
271
+ posterior_mean = (
272
+ extract(self.posterior_mean_coef1, t, x_t.shape) * x_start +
273
+ extract(self.posterior_mean_coef2, t, x_t.shape) * x_t
274
+ )
275
+ posterior_variance = extract(self.posterior_variance, t, x_t.shape)
276
+ posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape)
277
+ return posterior_mean, posterior_variance, posterior_log_variance_clipped
278
+
279
+ def p_mean_variance(self, x, t, cond):
280
+ noise_pred = self.denoise_fn(x, t, cond=cond)
281
+ x_recon = self.predict_start_from_noise(x, t=t, noise=noise_pred)
282
+
283
+ x_recon.clamp_(-1., 1.)
284
+
285
+ model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start=x_recon, x_t=x, t=t)
286
+ return model_mean, posterior_variance, posterior_log_variance
287
+
288
+ @torch.no_grad()
289
+ def p_sample(self, x, t, cond, clip_denoised=True, repeat_noise=False):
290
+ b, *_, device = *x.shape, x.device
291
+ model_mean, _, model_log_variance = self.p_mean_variance(x=x, t=t, cond=cond)
292
+ noise = noise_like(x.shape, device, repeat_noise)
293
+ # no noise when t == 0
294
+ nonzero_mask = (1 - (t == 0).float()).reshape(b, *((1,) * (len(x.shape) - 1)))
295
+ return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise
296
+
297
+ @torch.no_grad()
298
+ def p_sample_plms(self, x, t, interval, cond, clip_denoised=True, repeat_noise=False):
299
+ """
300
+ Use the PLMS method from
301
+ [Pseudo Numerical Methods for Diffusion Models on Manifolds](https://arxiv.org/abs/2202.09778).
302
+ """
303
+
304
+ def get_x_pred(x, noise_t, t):
305
+ a_t = extract(self.alphas_cumprod, t)
306
+ a_prev = extract(self.alphas_cumprod, torch.max(t - interval, torch.zeros_like(t)))
307
+ a_t_sq, a_prev_sq = a_t.sqrt(), a_prev.sqrt()
308
+
309
+ x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x - 1 / (
310
+ a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
311
+ x_pred = x + x_delta
312
+
313
+ return x_pred
314
+
315
+ noise_list = self.noise_list
316
+ noise_pred = self.denoise_fn(x, t, cond=cond)
317
+
318
+ if len(noise_list) == 0:
319
+ x_pred = get_x_pred(x, noise_pred, t)
320
+ noise_pred_prev = self.denoise_fn(x_pred, max(t - interval, 0), cond=cond)
321
+ noise_pred_prime = (noise_pred + noise_pred_prev) / 2
322
+ elif len(noise_list) == 1:
323
+ noise_pred_prime = (3 * noise_pred - noise_list[-1]) / 2
324
+ elif len(noise_list) == 2:
325
+ noise_pred_prime = (23 * noise_pred - 16 * noise_list[-1] + 5 * noise_list[-2]) / 12
326
+ else:
327
+ noise_pred_prime = (55 * noise_pred - 59 * noise_list[-1] + 37 * noise_list[-2] - 9 * noise_list[-3]) / 24
328
+
329
+ x_prev = get_x_pred(x, noise_pred_prime, t)
330
+ noise_list.append(noise_pred)
331
+
332
+ return x_prev
333
+
334
+ def q_sample(self, x_start, t, noise=None):
335
+ noise = default(noise, lambda: torch.randn_like(x_start))
336
+ return (
337
+ extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
338
+ extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise
339
+ )
340
+
341
+ def p_losses(self, x_start, t, cond, noise=None, loss_type='l2'):
342
+ noise = default(noise, lambda: torch.randn_like(x_start))
343
+
344
+ x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise)
345
+ x_recon = self.denoise_fn(x_noisy, t, cond)
346
+
347
+ if loss_type == 'l1':
348
+ loss = (noise - x_recon).abs().mean()
349
+ elif loss_type == 'l2':
350
+ loss = F.mse_loss(noise, x_recon)
351
+ else:
352
+ raise NotImplementedError()
353
+
354
+ return loss
355
+
356
+ def org_forward(self,
357
+ condition,
358
+ init_noise=None,
359
+ gt_spec=None,
360
+ infer=True,
361
+ infer_speedup=100,
362
+ method='pndm',
363
+ k_step=1000,
364
+ use_tqdm=True):
365
+ """
366
+ conditioning diffusion, use fastspeech2 encoder output as the condition
367
+ """
368
+ cond = condition
369
+ b, device = condition.shape[0], condition.device
370
+ if not infer:
371
+ spec = self.norm_spec(gt_spec)
372
+ t = torch.randint(0, self.k_step, (b,), device=device).long()
373
+ norm_spec = spec.transpose(1, 2)[:, None, :, :] # [B, 1, M, T]
374
+ return self.p_losses(norm_spec, t, cond=cond)
375
+ else:
376
+ shape = (cond.shape[0], 1, self.out_dims, cond.shape[2])
377
+
378
+ if gt_spec is None:
379
+ t = self.k_step
380
+ if init_noise is None:
381
+ x = torch.randn(shape, device=device)
382
+ else:
383
+ x = init_noise
384
+ else:
385
+ t = k_step
386
+ norm_spec = self.norm_spec(gt_spec)
387
+ norm_spec = norm_spec.transpose(1, 2)[:, None, :, :]
388
+ x = self.q_sample(x_start=norm_spec, t=torch.tensor([t - 1], device=device).long())
389
+
390
+ if method is not None and infer_speedup > 1:
391
+ if method == 'dpm-solver':
392
+ from .dpm_solver_pytorch import NoiseScheduleVP, model_wrapper, DPM_Solver
393
+ # 1. Define the noise schedule.
394
+ noise_schedule = NoiseScheduleVP(schedule='discrete', betas=self.betas[:t])
395
+
396
+ # 2. Convert your discrete-time `model` to the continuous-time
397
+ # noise prediction model. Here is an example for a diffusion model
398
+ # `model` with the noise prediction type ("noise") .
399
+ def my_wrapper(fn):
400
+ def wrapped(x, t, **kwargs):
401
+ ret = fn(x, t, **kwargs)
402
+ if use_tqdm:
403
+ self.bar.update(1)
404
+ return ret
405
+
406
+ return wrapped
407
+
408
+ model_fn = model_wrapper(
409
+ my_wrapper(self.denoise_fn),
410
+ noise_schedule,
411
+ model_type="noise", # or "x_start" or "v" or "score"
412
+ model_kwargs={"cond": cond}
413
+ )
414
+
415
+ # 3. Define dpm-solver and sample by singlestep DPM-Solver.
416
+ # (We recommend singlestep DPM-Solver for unconditional sampling)
417
+ # You can adjust the `steps` to balance the computation
418
+ # costs and the sample quality.
419
+ dpm_solver = DPM_Solver(model_fn, noise_schedule)
420
+
421
+ steps = t // infer_speedup
422
+ if use_tqdm:
423
+ self.bar = tqdm(desc="sample time step", total=steps)
424
+ x = dpm_solver.sample(
425
+ x,
426
+ steps=steps,
427
+ order=3,
428
+ skip_type="time_uniform",
429
+ method="singlestep",
430
+ )
431
+ if use_tqdm:
432
+ self.bar.close()
433
+ elif method == 'pndm':
434
+ self.noise_list = deque(maxlen=4)
435
+ if use_tqdm:
436
+ for i in tqdm(
437
+ reversed(range(0, t, infer_speedup)), desc='sample time step',
438
+ total=t // infer_speedup,
439
+ ):
440
+ x = self.p_sample_plms(
441
+ x, torch.full((b,), i, device=device, dtype=torch.long),
442
+ infer_speedup, cond=cond
443
+ )
444
+ else:
445
+ for i in reversed(range(0, t, infer_speedup)):
446
+ x = self.p_sample_plms(
447
+ x, torch.full((b,), i, device=device, dtype=torch.long),
448
+ infer_speedup, cond=cond
449
+ )
450
+ else:
451
+ raise NotImplementedError(method)
452
+ else:
453
+ if use_tqdm:
454
+ for i in tqdm(reversed(range(0, t)), desc='sample time step', total=t):
455
+ x = self.p_sample(x, torch.full((b,), i, device=device, dtype=torch.long), cond)
456
+ else:
457
+ for i in reversed(range(0, t)):
458
+ x = self.p_sample(x, torch.full((b,), i, device=device, dtype=torch.long), cond)
459
+ x = x.squeeze(1).transpose(1, 2) # [B, T, M]
460
+ return self.denorm_spec(x).transpose(2, 1)
461
+
462
+ def norm_spec(self, x):
463
+ return (x - self.spec_min) / (self.spec_max - self.spec_min) * 2 - 1
464
+
465
+ def denorm_spec(self, x):
466
+ return (x + 1) / 2 * (self.spec_max - self.spec_min) + self.spec_min
467
+
468
+ def get_x_pred(self, x_1, noise_t, t_1, t_prev):
469
+ a_t = extract(self.alphas_cumprod, t_1)
470
+ a_prev = extract(self.alphas_cumprod, t_prev)
471
+ a_t_sq, a_prev_sq = a_t.sqrt(), a_prev.sqrt()
472
+ x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x_1 - 1 / (
473
+ a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
474
+ x_pred = x_1 + x_delta
475
+ return x_pred
476
+
477
+ def OnnxExport(self, project_name=None, init_noise=None, hidden_channels=256, export_denoise=True, export_pred=True, export_after=True):
478
+ cond = torch.randn([1, self.n_hidden, 10]).cpu()
479
+ if init_noise is None:
480
+ x = torch.randn((1, 1, self.mel_bins, cond.shape[2]), dtype=torch.float32).cpu()
481
+ else:
482
+ x = init_noise
483
+ pndms = 100
484
+
485
+ org_y_x = self.org_forward(cond, init_noise=x)
486
+
487
+ device = cond.device
488
+ n_frames = cond.shape[2]
489
+ step_range = torch.arange(0, self.k_step, pndms, dtype=torch.long, device=device).flip(0)
490
+ plms_noise_stage = torch.tensor(0, dtype=torch.long, device=device)
491
+ noise_list = torch.zeros((0, 1, 1, self.mel_bins, n_frames), device=device)
492
+
493
+ ot = step_range[0]
494
+ ot_1 = torch.full((1,), ot, device=device, dtype=torch.long)
495
+ if export_denoise:
496
+ torch.onnx.export(
497
+ self.denoise_fn,
498
+ (x.cpu(), ot_1.cpu(), cond.cpu()),
499
+ f"{project_name}_denoise.onnx",
500
+ input_names=["noise", "time", "condition"],
501
+ output_names=["noise_pred"],
502
+ dynamic_axes={
503
+ "noise": [3],
504
+ "condition": [2]
505
+ },
506
+ opset_version=16
507
+ )
508
+
509
+ for t in step_range:
510
+ t_1 = torch.full((1,), t, device=device, dtype=torch.long)
511
+ noise_pred = self.denoise_fn(x, t_1, cond)
512
+ t_prev = t_1 - pndms
513
+ t_prev = t_prev * (t_prev > 0)
514
+ if plms_noise_stage == 0:
515
+ if export_pred:
516
+ torch.onnx.export(
517
+ self.xp,
518
+ (x.cpu(), noise_pred.cpu(), t_1.cpu(), t_prev.cpu()),
519
+ f"{project_name}_pred.onnx",
520
+ input_names=["noise", "noise_pred", "time", "time_prev"],
521
+ output_names=["noise_pred_o"],
522
+ dynamic_axes={
523
+ "noise": [3],
524
+ "noise_pred": [3]
525
+ },
526
+ opset_version=16
527
+ )
528
+
529
+ x_pred = self.get_x_pred(x, noise_pred, t_1, t_prev)
530
+ noise_pred_prev = self.denoise_fn(x_pred, t_prev, cond=cond)
531
+ noise_pred_prime = predict_stage0(noise_pred, noise_pred_prev)
532
+
533
+ elif plms_noise_stage == 1:
534
+ noise_pred_prime = predict_stage1(noise_pred, noise_list)
535
+
536
+ elif plms_noise_stage == 2:
537
+ noise_pred_prime = predict_stage2(noise_pred, noise_list)
538
+
539
+ else:
540
+ noise_pred_prime = predict_stage3(noise_pred, noise_list)
541
+
542
+ noise_pred = noise_pred.unsqueeze(0)
543
+
544
+ if plms_noise_stage < 3:
545
+ noise_list = torch.cat((noise_list, noise_pred), dim=0)
546
+ plms_noise_stage = plms_noise_stage + 1
547
+
548
+ else:
549
+ noise_list = torch.cat((noise_list[-2:], noise_pred), dim=0)
550
+
551
+ x = self.get_x_pred(x, noise_pred_prime, t_1, t_prev)
552
+ if export_after:
553
+ torch.onnx.export(
554
+ self.ad,
555
+ x.cpu(),
556
+ f"{project_name}_after.onnx",
557
+ input_names=["x"],
558
+ output_names=["mel_out"],
559
+ dynamic_axes={
560
+ "x": [3]
561
+ },
562
+ opset_version=16
563
+ )
564
+ x = self.ad(x)
565
+
566
+ print((x == org_y_x).all())
567
+ return x
568
+
569
+ def forward(self, condition=None, init_noise=None, pndms=None, k_step=None):
570
+ cond = condition
571
+ x = init_noise
572
+
573
+ device = cond.device
574
+ n_frames = cond.shape[2]
575
+ step_range = torch.arange(0, k_step.item(), pndms.item(), dtype=torch.long, device=device).flip(0)
576
+ plms_noise_stage = torch.tensor(0, dtype=torch.long, device=device)
577
+ noise_list = torch.zeros((0, 1, 1, self.mel_bins, n_frames), device=device)
578
+
579
+ ot = step_range[0]
580
+ ot_1 = torch.full((1,), ot, device=device, dtype=torch.long)
581
+
582
+ for t in step_range:
583
+ t_1 = torch.full((1,), t, device=device, dtype=torch.long)
584
+ noise_pred = self.denoise_fn(x, t_1, cond)
585
+ t_prev = t_1 - pndms
586
+ t_prev = t_prev * (t_prev > 0)
587
+ if plms_noise_stage == 0:
588
+ x_pred = self.get_x_pred(x, noise_pred, t_1, t_prev)
589
+ noise_pred_prev = self.denoise_fn(x_pred, t_prev, cond=cond)
590
+ noise_pred_prime = predict_stage0(noise_pred, noise_pred_prev)
591
+
592
+ elif plms_noise_stage == 1:
593
+ noise_pred_prime = predict_stage1(noise_pred, noise_list)
594
+
595
+ elif plms_noise_stage == 2:
596
+ noise_pred_prime = predict_stage2(noise_pred, noise_list)
597
+
598
+ else:
599
+ noise_pred_prime = predict_stage3(noise_pred, noise_list)
600
+
601
+ noise_pred = noise_pred.unsqueeze(0)
602
+
603
+ if plms_noise_stage < 3:
604
+ noise_list = torch.cat((noise_list, noise_pred), dim=0)
605
+ plms_noise_stage = plms_noise_stage + 1
606
+
607
+ else:
608
+ noise_list = torch.cat((noise_list[-2:], noise_pred), dim=0)
609
+
610
+ x = self.get_x_pred(x, noise_pred_prime, t_1, t_prev)
611
+ x = self.ad(x)
612
+ return x
diffusion/dpm_solver_pytorch.py ADDED
@@ -0,0 +1,1201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+
3
+ import torch
4
+
5
+
6
+ class NoiseScheduleVP:
7
+ def __init__(
8
+ self,
9
+ schedule='discrete',
10
+ betas=None,
11
+ alphas_cumprod=None,
12
+ continuous_beta_0=0.1,
13
+ continuous_beta_1=20.,
14
+ ):
15
+ """Create a wrapper class for the forward SDE (VP type).
16
+
17
+ ***
18
+ Update: We support discrete-time diffusion models by implementing a picewise linear interpolation for log_alpha_t.
19
+ We recommend to use schedule='discrete' for the discrete-time diffusion models, especially for high-resolution images.
20
+ ***
21
+
22
+ The forward SDE ensures that the condition distribution q_{t|0}(x_t | x_0) = N ( alpha_t * x_0, sigma_t^2 * I ).
23
+ We further define lambda_t = log(alpha_t) - log(sigma_t), which is the half-logSNR (described in the DPM-Solver paper).
24
+ Therefore, we implement the functions for computing alpha_t, sigma_t and lambda_t. For t in [0, T], we have:
25
+
26
+ log_alpha_t = self.marginal_log_mean_coeff(t)
27
+ sigma_t = self.marginal_std(t)
28
+ lambda_t = self.marginal_lambda(t)
29
+
30
+ Moreover, as lambda(t) is an invertible function, we also support its inverse function:
31
+
32
+ t = self.inverse_lambda(lambda_t)
33
+
34
+ ===============================================================
35
+
36
+ We support both discrete-time DPMs (trained on n = 0, 1, ..., N-1) and continuous-time DPMs (trained on t in [t_0, T]).
37
+
38
+ 1. For discrete-time DPMs:
39
+
40
+ For discrete-time DPMs trained on n = 0, 1, ..., N-1, we convert the discrete steps to continuous time steps by:
41
+ t_i = (i + 1) / N
42
+ e.g. for N = 1000, we have t_0 = 1e-3 and T = t_{N-1} = 1.
43
+ We solve the corresponding diffusion ODE from time T = 1 to time t_0 = 1e-3.
44
+
45
+ Args:
46
+ betas: A `torch.Tensor`. The beta array for the discrete-time DPM. (See the original DDPM paper for details)
47
+ alphas_cumprod: A `torch.Tensor`. The cumprod alphas for the discrete-time DPM. (See the original DDPM paper for details)
48
+
49
+ Note that we always have alphas_cumprod = cumprod(betas). Therefore, we only need to set one of `betas` and `alphas_cumprod`.
50
+
51
+ **Important**: Please pay special attention for the args for `alphas_cumprod`:
52
+ The `alphas_cumprod` is the \hat{alpha_n} arrays in the notations of DDPM. Specifically, DDPMs assume that
53
+ q_{t_n | 0}(x_{t_n} | x_0) = N ( \sqrt{\hat{alpha_n}} * x_0, (1 - \hat{alpha_n}) * I ).
54
+ Therefore, the notation \hat{alpha_n} is different from the notation alpha_t in DPM-Solver. In fact, we have
55
+ alpha_{t_n} = \sqrt{\hat{alpha_n}},
56
+ and
57
+ log(alpha_{t_n}) = 0.5 * log(\hat{alpha_n}).
58
+
59
+
60
+ 2. For continuous-time DPMs:
61
+
62
+ We support two types of VPSDEs: linear (DDPM) and cosine (improved-DDPM). The hyperparameters for the noise
63
+ schedule are the default settings in DDPM and improved-DDPM:
64
+
65
+ Args:
66
+ beta_min: A `float` number. The smallest beta for the linear schedule.
67
+ beta_max: A `float` number. The largest beta for the linear schedule.
68
+ cosine_s: A `float` number. The hyperparameter in the cosine schedule.
69
+ cosine_beta_max: A `float` number. The hyperparameter in the cosine schedule.
70
+ T: A `float` number. The ending time of the forward process.
71
+
72
+ ===============================================================
73
+
74
+ Args:
75
+ schedule: A `str`. The noise schedule of the forward SDE. 'discrete' for discrete-time DPMs,
76
+ 'linear' or 'cosine' for continuous-time DPMs.
77
+ Returns:
78
+ A wrapper object of the forward SDE (VP type).
79
+
80
+ ===============================================================
81
+
82
+ Example:
83
+
84
+ # For discrete-time DPMs, given betas (the beta array for n = 0, 1, ..., N - 1):
85
+ >>> ns = NoiseScheduleVP('discrete', betas=betas)
86
+
87
+ # For discrete-time DPMs, given alphas_cumprod (the \hat{alpha_n} array for n = 0, 1, ..., N - 1):
88
+ >>> ns = NoiseScheduleVP('discrete', alphas_cumprod=alphas_cumprod)
89
+
90
+ # For continuous-time DPMs (VPSDE), linear schedule:
91
+ >>> ns = NoiseScheduleVP('linear', continuous_beta_0=0.1, continuous_beta_1=20.)
92
+
93
+ """
94
+
95
+ if schedule not in ['discrete', 'linear', 'cosine']:
96
+ raise ValueError(
97
+ "Unsupported noise schedule {}. The schedule needs to be 'discrete' or 'linear' or 'cosine'".format(
98
+ schedule))
99
+
100
+ self.schedule = schedule
101
+ if schedule == 'discrete':
102
+ if betas is not None:
103
+ log_alphas = 0.5 * torch.log(1 - betas).cumsum(dim=0)
104
+ else:
105
+ assert alphas_cumprod is not None
106
+ log_alphas = 0.5 * torch.log(alphas_cumprod)
107
+ self.total_N = len(log_alphas)
108
+ self.T = 1.
109
+ self.t_array = torch.linspace(0., 1., self.total_N + 1)[1:].reshape((1, -1))
110
+ self.log_alpha_array = log_alphas.reshape((1, -1,))
111
+ else:
112
+ self.total_N = 1000
113
+ self.beta_0 = continuous_beta_0
114
+ self.beta_1 = continuous_beta_1
115
+ self.cosine_s = 0.008
116
+ self.cosine_beta_max = 999.
117
+ self.cosine_t_max = math.atan(self.cosine_beta_max * (1. + self.cosine_s) / math.pi) * 2. * (
118
+ 1. + self.cosine_s) / math.pi - self.cosine_s
119
+ self.cosine_log_alpha_0 = math.log(math.cos(self.cosine_s / (1. + self.cosine_s) * math.pi / 2.))
120
+ self.schedule = schedule
121
+ if schedule == 'cosine':
122
+ # For the cosine schedule, T = 1 will have numerical issues. So we manually set the ending time T.
123
+ # Note that T = 0.9946 may be not the optimal setting. However, we find it works well.
124
+ self.T = 0.9946
125
+ else:
126
+ self.T = 1.
127
+
128
+ def marginal_log_mean_coeff(self, t):
129
+ """
130
+ Compute log(alpha_t) of a given continuous-time label t in [0, T].
131
+ """
132
+ if self.schedule == 'discrete':
133
+ return interpolate_fn(t.reshape((-1, 1)), self.t_array.to(t.device),
134
+ self.log_alpha_array.to(t.device)).reshape((-1))
135
+ elif self.schedule == 'linear':
136
+ return -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0
137
+ elif self.schedule == 'cosine':
138
+ log_alpha_fn = lambda s: torch.log(torch.cos((s + self.cosine_s) / (1. + self.cosine_s) * math.pi / 2.))
139
+ log_alpha_t = log_alpha_fn(t) - self.cosine_log_alpha_0
140
+ return log_alpha_t
141
+
142
+ def marginal_alpha(self, t):
143
+ """
144
+ Compute alpha_t of a given continuous-time label t in [0, T].
145
+ """
146
+ return torch.exp(self.marginal_log_mean_coeff(t))
147
+
148
+ def marginal_std(self, t):
149
+ """
150
+ Compute sigma_t of a given continuous-time label t in [0, T].
151
+ """
152
+ return torch.sqrt(1. - torch.exp(2. * self.marginal_log_mean_coeff(t)))
153
+
154
+ def marginal_lambda(self, t):
155
+ """
156
+ Compute lambda_t = log(alpha_t) - log(sigma_t) of a given continuous-time label t in [0, T].
157
+ """
158
+ log_mean_coeff = self.marginal_log_mean_coeff(t)
159
+ log_std = 0.5 * torch.log(1. - torch.exp(2. * log_mean_coeff))
160
+ return log_mean_coeff - log_std
161
+
162
+ def inverse_lambda(self, lamb):
163
+ """
164
+ Compute the continuous-time label t in [0, T] of a given half-logSNR lambda_t.
165
+ """
166
+ if self.schedule == 'linear':
167
+ tmp = 2. * (self.beta_1 - self.beta_0) * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
168
+ Delta = self.beta_0 ** 2 + tmp
169
+ return tmp / (torch.sqrt(Delta) + self.beta_0) / (self.beta_1 - self.beta_0)
170
+ elif self.schedule == 'discrete':
171
+ log_alpha = -0.5 * torch.logaddexp(torch.zeros((1,)).to(lamb.device), -2. * lamb)
172
+ t = interpolate_fn(log_alpha.reshape((-1, 1)), torch.flip(self.log_alpha_array.to(lamb.device), [1]),
173
+ torch.flip(self.t_array.to(lamb.device), [1]))
174
+ return t.reshape((-1,))
175
+ else:
176
+ log_alpha = -0.5 * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
177
+ t_fn = lambda log_alpha_t: torch.arccos(torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2. * (
178
+ 1. + self.cosine_s) / math.pi - self.cosine_s
179
+ t = t_fn(log_alpha)
180
+ return t
181
+
182
+
183
+ def model_wrapper(
184
+ model,
185
+ noise_schedule,
186
+ model_type="noise",
187
+ model_kwargs={},
188
+ guidance_type="uncond",
189
+ condition=None,
190
+ unconditional_condition=None,
191
+ guidance_scale=1.,
192
+ classifier_fn=None,
193
+ classifier_kwargs={},
194
+ ):
195
+ """Create a wrapper function for the noise prediction model.
196
+
197
+ DPM-Solver needs to solve the continuous-time diffusion ODEs. For DPMs trained on discrete-time labels, we need to
198
+ firstly wrap the model function to a noise prediction model that accepts the continuous time as the input.
199
+
200
+ We support four types of the diffusion model by setting `model_type`:
201
+
202
+ 1. "noise": noise prediction model. (Trained by predicting noise).
203
+
204
+ 2. "x_start": data prediction model. (Trained by predicting the data x_0 at time 0).
205
+
206
+ 3. "v": velocity prediction model. (Trained by predicting the velocity).
207
+ The "v" prediction is derivation detailed in Appendix D of [1], and is used in Imagen-Video [2].
208
+
209
+ [1] Salimans, Tim, and Jonathan Ho. "Progressive distillation for fast sampling of diffusion models."
210
+ arXiv preprint arXiv:2202.00512 (2022).
211
+ [2] Ho, Jonathan, et al. "Imagen Video: High Definition Video Generation with Diffusion Models."
212
+ arXiv preprint arXiv:2210.02303 (2022).
213
+
214
+ 4. "score": marginal score function. (Trained by denoising score matching).
215
+ Note that the score function and the noise prediction model follows a simple relationship:
216
+ ```
217
+ noise(x_t, t) = -sigma_t * score(x_t, t)
218
+ ```
219
+
220
+ We support three types of guided sampling by DPMs by setting `guidance_type`:
221
+ 1. "uncond": unconditional sampling by DPMs.
222
+ The input `model` has the following format:
223
+ ``
224
+ model(x, t_input, **model_kwargs) -> noise | x_start | v | score
225
+ ``
226
+
227
+ 2. "classifier": classifier guidance sampling [3] by DPMs and another classifier.
228
+ The input `model` has the following format:
229
+ ``
230
+ model(x, t_input, **model_kwargs) -> noise | x_start | v | score
231
+ ``
232
+
233
+ The input `classifier_fn` has the following format:
234
+ ``
235
+ classifier_fn(x, t_input, cond, **classifier_kwargs) -> logits(x, t_input, cond)
236
+ ``
237
+
238
+ [3] P. Dhariwal and A. Q. Nichol, "Diffusion models beat GANs on image synthesis,"
239
+ in Advances in Neural Information Processing Systems, vol. 34, 2021, pp. 8780-8794.
240
+
241
+ 3. "classifier-free": classifier-free guidance sampling by conditional DPMs.
242
+ The input `model` has the following format:
243
+ ``
244
+ model(x, t_input, cond, **model_kwargs) -> noise | x_start | v | score
245
+ ``
246
+ And if cond == `unconditional_condition`, the model output is the unconditional DPM output.
247
+
248
+ [4] Ho, Jonathan, and Tim Salimans. "Classifier-free diffusion guidance."
249
+ arXiv preprint arXiv:2207.12598 (2022).
250
+
251
+
252
+ The `t_input` is the time label of the model, which may be discrete-time labels (i.e. 0 to 999)
253
+ or continuous-time labels (i.e. epsilon to T).
254
+
255
+ We wrap the model function to accept only `x` and `t_continuous` as inputs, and outputs the predicted noise:
256
+ ``
257
+ def model_fn(x, t_continuous) -> noise:
258
+ t_input = get_model_input_time(t_continuous)
259
+ return noise_pred(model, x, t_input, **model_kwargs)
260
+ ``
261
+ where `t_continuous` is the continuous time labels (i.e. epsilon to T). And we use `model_fn` for DPM-Solver.
262
+
263
+ ===============================================================
264
+
265
+ Args:
266
+ model: A diffusion model with the corresponding format described above.
267
+ noise_schedule: A noise schedule object, such as NoiseScheduleVP.
268
+ model_type: A `str`. The parameterization type of the diffusion model.
269
+ "noise" or "x_start" or "v" or "score".
270
+ model_kwargs: A `dict`. A dict for the other inputs of the model function.
271
+ guidance_type: A `str`. The type of the guidance for sampling.
272
+ "uncond" or "classifier" or "classifier-free".
273
+ condition: A pytorch tensor. The condition for the guided sampling.
274
+ Only used for "classifier" or "classifier-free" guidance type.
275
+ unconditional_condition: A pytorch tensor. The condition for the unconditional sampling.
276
+ Only used for "classifier-free" guidance type.
277
+ guidance_scale: A `float`. The scale for the guided sampling.
278
+ classifier_fn: A classifier function. Only used for the classifier guidance.
279
+ classifier_kwargs: A `dict`. A dict for the other inputs of the classifier function.
280
+ Returns:
281
+ A noise prediction model that accepts the noised data and the continuous time as the inputs.
282
+ """
283
+
284
+ def get_model_input_time(t_continuous):
285
+ """
286
+ Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time.
287
+ For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N].
288
+ For continuous-time DPMs, we just use `t_continuous`.
289
+ """
290
+ if noise_schedule.schedule == 'discrete':
291
+ return (t_continuous - 1. / noise_schedule.total_N) * noise_schedule.total_N
292
+ else:
293
+ return t_continuous
294
+
295
+ def noise_pred_fn(x, t_continuous, cond=None):
296
+ if t_continuous.reshape((-1,)).shape[0] == 1:
297
+ t_continuous = t_continuous.expand((x.shape[0]))
298
+ t_input = get_model_input_time(t_continuous)
299
+ if cond is None:
300
+ output = model(x, t_input, **model_kwargs)
301
+ else:
302
+ output = model(x, t_input, cond, **model_kwargs)
303
+ if model_type == "noise":
304
+ return output
305
+ elif model_type == "x_start":
306
+ alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
307
+ dims = x.dim()
308
+ return (x - expand_dims(alpha_t, dims) * output) / expand_dims(sigma_t, dims)
309
+ elif model_type == "v":
310
+ alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
311
+ dims = x.dim()
312
+ return expand_dims(alpha_t, dims) * output + expand_dims(sigma_t, dims) * x
313
+ elif model_type == "score":
314
+ sigma_t = noise_schedule.marginal_std(t_continuous)
315
+ dims = x.dim()
316
+ return -expand_dims(sigma_t, dims) * output
317
+
318
+ def cond_grad_fn(x, t_input):
319
+ """
320
+ Compute the gradient of the classifier, i.e. nabla_{x} log p_t(cond | x_t).
321
+ """
322
+ with torch.enable_grad():
323
+ x_in = x.detach().requires_grad_(True)
324
+ log_prob = classifier_fn(x_in, t_input, condition, **classifier_kwargs)
325
+ return torch.autograd.grad(log_prob.sum(), x_in)[0]
326
+
327
+ def model_fn(x, t_continuous):
328
+ """
329
+ The noise predicition model function that is used for DPM-Solver.
330
+ """
331
+ if t_continuous.reshape((-1,)).shape[0] == 1:
332
+ t_continuous = t_continuous.expand((x.shape[0]))
333
+ if guidance_type == "uncond":
334
+ return noise_pred_fn(x, t_continuous)
335
+ elif guidance_type == "classifier":
336
+ assert classifier_fn is not None
337
+ t_input = get_model_input_time(t_continuous)
338
+ cond_grad = cond_grad_fn(x, t_input)
339
+ sigma_t = noise_schedule.marginal_std(t_continuous)
340
+ noise = noise_pred_fn(x, t_continuous)
341
+ return noise - guidance_scale * expand_dims(sigma_t, dims=cond_grad.dim()) * cond_grad
342
+ elif guidance_type == "classifier-free":
343
+ if guidance_scale == 1. or unconditional_condition is None:
344
+ return noise_pred_fn(x, t_continuous, cond=condition)
345
+ else:
346
+ x_in = torch.cat([x] * 2)
347
+ t_in = torch.cat([t_continuous] * 2)
348
+ c_in = torch.cat([unconditional_condition, condition])
349
+ noise_uncond, noise = noise_pred_fn(x_in, t_in, cond=c_in).chunk(2)
350
+ return noise_uncond + guidance_scale * (noise - noise_uncond)
351
+
352
+ assert model_type in ["noise", "x_start", "v"]
353
+ assert guidance_type in ["uncond", "classifier", "classifier-free"]
354
+ return model_fn
355
+
356
+
357
+ class DPM_Solver:
358
+ def __init__(self, model_fn, noise_schedule, predict_x0=False, thresholding=False, max_val=1.):
359
+ """Construct a DPM-Solver.
360
+
361
+ We support both the noise prediction model ("predicting epsilon") and the data prediction model ("predicting x0").
362
+ If `predict_x0` is False, we use the solver for the noise prediction model (DPM-Solver).
363
+ If `predict_x0` is True, we use the solver for the data prediction model (DPM-Solver++).
364
+ In such case, we further support the "dynamic thresholding" in [1] when `thresholding` is True.
365
+ The "dynamic thresholding" can greatly improve the sample quality for pixel-space DPMs with large guidance scales.
366
+
367
+ Args:
368
+ model_fn: A noise prediction model function which accepts the continuous-time input (t in [epsilon, T]):
369
+ ``
370
+ def model_fn(x, t_continuous):
371
+ return noise
372
+ ``
373
+ noise_schedule: A noise schedule object, such as NoiseScheduleVP.
374
+ predict_x0: A `bool`. If true, use the data prediction model; else, use the noise prediction model.
375
+ thresholding: A `bool`. Valid when `predict_x0` is True. Whether to use the "dynamic thresholding" in [1].
376
+ max_val: A `float`. Valid when both `predict_x0` and `thresholding` are True. The max value for thresholding.
377
+
378
+ [1] Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemipour, Burcu Karagol Ayan, S Sara Mahdavi, Rapha Gontijo Lopes, et al. Photorealistic text-to-image diffusion models with deep language understanding. arXiv preprint arXiv:2205.11487, 2022b.
379
+ """
380
+ self.model = model_fn
381
+ self.noise_schedule = noise_schedule
382
+ self.predict_x0 = predict_x0
383
+ self.thresholding = thresholding
384
+ self.max_val = max_val
385
+
386
+ def noise_prediction_fn(self, x, t):
387
+ """
388
+ Return the noise prediction model.
389
+ """
390
+ return self.model(x, t)
391
+
392
+ def data_prediction_fn(self, x, t):
393
+ """
394
+ Return the data prediction model (with thresholding).
395
+ """
396
+ noise = self.noise_prediction_fn(x, t)
397
+ dims = x.dim()
398
+ alpha_t, sigma_t = self.noise_schedule.marginal_alpha(t), self.noise_schedule.marginal_std(t)
399
+ x0 = (x - expand_dims(sigma_t, dims) * noise) / expand_dims(alpha_t, dims)
400
+ if self.thresholding:
401
+ p = 0.995 # A hyperparameter in the paper of "Imagen" [1].
402
+ s = torch.quantile(torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1)
403
+ s = expand_dims(torch.maximum(s, self.max_val * torch.ones_like(s).to(s.device)), dims)
404
+ x0 = torch.clamp(x0, -s, s) / s
405
+ return x0
406
+
407
+ def model_fn(self, x, t):
408
+ """
409
+ Convert the model to the noise prediction model or the data prediction model.
410
+ """
411
+ if self.predict_x0:
412
+ return self.data_prediction_fn(x, t)
413
+ else:
414
+ return self.noise_prediction_fn(x, t)
415
+
416
+ def get_time_steps(self, skip_type, t_T, t_0, N, device):
417
+ """Compute the intermediate time steps for sampling.
418
+
419
+ Args:
420
+ skip_type: A `str`. The type for the spacing of the time steps. We support three types:
421
+ - 'logSNR': uniform logSNR for the time steps.
422
+ - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.)
423
+ - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.)
424
+ t_T: A `float`. The starting time of the sampling (default is T).
425
+ t_0: A `float`. The ending time of the sampling (default is epsilon).
426
+ N: A `int`. The total number of the spacing of the time steps.
427
+ device: A torch device.
428
+ Returns:
429
+ A pytorch tensor of the time steps, with the shape (N + 1,).
430
+ """
431
+ if skip_type == 'logSNR':
432
+ lambda_T = self.noise_schedule.marginal_lambda(torch.tensor(t_T).to(device))
433
+ lambda_0 = self.noise_schedule.marginal_lambda(torch.tensor(t_0).to(device))
434
+ logSNR_steps = torch.linspace(lambda_T.cpu().item(), lambda_0.cpu().item(), N + 1).to(device)
435
+ return self.noise_schedule.inverse_lambda(logSNR_steps)
436
+ elif skip_type == 'time_uniform':
437
+ return torch.linspace(t_T, t_0, N + 1).to(device)
438
+ elif skip_type == 'time_quadratic':
439
+ t_order = 2
440
+ t = torch.linspace(t_T ** (1. / t_order), t_0 ** (1. / t_order), N + 1).pow(t_order).to(device)
441
+ return t
442
+ else:
443
+ raise ValueError(
444
+ "Unsupported skip_type {}, need to be 'logSNR' or 'time_uniform' or 'time_quadratic'".format(skip_type))
445
+
446
+ def get_orders_and_timesteps_for_singlestep_solver(self, steps, order, skip_type, t_T, t_0, device):
447
+ """
448
+ Get the order of each step for sampling by the singlestep DPM-Solver.
449
+
450
+ We combine both DPM-Solver-1,2,3 to use all the function evaluations, which is named as "DPM-Solver-fast".
451
+ Given a fixed number of function evaluations by `steps`, the sampling procedure by DPM-Solver-fast is:
452
+ - If order == 1:
453
+ We take `steps` of DPM-Solver-1 (i.e. DDIM).
454
+ - If order == 2:
455
+ - Denote K = (steps // 2). We take K or (K + 1) intermediate time steps for sampling.
456
+ - If steps % 2 == 0, we use K steps of DPM-Solver-2.
457
+ - If steps % 2 == 1, we use K steps of DPM-Solver-2 and 1 step of DPM-Solver-1.
458
+ - If order == 3:
459
+ - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling.
460
+ - If steps % 3 == 0, we use (K - 2) steps of DPM-Solver-3, and 1 step of DPM-Solver-2 and 1 step of DPM-Solver-1.
461
+ - If steps % 3 == 1, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-1.
462
+ - If steps % 3 == 2, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-2.
463
+
464
+ ============================================
465
+ Args:
466
+ order: A `int`. The max order for the solver (2 or 3).
467
+ steps: A `int`. The total number of function evaluations (NFE).
468
+ skip_type: A `str`. The type for the spacing of the time steps. We support three types:
469
+ - 'logSNR': uniform logSNR for the time steps.
470
+ - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.)
471
+ - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.)
472
+ t_T: A `float`. The starting time of the sampling (default is T).
473
+ t_0: A `float`. The ending time of the sampling (default is epsilon).
474
+ device: A torch device.
475
+ Returns:
476
+ orders: A list of the solver order of each step.
477
+ """
478
+ if order == 3:
479
+ K = steps // 3 + 1
480
+ if steps % 3 == 0:
481
+ orders = [3, ] * (K - 2) + [2, 1]
482
+ elif steps % 3 == 1:
483
+ orders = [3, ] * (K - 1) + [1]
484
+ else:
485
+ orders = [3, ] * (K - 1) + [2]
486
+ elif order == 2:
487
+ if steps % 2 == 0:
488
+ K = steps // 2
489
+ orders = [2, ] * K
490
+ else:
491
+ K = steps // 2 + 1
492
+ orders = [2, ] * (K - 1) + [1]
493
+ elif order == 1:
494
+ K = 1
495
+ orders = [1, ] * steps
496
+ else:
497
+ raise ValueError("'order' must be '1' or '2' or '3'.")
498
+ if skip_type == 'logSNR':
499
+ # To reproduce the results in DPM-Solver paper
500
+ timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, K, device)
501
+ else:
502
+ timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, steps, device)[
503
+ torch.cumsum(torch.tensor([0, ] + orders), dim=0).to(device)]
504
+ return timesteps_outer, orders
505
+
506
+ def denoise_fn(self, x, s):
507
+ """
508
+ Denoise at the final step, which is equivalent to solve the ODE from lambda_s to infty by first-order discretization.
509
+ """
510
+ return self.data_prediction_fn(x, s)
511
+
512
+ def dpm_solver_first_update(self, x, s, t, model_s=None, return_intermediate=False):
513
+ """
514
+ DPM-Solver-1 (equivalent to DDIM) from time `s` to time `t`.
515
+
516
+ Args:
517
+ x: A pytorch tensor. The initial value at time `s`.
518
+ s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
519
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
520
+ model_s: A pytorch tensor. The model function evaluated at time `s`.
521
+ If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
522
+ return_intermediate: A `bool`. If true, also return the model value at time `s`.
523
+ Returns:
524
+ x_t: A pytorch tensor. The approximated solution at time `t`.
525
+ """
526
+ ns = self.noise_schedule
527
+ dims = x.dim()
528
+ lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
529
+ h = lambda_t - lambda_s
530
+ log_alpha_s, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(t)
531
+ sigma_s, sigma_t = ns.marginal_std(s), ns.marginal_std(t)
532
+ alpha_t = torch.exp(log_alpha_t)
533
+
534
+ if self.predict_x0:
535
+ phi_1 = torch.expm1(-h)
536
+ if model_s is None:
537
+ model_s = self.model_fn(x, s)
538
+ x_t = (
539
+ expand_dims(sigma_t / sigma_s, dims) * x
540
+ - expand_dims(alpha_t * phi_1, dims) * model_s
541
+ )
542
+ if return_intermediate:
543
+ return x_t, {'model_s': model_s}
544
+ else:
545
+ return x_t
546
+ else:
547
+ phi_1 = torch.expm1(h)
548
+ if model_s is None:
549
+ model_s = self.model_fn(x, s)
550
+ x_t = (
551
+ expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
552
+ - expand_dims(sigma_t * phi_1, dims) * model_s
553
+ )
554
+ if return_intermediate:
555
+ return x_t, {'model_s': model_s}
556
+ else:
557
+ return x_t
558
+
559
+ def singlestep_dpm_solver_second_update(self, x, s, t, r1=0.5, model_s=None, return_intermediate=False,
560
+ solver_type='dpm_solver'):
561
+ """
562
+ Singlestep solver DPM-Solver-2 from time `s` to time `t`.
563
+
564
+ Args:
565
+ x: A pytorch tensor. The initial value at time `s`.
566
+ s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
567
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
568
+ r1: A `float`. The hyperparameter of the second-order solver.
569
+ model_s: A pytorch tensor. The model function evaluated at time `s`.
570
+ If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
571
+ return_intermediate: A `bool`. If true, also return the model value at time `s` and `s1` (the intermediate time).
572
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
573
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
574
+ Returns:
575
+ x_t: A pytorch tensor. The approximated solution at time `t`.
576
+ """
577
+ if solver_type not in ['dpm_solver', 'taylor']:
578
+ raise ValueError("'solver_type' must be either 'dpm_solver' or 'taylor', got {}".format(solver_type))
579
+ if r1 is None:
580
+ r1 = 0.5
581
+ ns = self.noise_schedule
582
+ dims = x.dim()
583
+ lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
584
+ h = lambda_t - lambda_s
585
+ lambda_s1 = lambda_s + r1 * h
586
+ s1 = ns.inverse_lambda(lambda_s1)
587
+ log_alpha_s, log_alpha_s1, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(
588
+ s1), ns.marginal_log_mean_coeff(t)
589
+ sigma_s, sigma_s1, sigma_t = ns.marginal_std(s), ns.marginal_std(s1), ns.marginal_std(t)
590
+ alpha_s1, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_t)
591
+
592
+ if self.predict_x0:
593
+ phi_11 = torch.expm1(-r1 * h)
594
+ phi_1 = torch.expm1(-h)
595
+
596
+ if model_s is None:
597
+ model_s = self.model_fn(x, s)
598
+ x_s1 = (
599
+ expand_dims(sigma_s1 / sigma_s, dims) * x
600
+ - expand_dims(alpha_s1 * phi_11, dims) * model_s
601
+ )
602
+ model_s1 = self.model_fn(x_s1, s1)
603
+ if solver_type == 'dpm_solver':
604
+ x_t = (
605
+ expand_dims(sigma_t / sigma_s, dims) * x
606
+ - expand_dims(alpha_t * phi_1, dims) * model_s
607
+ - (0.5 / r1) * expand_dims(alpha_t * phi_1, dims) * (model_s1 - model_s)
608
+ )
609
+ elif solver_type == 'taylor':
610
+ x_t = (
611
+ expand_dims(sigma_t / sigma_s, dims) * x
612
+ - expand_dims(alpha_t * phi_1, dims) * model_s
613
+ + (1. / r1) * expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) * (
614
+ model_s1 - model_s)
615
+ )
616
+ else:
617
+ phi_11 = torch.expm1(r1 * h)
618
+ phi_1 = torch.expm1(h)
619
+
620
+ if model_s is None:
621
+ model_s = self.model_fn(x, s)
622
+ x_s1 = (
623
+ expand_dims(torch.exp(log_alpha_s1 - log_alpha_s), dims) * x
624
+ - expand_dims(sigma_s1 * phi_11, dims) * model_s
625
+ )
626
+ model_s1 = self.model_fn(x_s1, s1)
627
+ if solver_type == 'dpm_solver':
628
+ x_t = (
629
+ expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
630
+ - expand_dims(sigma_t * phi_1, dims) * model_s
631
+ - (0.5 / r1) * expand_dims(sigma_t * phi_1, dims) * (model_s1 - model_s)
632
+ )
633
+ elif solver_type == 'taylor':
634
+ x_t = (
635
+ expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
636
+ - expand_dims(sigma_t * phi_1, dims) * model_s
637
+ - (1. / r1) * expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) * (model_s1 - model_s)
638
+ )
639
+ if return_intermediate:
640
+ return x_t, {'model_s': model_s, 'model_s1': model_s1}
641
+ else:
642
+ return x_t
643
+
644
+ def singlestep_dpm_solver_third_update(self, x, s, t, r1=1. / 3., r2=2. / 3., model_s=None, model_s1=None,
645
+ return_intermediate=False, solver_type='dpm_solver'):
646
+ """
647
+ Singlestep solver DPM-Solver-3 from time `s` to time `t`.
648
+
649
+ Args:
650
+ x: A pytorch tensor. The initial value at time `s`.
651
+ s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
652
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
653
+ r1: A `float`. The hyperparameter of the third-order solver.
654
+ r2: A `float`. The hyperparameter of the third-order solver.
655
+ model_s: A pytorch tensor. The model function evaluated at time `s`.
656
+ If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
657
+ model_s1: A pytorch tensor. The model function evaluated at time `s1` (the intermediate time given by `r1`).
658
+ If `model_s1` is None, we evaluate the model at `s1`; otherwise we directly use it.
659
+ return_intermediate: A `bool`. If true, also return the model value at time `s`, `s1` and `s2` (the intermediate times).
660
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
661
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
662
+ Returns:
663
+ x_t: A pytorch tensor. The approximated solution at time `t`.
664
+ """
665
+ if solver_type not in ['dpm_solver', 'taylor']:
666
+ raise ValueError("'solver_type' must be either 'dpm_solver' or 'taylor', got {}".format(solver_type))
667
+ if r1 is None:
668
+ r1 = 1. / 3.
669
+ if r2 is None:
670
+ r2 = 2. / 3.
671
+ ns = self.noise_schedule
672
+ dims = x.dim()
673
+ lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
674
+ h = lambda_t - lambda_s
675
+ lambda_s1 = lambda_s + r1 * h
676
+ lambda_s2 = lambda_s + r2 * h
677
+ s1 = ns.inverse_lambda(lambda_s1)
678
+ s2 = ns.inverse_lambda(lambda_s2)
679
+ log_alpha_s, log_alpha_s1, log_alpha_s2, log_alpha_t = ns.marginal_log_mean_coeff(
680
+ s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff(s2), ns.marginal_log_mean_coeff(t)
681
+ sigma_s, sigma_s1, sigma_s2, sigma_t = ns.marginal_std(s), ns.marginal_std(s1), ns.marginal_std(
682
+ s2), ns.marginal_std(t)
683
+ alpha_s1, alpha_s2, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_s2), torch.exp(log_alpha_t)
684
+
685
+ if self.predict_x0:
686
+ phi_11 = torch.expm1(-r1 * h)
687
+ phi_12 = torch.expm1(-r2 * h)
688
+ phi_1 = torch.expm1(-h)
689
+ phi_22 = torch.expm1(-r2 * h) / (r2 * h) + 1.
690
+ phi_2 = phi_1 / h + 1.
691
+ phi_3 = phi_2 / h - 0.5
692
+
693
+ if model_s is None:
694
+ model_s = self.model_fn(x, s)
695
+ if model_s1 is None:
696
+ x_s1 = (
697
+ expand_dims(sigma_s1 / sigma_s, dims) * x
698
+ - expand_dims(alpha_s1 * phi_11, dims) * model_s
699
+ )
700
+ model_s1 = self.model_fn(x_s1, s1)
701
+ x_s2 = (
702
+ expand_dims(sigma_s2 / sigma_s, dims) * x
703
+ - expand_dims(alpha_s2 * phi_12, dims) * model_s
704
+ + r2 / r1 * expand_dims(alpha_s2 * phi_22, dims) * (model_s1 - model_s)
705
+ )
706
+ model_s2 = self.model_fn(x_s2, s2)
707
+ if solver_type == 'dpm_solver':
708
+ x_t = (
709
+ expand_dims(sigma_t / sigma_s, dims) * x
710
+ - expand_dims(alpha_t * phi_1, dims) * model_s
711
+ + (1. / r2) * expand_dims(alpha_t * phi_2, dims) * (model_s2 - model_s)
712
+ )
713
+ elif solver_type == 'taylor':
714
+ D1_0 = (1. / r1) * (model_s1 - model_s)
715
+ D1_1 = (1. / r2) * (model_s2 - model_s)
716
+ D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1)
717
+ D2 = 2. * (D1_1 - D1_0) / (r2 - r1)
718
+ x_t = (
719
+ expand_dims(sigma_t / sigma_s, dims) * x
720
+ - expand_dims(alpha_t * phi_1, dims) * model_s
721
+ + expand_dims(alpha_t * phi_2, dims) * D1
722
+ - expand_dims(alpha_t * phi_3, dims) * D2
723
+ )
724
+ else:
725
+ phi_11 = torch.expm1(r1 * h)
726
+ phi_12 = torch.expm1(r2 * h)
727
+ phi_1 = torch.expm1(h)
728
+ phi_22 = torch.expm1(r2 * h) / (r2 * h) - 1.
729
+ phi_2 = phi_1 / h - 1.
730
+ phi_3 = phi_2 / h - 0.5
731
+
732
+ if model_s is None:
733
+ model_s = self.model_fn(x, s)
734
+ if model_s1 is None:
735
+ x_s1 = (
736
+ expand_dims(torch.exp(log_alpha_s1 - log_alpha_s), dims) * x
737
+ - expand_dims(sigma_s1 * phi_11, dims) * model_s
738
+ )
739
+ model_s1 = self.model_fn(x_s1, s1)
740
+ x_s2 = (
741
+ expand_dims(torch.exp(log_alpha_s2 - log_alpha_s), dims) * x
742
+ - expand_dims(sigma_s2 * phi_12, dims) * model_s
743
+ - r2 / r1 * expand_dims(sigma_s2 * phi_22, dims) * (model_s1 - model_s)
744
+ )
745
+ model_s2 = self.model_fn(x_s2, s2)
746
+ if solver_type == 'dpm_solver':
747
+ x_t = (
748
+ expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
749
+ - expand_dims(sigma_t * phi_1, dims) * model_s
750
+ - (1. / r2) * expand_dims(sigma_t * phi_2, dims) * (model_s2 - model_s)
751
+ )
752
+ elif solver_type == 'taylor':
753
+ D1_0 = (1. / r1) * (model_s1 - model_s)
754
+ D1_1 = (1. / r2) * (model_s2 - model_s)
755
+ D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1)
756
+ D2 = 2. * (D1_1 - D1_0) / (r2 - r1)
757
+ x_t = (
758
+ expand_dims(torch.exp(log_alpha_t - log_alpha_s), dims) * x
759
+ - expand_dims(sigma_t * phi_1, dims) * model_s
760
+ - expand_dims(sigma_t * phi_2, dims) * D1
761
+ - expand_dims(sigma_t * phi_3, dims) * D2
762
+ )
763
+
764
+ if return_intermediate:
765
+ return x_t, {'model_s': model_s, 'model_s1': model_s1, 'model_s2': model_s2}
766
+ else:
767
+ return x_t
768
+
769
+ def multistep_dpm_solver_second_update(self, x, model_prev_list, t_prev_list, t, solver_type="dpm_solver"):
770
+ """
771
+ Multistep solver DPM-Solver-2 from time `t_prev_list[-1]` to time `t`.
772
+
773
+ Args:
774
+ x: A pytorch tensor. The initial value at time `s`.
775
+ model_prev_list: A list of pytorch tensor. The previous computed model values.
776
+ t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (x.shape[0],)
777
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
778
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
779
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
780
+ Returns:
781
+ x_t: A pytorch tensor. The approximated solution at time `t`.
782
+ """
783
+ if solver_type not in ['dpm_solver', 'taylor']:
784
+ raise ValueError("'solver_type' must be either 'dpm_solver' or 'taylor', got {}".format(solver_type))
785
+ ns = self.noise_schedule
786
+ dims = x.dim()
787
+ model_prev_1, model_prev_0 = model_prev_list
788
+ t_prev_1, t_prev_0 = t_prev_list
789
+ lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda(t_prev_1), ns.marginal_lambda(
790
+ t_prev_0), ns.marginal_lambda(t)
791
+ log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
792
+ sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
793
+ alpha_t = torch.exp(log_alpha_t)
794
+
795
+ h_0 = lambda_prev_0 - lambda_prev_1
796
+ h = lambda_t - lambda_prev_0
797
+ r0 = h_0 / h
798
+ D1_0 = expand_dims(1. / r0, dims) * (model_prev_0 - model_prev_1)
799
+ if self.predict_x0:
800
+ if solver_type == 'dpm_solver':
801
+ x_t = (
802
+ expand_dims(sigma_t / sigma_prev_0, dims) * x
803
+ - expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * model_prev_0
804
+ - 0.5 * expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * D1_0
805
+ )
806
+ elif solver_type == 'taylor':
807
+ x_t = (
808
+ expand_dims(sigma_t / sigma_prev_0, dims) * x
809
+ - expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * model_prev_0
810
+ + expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) * D1_0
811
+ )
812
+ else:
813
+ if solver_type == 'dpm_solver':
814
+ x_t = (
815
+ expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
816
+ - expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * model_prev_0
817
+ - 0.5 * expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * D1_0
818
+ )
819
+ elif solver_type == 'taylor':
820
+ x_t = (
821
+ expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
822
+ - expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * model_prev_0
823
+ - expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) * D1_0
824
+ )
825
+ return x_t
826
+
827
+ def multistep_dpm_solver_third_update(self, x, model_prev_list, t_prev_list, t, solver_type='dpm_solver'):
828
+ """
829
+ Multistep solver DPM-Solver-3 from time `t_prev_list[-1]` to time `t`.
830
+
831
+ Args:
832
+ x: A pytorch tensor. The initial value at time `s`.
833
+ model_prev_list: A list of pytorch tensor. The previous computed model values.
834
+ t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (x.shape[0],)
835
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
836
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
837
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
838
+ Returns:
839
+ x_t: A pytorch tensor. The approximated solution at time `t`.
840
+ """
841
+ ns = self.noise_schedule
842
+ dims = x.dim()
843
+ model_prev_2, model_prev_1, model_prev_0 = model_prev_list
844
+ t_prev_2, t_prev_1, t_prev_0 = t_prev_list
845
+ lambda_prev_2, lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda(t_prev_2), ns.marginal_lambda(
846
+ t_prev_1), ns.marginal_lambda(t_prev_0), ns.marginal_lambda(t)
847
+ log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
848
+ sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
849
+ alpha_t = torch.exp(log_alpha_t)
850
+
851
+ h_1 = lambda_prev_1 - lambda_prev_2
852
+ h_0 = lambda_prev_0 - lambda_prev_1
853
+ h = lambda_t - lambda_prev_0
854
+ r0, r1 = h_0 / h, h_1 / h
855
+ D1_0 = expand_dims(1. / r0, dims) * (model_prev_0 - model_prev_1)
856
+ D1_1 = expand_dims(1. / r1, dims) * (model_prev_1 - model_prev_2)
857
+ D1 = D1_0 + expand_dims(r0 / (r0 + r1), dims) * (D1_0 - D1_1)
858
+ D2 = expand_dims(1. / (r0 + r1), dims) * (D1_0 - D1_1)
859
+ if self.predict_x0:
860
+ x_t = (
861
+ expand_dims(sigma_t / sigma_prev_0, dims) * x
862
+ - expand_dims(alpha_t * (torch.exp(-h) - 1.), dims) * model_prev_0
863
+ + expand_dims(alpha_t * ((torch.exp(-h) - 1.) / h + 1.), dims) * D1
864
+ - expand_dims(alpha_t * ((torch.exp(-h) - 1. + h) / h ** 2 - 0.5), dims) * D2
865
+ )
866
+ else:
867
+ x_t = (
868
+ expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
869
+ - expand_dims(sigma_t * (torch.exp(h) - 1.), dims) * model_prev_0
870
+ - expand_dims(sigma_t * ((torch.exp(h) - 1.) / h - 1.), dims) * D1
871
+ - expand_dims(sigma_t * ((torch.exp(h) - 1. - h) / h ** 2 - 0.5), dims) * D2
872
+ )
873
+ return x_t
874
+
875
+ def singlestep_dpm_solver_update(self, x, s, t, order, return_intermediate=False, solver_type='dpm_solver', r1=None,
876
+ r2=None):
877
+ """
878
+ Singlestep DPM-Solver with the order `order` from time `s` to time `t`.
879
+
880
+ Args:
881
+ x: A pytorch tensor. The initial value at time `s`.
882
+ s: A pytorch tensor. The starting time, with the shape (x.shape[0],).
883
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
884
+ order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3.
885
+ return_intermediate: A `bool`. If true, also return the model value at time `s`, `s1` and `s2` (the intermediate times).
886
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
887
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
888
+ r1: A `float`. The hyperparameter of the second-order or third-order solver.
889
+ r2: A `float`. The hyperparameter of the third-order solver.
890
+ Returns:
891
+ x_t: A pytorch tensor. The approximated solution at time `t`.
892
+ """
893
+ if order == 1:
894
+ return self.dpm_solver_first_update(x, s, t, return_intermediate=return_intermediate)
895
+ elif order == 2:
896
+ return self.singlestep_dpm_solver_second_update(x, s, t, return_intermediate=return_intermediate,
897
+ solver_type=solver_type, r1=r1)
898
+ elif order == 3:
899
+ return self.singlestep_dpm_solver_third_update(x, s, t, return_intermediate=return_intermediate,
900
+ solver_type=solver_type, r1=r1, r2=r2)
901
+ else:
902
+ raise ValueError("Solver order must be 1 or 2 or 3, got {}".format(order))
903
+
904
+ def multistep_dpm_solver_update(self, x, model_prev_list, t_prev_list, t, order, solver_type='dpm_solver'):
905
+ """
906
+ Multistep DPM-Solver with the order `order` from time `t_prev_list[-1]` to time `t`.
907
+
908
+ Args:
909
+ x: A pytorch tensor. The initial value at time `s`.
910
+ model_prev_list: A list of pytorch tensor. The previous computed model values.
911
+ t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (x.shape[0],)
912
+ t: A pytorch tensor. The ending time, with the shape (x.shape[0],).
913
+ order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3.
914
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
915
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
916
+ Returns:
917
+ x_t: A pytorch tensor. The approximated solution at time `t`.
918
+ """
919
+ if order == 1:
920
+ return self.dpm_solver_first_update(x, t_prev_list[-1], t, model_s=model_prev_list[-1])
921
+ elif order == 2:
922
+ return self.multistep_dpm_solver_second_update(x, model_prev_list, t_prev_list, t, solver_type=solver_type)
923
+ elif order == 3:
924
+ return self.multistep_dpm_solver_third_update(x, model_prev_list, t_prev_list, t, solver_type=solver_type)
925
+ else:
926
+ raise ValueError("Solver order must be 1 or 2 or 3, got {}".format(order))
927
+
928
+ def dpm_solver_adaptive(self, x, order, t_T, t_0, h_init=0.05, atol=0.0078, rtol=0.05, theta=0.9, t_err=1e-5,
929
+ solver_type='dpm_solver'):
930
+ """
931
+ The adaptive step size solver based on singlestep DPM-Solver.
932
+
933
+ Args:
934
+ x: A pytorch tensor. The initial value at time `t_T`.
935
+ order: A `int`. The (higher) order of the solver. We only support order == 2 or 3.
936
+ t_T: A `float`. The starting time of the sampling (default is T).
937
+ t_0: A `float`. The ending time of the sampling (default is epsilon).
938
+ h_init: A `float`. The initial step size (for logSNR).
939
+ atol: A `float`. The absolute tolerance of the solver. For image data, the default setting is 0.0078, followed [1].
940
+ rtol: A `float`. The relative tolerance of the solver. The default setting is 0.05.
941
+ theta: A `float`. The safety hyperparameter for adapting the step size. The default setting is 0.9, followed [1].
942
+ t_err: A `float`. The tolerance for the time. We solve the diffusion ODE until the absolute error between the
943
+ current time and `t_0` is less than `t_err`. The default setting is 1e-5.
944
+ solver_type: either 'dpm_solver' or 'taylor'. The type for the high-order solvers.
945
+ The type slightly impacts the performance. We recommend to use 'dpm_solver' type.
946
+ Returns:
947
+ x_0: A pytorch tensor. The approximated solution at time `t_0`.
948
+
949
+ [1] A. Jolicoeur-Martineau, K. Li, R. Piché-Taillefer, T. Kachman, and I. Mitliagkas, "Gotta go fast when generating data with score-based models," arXiv preprint arXiv:2105.14080, 2021.
950
+ """
951
+ ns = self.noise_schedule
952
+ s = t_T * torch.ones((x.shape[0],)).to(x)
953
+ lambda_s = ns.marginal_lambda(s)
954
+ lambda_0 = ns.marginal_lambda(t_0 * torch.ones_like(s).to(x))
955
+ h = h_init * torch.ones_like(s).to(x)
956
+ x_prev = x
957
+ nfe = 0
958
+ if order == 2:
959
+ r1 = 0.5
960
+ lower_update = lambda x, s, t: self.dpm_solver_first_update(x, s, t, return_intermediate=True)
961
+ higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_second_update(x, s, t, r1=r1,
962
+ solver_type=solver_type,
963
+ **kwargs)
964
+ elif order == 3:
965
+ r1, r2 = 1. / 3., 2. / 3.
966
+ lower_update = lambda x, s, t: self.singlestep_dpm_solver_second_update(x, s, t, r1=r1,
967
+ return_intermediate=True,
968
+ solver_type=solver_type)
969
+ higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_third_update(x, s, t, r1=r1, r2=r2,
970
+ solver_type=solver_type,
971
+ **kwargs)
972
+ else:
973
+ raise ValueError("For adaptive step size solver, order must be 2 or 3, got {}".format(order))
974
+ while torch.abs((s - t_0)).mean() > t_err:
975
+ t = ns.inverse_lambda(lambda_s + h)
976
+ x_lower, lower_noise_kwargs = lower_update(x, s, t)
977
+ x_higher = higher_update(x, s, t, **lower_noise_kwargs)
978
+ delta = torch.max(torch.ones_like(x).to(x) * atol, rtol * torch.max(torch.abs(x_lower), torch.abs(x_prev)))
979
+ norm_fn = lambda v: torch.sqrt(torch.square(v.reshape((v.shape[0], -1))).mean(dim=-1, keepdim=True))
980
+ E = norm_fn((x_higher - x_lower) / delta).max()
981
+ if torch.all(E <= 1.):
982
+ x = x_higher
983
+ s = t
984
+ x_prev = x_lower
985
+ lambda_s = ns.marginal_lambda(s)
986
+ h = torch.min(theta * h * torch.float_power(E, -1. / order).float(), lambda_0 - lambda_s)
987
+ nfe += order
988
+ print('adaptive solver nfe', nfe)
989
+ return x
990
+
991
+ def sample(self, x, steps=20, t_start=None, t_end=None, order=3, skip_type='time_uniform',
992
+ method='singlestep', denoise=False, solver_type='dpm_solver', atol=0.0078,
993
+ rtol=0.05,
994
+ ):
995
+ """
996
+ Compute the sample at time `t_end` by DPM-Solver, given the initial `x` at time `t_start`.
997
+
998
+ =====================================================
999
+
1000
+ We support the following algorithms for both noise prediction model and data prediction model:
1001
+ - 'singlestep':
1002
+ Singlestep DPM-Solver (i.e. "DPM-Solver-fast" in the paper), which combines different orders of singlestep DPM-Solver.
1003
+ We combine all the singlestep solvers with order <= `order` to use up all the function evaluations (steps).
1004
+ The total number of function evaluations (NFE) == `steps`.
1005
+ Given a fixed NFE == `steps`, the sampling procedure is:
1006
+ - If `order` == 1:
1007
+ - Denote K = steps. We use K steps of DPM-Solver-1 (i.e. DDIM).
1008
+ - If `order` == 2:
1009
+ - Denote K = (steps // 2) + (steps % 2). We take K intermediate time steps for sampling.
1010
+ - If steps % 2 == 0, we use K steps of singlestep DPM-Solver-2.
1011
+ - If steps % 2 == 1, we use (K - 1) steps of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1.
1012
+ - If `order` == 3:
1013
+ - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling.
1014
+ - If steps % 3 == 0, we use (K - 2) steps of singlestep DPM-Solver-3, and 1 step of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1.
1015
+ - If steps % 3 == 1, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of DPM-Solver-1.
1016
+ - If steps % 3 == 2, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of singlestep DPM-Solver-2.
1017
+ - 'multistep':
1018
+ Multistep DPM-Solver with the order of `order`. The total number of function evaluations (NFE) == `steps`.
1019
+ We initialize the first `order` values by lower order multistep solvers.
1020
+ Given a fixed NFE == `steps`, the sampling procedure is:
1021
+ Denote K = steps.
1022
+ - If `order` == 1:
1023
+ - We use K steps of DPM-Solver-1 (i.e. DDIM).
1024
+ - If `order` == 2:
1025
+ - We firstly use 1 step of DPM-Solver-1, then use (K - 1) step of multistep DPM-Solver-2.
1026
+ - If `order` == 3:
1027
+ - We firstly use 1 step of DPM-Solver-1, then 1 step of multistep DPM-Solver-2, then (K - 2) step of multistep DPM-Solver-3.
1028
+ - 'singlestep_fixed':
1029
+ Fixed order singlestep DPM-Solver (i.e. DPM-Solver-1 or singlestep DPM-Solver-2 or singlestep DPM-Solver-3).
1030
+ We use singlestep DPM-Solver-`order` for `order`=1 or 2 or 3, with total [`steps` // `order`] * `order` NFE.
1031
+ - 'adaptive':
1032
+ Adaptive step size DPM-Solver (i.e. "DPM-Solver-12" and "DPM-Solver-23" in the paper).
1033
+ We ignore `steps` and use adaptive step size DPM-Solver with a higher order of `order`.
1034
+ You can adjust the absolute tolerance `atol` and the relative tolerance `rtol` to balance the computatation costs
1035
+ (NFE) and the sample quality.
1036
+ - If `order` == 2, we use DPM-Solver-12 which combines DPM-Solver-1 and singlestep DPM-Solver-2.
1037
+ - If `order` == 3, we use DPM-Solver-23 which combines singlestep DPM-Solver-2 and singlestep DPM-Solver-3.
1038
+
1039
+ =====================================================
1040
+
1041
+ Some advices for choosing the algorithm:
1042
+ - For **unconditional sampling** or **guided sampling with small guidance scale** by DPMs:
1043
+ Use singlestep DPM-Solver ("DPM-Solver-fast" in the paper) with `order = 3`.
1044
+ e.g.
1045
+ >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, predict_x0=False)
1046
+ >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=3,
1047
+ skip_type='time_uniform', method='singlestep')
1048
+ - For **guided sampling with large guidance scale** by DPMs:
1049
+ Use multistep DPM-Solver with `predict_x0 = True` and `order = 2`.
1050
+ e.g.
1051
+ >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, predict_x0=True)
1052
+ >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=2,
1053
+ skip_type='time_uniform', method='multistep')
1054
+
1055
+ We support three types of `skip_type`:
1056
+ - 'logSNR': uniform logSNR for the time steps. **Recommended for low-resolutional images**
1057
+ - 'time_uniform': uniform time for the time steps. **Recommended for high-resolutional images**.
1058
+ - 'time_quadratic': quadratic time for the time steps.
1059
+
1060
+ =====================================================
1061
+ Args:
1062
+ x: A pytorch tensor. The initial value at time `t_start`
1063
+ e.g. if `t_start` == T, then `x` is a sample from the standard normal distribution.
1064
+ steps: A `int`. The total number of function evaluations (NFE).
1065
+ t_start: A `float`. The starting time of the sampling.
1066
+ If `T` is None, we use self.noise_schedule.T (default is 1.0).
1067
+ t_end: A `float`. The ending time of the sampling.
1068
+ If `t_end` is None, we use 1. / self.noise_schedule.total_N.
1069
+ e.g. if total_N == 1000, we have `t_end` == 1e-3.
1070
+ For discrete-time DPMs:
1071
+ - We recommend `t_end` == 1. / self.noise_schedule.total_N.
1072
+ For continuous-time DPMs:
1073
+ - We recommend `t_end` == 1e-3 when `steps` <= 15; and `t_end` == 1e-4 when `steps` > 15.
1074
+ order: A `int`. The order of DPM-Solver.
1075
+ skip_type: A `str`. The type for the spacing of the time steps. 'time_uniform' or 'logSNR' or 'time_quadratic'.
1076
+ method: A `str`. The method for sampling. 'singlestep' or 'multistep' or 'singlestep_fixed' or 'adaptive'.
1077
+ denoise: A `bool`. Whether to denoise at the final step. Default is False.
1078
+ If `denoise` is True, the total NFE is (`steps` + 1).
1079
+ solver_type: A `str`. The taylor expansion type for the solver. `dpm_solver` or `taylor`. We recommend `dpm_solver`.
1080
+ atol: A `float`. The absolute tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'.
1081
+ rtol: A `float`. The relative tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'.
1082
+ Returns:
1083
+ x_end: A pytorch tensor. The approximated solution at time `t_end`.
1084
+
1085
+ """
1086
+ t_0 = 1. / self.noise_schedule.total_N if t_end is None else t_end
1087
+ t_T = self.noise_schedule.T if t_start is None else t_start
1088
+ device = x.device
1089
+ if method == 'adaptive':
1090
+ with torch.no_grad():
1091
+ x = self.dpm_solver_adaptive(x, order=order, t_T=t_T, t_0=t_0, atol=atol, rtol=rtol,
1092
+ solver_type=solver_type)
1093
+ elif method == 'multistep':
1094
+ assert steps >= order
1095
+ timesteps = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=steps, device=device)
1096
+ assert timesteps.shape[0] - 1 == steps
1097
+ with torch.no_grad():
1098
+ vec_t = timesteps[0].expand((x.shape[0]))
1099
+ model_prev_list = [self.model_fn(x, vec_t)]
1100
+ t_prev_list = [vec_t]
1101
+ # Init the first `order` values by lower order multistep DPM-Solver.
1102
+ for init_order in range(1, order):
1103
+ vec_t = timesteps[init_order].expand(x.shape[0])
1104
+ x = self.multistep_dpm_solver_update(x, model_prev_list, t_prev_list, vec_t, init_order,
1105
+ solver_type=solver_type)
1106
+ model_prev_list.append(self.model_fn(x, vec_t))
1107
+ t_prev_list.append(vec_t)
1108
+ # Compute the remaining values by `order`-th order multistep DPM-Solver.
1109
+ for step in range(order, steps + 1):
1110
+ vec_t = timesteps[step].expand(x.shape[0])
1111
+ x = self.multistep_dpm_solver_update(x, model_prev_list, t_prev_list, vec_t, order,
1112
+ solver_type=solver_type)
1113
+ for i in range(order - 1):
1114
+ t_prev_list[i] = t_prev_list[i + 1]
1115
+ model_prev_list[i] = model_prev_list[i + 1]
1116
+ t_prev_list[-1] = vec_t
1117
+ # We do not need to evaluate the final model value.
1118
+ if step < steps:
1119
+ model_prev_list[-1] = self.model_fn(x, vec_t)
1120
+ elif method in ['singlestep', 'singlestep_fixed']:
1121
+ if method == 'singlestep':
1122
+ timesteps_outer, orders = self.get_orders_and_timesteps_for_singlestep_solver(steps=steps, order=order,
1123
+ skip_type=skip_type,
1124
+ t_T=t_T, t_0=t_0,
1125
+ device=device)
1126
+ elif method == 'singlestep_fixed':
1127
+ K = steps // order
1128
+ orders = [order, ] * K
1129
+ timesteps_outer = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=K, device=device)
1130
+ for i, order in enumerate(orders):
1131
+ t_T_inner, t_0_inner = timesteps_outer[i], timesteps_outer[i + 1]
1132
+ timesteps_inner = self.get_time_steps(skip_type=skip_type, t_T=t_T_inner.item(), t_0=t_0_inner.item(),
1133
+ N=order, device=device)
1134
+ lambda_inner = self.noise_schedule.marginal_lambda(timesteps_inner)
1135
+ vec_s, vec_t = t_T_inner.repeat(x.shape[0]), t_0_inner.repeat(x.shape[0])
1136
+ h = lambda_inner[-1] - lambda_inner[0]
1137
+ r1 = None if order <= 1 else (lambda_inner[1] - lambda_inner[0]) / h
1138
+ r2 = None if order <= 2 else (lambda_inner[2] - lambda_inner[0]) / h
1139
+ x = self.singlestep_dpm_solver_update(x, vec_s, vec_t, order, solver_type=solver_type, r1=r1, r2=r2)
1140
+ if denoise:
1141
+ x = self.denoise_fn(x, torch.ones((x.shape[0],)).to(device) * t_0)
1142
+ return x
1143
+
1144
+
1145
+ #############################################################
1146
+ # other utility functions
1147
+ #############################################################
1148
+
1149
+ def interpolate_fn(x, xp, yp):
1150
+ """
1151
+ A piecewise linear function y = f(x), using xp and yp as keypoints.
1152
+ We implement f(x) in a differentiable way (i.e. applicable for autograd).
1153
+ The function f(x) is well-defined for all x-axis. (For x beyond the bounds of xp, we use the outmost points of xp to define the linear function.)
1154
+
1155
+ Args:
1156
+ x: PyTorch tensor with shape [N, C], where N is the batch size, C is the number of channels (we use C = 1 for DPM-Solver).
1157
+ xp: PyTorch tensor with shape [C, K], where K is the number of keypoints.
1158
+ yp: PyTorch tensor with shape [C, K].
1159
+ Returns:
1160
+ The function values f(x), with shape [N, C].
1161
+ """
1162
+ N, K = x.shape[0], xp.shape[1]
1163
+ all_x = torch.cat([x.unsqueeze(2), xp.unsqueeze(0).repeat((N, 1, 1))], dim=2)
1164
+ sorted_all_x, x_indices = torch.sort(all_x, dim=2)
1165
+ x_idx = torch.argmin(x_indices, dim=2)
1166
+ cand_start_idx = x_idx - 1
1167
+ start_idx = torch.where(
1168
+ torch.eq(x_idx, 0),
1169
+ torch.tensor(1, device=x.device),
1170
+ torch.where(
1171
+ torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
1172
+ ),
1173
+ )
1174
+ end_idx = torch.where(torch.eq(start_idx, cand_start_idx), start_idx + 2, start_idx + 1)
1175
+ start_x = torch.gather(sorted_all_x, dim=2, index=start_idx.unsqueeze(2)).squeeze(2)
1176
+ end_x = torch.gather(sorted_all_x, dim=2, index=end_idx.unsqueeze(2)).squeeze(2)
1177
+ start_idx2 = torch.where(
1178
+ torch.eq(x_idx, 0),
1179
+ torch.tensor(0, device=x.device),
1180
+ torch.where(
1181
+ torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
1182
+ ),
1183
+ )
1184
+ y_positions_expanded = yp.unsqueeze(0).expand(N, -1, -1)
1185
+ start_y = torch.gather(y_positions_expanded, dim=2, index=start_idx2.unsqueeze(2)).squeeze(2)
1186
+ end_y = torch.gather(y_positions_expanded, dim=2, index=(start_idx2 + 1).unsqueeze(2)).squeeze(2)
1187
+ cand = start_y + (x - start_x) * (end_y - start_y) / (end_x - start_x)
1188
+ return cand
1189
+
1190
+
1191
+ def expand_dims(v, dims):
1192
+ """
1193
+ Expand the tensor `v` to the dim `dims`.
1194
+
1195
+ Args:
1196
+ `v`: a PyTorch tensor with shape [N].
1197
+ `dim`: a `int`.
1198
+ Returns:
1199
+ a PyTorch tensor with shape [N, 1, 1, ..., 1] and the total dimension is `dims`.
1200
+ """
1201
+ return v[(...,) + (None,) * (dims - 1)]
diffusion/how to export onnx.md ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ - Open [onnx_export](onnx_export.py)
2
+ - project_name = "dddsp" change "project_name" to your project name
3
+ - model_path = f'{project_name}/model_500000.pt' change "model_path" to your model path
4
+ - Run
diffusion/infer_gt_mel.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from diffusion.unit2mel import load_model_vocoder
5
+
6
+
7
+ class DiffGtMel:
8
+ def __init__(self, project_path=None, device=None):
9
+ self.project_path = project_path
10
+ if device is not None:
11
+ self.device = device
12
+ else:
13
+ self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
14
+ self.model = None
15
+ self.vocoder = None
16
+ self.args = None
17
+
18
+ def flush_model(self, project_path, ddsp_config=None):
19
+ if (self.model is None) or (project_path != self.project_path):
20
+ model, vocoder, args = load_model_vocoder(project_path, device=self.device)
21
+ if self.check_args(ddsp_config, args):
22
+ self.model = model
23
+ self.vocoder = vocoder
24
+ self.args = args
25
+
26
+ def check_args(self, args1, args2):
27
+ if args1.data.block_size != args2.data.block_size:
28
+ raise ValueError("DDSP与DIFF模型的block_size不一致")
29
+ if args1.data.sampling_rate != args2.data.sampling_rate:
30
+ raise ValueError("DDSP与DIFF模型的sampling_rate不一致")
31
+ if args1.data.encoder != args2.data.encoder:
32
+ raise ValueError("DDSP与DIFF模型的encoder不一致")
33
+ return True
34
+
35
+ def __call__(self, audio, f0, hubert, volume, acc=1, spk_id=1, k_step=0, method='pndm',
36
+ spk_mix_dict=None, start_frame=0):
37
+ input_mel = self.vocoder.extract(audio, self.args.data.sampling_rate)
38
+ out_mel = self.model(
39
+ hubert,
40
+ f0,
41
+ volume,
42
+ spk_id=spk_id,
43
+ spk_mix_dict=spk_mix_dict,
44
+ gt_spec=input_mel,
45
+ infer=True,
46
+ infer_speedup=acc,
47
+ method=method,
48
+ k_step=k_step,
49
+ use_tqdm=False)
50
+ if start_frame > 0:
51
+ out_mel = out_mel[:, start_frame:, :]
52
+ f0 = f0[:, start_frame:, :]
53
+ output = self.vocoder.infer(out_mel, f0)
54
+ if start_frame > 0:
55
+ output = F.pad(output, (start_frame * self.vocoder.vocoder_hop_size, 0))
56
+ return output
57
+
58
+ def infer(self, audio, f0, hubert, volume, acc=1, spk_id=1, k_step=0, method='pndm', silence_front=0,
59
+ use_silence=False, spk_mix_dict=None):
60
+ start_frame = int(silence_front * self.vocoder.vocoder_sample_rate / self.vocoder.vocoder_hop_size)
61
+ if use_silence:
62
+ audio = audio[:, start_frame * self.vocoder.vocoder_hop_size:]
63
+ f0 = f0[:, start_frame:, :]
64
+ hubert = hubert[:, start_frame:, :]
65
+ volume = volume[:, start_frame:, :]
66
+ _start_frame = 0
67
+ else:
68
+ _start_frame = start_frame
69
+ audio = self.__call__(audio, f0, hubert, volume, acc=acc, spk_id=spk_id, k_step=k_step,
70
+ method=method, spk_mix_dict=spk_mix_dict, start_frame=_start_frame)
71
+ if use_silence:
72
+ if start_frame > 0:
73
+ audio = F.pad(audio, (start_frame * self.vocoder.vocoder_hop_size, 0))
74
+ return audio
diffusion/logger/__init__.py ADDED
File without changes
diffusion/logger/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (148 Bytes). View file
 
diffusion/logger/__pycache__/utils.cpython-38.pyc ADDED
Binary file (3.81 kB). View file
 
diffusion/logger/saver.py ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ '''
2
+ author: wayn391@mastertones
3
+ '''
4
+
5
+ import os
6
+ import json
7
+ import time
8
+ import yaml
9
+ import datetime
10
+ import torch
11
+ import matplotlib.pyplot as plt
12
+ from . import utils
13
+ from torch.utils.tensorboard import SummaryWriter
14
+
15
+ class Saver(object):
16
+ def __init__(
17
+ self,
18
+ args,
19
+ initial_global_step=-1):
20
+
21
+ self.expdir = args.env.expdir
22
+ self.sample_rate = args.data.sampling_rate
23
+
24
+ # cold start
25
+ self.global_step = initial_global_step
26
+ self.init_time = time.time()
27
+ self.last_time = time.time()
28
+
29
+ # makedirs
30
+ os.makedirs(self.expdir, exist_ok=True)
31
+
32
+ # path
33
+ self.path_log_info = os.path.join(self.expdir, 'log_info.txt')
34
+
35
+ # ckpt
36
+ os.makedirs(self.expdir, exist_ok=True)
37
+
38
+ # writer
39
+ self.writer = SummaryWriter(os.path.join(self.expdir, 'logs'))
40
+
41
+ # save config
42
+ path_config = os.path.join(self.expdir, 'config.yaml')
43
+ with open(path_config, "w") as out_config:
44
+ yaml.dump(dict(args), out_config)
45
+
46
+
47
+ def log_info(self, msg):
48
+ '''log method'''
49
+ if isinstance(msg, dict):
50
+ msg_list = []
51
+ for k, v in msg.items():
52
+ tmp_str = ''
53
+ if isinstance(v, int):
54
+ tmp_str = '{}: {:,}'.format(k, v)
55
+ else:
56
+ tmp_str = '{}: {}'.format(k, v)
57
+
58
+ msg_list.append(tmp_str)
59
+ msg_str = '\n'.join(msg_list)
60
+ else:
61
+ msg_str = msg
62
+
63
+ # dsplay
64
+ print(msg_str)
65
+
66
+ # save
67
+ with open(self.path_log_info, 'a') as fp:
68
+ fp.write(msg_str+'\n')
69
+
70
+ def log_value(self, dict):
71
+ for k, v in dict.items():
72
+ self.writer.add_scalar(k, v, self.global_step)
73
+
74
+ def log_spec(self, name, spec, spec_out, vmin=-14, vmax=3.5):
75
+ spec_cat = torch.cat([(spec_out - spec).abs() + vmin, spec, spec_out], -1)
76
+ spec = spec_cat[0]
77
+ if isinstance(spec, torch.Tensor):
78
+ spec = spec.cpu().numpy()
79
+ fig = plt.figure(figsize=(12, 9))
80
+ plt.pcolor(spec.T, vmin=vmin, vmax=vmax)
81
+ plt.tight_layout()
82
+ self.writer.add_figure(name, fig, self.global_step)
83
+
84
+ def log_audio(self, dict):
85
+ for k, v in dict.items():
86
+ self.writer.add_audio(k, v, global_step=self.global_step, sample_rate=self.sample_rate)
87
+
88
+ def get_interval_time(self, update=True):
89
+ cur_time = time.time()
90
+ time_interval = cur_time - self.last_time
91
+ if update:
92
+ self.last_time = cur_time
93
+ return time_interval
94
+
95
+ def get_total_time(self, to_str=True):
96
+ total_time = time.time() - self.init_time
97
+ if to_str:
98
+ total_time = str(datetime.timedelta(
99
+ seconds=total_time))[:-5]
100
+ return total_time
101
+
102
+ def save_model(
103
+ self,
104
+ model,
105
+ optimizer,
106
+ name='model',
107
+ postfix='',
108
+ to_json=False):
109
+ # path
110
+ if postfix:
111
+ postfix = '_' + postfix
112
+ path_pt = os.path.join(
113
+ self.expdir , name+postfix+'.pt')
114
+
115
+ # check
116
+ print(' [*] model checkpoint saved: {}'.format(path_pt))
117
+
118
+ # save
119
+ if optimizer is not None:
120
+ torch.save({
121
+ 'global_step': self.global_step,
122
+ 'model': model.state_dict(),
123
+ 'optimizer': optimizer.state_dict()}, path_pt)
124
+ else:
125
+ torch.save({
126
+ 'global_step': self.global_step,
127
+ 'model': model.state_dict()}, path_pt)
128
+
129
+ # to json
130
+ if to_json:
131
+ path_json = os.path.join(
132
+ self.expdir , name+'.json')
133
+ utils.to_json(path_params, path_json)
134
+
135
+ def delete_model(self, name='model', postfix=''):
136
+ # path
137
+ if postfix:
138
+ postfix = '_' + postfix
139
+ path_pt = os.path.join(
140
+ self.expdir , name+postfix+'.pt')
141
+
142
+ # delete
143
+ if os.path.exists(path_pt):
144
+ os.remove(path_pt)
145
+ print(' [*] model checkpoint deleted: {}'.format(path_pt))
146
+
147
+ def global_step_increment(self):
148
+ self.global_step += 1
149
+
150
+
diffusion/logger/utils.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import yaml
3
+ import json
4
+ import pickle
5
+ import torch
6
+
7
+ def traverse_dir(
8
+ root_dir,
9
+ extensions,
10
+ amount=None,
11
+ str_include=None,
12
+ str_exclude=None,
13
+ is_pure=False,
14
+ is_sort=False,
15
+ is_ext=True):
16
+
17
+ file_list = []
18
+ cnt = 0
19
+ for root, _, files in os.walk(root_dir):
20
+ for file in files:
21
+ if any([file.endswith(f".{ext}") for ext in extensions]):
22
+ # path
23
+ mix_path = os.path.join(root, file)
24
+ pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
25
+
26
+ # amount
27
+ if (amount is not None) and (cnt == amount):
28
+ if is_sort:
29
+ file_list.sort()
30
+ return file_list
31
+
32
+ # check string
33
+ if (str_include is not None) and (str_include not in pure_path):
34
+ continue
35
+ if (str_exclude is not None) and (str_exclude in pure_path):
36
+ continue
37
+
38
+ if not is_ext:
39
+ ext = pure_path.split('.')[-1]
40
+ pure_path = pure_path[:-(len(ext)+1)]
41
+ file_list.append(pure_path)
42
+ cnt += 1
43
+ if is_sort:
44
+ file_list.sort()
45
+ return file_list
46
+
47
+
48
+
49
+ class DotDict(dict):
50
+ def __getattr__(*args):
51
+ val = dict.get(*args)
52
+ return DotDict(val) if type(val) is dict else val
53
+
54
+ __setattr__ = dict.__setitem__
55
+ __delattr__ = dict.__delitem__
56
+
57
+
58
+ def get_network_paras_amount(model_dict):
59
+ info = dict()
60
+ for model_name, model in model_dict.items():
61
+ # all_params = sum(p.numel() for p in model.parameters())
62
+ trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
63
+
64
+ info[model_name] = trainable_params
65
+ return info
66
+
67
+
68
+ def load_config(path_config):
69
+ with open(path_config, "r") as config:
70
+ args = yaml.safe_load(config)
71
+ args = DotDict(args)
72
+ # print(args)
73
+ return args
74
+
75
+ def save_config(path_config,config):
76
+ config = dict(config)
77
+ with open(path_config, "w") as f:
78
+ yaml.dump(config, f)
79
+
80
+ def to_json(path_params, path_json):
81
+ params = torch.load(path_params, map_location=torch.device('cpu'))
82
+ raw_state_dict = {}
83
+ for k, v in params.items():
84
+ val = v.flatten().numpy().tolist()
85
+ raw_state_dict[k] = val
86
+
87
+ with open(path_json, 'w') as outfile:
88
+ json.dump(raw_state_dict, outfile,indent= "\t")
89
+
90
+
91
+ def convert_tensor_to_numpy(tensor, is_squeeze=True):
92
+ if is_squeeze:
93
+ tensor = tensor.squeeze()
94
+ if tensor.requires_grad:
95
+ tensor = tensor.detach()
96
+ if tensor.is_cuda:
97
+ tensor = tensor.cpu()
98
+ return tensor.numpy()
99
+
100
+
101
+ def load_model(
102
+ expdir,
103
+ model,
104
+ optimizer,
105
+ name='model',
106
+ postfix='',
107
+ device='cpu'):
108
+ if postfix == '':
109
+ postfix = '_' + postfix
110
+ path = os.path.join(expdir, name+postfix)
111
+ path_pt = traverse_dir(expdir, ['pt'], is_ext=False)
112
+ global_step = 0
113
+ if len(path_pt) > 0:
114
+ steps = [s[len(path):] for s in path_pt]
115
+ maxstep = max([int(s) if s.isdigit() else 0 for s in steps])
116
+ if maxstep >= 0:
117
+ path_pt = path+str(maxstep)+'.pt'
118
+ else:
119
+ path_pt = path+'best.pt'
120
+ print(' [*] restoring model from', path_pt)
121
+ ckpt = torch.load(path_pt, map_location=torch.device(device))
122
+ global_step = ckpt['global_step']
123
+ model.load_state_dict(ckpt['model'], strict=False)
124
+ if ckpt.get('optimizer') != None:
125
+ optimizer.load_state_dict(ckpt['optimizer'])
126
+ return global_step, model, optimizer
diffusion/onnx_export.py ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from diffusion_onnx import GaussianDiffusion
2
+ import os
3
+ import yaml
4
+ import torch
5
+ import torch.nn as nn
6
+ import numpy as np
7
+ from wavenet import WaveNet
8
+ import torch.nn.functional as F
9
+ import diffusion
10
+
11
+ class DotDict(dict):
12
+ def __getattr__(*args):
13
+ val = dict.get(*args)
14
+ return DotDict(val) if type(val) is dict else val
15
+
16
+ __setattr__ = dict.__setitem__
17
+ __delattr__ = dict.__delitem__
18
+
19
+
20
+ def load_model_vocoder(
21
+ model_path,
22
+ device='cpu'):
23
+ config_file = os.path.join(os.path.split(model_path)[0], 'config.yaml')
24
+ with open(config_file, "r") as config:
25
+ args = yaml.safe_load(config)
26
+ args = DotDict(args)
27
+
28
+ # load model
29
+ model = Unit2Mel(
30
+ args.data.encoder_out_channels,
31
+ args.model.n_spk,
32
+ args.model.use_pitch_aug,
33
+ 128,
34
+ args.model.n_layers,
35
+ args.model.n_chans,
36
+ args.model.n_hidden)
37
+
38
+ print(' [Loading] ' + model_path)
39
+ ckpt = torch.load(model_path, map_location=torch.device(device))
40
+ model.to(device)
41
+ model.load_state_dict(ckpt['model'])
42
+ model.eval()
43
+ return model, args
44
+
45
+
46
+ class Unit2Mel(nn.Module):
47
+ def __init__(
48
+ self,
49
+ input_channel,
50
+ n_spk,
51
+ use_pitch_aug=False,
52
+ out_dims=128,
53
+ n_layers=20,
54
+ n_chans=384,
55
+ n_hidden=256):
56
+ super().__init__()
57
+ self.unit_embed = nn.Linear(input_channel, n_hidden)
58
+ self.f0_embed = nn.Linear(1, n_hidden)
59
+ self.volume_embed = nn.Linear(1, n_hidden)
60
+ if use_pitch_aug:
61
+ self.aug_shift_embed = nn.Linear(1, n_hidden, bias=False)
62
+ else:
63
+ self.aug_shift_embed = None
64
+ self.n_spk = n_spk
65
+ if n_spk is not None and n_spk > 1:
66
+ self.spk_embed = nn.Embedding(n_spk, n_hidden)
67
+
68
+ # diffusion
69
+ self.decoder = GaussianDiffusion(out_dims, n_layers, n_chans, n_hidden)
70
+ self.hidden_size = n_hidden
71
+ self.speaker_map = torch.zeros((self.n_spk,1,1,n_hidden))
72
+
73
+
74
+
75
+ def forward(self, units, mel2ph, f0, volume, g = None):
76
+
77
+ '''
78
+ input:
79
+ B x n_frames x n_unit
80
+ return:
81
+ dict of B x n_frames x feat
82
+ '''
83
+
84
+ decoder_inp = F.pad(units, [0, 0, 1, 0])
85
+ mel2ph_ = mel2ph.unsqueeze(2).repeat([1, 1, units.shape[-1]])
86
+ units = torch.gather(decoder_inp, 1, mel2ph_) # [B, T, H]
87
+
88
+ x = self.unit_embed(units) + self.f0_embed((1 + f0.unsqueeze(-1) / 700).log()) + self.volume_embed(volume.unsqueeze(-1))
89
+
90
+ if self.n_spk is not None and self.n_spk > 1: # [N, S] * [S, B, 1, H]
91
+ g = g.reshape((g.shape[0], g.shape[1], 1, 1, 1)) # [N, S, B, 1, 1]
92
+ g = g * self.speaker_map # [N, S, B, 1, H]
93
+ g = torch.sum(g, dim=1) # [N, 1, B, 1, H]
94
+ g = g.transpose(0, -1).transpose(0, -2).squeeze(0) # [B, H, N]
95
+ x = x.transpose(1, 2) + g
96
+ return x
97
+ else:
98
+ return x.transpose(1, 2)
99
+
100
+
101
+ def init_spkembed(self, units, f0, volume, spk_id = None, spk_mix_dict = None, aug_shift = None,
102
+ gt_spec=None, infer=True, infer_speedup=10, method='dpm-solver', k_step=300, use_tqdm=True):
103
+
104
+ '''
105
+ input:
106
+ B x n_frames x n_unit
107
+ return:
108
+ dict of B x n_frames x feat
109
+ '''
110
+ x = self.unit_embed(units) + self.f0_embed((1+ f0 / 700).log()) + self.volume_embed(volume)
111
+ if self.n_spk is not None and self.n_spk > 1:
112
+ if spk_mix_dict is not None:
113
+ spk_embed_mix = torch.zeros((1,1,self.hidden_size))
114
+ for k, v in spk_mix_dict.items():
115
+ spk_id_torch = torch.LongTensor(np.array([[k]])).to(units.device)
116
+ spk_embeddd = self.spk_embed(spk_id_torch)
117
+ self.speaker_map[k] = spk_embeddd
118
+ spk_embed_mix = spk_embed_mix + v * spk_embeddd
119
+ x = x + spk_embed_mix
120
+ else:
121
+ x = x + self.spk_embed(spk_id - 1)
122
+ self.speaker_map = self.speaker_map.unsqueeze(0)
123
+ self.speaker_map = self.speaker_map.detach()
124
+ return x.transpose(1, 2)
125
+
126
+ def OnnxExport(self, project_name=None, init_noise=None, export_encoder=True, export_denoise=True, export_pred=True, export_after=True):
127
+ hubert_hidden_size = 768
128
+ n_frames = 100
129
+ hubert = torch.randn((1, n_frames, hubert_hidden_size))
130
+ mel2ph = torch.arange(end=n_frames).unsqueeze(0).long()
131
+ f0 = torch.randn((1, n_frames))
132
+ volume = torch.randn((1, n_frames))
133
+ spk_mix = []
134
+ spks = {}
135
+ if self.n_spk is not None and self.n_spk > 1:
136
+ for i in range(self.n_spk):
137
+ spk_mix.append(1.0/float(self.n_spk))
138
+ spks.update({i:1.0/float(self.n_spk)})
139
+ spk_mix = torch.tensor(spk_mix)
140
+ spk_mix = spk_mix.repeat(n_frames, 1)
141
+ orgouttt = self.init_spkembed(hubert, f0.unsqueeze(-1), volume.unsqueeze(-1), spk_mix_dict=spks)
142
+ outtt = self.forward(hubert, mel2ph, f0, volume, spk_mix)
143
+ if export_encoder:
144
+ torch.onnx.export(
145
+ self,
146
+ (hubert, mel2ph, f0, volume, spk_mix),
147
+ f"{project_name}_encoder.onnx",
148
+ input_names=["hubert", "mel2ph", "f0", "volume", "spk_mix"],
149
+ output_names=["mel_pred"],
150
+ dynamic_axes={
151
+ "hubert": [1],
152
+ "f0": [1],
153
+ "volume": [1],
154
+ "mel2ph": [1],
155
+ "spk_mix": [0],
156
+ },
157
+ opset_version=16
158
+ )
159
+
160
+ self.decoder.OnnxExport(project_name, init_noise=init_noise, export_denoise=export_denoise, export_pred=export_pred, export_after=export_after)
161
+
162
+ def ExportOnnx(self, project_name=None):
163
+ hubert_hidden_size = 768
164
+ n_frames = 100
165
+ hubert = torch.randn((1, n_frames, hubert_hidden_size))
166
+ mel2ph = torch.arange(end=n_frames).unsqueeze(0).long()
167
+ f0 = torch.randn((1, n_frames))
168
+ volume = torch.randn((1, n_frames))
169
+ spk_mix = []
170
+ spks = {}
171
+ if self.n_spk is not None and self.n_spk > 1:
172
+ for i in range(self.n_spk):
173
+ spk_mix.append(1.0/float(self.n_spk))
174
+ spks.update({i:1.0/float(self.n_spk)})
175
+ spk_mix = torch.tensor(spk_mix)
176
+ orgouttt = self.orgforward(hubert, f0.unsqueeze(-1), volume.unsqueeze(-1), spk_mix_dict=spks)
177
+ outtt = self.forward(hubert, mel2ph, f0, volume, spk_mix)
178
+
179
+ torch.onnx.export(
180
+ self,
181
+ (hubert, mel2ph, f0, volume, spk_mix),
182
+ f"{project_name}_encoder.onnx",
183
+ input_names=["hubert", "mel2ph", "f0", "volume", "spk_mix"],
184
+ output_names=["mel_pred"],
185
+ dynamic_axes={
186
+ "hubert": [1],
187
+ "f0": [1],
188
+ "volume": [1],
189
+ "mel2ph": [1]
190
+ },
191
+ opset_version=16
192
+ )
193
+
194
+ condition = torch.randn(1,self.decoder.n_hidden,n_frames)
195
+ noise = torch.randn((1, 1, self.decoder.mel_bins, condition.shape[2]), dtype=torch.float32)
196
+ pndm_speedup = torch.LongTensor([100])
197
+ K_steps = torch.LongTensor([1000])
198
+ self.decoder = torch.jit.script(self.decoder)
199
+ self.decoder(condition, noise, pndm_speedup, K_steps)
200
+
201
+ torch.onnx.export(
202
+ self.decoder,
203
+ (condition, noise, pndm_speedup, K_steps),
204
+ f"{project_name}_diffusion.onnx",
205
+ input_names=["condition", "noise", "pndm_speedup", "K_steps"],
206
+ output_names=["mel"],
207
+ dynamic_axes={
208
+ "condition": [2],
209
+ "noise": [3],
210
+ },
211
+ opset_version=16
212
+ )
213
+
214
+
215
+ if __name__ == "__main__":
216
+ project_name = "dddsp"
217
+ model_path = f'{project_name}/model_500000.pt'
218
+
219
+ model, _ = load_model_vocoder(model_path)
220
+
221
+ # 分开Diffusion导出(需要使用MoeSS/MoeVoiceStudio或者自己编写Pndm/Dpm采样)
222
+ model.OnnxExport(project_name, export_encoder=True, export_denoise=True, export_pred=True, export_after=True)
223
+
224
+ # 合并Diffusion导出(Encoder和Diffusion分开,直接将Encoder的结果和初始噪声输入Diffusion即可)
225
+ # model.ExportOnnx(project_name)
226
+
diffusion/solver.py ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import time
3
+ import numpy as np
4
+ import torch
5
+ import librosa
6
+ from diffusion.logger.saver import Saver
7
+ from diffusion.logger import utils
8
+ from torch import autocast
9
+ from torch.cuda.amp import GradScaler
10
+
11
+ def test(args, model, vocoder, loader_test, saver):
12
+ print(' [*] testing...')
13
+ model.eval()
14
+
15
+ # losses
16
+ test_loss = 0.
17
+
18
+ # intialization
19
+ num_batches = len(loader_test)
20
+ rtf_all = []
21
+
22
+ # run
23
+ with torch.no_grad():
24
+ for bidx, data in enumerate(loader_test):
25
+ fn = data['name'][0].split("/")[-1]
26
+ speaker = data['name'][0].split("/")[-2]
27
+ print('--------')
28
+ print('{}/{} - {}'.format(bidx, num_batches, fn))
29
+
30
+ # unpack data
31
+ for k in data.keys():
32
+ if not k.startswith('name'):
33
+ data[k] = data[k].to(args.device)
34
+ print('>>', data['name'][0])
35
+
36
+ # forward
37
+ st_time = time.time()
38
+ mel = model(
39
+ data['units'],
40
+ data['f0'],
41
+ data['volume'],
42
+ data['spk_id'],
43
+ gt_spec=None,
44
+ infer=True,
45
+ infer_speedup=args.infer.speedup,
46
+ method=args.infer.method)
47
+ signal = vocoder.infer(mel, data['f0'])
48
+ ed_time = time.time()
49
+
50
+ # RTF
51
+ run_time = ed_time - st_time
52
+ song_time = signal.shape[-1] / args.data.sampling_rate
53
+ rtf = run_time / song_time
54
+ print('RTF: {} | {} / {}'.format(rtf, run_time, song_time))
55
+ rtf_all.append(rtf)
56
+
57
+ # loss
58
+ for i in range(args.train.batch_size):
59
+ loss = model(
60
+ data['units'],
61
+ data['f0'],
62
+ data['volume'],
63
+ data['spk_id'],
64
+ gt_spec=data['mel'],
65
+ infer=False)
66
+ test_loss += loss.item()
67
+
68
+ # log mel
69
+ saver.log_spec(f"{speaker}_{fn}.wav", data['mel'], mel)
70
+
71
+ # log audi
72
+ path_audio = data['name_ext'][0]
73
+ audio, sr = librosa.load(path_audio, sr=args.data.sampling_rate)
74
+ if len(audio.shape) > 1:
75
+ audio = librosa.to_mono(audio)
76
+ audio = torch.from_numpy(audio).unsqueeze(0).to(signal)
77
+ saver.log_audio({f"{speaker}_{fn}_gt.wav": audio,f"{speaker}_{fn}_pred.wav": signal})
78
+ # report
79
+ test_loss /= args.train.batch_size
80
+ test_loss /= num_batches
81
+
82
+ # check
83
+ print(' [test_loss] test_loss:', test_loss)
84
+ print(' Real Time Factor', np.mean(rtf_all))
85
+ return test_loss
86
+
87
+
88
+ def train(args, initial_global_step, model, optimizer, scheduler, vocoder, loader_train, loader_test):
89
+ # saver
90
+ saver = Saver(args, initial_global_step=initial_global_step)
91
+
92
+ # model size
93
+ params_count = utils.get_network_paras_amount({'model': model})
94
+ saver.log_info('--- model size ---')
95
+ saver.log_info(params_count)
96
+
97
+ # run
98
+ num_batches = len(loader_train)
99
+ model.train()
100
+ saver.log_info('======= start training =======')
101
+ scaler = GradScaler()
102
+ if args.train.amp_dtype == 'fp32':
103
+ dtype = torch.float32
104
+ elif args.train.amp_dtype == 'fp16':
105
+ dtype = torch.float16
106
+ elif args.train.amp_dtype == 'bf16':
107
+ dtype = torch.bfloat16
108
+ else:
109
+ raise ValueError(' [x] Unknown amp_dtype: ' + args.train.amp_dtype)
110
+ saver.log_info("epoch|batch_idx/num_batches|output_dir|batch/s|lr|time|step")
111
+ for epoch in range(args.train.epochs):
112
+ for batch_idx, data in enumerate(loader_train):
113
+ saver.global_step_increment()
114
+ optimizer.zero_grad()
115
+
116
+ # unpack data
117
+ for k in data.keys():
118
+ if not k.startswith('name'):
119
+ data[k] = data[k].to(args.device)
120
+
121
+ # forward
122
+ if dtype == torch.float32:
123
+ loss = model(data['units'].float(), data['f0'], data['volume'], data['spk_id'],
124
+ aug_shift = data['aug_shift'], gt_spec=data['mel'].float(), infer=False)
125
+ else:
126
+ with autocast(device_type=args.device, dtype=dtype):
127
+ loss = model(data['units'], data['f0'], data['volume'], data['spk_id'],
128
+ aug_shift = data['aug_shift'], gt_spec=data['mel'], infer=False)
129
+
130
+ # handle nan loss
131
+ if torch.isnan(loss):
132
+ raise ValueError(' [x] nan loss ')
133
+ else:
134
+ # backpropagate
135
+ if dtype == torch.float32:
136
+ loss.backward()
137
+ optimizer.step()
138
+ else:
139
+ scaler.scale(loss).backward()
140
+ scaler.step(optimizer)
141
+ scaler.update()
142
+ scheduler.step()
143
+
144
+ # log loss
145
+ if saver.global_step % args.train.interval_log == 0:
146
+ current_lr = optimizer.param_groups[0]['lr']
147
+ saver.log_info(
148
+ 'epoch: {} | {:3d}/{:3d} | {} | batch/s: {:.2f} | lr: {:.6} | loss: {:.3f} | time: {} | step: {}'.format(
149
+ epoch,
150
+ batch_idx,
151
+ num_batches,
152
+ args.env.expdir,
153
+ args.train.interval_log/saver.get_interval_time(),
154
+ current_lr,
155
+ loss.item(),
156
+ saver.get_total_time(),
157
+ saver.global_step
158
+ )
159
+ )
160
+
161
+ saver.log_value({
162
+ 'train/loss': loss.item()
163
+ })
164
+
165
+ saver.log_value({
166
+ 'train/lr': current_lr
167
+ })
168
+
169
+ # validation
170
+ if saver.global_step % args.train.interval_val == 0:
171
+ optimizer_save = optimizer if args.train.save_opt else None
172
+
173
+ # save latest
174
+ saver.save_model(model, optimizer_save, postfix=f'{saver.global_step}')
175
+ last_val_step = saver.global_step - args.train.interval_val
176
+ if last_val_step % args.train.interval_force_save != 0:
177
+ saver.delete_model(postfix=f'{last_val_step}')
178
+
179
+ # run testing set
180
+ test_loss = test(args, model, vocoder, loader_test, saver)
181
+
182
+ # log loss
183
+ saver.log_info(
184
+ ' --- <validation> --- \nloss: {:.3f}. '.format(
185
+ test_loss,
186
+ )
187
+ )
188
+
189
+ saver.log_value({
190
+ 'validation/loss': test_loss
191
+ })
192
+
193
+ model.train()
194
+
195
+
diffusion/unit2mel.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import yaml
3
+ import torch
4
+ import torch.nn as nn
5
+ import numpy as np
6
+ from .diffusion import GaussianDiffusion
7
+ from .wavenet import WaveNet
8
+ from .vocoder import Vocoder
9
+
10
+ class DotDict(dict):
11
+ def __getattr__(*args):
12
+ val = dict.get(*args)
13
+ return DotDict(val) if type(val) is dict else val
14
+
15
+ __setattr__ = dict.__setitem__
16
+ __delattr__ = dict.__delitem__
17
+
18
+
19
+ def load_model_vocoder(
20
+ model_path,
21
+ device='cpu',
22
+ config_path = None
23
+ ):
24
+ if config_path is None: config_file = os.path.join(os.path.split(model_path)[0], 'config.yaml')
25
+ else: config_file = config_path
26
+
27
+ with open(config_file, "r") as config:
28
+ args = yaml.safe_load(config)
29
+ args = DotDict(args)
30
+
31
+ # load vocoder
32
+ vocoder = Vocoder(args.vocoder.type, args.vocoder.ckpt, device=device)
33
+
34
+ # load model
35
+ model = Unit2Mel(
36
+ args.data.encoder_out_channels,
37
+ args.model.n_spk,
38
+ args.model.use_pitch_aug,
39
+ vocoder.dimension,
40
+ args.model.n_layers,
41
+ args.model.n_chans,
42
+ args.model.n_hidden)
43
+
44
+ print(' [Loading] ' + model_path)
45
+ ckpt = torch.load(model_path, map_location=torch.device(device))
46
+ model.to(device)
47
+ model.load_state_dict(ckpt['model'])
48
+ model.eval()
49
+ return model, vocoder, args
50
+
51
+
52
+ class Unit2Mel(nn.Module):
53
+ def __init__(
54
+ self,
55
+ input_channel,
56
+ n_spk,
57
+ use_pitch_aug=False,
58
+ out_dims=128,
59
+ n_layers=20,
60
+ n_chans=384,
61
+ n_hidden=256):
62
+ super().__init__()
63
+ self.unit_embed = nn.Linear(input_channel, n_hidden)
64
+ self.f0_embed = nn.Linear(1, n_hidden)
65
+ self.volume_embed = nn.Linear(1, n_hidden)
66
+ if use_pitch_aug:
67
+ self.aug_shift_embed = nn.Linear(1, n_hidden, bias=False)
68
+ else:
69
+ self.aug_shift_embed = None
70
+ self.n_spk = n_spk
71
+ if n_spk is not None and n_spk > 1:
72
+ self.spk_embed = nn.Embedding(n_spk, n_hidden)
73
+
74
+ self.n_hidden = n_hidden
75
+ # diffusion
76
+ self.decoder = GaussianDiffusion(WaveNet(out_dims, n_layers, n_chans, n_hidden), out_dims=out_dims)
77
+ self.input_channel = input_channel
78
+
79
+ def init_spkembed(self, units, f0, volume, spk_id = None, spk_mix_dict = None, aug_shift = None,
80
+ gt_spec=None, infer=True, infer_speedup=10, method='dpm-solver', k_step=300, use_tqdm=True):
81
+
82
+ '''
83
+ input:
84
+ B x n_frames x n_unit
85
+ return:
86
+ dict of B x n_frames x feat
87
+ '''
88
+ x = self.unit_embed(units) + self.f0_embed((1+ f0 / 700).log()) + self.volume_embed(volume)
89
+ if self.n_spk is not None and self.n_spk > 1:
90
+ if spk_mix_dict is not None:
91
+ spk_embed_mix = torch.zeros((1,1,self.hidden_size))
92
+ for k, v in spk_mix_dict.items():
93
+ spk_id_torch = torch.LongTensor(np.array([[k]])).to(units.device)
94
+ spk_embeddd = self.spk_embed(spk_id_torch)
95
+ self.speaker_map[k] = spk_embeddd
96
+ spk_embed_mix = spk_embed_mix + v * spk_embeddd
97
+ x = x + spk_embed_mix
98
+ else:
99
+ x = x + self.spk_embed(spk_id - 1)
100
+ self.speaker_map = self.speaker_map.unsqueeze(0)
101
+ self.speaker_map = self.speaker_map.detach()
102
+ return x.transpose(1, 2)
103
+
104
+ def init_spkmix(self, n_spk):
105
+ self.speaker_map = torch.zeros((n_spk,1,1,self.n_hidden))
106
+ hubert_hidden_size = self.input_channel
107
+ n_frames = 10
108
+ hubert = torch.randn((1, n_frames, hubert_hidden_size))
109
+ mel2ph = torch.arange(end=n_frames).unsqueeze(0).long()
110
+ f0 = torch.randn((1, n_frames))
111
+ volume = torch.randn((1, n_frames))
112
+ spks = {}
113
+ for i in range(n_spk):
114
+ spks.update({i:1.0/float(self.n_spk)})
115
+ orgouttt = self.init_spkembed(hubert, f0.unsqueeze(-1), volume.unsqueeze(-1), spk_mix_dict=spks)
116
+
117
+ def forward(self, units, f0, volume, spk_id = None, spk_mix_dict = None, aug_shift = None,
118
+ gt_spec=None, infer=True, infer_speedup=10, method='dpm-solver', k_step=300, use_tqdm=True):
119
+
120
+ '''
121
+ input:
122
+ B x n_frames x n_unit
123
+ return:
124
+ dict of B x n_frames x feat
125
+ '''
126
+
127
+ x = self.unit_embed(units) + self.f0_embed((1+ f0 / 700).log()) + self.volume_embed(volume)
128
+ if self.n_spk is not None and self.n_spk > 1:
129
+ if spk_mix_dict is not None:
130
+ for k, v in spk_mix_dict.items():
131
+ spk_id_torch = torch.LongTensor(np.array([[k]])).to(units.device)
132
+ x = x + v * self.spk_embed(spk_id_torch)
133
+ else:
134
+ if spk_id.shape[1] > 1:
135
+ g = spk_id.reshape((spk_id.shape[0], spk_id.shape[1], 1, 1, 1)) # [N, S, B, 1, 1]
136
+ g = g * self.speaker_map # [N, S, B, 1, H]
137
+ g = torch.sum(g, dim=1) # [N, 1, B, 1, H]
138
+ g = g.transpose(0, -1).transpose(0, -2).squeeze(0) # [B, H, N]
139
+ x = x + g
140
+ else:
141
+ x = x + self.spk_embed(spk_id)
142
+ if self.aug_shift_embed is not None and aug_shift is not None:
143
+ x = x + self.aug_shift_embed(aug_shift / 5)
144
+ x = self.decoder(x, gt_spec=gt_spec, infer=infer, infer_speedup=infer_speedup, method=method, k_step=k_step, use_tqdm=use_tqdm)
145
+
146
+ return x
147
+
diffusion/vocoder.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from vdecoder.nsf_hifigan.nvSTFT import STFT
3
+ from vdecoder.nsf_hifigan.models import load_model,load_config
4
+ from torchaudio.transforms import Resample
5
+
6
+
7
+ class Vocoder:
8
+ def __init__(self, vocoder_type, vocoder_ckpt, device = None):
9
+ if device is None:
10
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
11
+ self.device = device
12
+
13
+ if vocoder_type == 'nsf-hifigan':
14
+ self.vocoder = NsfHifiGAN(vocoder_ckpt, device = device)
15
+ elif vocoder_type == 'nsf-hifigan-log10':
16
+ self.vocoder = NsfHifiGANLog10(vocoder_ckpt, device = device)
17
+ else:
18
+ raise ValueError(f" [x] Unknown vocoder: {vocoder_type}")
19
+
20
+ self.resample_kernel = {}
21
+ self.vocoder_sample_rate = self.vocoder.sample_rate()
22
+ self.vocoder_hop_size = self.vocoder.hop_size()
23
+ self.dimension = self.vocoder.dimension()
24
+
25
+ def extract(self, audio, sample_rate, keyshift=0):
26
+
27
+ # resample
28
+ if sample_rate == self.vocoder_sample_rate:
29
+ audio_res = audio
30
+ else:
31
+ key_str = str(sample_rate)
32
+ if key_str not in self.resample_kernel:
33
+ self.resample_kernel[key_str] = Resample(sample_rate, self.vocoder_sample_rate, lowpass_filter_width = 128).to(self.device)
34
+ audio_res = self.resample_kernel[key_str](audio)
35
+
36
+ # extract
37
+ mel = self.vocoder.extract(audio_res, keyshift=keyshift) # B, n_frames, bins
38
+ return mel
39
+
40
+ def infer(self, mel, f0):
41
+ f0 = f0[:,:mel.size(1),0] # B, n_frames
42
+ audio = self.vocoder(mel, f0)
43
+ return audio
44
+
45
+
46
+ class NsfHifiGAN(torch.nn.Module):
47
+ def __init__(self, model_path, device=None):
48
+ super().__init__()
49
+ if device is None:
50
+ device = 'cuda' if torch.cuda.is_available() else 'cpu'
51
+ self.device = device
52
+ self.model_path = model_path
53
+ self.model = None
54
+ self.h = load_config(model_path)
55
+ self.stft = STFT(
56
+ self.h.sampling_rate,
57
+ self.h.num_mels,
58
+ self.h.n_fft,
59
+ self.h.win_size,
60
+ self.h.hop_size,
61
+ self.h.fmin,
62
+ self.h.fmax)
63
+
64
+ def sample_rate(self):
65
+ return self.h.sampling_rate
66
+
67
+ def hop_size(self):
68
+ return self.h.hop_size
69
+
70
+ def dimension(self):
71
+ return self.h.num_mels
72
+
73
+ def extract(self, audio, keyshift=0):
74
+ mel = self.stft.get_mel(audio, keyshift=keyshift).transpose(1, 2) # B, n_frames, bins
75
+ return mel
76
+
77
+ def forward(self, mel, f0):
78
+ if self.model is None:
79
+ print('| Load HifiGAN: ', self.model_path)
80
+ self.model, self.h = load_model(self.model_path, device=self.device)
81
+ with torch.no_grad():
82
+ c = mel.transpose(1, 2)
83
+ audio = self.model(c, f0)
84
+ return audio
85
+
86
+ class NsfHifiGANLog10(NsfHifiGAN):
87
+ def forward(self, mel, f0):
88
+ if self.model is None:
89
+ print('| Load HifiGAN: ', self.model_path)
90
+ self.model, self.h = load_model(self.model_path, device=self.device)
91
+ with torch.no_grad():
92
+ c = 0.434294 * mel.transpose(1, 2)
93
+ audio = self.model(c, f0)
94
+ return audio
diffusion/wavenet.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from math import sqrt
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ from torch.nn import Mish
8
+
9
+
10
+ class Conv1d(torch.nn.Conv1d):
11
+ def __init__(self, *args, **kwargs):
12
+ super().__init__(*args, **kwargs)
13
+ nn.init.kaiming_normal_(self.weight)
14
+
15
+
16
+ class SinusoidalPosEmb(nn.Module):
17
+ def __init__(self, dim):
18
+ super().__init__()
19
+ self.dim = dim
20
+
21
+ def forward(self, x):
22
+ device = x.device
23
+ half_dim = self.dim // 2
24
+ emb = math.log(10000) / (half_dim - 1)
25
+ emb = torch.exp(torch.arange(half_dim, device=device) * -emb)
26
+ emb = x[:, None] * emb[None, :]
27
+ emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
28
+ return emb
29
+
30
+
31
+ class ResidualBlock(nn.Module):
32
+ def __init__(self, encoder_hidden, residual_channels, dilation):
33
+ super().__init__()
34
+ self.residual_channels = residual_channels
35
+ self.dilated_conv = nn.Conv1d(
36
+ residual_channels,
37
+ 2 * residual_channels,
38
+ kernel_size=3,
39
+ padding=dilation,
40
+ dilation=dilation
41
+ )
42
+ self.diffusion_projection = nn.Linear(residual_channels, residual_channels)
43
+ self.conditioner_projection = nn.Conv1d(encoder_hidden, 2 * residual_channels, 1)
44
+ self.output_projection = nn.Conv1d(residual_channels, 2 * residual_channels, 1)
45
+
46
+ def forward(self, x, conditioner, diffusion_step):
47
+ diffusion_step = self.diffusion_projection(diffusion_step).unsqueeze(-1)
48
+ conditioner = self.conditioner_projection(conditioner)
49
+ y = x + diffusion_step
50
+
51
+ y = self.dilated_conv(y) + conditioner
52
+
53
+ # Using torch.split instead of torch.chunk to avoid using onnx::Slice
54
+ gate, filter = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
55
+ y = torch.sigmoid(gate) * torch.tanh(filter)
56
+
57
+ y = self.output_projection(y)
58
+
59
+ # Using torch.split instead of torch.chunk to avoid using onnx::Slice
60
+ residual, skip = torch.split(y, [self.residual_channels, self.residual_channels], dim=1)
61
+ return (x + residual) / math.sqrt(2.0), skip
62
+
63
+
64
+ class WaveNet(nn.Module):
65
+ def __init__(self, in_dims=128, n_layers=20, n_chans=384, n_hidden=256):
66
+ super().__init__()
67
+ self.input_projection = Conv1d(in_dims, n_chans, 1)
68
+ self.diffusion_embedding = SinusoidalPosEmb(n_chans)
69
+ self.mlp = nn.Sequential(
70
+ nn.Linear(n_chans, n_chans * 4),
71
+ Mish(),
72
+ nn.Linear(n_chans * 4, n_chans)
73
+ )
74
+ self.residual_layers = nn.ModuleList([
75
+ ResidualBlock(
76
+ encoder_hidden=n_hidden,
77
+ residual_channels=n_chans,
78
+ dilation=1
79
+ )
80
+ for i in range(n_layers)
81
+ ])
82
+ self.skip_projection = Conv1d(n_chans, n_chans, 1)
83
+ self.output_projection = Conv1d(n_chans, in_dims, 1)
84
+ nn.init.zeros_(self.output_projection.weight)
85
+
86
+ def forward(self, spec, diffusion_step, cond):
87
+ """
88
+ :param spec: [B, 1, M, T]
89
+ :param diffusion_step: [B, 1]
90
+ :param cond: [B, M, T]
91
+ :return:
92
+ """
93
+ x = spec.squeeze(1)
94
+ x = self.input_projection(x) # [B, residual_channel, T]
95
+
96
+ x = F.relu(x)
97
+ diffusion_step = self.diffusion_embedding(diffusion_step)
98
+ diffusion_step = self.mlp(diffusion_step)
99
+ skip = []
100
+ for layer in self.residual_layers:
101
+ x, skip_connection = layer(x, cond, diffusion_step)
102
+ skip.append(skip_connection)
103
+
104
+ x = torch.sum(torch.stack(skip), dim=0) / sqrt(len(self.residual_layers))
105
+ x = self.skip_projection(x)
106
+ x = F.relu(x)
107
+ x = self.output_projection(x) # [B, mel_bins, T]
108
+ return x[:, None, :, :]
filelists/test.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ ./dataset/44k/taffy/000562.wav
2
+ ./dataset/44k/nyaru/000011.wav
3
+ ./dataset/44k/nyaru/000008.wav
4
+ ./dataset/44k/taffy/000563.wav
filelists/train.txt ADDED
@@ -0,0 +1,924 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_56.wav
2
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006211_35.wav
3
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_25.wav
4
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006221_19.wav
5
+ ./dataset/44k/1001/story_data_04_1052_storytimeline_041052002_35.wav
6
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_35.wav
7
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_33.wav
8
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_26.wav
9
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_47.wav
10
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_14.wav
11
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_17.wav
12
+ ./dataset/44k/1001/story_data_10_0003_storytimeline_100003001_41.wav
13
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_0.wav
14
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_17.wav
15
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_45.wav
16
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_41.wav
17
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_24.wav
18
+ ./dataset/44k/1001/story_data_04_1020_storytimeline_041020004_43.wav
19
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011010_71.wav
20
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_43.wav
21
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_55.wav
22
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_26.wav
23
+ ./dataset/44k/1001/story_data_10_0002_storytimeline_100002001_1.wav
24
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_55.wav
25
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005002_3.wav
26
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_3.wav
27
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_18.wav
28
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_24.wav
29
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_51.wav
30
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021005_39.wav
31
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_31.wav
32
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_54.wav
33
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_34.wav
34
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_57.wav
35
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_26.wav
36
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_41.wav
37
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_4.wav
38
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005131_0.wav
39
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_8.wav
40
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_33.wav
41
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100102_7.wav
42
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_2.wav
43
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_26.wav
44
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_25.wav
45
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_26.wav
46
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_25.wav
47
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_19.wav
48
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_16.wav
49
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_12.wav
50
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_21.wav
51
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_47.wav
52
+ ./dataset/44k/1001/story_data_04_1052_storytimeline_041052002_34.wav
53
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_14.wav
54
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_12.wav
55
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090011_3.wav
56
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_gacha_100103_0.wav
57
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_5.wav
58
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_8.wav
59
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_11.wav
60
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_8.wav
61
+ ./dataset/44k/1001/story_data_04_1052_storytimeline_041052004_8.wav
62
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_2.wav
63
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001006_2.wav
64
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_33.wav
65
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_50.wav
66
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_12.wav
67
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_race_100101_2.wav
68
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_42.wav
69
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_1.wav
70
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_17.wav
71
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_26.wav
72
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_race_100102_2.wav
73
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_7.wav
74
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011007_6.wav
75
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_47.wav
76
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_26.wav
77
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021005_27.wav
78
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021003_37.wav
79
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_33.wav
80
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_10.wav
81
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_40.wav
82
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_36.wav
83
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006211_90.wav
84
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011010_4.wav
85
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001005_3.wav
86
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_22.wav
87
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_27.wav
88
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011003_10.wav
89
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_37.wav
90
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_33.wav
91
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_32.wav
92
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_39.wav
93
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_21.wav
94
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_28.wav
95
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_36.wav
96
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_14.wav
97
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100102_6.wav
98
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_6.wav
99
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000007_8.wav
100
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_39.wav
101
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_36.wav
102
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_36.wav
103
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021005_8.wav
104
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_0.wav
105
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_0.wav
106
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_13.wav
107
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_5.wav
108
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_2.wav
109
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_54.wav
110
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_37.wav
111
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_37.wav
112
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_31.wav
113
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_33.wav
114
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_25.wav
115
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_8.wav
116
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_28.wav
117
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005021_45.wav
118
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_39.wav
119
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_1.wav
120
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011001_25.wav
121
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_31.wav
122
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_32.wav
123
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_22.wav
124
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_24.wav
125
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_39.wav
126
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_10.wav
127
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_33.wav
128
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_15.wav
129
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_6.wav
130
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_0.wav
131
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_27.wav
132
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_26.wav
133
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_44.wav
134
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_11.wav
135
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_34.wav
136
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_2.wav
137
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_9.wav
138
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_outgame_100100_17.wav
139
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_10.wav
140
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011003_12.wav
141
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_9.wav
142
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_30.wav
143
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_1.wav
144
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_31.wav
145
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_23.wav
146
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_1.wav
147
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_12.wav
148
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_11.wav
149
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_3.wav
150
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100101_7.wav
151
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_33.wav
152
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_20.wav
153
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_11.wav
154
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_19.wav
155
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_3.wav
156
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_12.wav
157
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_3.wav
158
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_22.wav
159
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_15.wav
160
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_6.wav
161
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_32.wav
162
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_28.wav
163
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021002_33.wav
164
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_13.wav
165
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_17.wav
166
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_10.wav
167
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_23.wav
168
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_14.wav
169
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006131_9.wav
170
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_0.wav
171
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_5.wav
172
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006211_61.wav
173
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_54.wav
174
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_17.wav
175
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_32.wav
176
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_8.wav
177
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_27.wav
178
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_20.wav
179
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_37.wav
180
+ ./dataset/44k/1001/story_data_04_1020_storytimeline_041020004_6.wav
181
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_42.wav
182
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_18.wav
183
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_0.wav
184
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_13.wav
185
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005091_1.wav
186
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_20.wav
187
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_38.wav
188
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_1.wav
189
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_28.wav
190
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_3.wav
191
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_14.wav
192
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_42.wav
193
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_23.wav
194
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_21.wav
195
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_52.wav
196
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_41.wav
197
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_6.wav
198
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_35.wav
199
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_41.wav
200
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_20.wav
201
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_28.wav
202
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_17.wav
203
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100103_4.wav
204
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_48.wav
205
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_7.wav
206
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_20.wav
207
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_12.wav
208
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_24.wav
209
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_29.wav
210
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_49.wav
211
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_65.wav
212
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_21.wav
213
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_15.wav
214
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_46.wav
215
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_14.wav
216
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001005_8.wav
217
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_0.wav
218
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_16.wav
219
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_37.wav
220
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_race_100103_2.wav
221
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_35.wav
222
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011010_37.wav
223
+ ./dataset/44k/1001/story_data_04_1020_storytimeline_041020004_5.wav
224
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021002_31.wav
225
+ ./dataset/44k/1001/story_data_04_1011_storytimeline_041011002_4.wav
226
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011005_51.wav
227
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_6.wav
228
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_12.wav
229
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_35.wav
230
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_20.wav
231
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_37.wav
232
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021002_27.wav
233
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_12.wav
234
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_8.wav
235
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_16.wav
236
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_17.wav
237
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_43.wav
238
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_45.wav
239
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_3.wav
240
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_34.wav
241
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_13.wav
242
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_2.wav
243
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_44.wav
244
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005021_40.wav
245
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_26.wav
246
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000007_19.wav
247
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_5.wav
248
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_20.wav
249
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_24.wav
250
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_42.wav
251
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005002_2.wav
252
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001005_9.wav
253
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_8.wav
254
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_24.wav
255
+ ./dataset/44k/1001/story_data_04_1061_storytimeline_041061001_39.wav
256
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_6.wav
257
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_52.wav
258
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000007_20.wav
259
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_5.wav
260
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_41.wav
261
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_25.wav
262
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_16.wav
263
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_1.wav
264
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_2.wav
265
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_13.wav
266
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_35.wav
267
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_21.wav
268
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_22.wav
269
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_60.wav
270
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100102_8.wav
271
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_38.wav
272
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_17.wav
273
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_33.wav
274
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006171_3.wav
275
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_17.wav
276
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_13.wav
277
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_live_100100_1.wav
278
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_43.wav
279
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_11.wav
280
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_37.wav
281
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005021_46.wav
282
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_31.wav
283
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_53.wav
284
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005133_3.wav
285
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_34.wav
286
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_44.wav
287
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_10.wav
288
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_6.wav
289
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_13.wav
290
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_25.wav
291
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021003_30.wav
292
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_19.wav
293
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_54.wav
294
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_15.wav
295
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_33.wav
296
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_19.wav
297
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_12.wav
298
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_34.wav
299
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_34.wav
300
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_33.wav
301
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_39.wav
302
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_21.wav
303
+ ./dataset/44k/1001/story_data_10_0003_storytimeline_100003001_43.wav
304
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_23.wav
305
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_33.wav
306
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_34.wav
307
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011001_2.wav
308
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_17.wav
309
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_39.wav
310
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_18.wav
311
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_9.wav
312
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_49.wav
313
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_20.wav
314
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_5.wav
315
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_17.wav
316
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_21.wav
317
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_18.wav
318
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_race_9100101_0.wav
319
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_34.wav
320
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_41.wav
321
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_21.wav
322
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_outgame_100100_20.wav
323
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_33.wav
324
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_71.wav
325
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_31.wav
326
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_37.wav
327
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_11.wav
328
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_2.wav
329
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_40.wav
330
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_33.wav
331
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_7.wav
332
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_20.wav
333
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_5.wav
334
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021008_30.wav
335
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_27.wav
336
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100103_2.wav
337
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_19.wav
338
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006201_57.wav
339
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_8.wav
340
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_1.wav
341
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_45.wav
342
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_35.wav
343
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_9.wav
344
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_46.wav
345
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_21.wav
346
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_28.wav
347
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_18.wav
348
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_46.wav
349
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001005_11.wav
350
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_15.wav
351
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_24.wav
352
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_18.wav
353
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_4.wav
354
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_23.wav
355
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_40.wav
356
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_2.wav
357
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_37.wav
358
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_52.wav
359
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_15.wav
360
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_15.wav
361
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_4.wav
362
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_16.wav
363
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_19.wav
364
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_25.wav
365
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_45.wav
366
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_10.wav
367
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_4.wav
368
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_5.wav
369
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_15.wav
370
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_32.wav
371
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_25.wav
372
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100101_4.wav
373
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005112_10.wav
374
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_2.wav
375
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_53.wav
376
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_19.wav
377
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_3.wav
378
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090005_0.wav
379
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011001_1.wav
380
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005061_7.wav
381
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_49.wav
382
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_48.wav
383
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_10.wav
384
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_38.wav
385
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_29.wav
386
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_9.wav
387
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_27.wav
388
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_29.wav
389
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_42.wav
390
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_outgame_100100_0.wav
391
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_30.wav
392
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_26.wav
393
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_0.wav
394
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100101_5.wav
395
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_8.wav
396
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_21.wav
397
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_33.wav
398
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_42.wav
399
+ ./dataset/44k/1001/story_data_04_1020_storytimeline_041020002_18.wav
400
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_23.wav
401
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_34.wav
402
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_3.wav
403
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_38.wav
404
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_23.wav
405
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_1.wav
406
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_45.wav
407
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_5.wav
408
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_16.wav
409
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_51.wav
410
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_52.wav
411
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_4.wav
412
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_31.wav
413
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_21.wav
414
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005101_9.wav
415
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011010_34.wav
416
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_25.wav
417
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000007_9.wav
418
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_6.wav
419
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_9.wav
420
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021008_47.wav
421
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_10.wav
422
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_5.wav
423
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_3.wav
424
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_39.wav
425
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_7.wav
426
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_31.wav
427
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006171_2.wav
428
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_48.wav
429
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090021_1.wav
430
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_51.wav
431
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006201_55.wav
432
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_13.wav
433
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_5.wav
434
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_37.wav
435
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_38.wav
436
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_8.wav
437
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_18.wav
438
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011010_69.wav
439
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_48.wav
440
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_7.wav
441
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_7.wav
442
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_12.wav
443
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_11.wav
444
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_30.wav
445
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_51.wav
446
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_13.wav
447
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_9.wav
448
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_4.wav
449
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_18.wav
450
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_19.wav
451
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_12.wav
452
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_16.wav
453
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_47.wav
454
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006201_40.wav
455
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_27.wav
456
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_41.wav
457
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_22.wav
458
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_4.wav
459
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_30.wav
460
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_53.wav
461
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_9.wav
462
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_7.wav
463
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_31.wav
464
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_3.wav
465
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011005_39.wav
466
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_24.wav
467
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_44.wav
468
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_27.wav
469
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005131_2.wav
470
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_16.wav
471
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_6.wav
472
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_12.wav
473
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_13.wav
474
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_22.wav
475
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_30.wav
476
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011001_26.wav
477
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_7.wav
478
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001006_26.wav
479
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_55.wav
480
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_56.wav
481
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_39.wav
482
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_37.wav
483
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_17.wav
484
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001006_4.wav
485
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_3.wav
486
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_37.wav
487
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_59.wav
488
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_49.wav
489
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_24.wav
490
+ ./dataset/44k/1001/story_data_04_1052_storytimeline_041052002_32.wav
491
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_35.wav
492
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100101_6.wav
493
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_24.wav
494
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_22.wav
495
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_10.wav
496
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_15.wav
497
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_41.wav
498
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_6.wav
499
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_28.wav
500
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_49.wav
501
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_41.wav
502
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_1.wav
503
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005101_10.wav
504
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001005_7.wav
505
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_34.wav
506
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_24.wav
507
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_11.wav
508
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_49.wav
509
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001006_40.wav
510
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_9.wav
511
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_16.wav
512
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_21.wav
513
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_9.wav
514
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_42.wav
515
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_34.wav
516
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_15.wav
517
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006091_38.wav
518
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_45.wav
519
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_10.wav
520
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_4.wav
521
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_39.wav
522
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000008_3.wav
523
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_11.wav
524
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021003_1.wav
525
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_28.wav
526
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_19.wav
527
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_42.wav
528
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_42.wav
529
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_outgame_100100_19.wav
530
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_29.wav
531
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_22.wav
532
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_15.wav
533
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_27.wav
534
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_25.wav
535
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021007_3.wav
536
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_51.wav
537
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_34.wav
538
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_0.wav
539
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_32.wav
540
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_45.wav
541
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005002_4.wav
542
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_24.wav
543
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_22.wav
544
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_1.wav
545
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_13.wav
546
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_7.wav
547
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011003_19.wav
548
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_34.wav
549
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_2.wav
550
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021002_34.wav
551
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_20.wav
552
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005101_0.wav
553
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_34.wav
554
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_6.wav
555
+ ./dataset/44k/1001/story_data_04_1061_storytimeline_041061002_6.wav
556
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_19.wav
557
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_31.wav
558
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_39.wav
559
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006221_30.wav
560
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_2.wav
561
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_40.wav
562
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_22.wav
563
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006232_11.wav
564
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_21.wav
565
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_9.wav
566
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_9.wav
567
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_20.wav
568
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_31.wav
569
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_37.wav
570
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_47.wav
571
+ ./dataset/44k/1001/story_data_10_0002_storytimeline_100002001_27.wav
572
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_22.wav
573
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_50.wav
574
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_39.wav
575
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_24.wav
576
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006131_15.wav
577
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_25.wav
578
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_20.wav
579
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005112_0.wav
580
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_30.wav
581
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_44.wav
582
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100101_2.wav
583
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_29.wav
584
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_21.wav
585
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_46.wav
586
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_31.wav
587
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100103_3.wav
588
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_15.wav
589
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_6.wav
590
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090021_14.wav
591
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_12.wav
592
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_18.wav
593
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_24.wav
594
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_30.wav
595
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_35.wav
596
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_11.wav
597
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_43.wav
598
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_46.wav
599
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_13.wav
600
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_3.wav
601
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_36.wav
602
+ ./dataset/44k/1001/story_data_10_0003_storytimeline_100003001_34.wav
603
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_22.wav
604
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011007_9.wav
605
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_22.wav
606
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_8.wav
607
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_race_100102_1.wav
608
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_50.wav
609
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_51.wav
610
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_14.wav
611
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_15.wav
612
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_18.wav
613
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_15.wav
614
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_10.wav
615
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_28.wav
616
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_20.wav
617
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_43.wav
618
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_30.wav
619
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_39.wav
620
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_51.wav
621
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_30.wav
622
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006211_37.wav
623
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_52.wav
624
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_39.wav
625
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_35.wav
626
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006232_12.wav
627
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_27.wav
628
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_0.wav
629
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000008_2.wav
630
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_30.wav
631
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_4.wav
632
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_35.wav
633
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_33.wav
634
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021005_16.wav
635
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_13.wav
636
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006171_1.wav
637
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005101_5.wav
638
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_3.wav
639
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_11.wav
640
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_32.wav
641
+ ./dataset/44k/1001/story_data_04_1011_storytimeline_041011002_10.wav
642
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_32.wav
643
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_12.wav
644
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006211_69.wav
645
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_30.wav
646
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_8.wav
647
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_19.wav
648
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_4.wav
649
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_53.wav
650
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_29.wav
651
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_44.wav
652
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006232_18.wav
653
+ ./dataset/44k/1001/story_data_04_1011_storytimeline_041011004_40.wav
654
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_43.wav
655
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_43.wav
656
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_25.wav
657
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_17.wav
658
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_60.wav
659
+ ./dataset/44k/1001/story_data_04_1011_storytimeline_041011002_20.wav
660
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_54.wav
661
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_45.wav
662
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_9.wav
663
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006201_56.wav
664
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001005_6.wav
665
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011003_8.wav
666
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006133_22.wav
667
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_0.wav
668
+ ./dataset/44k/1001/story_data_04_1020_storytimeline_041020004_44.wav
669
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_38.wav
670
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_41.wav
671
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_62.wav
672
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_15.wav
673
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_51.wav
674
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_29.wav
675
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_6.wav
676
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_57.wav
677
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_35.wav
678
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_26.wav
679
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_0.wav
680
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_23.wav
681
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_3.wav
682
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_26.wav
683
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005021_43.wav
684
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_9.wav
685
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006091_39.wav
686
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_29.wav
687
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_14.wav
688
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005101_3.wav
689
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005005_5.wav
690
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_23.wav
691
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_8.wav
692
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_46.wav
693
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_16.wav
694
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006121_50.wav
695
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_0.wav
696
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100101_0.wav
697
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_63.wav
698
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_11.wav
699
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_30.wav
700
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_25.wav
701
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_10.wav
702
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_43.wav
703
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_38.wav
704
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005021_42.wav
705
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021005_10.wav
706
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_45.wav
707
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_26.wav
708
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021004_12.wav
709
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_34.wav
710
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_1.wav
711
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_1.wav
712
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_72.wav
713
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_20.wav
714
+ ./dataset/44k/1001/story_data_04_1052_storytimeline_041052002_33.wav
715
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_10.wav
716
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_4.wav
717
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_48.wav
718
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_2.wav
719
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_32.wav
720
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_48.wav
721
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_51.wav
722
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_49.wav
723
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_8.wav
724
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_39.wav
725
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_5.wav
726
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_7.wav
727
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_11.wav
728
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_29.wav
729
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_25.wav
730
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100103_1.wav
731
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_11.wav
732
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_25.wav
733
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_39.wav
734
+ ./dataset/44k/1001/story_data_10_0002_storytimeline_100002001_0.wav
735
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_24.wav
736
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_37.wav
737
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_7.wav
738
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_7.wav
739
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_18.wav
740
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_28.wav
741
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_1.wav
742
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005005_1.wav
743
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006181_17.wav
744
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001003_47.wav
745
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_30.wav
746
+ ./dataset/44k/1001/story_data_04_1011_storytimeline_041011002_6.wav
747
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_22.wav
748
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_43.wav
749
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001006_2.wav
750
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_8.wav
751
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_56.wav
752
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_17.wav
753
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_16.wav
754
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011008_14.wav
755
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_18.wav
756
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_10.wav
757
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_19.wav
758
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_outgame_100100_6.wav
759
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_6.wav
760
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_44.wav
761
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_27.wav
762
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_64.wav
763
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_0.wav
764
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_20.wav
765
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005005_29.wav
766
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_27.wav
767
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_22.wav
768
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100102_5.wav
769
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_14.wav
770
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_4.wav
771
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_19.wav
772
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021004_19.wav
773
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_4.wav
774
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_6.wav
775
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_50.wav
776
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_45.wav
777
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_20.wav
778
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_9.wav
779
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_40.wav
780
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_9.wav
781
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_1.wav
782
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_24.wav
783
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_23.wav
784
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_17.wav
785
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_31.wav
786
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_11.wav
787
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_52.wav
788
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_40.wav
789
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_35.wav
790
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000007_12.wav
791
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_20.wav
792
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_16.wav
793
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_41.wav
794
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_2.wav
795
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_4.wav
796
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090021_11.wav
797
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_23.wav
798
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_53.wav
799
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_18.wav
800
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_5.wav
801
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_52.wav
802
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_39.wav
803
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_9.wav
804
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011005_45.wav
805
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_5.wav
806
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_9.wav
807
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_31.wav
808
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100103_1.wav
809
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_45.wav
810
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011007_8.wav
811
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_19.wav
812
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_13.wav
813
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_51.wav
814
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_23.wav
815
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_47.wav
816
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_34.wav
817
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021008_41.wav
818
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005031_32.wav
819
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_outgame_100102_2.wav
820
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_35.wav
821
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_23.wav
822
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_21.wav
823
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_52.wav
824
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_14.wav
825
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_9.wav
826
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006021_28.wav
827
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001003_8.wav
828
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001007_36.wav
829
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_18.wav
830
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_19.wav
831
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_2.wav
832
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005008_46.wav
833
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005081_8.wav
834
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_16.wav
835
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006091_10.wav
836
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_64.wav
837
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_16.wav
838
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_40.wav
839
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_13.wav
840
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006051_3.wav
841
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011010_70.wav
842
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_38.wav
843
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005007_13.wav
844
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_19.wav
845
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_23.wav
846
+ ./dataset/44k/1001/story_data_10_0002_storytimeline_100002001_20.wav
847
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006041_30.wav
848
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006161_25.wav
849
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005001_41.wav
850
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_14.wav
851
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_0.wav
852
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006091_4.wav
853
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006061_27.wav
854
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_8.wav
855
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_32.wav
856
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006232_19.wav
857
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011003_17.wav
858
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006071_45.wav
859
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_7.wav
860
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_41.wav
861
+ ./dataset/44k/1001/story_data_10_0002_storytimeline_100002001_35.wav
862
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006221_29.wav
863
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_14.wav
864
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_34.wav
865
+ ./dataset/44k/1001/story_data_04_1004_storytimeline_041004004_41.wav
866
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005051_29.wav
867
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100103_5.wav
868
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_10.wav
869
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_10.wav
870
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006031_5.wav
871
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_18.wav
872
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_26.wav
873
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005003_37.wav
874
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_52.wav
875
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000008_5.wav
876
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_0.wav
877
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_47.wav
878
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021007_21.wav
879
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006011_41.wav
880
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001007_18.wav
881
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011006_38.wav
882
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001004_0.wav
883
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_32.wav
884
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_35.wav
885
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_13.wav
886
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_18.wav
887
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006081_25.wav
888
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_28.wav
889
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_6.wav
890
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_29.wav
891
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100103_6.wav
892
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_55.wav
893
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001005_21.wav
894
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005071_25.wav
895
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001002_30.wav
896
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100101_2.wav
897
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006191_36.wav
898
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006213_25.wav
899
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100102_14.wav
900
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005101_4.wav
901
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_54.wav
902
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021005_41.wav
903
+ ./dataset/44k/1001/story_data_02_0005_storytimeline_020005041_10.wav
904
+ ./dataset/44k/1001/story_data_10_0000_storytimeline_100000008_8.wav
905
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001006_18.wav
906
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006141_36.wav
907
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_13.wav
908
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005004_21.wav
909
+ ./dataset/44k/1001/story_data_10_0002_storytimeline_100002001_4.wav
910
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_14.wav
911
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001008_42.wav
912
+ ./dataset/44k/1001/story_data_09_0011_storytimeline_090011001_3.wav
913
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006063_28.wav
914
+ ./dataset/44k/1001/story_data_09_0005_storytimeline_090005005_9.wav
915
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001002_1.wav
916
+ ./dataset/44k/1001/story_data_09_0021_storytimeline_090021006_39.wav
917
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006231_23.wav
918
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_event_090001_5.wav
919
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001001_39.wav
920
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006101_29.wav
921
+ ./dataset/44k/1001/story_data_04_1001_storytimeline_041001001_36.wav
922
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_home_100100_9.wav
923
+ ./dataset/44k/1001/character_system_text_1001_snd_voi_training_100100_21.wav
924
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006151_11.wav
filelists/val.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ ./dataset/44k/1001/story_data_09_0001_storytimeline_090001004_19.wav
2
+ ./dataset/44k/1001/story_data_02_0006_storytimeline_020006211_88.wav
flask_api.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import io
2
+ import logging
3
+
4
+ import soundfile
5
+ import torch
6
+ import torchaudio
7
+ from flask import Flask, request, send_file
8
+ from flask_cors import CORS
9
+
10
+ from inference.infer_tool import Svc, RealTimeVC
11
+
12
+ app = Flask(__name__)
13
+
14
+ CORS(app)
15
+
16
+ logging.getLogger('numba').setLevel(logging.WARNING)
17
+
18
+
19
+ @app.route("/voiceChangeModel", methods=["POST"])
20
+ def voice_change_model():
21
+ request_form = request.form
22
+ wave_file = request.files.get("sample", None)
23
+ # 变调信息
24
+ f_pitch_change = float(request_form.get("fPitchChange", 0))
25
+ # DAW所需的采样率
26
+ daw_sample = int(float(request_form.get("sampleRate", 0)))
27
+ speaker_id = int(float(request_form.get("sSpeakId", 0)))
28
+ # http获得wav文件并转换
29
+ input_wav_path = io.BytesIO(wave_file.read())
30
+
31
+ # 模型推理
32
+ if raw_infer:
33
+ # out_audio, out_sr = svc_model.infer(speaker_id, f_pitch_change, input_wav_path)
34
+ out_audio, out_sr = svc_model.infer(speaker_id, f_pitch_change, input_wav_path, cluster_infer_ratio=0,
35
+ auto_predict_f0=False, noice_scale=0.4, f0_filter=False)
36
+ tar_audio = torchaudio.functional.resample(out_audio, svc_model.target_sample, daw_sample)
37
+ else:
38
+ out_audio = svc.process(svc_model, speaker_id, f_pitch_change, input_wav_path, cluster_infer_ratio=0,
39
+ auto_predict_f0=False, noice_scale=0.4, f0_filter=False)
40
+ tar_audio = torchaudio.functional.resample(torch.from_numpy(out_audio), svc_model.target_sample, daw_sample)
41
+ # 返回音频
42
+ out_wav_path = io.BytesIO()
43
+ soundfile.write(out_wav_path, tar_audio.cpu().numpy(), daw_sample, format="wav")
44
+ out_wav_path.seek(0)
45
+ return send_file(out_wav_path, download_name="temp.wav", as_attachment=True)
46
+
47
+
48
+ if __name__ == '__main__':
49
+ # 启用则为直接切片合成,False为交叉淡化方式
50
+ # vst插件调整0.3-0.5s切片时间可以降低延迟,直接切片方法会有连接处爆音、交叉淡化会有轻微重叠声音
51
+ # 自行选择能接受的方法,或将vst最大切片时间调整为1s,此处设为Ture,延迟大音质稳定一些
52
+ raw_infer = True
53
+ # 每个模型和config是唯一对应的
54
+ model_name = "logs/32k/G_174000-Copy1.pth"
55
+ config_name = "configs/config.json"
56
+ cluster_model_path = "logs/44k/kmeans_10000.pt"
57
+ svc_model = Svc(model_name, config_name, cluster_model_path=cluster_model_path)
58
+ svc = RealTimeVC()
59
+ # 此处与vst插件对应,不建议更改
60
+ app.run(port=6842, host="0.0.0.0", debug=False, threaded=False)
flask_api_full_song.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import io
2
+ import numpy as np
3
+ import soundfile
4
+ from flask import Flask, request, send_file
5
+
6
+ from inference import infer_tool
7
+ from inference import slicer
8
+
9
+ app = Flask(__name__)
10
+
11
+
12
+ @app.route("/wav2wav", methods=["POST"])
13
+ def wav2wav():
14
+ request_form = request.form
15
+ audio_path = request_form.get("audio_path", None) # wav文件地址
16
+ tran = int(float(request_form.get("tran", 0))) # 音调
17
+ spk = request_form.get("spk", 0) # 说话人(id或者name都可以,具体看你的config)
18
+ wav_format = request_form.get("wav_format", 'wav') # 范围文件格式
19
+ infer_tool.format_wav(audio_path)
20
+ chunks = slicer.cut(audio_path, db_thresh=-40)
21
+ audio_data, audio_sr = slicer.chunks2audio(audio_path, chunks)
22
+
23
+ audio = []
24
+ for (slice_tag, data) in audio_data:
25
+ print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
26
+
27
+ length = int(np.ceil(len(data) / audio_sr * svc_model.target_sample))
28
+ if slice_tag:
29
+ print('jump empty segment')
30
+ _audio = np.zeros(length)
31
+ else:
32
+ # padd
33
+ pad_len = int(audio_sr * 0.5)
34
+ data = np.concatenate([np.zeros([pad_len]), data, np.zeros([pad_len])])
35
+ raw_path = io.BytesIO()
36
+ soundfile.write(raw_path, data, audio_sr, format="wav")
37
+ raw_path.seek(0)
38
+ out_audio, out_sr = svc_model.infer(spk, tran, raw_path)
39
+ svc_model.clear_empty()
40
+ _audio = out_audio.cpu().numpy()
41
+ pad_len = int(svc_model.target_sample * 0.5)
42
+ _audio = _audio[pad_len:-pad_len]
43
+
44
+ audio.extend(list(infer_tool.pad_array(_audio, length)))
45
+ out_wav_path = io.BytesIO()
46
+ soundfile.write(out_wav_path, audio, svc_model.target_sample, format=wav_format)
47
+ out_wav_path.seek(0)
48
+ return send_file(out_wav_path, download_name=f"temp.{wav_format}", as_attachment=True)
49
+
50
+
51
+ if __name__ == '__main__':
52
+ model_name = "logs/44k/G_60000.pth" # 模型地址
53
+ config_name = "configs/config.json" # config地址
54
+ svc_model = infer_tool.Svc(model_name, config_name)
55
+ app.run(port=1145, host="0.0.0.0", debug=False, threaded=False)
inference/__init__.py ADDED
File without changes
inference/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (130 Bytes). View file
 
inference/__pycache__/infer_tool.cpython-38.pyc ADDED
Binary file (14.8 kB). View file
 
inference/__pycache__/slicer.cpython-38.pyc ADDED
Binary file (3.83 kB). View file
 
inference/infer_tool.py ADDED
@@ -0,0 +1,529 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import io
3
+ import json
4
+ import logging
5
+ import os
6
+ import time
7
+ from pathlib import Path
8
+ from inference import slicer
9
+ import gc
10
+
11
+ import librosa
12
+ import numpy as np
13
+ # import onnxruntime
14
+ import soundfile
15
+ import torch
16
+ import torchaudio
17
+
18
+ import cluster
19
+ import utils
20
+ from models import SynthesizerTrn
21
+ import pickle
22
+
23
+ from diffusion.unit2mel import load_model_vocoder
24
+ import yaml
25
+
26
+ logging.getLogger('matplotlib').setLevel(logging.WARNING)
27
+
28
+
29
+ def read_temp(file_name):
30
+ if not os.path.exists(file_name):
31
+ with open(file_name, "w") as f:
32
+ f.write(json.dumps({"info": "temp_dict"}))
33
+ return {}
34
+ else:
35
+ try:
36
+ with open(file_name, "r") as f:
37
+ data = f.read()
38
+ data_dict = json.loads(data)
39
+ if os.path.getsize(file_name) > 50 * 1024 * 1024:
40
+ f_name = file_name.replace("\\", "/").split("/")[-1]
41
+ print(f"clean {f_name}")
42
+ for wav_hash in list(data_dict.keys()):
43
+ if int(time.time()) - int(data_dict[wav_hash]["time"]) > 14 * 24 * 3600:
44
+ del data_dict[wav_hash]
45
+ except Exception as e:
46
+ print(e)
47
+ print(f"{file_name} error,auto rebuild file")
48
+ data_dict = {"info": "temp_dict"}
49
+ return data_dict
50
+
51
+
52
+ def write_temp(file_name, data):
53
+ with open(file_name, "w") as f:
54
+ f.write(json.dumps(data))
55
+
56
+
57
+ def timeit(func):
58
+ def run(*args, **kwargs):
59
+ t = time.time()
60
+ res = func(*args, **kwargs)
61
+ print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
62
+ return res
63
+
64
+ return run
65
+
66
+
67
+ def format_wav(audio_path):
68
+ if Path(audio_path).suffix == '.wav':
69
+ return
70
+ raw_audio, raw_sample_rate = librosa.load(audio_path, mono=True, sr=None)
71
+ soundfile.write(Path(audio_path).with_suffix(".wav"), raw_audio, raw_sample_rate)
72
+
73
+
74
+ def get_end_file(dir_path, end):
75
+ file_lists = []
76
+ for root, dirs, files in os.walk(dir_path):
77
+ files = [f for f in files if f[0] != '.']
78
+ dirs[:] = [d for d in dirs if d[0] != '.']
79
+ for f_file in files:
80
+ if f_file.endswith(end):
81
+ file_lists.append(os.path.join(root, f_file).replace("\\", "/"))
82
+ return file_lists
83
+
84
+
85
+ def get_md5(content):
86
+ return hashlib.new("md5", content).hexdigest()
87
+
88
+ def fill_a_to_b(a, b):
89
+ if len(a) < len(b):
90
+ for _ in range(0, len(b) - len(a)):
91
+ a.append(a[0])
92
+
93
+ def mkdir(paths: list):
94
+ for path in paths:
95
+ if not os.path.exists(path):
96
+ os.mkdir(path)
97
+
98
+ def pad_array(arr, target_length):
99
+ current_length = arr.shape[0]
100
+ if current_length >= target_length:
101
+ return arr
102
+ else:
103
+ pad_width = target_length - current_length
104
+ pad_left = pad_width // 2
105
+ pad_right = pad_width - pad_left
106
+ padded_arr = np.pad(arr, (pad_left, pad_right), 'constant', constant_values=(0, 0))
107
+ return padded_arr
108
+
109
+ def split_list_by_n(list_collection, n, pre=0):
110
+ for i in range(0, len(list_collection), n):
111
+ yield list_collection[i-pre if i-pre>=0 else i: i + n]
112
+
113
+
114
+ class F0FilterException(Exception):
115
+ pass
116
+
117
+ class Svc(object):
118
+ def __init__(self, net_g_path, config_path,
119
+ device=None,
120
+ cluster_model_path="logs/44k/kmeans_10000.pt",
121
+ nsf_hifigan_enhance = False,
122
+ diffusion_model_path="logs/44k/diffusion/model_0.pt",
123
+ diffusion_config_path="configs/diffusion.yaml",
124
+ shallow_diffusion = False,
125
+ only_diffusion = False,
126
+ spk_mix_enable = False,
127
+ feature_retrieval = False
128
+ ):
129
+ self.net_g_path = net_g_path
130
+ self.only_diffusion = only_diffusion
131
+ self.shallow_diffusion = shallow_diffusion
132
+ self.feature_retrieval = feature_retrieval
133
+ if device is None:
134
+ self.dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
135
+ else:
136
+ self.dev = torch.device(device)
137
+ self.net_g_ms = None
138
+ if not self.only_diffusion:
139
+ self.hps_ms = utils.get_hparams_from_file(config_path,True)
140
+ self.target_sample = self.hps_ms.data.sampling_rate
141
+ self.hop_size = self.hps_ms.data.hop_length
142
+ self.spk2id = self.hps_ms.spk
143
+ self.unit_interpolate_mode = self.hps_ms.data.unit_interpolate_mode if self.hps_ms.data.unit_interpolate_mode is not None else 'left'
144
+ self.vol_embedding = self.hps_ms.model.vol_embedding if self.hps_ms.model.vol_embedding is not None else False
145
+ self.speech_encoder = self.hps_ms.model.speech_encoder if self.hps_ms.model.speech_encoder is not None else 'vec768l12'
146
+
147
+ self.nsf_hifigan_enhance = nsf_hifigan_enhance
148
+ if self.shallow_diffusion or self.only_diffusion:
149
+ if os.path.exists(diffusion_model_path) and os.path.exists(diffusion_model_path):
150
+ self.diffusion_model,self.vocoder,self.diffusion_args = load_model_vocoder(diffusion_model_path,self.dev,config_path=diffusion_config_path)
151
+ if self.only_diffusion:
152
+ self.target_sample = self.diffusion_args.data.sampling_rate
153
+ self.hop_size = self.diffusion_args.data.block_size
154
+ self.spk2id = self.diffusion_args.spk
155
+ self.speech_encoder = self.diffusion_args.data.encoder
156
+ self.unit_interpolate_mode = self.diffusion_args.data.unit_interpolate_mode if self.diffusion_args.data.unit_interpolate_mode!=None else 'left'
157
+ if spk_mix_enable:
158
+ self.diffusion_model.init_spkmix(len(self.spk2id))
159
+ else:
160
+ print("No diffusion model or config found. Shallow diffusion mode will False")
161
+ self.shallow_diffusion = self.only_diffusion = False
162
+
163
+ # load hubert and model
164
+ if not self.only_diffusion:
165
+ self.load_model(spk_mix_enable)
166
+ self.hubert_model = utils.get_speech_encoder(self.speech_encoder,device=self.dev)
167
+ self.volume_extractor = utils.Volume_Extractor(self.hop_size)
168
+ else:
169
+ self.hubert_model = utils.get_speech_encoder(self.diffusion_args.data.encoder,device=self.dev)
170
+ self.volume_extractor = utils.Volume_Extractor(self.diffusion_args.data.block_size)
171
+
172
+ if os.path.exists(cluster_model_path):
173
+ if self.feature_retrieval:
174
+ with open(cluster_model_path,"rb") as f:
175
+ self.cluster_model = pickle.load(f)
176
+ self.big_npy = None
177
+ self.now_spk_id = -1
178
+ else:
179
+ self.cluster_model = cluster.get_cluster_model(cluster_model_path)
180
+ else:
181
+ self.feature_retrieval=False
182
+
183
+ if self.shallow_diffusion : self.nsf_hifigan_enhance = False
184
+ if self.nsf_hifigan_enhance:
185
+ from modules.enhancer import Enhancer
186
+ self.enhancer = Enhancer('nsf-hifigan', 'pretrain/nsf_hifigan/model',device=self.dev)
187
+
188
+ def load_model(self, spk_mix_enable=False):
189
+ # get model configuration
190
+ self.net_g_ms = SynthesizerTrn(
191
+ self.hps_ms.data.filter_length // 2 + 1,
192
+ self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
193
+ **self.hps_ms.model)
194
+ _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
195
+ if "half" in self.net_g_path and torch.cuda.is_available():
196
+ _ = self.net_g_ms.half().eval().to(self.dev)
197
+ else:
198
+ _ = self.net_g_ms.eval().to(self.dev)
199
+ if spk_mix_enable:
200
+ self.net_g_ms.EnableCharacterMix(len(self.spk2id), self.dev)
201
+
202
+ def get_unit_f0(self, wav, tran, cluster_infer_ratio, speaker, f0_filter ,f0_predictor,cr_threshold=0.05):
203
+
204
+ f0_predictor_object = utils.get_f0_predictor(f0_predictor,hop_length=self.hop_size,sampling_rate=self.target_sample,device=self.dev,threshold=cr_threshold)
205
+
206
+ f0, uv = f0_predictor_object.compute_f0_uv(wav)
207
+ if f0_filter and sum(f0) == 0:
208
+ raise F0FilterException("No voice detected")
209
+ f0 = torch.FloatTensor(f0).to(self.dev)
210
+ uv = torch.FloatTensor(uv).to(self.dev)
211
+
212
+ f0 = f0 * 2 ** (tran / 12)
213
+ f0 = f0.unsqueeze(0)
214
+ uv = uv.unsqueeze(0)
215
+
216
+ wav16k = librosa.resample(wav, orig_sr=self.target_sample, target_sr=16000)
217
+ wav16k = torch.from_numpy(wav16k).to(self.dev)
218
+ c = self.hubert_model.encoder(wav16k)
219
+ c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1],self.unit_interpolate_mode)
220
+
221
+ if cluster_infer_ratio !=0:
222
+ if self.feature_retrieval:
223
+ speaker_id = self.spk2id.get(speaker)
224
+ if speaker_id is None:
225
+ raise RuntimeError("The name you entered is not in the speaker list!")
226
+ if not speaker_id and type(speaker) is int:
227
+ if len(self.spk2id.__dict__) >= speaker:
228
+ speaker_id = speaker
229
+ feature_index = self.cluster_model[speaker_id]
230
+ feat_np = c.transpose(0,1).cpu().numpy()
231
+ if self.big_npy is None or self.now_spk_id != speaker_id:
232
+ self.big_npy = feature_index.reconstruct_n(0, feature_index.ntotal)
233
+ self.now_spk_id = speaker_id
234
+ print("starting feature retrieval...")
235
+ score, ix = feature_index.search(feat_np, k=8)
236
+ weight = np.square(1 / score)
237
+ weight /= weight.sum(axis=1, keepdims=True)
238
+ npy = np.sum(self.big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)
239
+ c = cluster_infer_ratio * npy + (1 - cluster_infer_ratio) * feat_np
240
+ c = torch.FloatTensor(c).to(self.dev).transpose(0,1)
241
+ print("end feature retrieval...")
242
+ else:
243
+ cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, speaker).T
244
+ cluster_c = torch.FloatTensor(cluster_c).to(self.dev)
245
+ c = cluster_infer_ratio * cluster_c + (1 - cluster_infer_ratio) * c
246
+
247
+ c = c.unsqueeze(0)
248
+ return c, f0, uv
249
+
250
+ def infer(self, speaker, tran, raw_path,
251
+ cluster_infer_ratio=0,
252
+ auto_predict_f0=False,
253
+ noice_scale=0.4,
254
+ f0_filter=False,
255
+ f0_predictor='pm',
256
+ enhancer_adaptive_key = 0,
257
+ cr_threshold = 0.05,
258
+ k_step = 100,
259
+ frame = 0,
260
+ spk_mix = False,
261
+ second_encoding = False,
262
+ loudness_envelope_adjustment = 1
263
+ ):
264
+ wav, sr = librosa.load(raw_path, sr=self.target_sample)
265
+ if spk_mix:
266
+ c, f0, uv = self.get_unit_f0(wav, tran, 0, None, f0_filter,f0_predictor,cr_threshold=cr_threshold)
267
+ n_frames = f0.size(1)
268
+ sid = speaker[:, frame:frame+n_frames].transpose(0,1)
269
+ else:
270
+ speaker_id = self.spk2id.get(speaker)
271
+ if not speaker_id and type(speaker) is int:
272
+ if len(self.spk2id.__dict__) >= speaker:
273
+ speaker_id = speaker
274
+ if speaker_id is None:
275
+ raise RuntimeError("The name you entered is not in the speaker list!")
276
+ sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
277
+ c, f0, uv = self.get_unit_f0(wav, tran, cluster_infer_ratio, speaker, f0_filter,f0_predictor,cr_threshold=cr_threshold)
278
+ n_frames = f0.size(1)
279
+ if "half" in self.net_g_path and torch.cuda.is_available():
280
+ c = c.half()
281
+ with torch.no_grad():
282
+ start = time.time()
283
+ vol = None
284
+ if not self.only_diffusion:
285
+ vol = self.volume_extractor.extract(torch.FloatTensor(wav).to(self.dev)[None,:])[None,:].to(self.dev) if self.vol_embedding else None
286
+ audio,f0 = self.net_g_ms.infer(c, f0=f0, g=sid, uv=uv, predict_f0=auto_predict_f0, noice_scale=noice_scale,vol=vol)
287
+ audio = audio[0,0].data.float()
288
+ audio_mel = self.vocoder.extract(audio[None,:],self.target_sample) if self.shallow_diffusion else None
289
+ else:
290
+ audio = torch.FloatTensor(wav).to(self.dev)
291
+ audio_mel = None
292
+ if self.only_diffusion or self.shallow_diffusion:
293
+ vol = self.volume_extractor.extract(audio[None,:])[None,:,None].to(self.dev) if vol==None else vol[:,:,None]
294
+ if self.shallow_diffusion and second_encoding:
295
+ audio16k = librosa.resample(audio.detach().cpu().numpy(), orig_sr=self.target_sample, target_sr=16000)
296
+ audio16k = torch.from_numpy(audio16k).to(self.dev)
297
+ c = self.hubert_model.encoder(audio16k)
298
+ c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1],self.unit_interpolate_mode)
299
+ f0 = f0[:,:,None]
300
+ c = c.transpose(-1,-2)
301
+ audio_mel = self.diffusion_model(
302
+ c,
303
+ f0,
304
+ vol,
305
+ spk_id = sid,
306
+ spk_mix_dict = None,
307
+ gt_spec=audio_mel,
308
+ infer=True,
309
+ infer_speedup=self.diffusion_args.infer.speedup,
310
+ method=self.diffusion_args.infer.method,
311
+ k_step=k_step)
312
+ audio = self.vocoder.infer(audio_mel, f0).squeeze()
313
+ if self.nsf_hifigan_enhance:
314
+ audio, _ = self.enhancer.enhance(
315
+ audio[None,:],
316
+ self.target_sample,
317
+ f0[:,:,None],
318
+ self.hps_ms.data.hop_length,
319
+ adaptive_key = enhancer_adaptive_key)
320
+ if loudness_envelope_adjustment != 1:
321
+ audio = utils.change_rms(wav,self.target_sample,audio,self.target_sample,loudness_envelope_adjustment)
322
+ use_time = time.time() - start
323
+ print("vits use time:{}".format(use_time))
324
+ return audio, audio.shape[-1], n_frames
325
+
326
+ def clear_empty(self):
327
+ # clean up vram
328
+ torch.cuda.empty_cache()
329
+
330
+ def unload_model(self):
331
+ # unload model
332
+ self.net_g_ms = self.net_g_ms.to("cpu")
333
+ del self.net_g_ms
334
+ if hasattr(self,"enhancer"):
335
+ self.enhancer.enhancer = self.enhancer.enhancer.to("cpu")
336
+ del self.enhancer.enhancer
337
+ del self.enhancer
338
+ gc.collect()
339
+
340
+ def slice_inference(self,
341
+ raw_audio_path,
342
+ spk,
343
+ tran,
344
+ slice_db,
345
+ cluster_infer_ratio,
346
+ auto_predict_f0,
347
+ noice_scale,
348
+ pad_seconds=0.5,
349
+ clip_seconds=0,
350
+ lg_num=0,
351
+ lgr_num =0.75,
352
+ f0_predictor='pm',
353
+ enhancer_adaptive_key = 0,
354
+ cr_threshold = 0.05,
355
+ k_step = 100,
356
+ use_spk_mix = False,
357
+ second_encoding = False,
358
+ loudness_envelope_adjustment = 1
359
+ ):
360
+ if use_spk_mix:
361
+ if len(self.spk2id) == 1:
362
+ spk = self.spk2id.keys()[0]
363
+ use_spk_mix = False
364
+ wav_path = Path(raw_audio_path).with_suffix('.wav')
365
+ chunks = slicer.cut(wav_path, db_thresh=slice_db)
366
+ audio_data, audio_sr = slicer.chunks2audio(wav_path, chunks)
367
+ per_size = int(clip_seconds*audio_sr)
368
+ lg_size = int(lg_num*audio_sr)
369
+ lg_size_r = int(lg_size*lgr_num)
370
+ lg_size_c_l = (lg_size-lg_size_r)//2
371
+ lg_size_c_r = lg_size-lg_size_r-lg_size_c_l
372
+ lg = np.linspace(0,1,lg_size_r) if lg_size!=0 else 0
373
+
374
+ if use_spk_mix:
375
+ assert len(self.spk2id) == len(spk)
376
+ audio_length = 0
377
+ for (slice_tag, data) in audio_data:
378
+ aud_length = int(np.ceil(len(data) / audio_sr * self.target_sample))
379
+ if slice_tag:
380
+ audio_length += aud_length // self.hop_size
381
+ continue
382
+ if per_size != 0:
383
+ datas = split_list_by_n(data, per_size,lg_size)
384
+ else:
385
+ datas = [data]
386
+ for k,dat in enumerate(datas):
387
+ pad_len = int(audio_sr * pad_seconds)
388
+ per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample))
389
+ a_length = per_length + 2 * pad_len
390
+ audio_length += a_length // self.hop_size
391
+ audio_length += len(audio_data)
392
+ spk_mix_tensor = torch.zeros(size=(len(spk), audio_length)).to(self.dev)
393
+ for i in range(len(spk)):
394
+ last_end = None
395
+ for mix in spk[i]:
396
+ if mix[3]<0. or mix[2]<0.:
397
+ raise RuntimeError("mix value must higer Than zero!")
398
+ begin = int(audio_length * mix[0])
399
+ end = int(audio_length * mix[1])
400
+ length = end - begin
401
+ if length<=0:
402
+ raise RuntimeError("begin Must lower Than end!")
403
+ step = (mix[3] - mix[2])/length
404
+ if last_end is not None:
405
+ if last_end != begin:
406
+ raise RuntimeError("[i]EndTime Must Equal [i+1]BeginTime!")
407
+ last_end = end
408
+ if step == 0.:
409
+ spk_mix_data = torch.zeros(length).to(self.dev) + mix[2]
410
+ else:
411
+ spk_mix_data = torch.arange(mix[2],mix[3],step).to(self.dev)
412
+ if(len(spk_mix_data)<length):
413
+ num_pad = length - len(spk_mix_data)
414
+ spk_mix_data = torch.nn.functional.pad(spk_mix_data, [0, num_pad], mode="reflect").to(self.dev)
415
+ spk_mix_tensor[i][begin:end] = spk_mix_data[:length]
416
+
417
+ spk_mix_ten = torch.sum(spk_mix_tensor,dim=0).unsqueeze(0).to(self.dev)
418
+ # spk_mix_tensor[0][spk_mix_ten<0.001] = 1.0
419
+ for i, x in enumerate(spk_mix_ten[0]):
420
+ if x == 0.0:
421
+ spk_mix_ten[0][i] = 1.0
422
+ spk_mix_tensor[:,i] = 1.0 / len(spk)
423
+ spk_mix_tensor = spk_mix_tensor / spk_mix_ten
424
+ if not ((torch.sum(spk_mix_tensor,dim=0) - 1.)<0.0001).all():
425
+ raise RuntimeError("sum(spk_mix_tensor) not equal 1")
426
+ spk = spk_mix_tensor
427
+
428
+ global_frame = 0
429
+ audio = []
430
+ for (slice_tag, data) in audio_data:
431
+ print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
432
+ # padd
433
+ length = int(np.ceil(len(data) / audio_sr * self.target_sample))
434
+ if slice_tag:
435
+ print('jump empty segment')
436
+ _audio = np.zeros(length)
437
+ audio.extend(list(pad_array(_audio, length)))
438
+ global_frame += length // self.hop_size
439
+ continue
440
+ if per_size != 0:
441
+ datas = split_list_by_n(data, per_size,lg_size)
442
+ else:
443
+ datas = [data]
444
+ for k,dat in enumerate(datas):
445
+ per_length = int(np.ceil(len(dat) / audio_sr * self.target_sample)) if clip_seconds!=0 else length
446
+ if clip_seconds!=0: print(f'###=====segment clip start, {round(len(dat) / audio_sr, 3)}s======')
447
+ # padd
448
+ pad_len = int(audio_sr * pad_seconds)
449
+ dat = np.concatenate([np.zeros([pad_len]), dat, np.zeros([pad_len])])
450
+ raw_path = io.BytesIO()
451
+ soundfile.write(raw_path, dat, audio_sr, format="wav")
452
+ raw_path.seek(0)
453
+ out_audio, out_sr, out_frame = self.infer(spk, tran, raw_path,
454
+ cluster_infer_ratio=cluster_infer_ratio,
455
+ auto_predict_f0=auto_predict_f0,
456
+ noice_scale=noice_scale,
457
+ f0_predictor = f0_predictor,
458
+ enhancer_adaptive_key = enhancer_adaptive_key,
459
+ cr_threshold = cr_threshold,
460
+ k_step = k_step,
461
+ frame = global_frame,
462
+ spk_mix = use_spk_mix,
463
+ second_encoding = second_encoding,
464
+ loudness_envelope_adjustment = loudness_envelope_adjustment
465
+ )
466
+ global_frame += out_frame
467
+ _audio = out_audio.cpu().numpy()
468
+ pad_len = int(self.target_sample * pad_seconds)
469
+ _audio = _audio[pad_len:-pad_len]
470
+ _audio = pad_array(_audio, per_length)
471
+ if lg_size!=0 and k!=0:
472
+ lg1 = audio[-(lg_size_r+lg_size_c_r):-lg_size_c_r] if lgr_num != 1 else audio[-lg_size:]
473
+ lg2 = _audio[lg_size_c_l:lg_size_c_l+lg_size_r] if lgr_num != 1 else _audio[0:lg_size]
474
+ lg_pre = lg1*(1-lg)+lg2*lg
475
+ audio = audio[0:-(lg_size_r+lg_size_c_r)] if lgr_num != 1 else audio[0:-lg_size]
476
+ audio.extend(lg_pre)
477
+ _audio = _audio[lg_size_c_l+lg_size_r:] if lgr_num != 1 else _audio[lg_size:]
478
+ audio.extend(list(_audio))
479
+ return np.array(audio)
480
+
481
+ class RealTimeVC:
482
+ def __init__(self):
483
+ self.last_chunk = None
484
+ self.last_o = None
485
+ self.chunk_len = 16000 # chunk length
486
+ self.pre_len = 3840 # cross fade length, multiples of 640
487
+
488
+ # Input and output are 1-dimensional numpy waveform arrays
489
+
490
+ def process(self, svc_model, speaker_id, f_pitch_change, input_wav_path,
491
+ cluster_infer_ratio=0,
492
+ auto_predict_f0=False,
493
+ noice_scale=0.4,
494
+ f0_filter=False):
495
+
496
+ import maad
497
+ audio, sr = torchaudio.load(input_wav_path)
498
+ audio = audio.cpu().numpy()[0]
499
+ temp_wav = io.BytesIO()
500
+ if self.last_chunk is None:
501
+ input_wav_path.seek(0)
502
+
503
+ audio, sr = svc_model.infer(speaker_id, f_pitch_change, input_wav_path,
504
+ cluster_infer_ratio=cluster_infer_ratio,
505
+ auto_predict_f0=auto_predict_f0,
506
+ noice_scale=noice_scale,
507
+ f0_filter=f0_filter)
508
+
509
+ audio = audio.cpu().numpy()
510
+ self.last_chunk = audio[-self.pre_len:]
511
+ self.last_o = audio
512
+ return audio[-self.chunk_len:]
513
+ else:
514
+ audio = np.concatenate([self.last_chunk, audio])
515
+ soundfile.write(temp_wav, audio, sr, format="wav")
516
+ temp_wav.seek(0)
517
+
518
+ audio, sr = svc_model.infer(speaker_id, f_pitch_change, temp_wav,
519
+ cluster_infer_ratio=cluster_infer_ratio,
520
+ auto_predict_f0=auto_predict_f0,
521
+ noice_scale=noice_scale,
522
+ f0_filter=f0_filter)
523
+
524
+ audio = audio.cpu().numpy()
525
+ ret = maad.util.crossfade(self.last_o, audio, self.pre_len)
526
+ self.last_chunk = audio[-self.pre_len:]
527
+ self.last_o = audio
528
+ return ret[self.chunk_len:2 * self.chunk_len]
529
+