Update app.py

app.py

@@ -1,4 +1,3 @@
-import spaces
 import gradio as gr
 import torch
 import torchaudio
@@ -15,8 +14,6 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
                                                 "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
                                                 "config_dit_mel_seed_uvit_whisper_small_wavenet.yml")
-# dit_checkpoint_path = "E:/DiT_epoch_00018_step_801000.pth"
-# dit_config_path = "configs/config_dit_mel_seed_uvit_whisper_small_encoder_wavenet.yml"
 config = yaml.safe_load(open(dit_config_path, 'r'))
 model_params = recursive_munch(config['model_params'])
 model = build_model(model_params, stage='DiT')
@@ -48,19 +45,6 @@ bigvgan_model = bigvgan.BigVGAN.from_pretrained('nvidia/bigvgan_v2_22khz_80band_
 bigvgan_model.remove_weight_norm()
 bigvgan_model = bigvgan_model.eval().to(device)
 
-ckpt_path, config_path = load_custom_model_from_hf("Plachta/FAcodec", 'pytorch_model.bin', 'config.yml')
-
-codec_config = yaml.safe_load(open(config_path))
-codec_model_params = recursive_munch(codec_config['model_params'])
-codec_encoder = build_model(codec_model_params, stage="codec")
-
-ckpt_params = torch.load(ckpt_path, map_location="cpu")
-
-for key in codec_encoder:
-    codec_encoder[key].load_state_dict(ckpt_params[key], strict=False)
-_ = [codec_encoder[key].eval() for key in codec_encoder]
-_ = [codec_encoder[key].to(device) for key in codec_encoder]
-
 # whisper
 from transformers import AutoFeatureExtractor, WhisperModel
 
@@ -134,16 +118,16 @@ def adjust_f0_semitones(f0_sequence, n_semitones):
 def crossfade(chunk1, chunk2, overlap):
     fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
     fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
-    chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
+    if len(chunk2) < overlap:
+        chunk2[:overlap] = chunk2[:overlap] * fade_in[:len(chunk2)] + (chunk1[-overlap:] * fade_out)[:len(chunk2)]
+    else:
+        chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
     return chunk2
 
 # streaming and chunk processing related params
-max_context_window = sr // hop_length * 30
 overlap_frame_len = 16
-overlap_wave_len = overlap_frame_len * hop_length
 bitrate = "320k"
 
-@spaces.GPU
 @torch.no_grad()
 @torch.inference_mode()
 def voice_conversion(source, target, diffusion_steps, length_adjust, inference_cfg_rate, f0_condition, auto_f0_adjust, pitch_shift):
@@ -151,6 +135,9 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
     mel_fn = to_mel if not f0_condition else to_mel_f0
     bigvgan_fn = bigvgan_model if not f0_condition else bigvgan_44k_model
     sr = 22050 if not f0_condition else 44100
+    hop_length = 256 if not f0_condition else 512
+    max_context_window = sr // hop_length * 30
+    overlap_wave_len = overlap_frame_len * hop_length
     # Load audio
     source_audio = librosa.load(source, sr=sr)[0]
     ref_audio = librosa.load(target, sr=sr)[0]
@@ -243,8 +230,8 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
     style2 = campplus_model(feat2.unsqueeze(0))
 
     if f0_condition:
-        F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.5)
-        F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.5)
+        F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.03)
+        F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.03)
 
         F0_ori = torch.from_numpy(F0_ori).to(device)[None]
         F0_alt = torch.from_numpy(F0_alt).to(device)[None]
@@ -283,7 +270,7 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
         chunk_cond = cond[:, processed_frames:processed_frames + max_source_window]
         is_last_chunk = processed_frames + max_source_window >= cond.size(1)
         cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
-        with torch.autocast(device_type='cuda', dtype=torch.float16):
+        with torch.autocast(device_type=device.type, dtype=torch.float16):
             # Voice Conversion
             vc_target = inference_module.cfm.inference(cat_condition,
                                                        torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
@@ -337,7 +324,7 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
 
 
 if __name__ == "__main__":
-    description = ("State-of-the-Art zero-shot voice conversion/singing voice conversion. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
+    description = ("Zero-shot voice conversion with in-context learning. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
                    "for details and updates.<br>Note that any reference audio will be forcefully clipped to 25s if beyond this length.<br> "
                    "If total duration of source and reference audio exceeds 30s, source audio will be processed in chunks.<br> "
                    "无需训练的 zero-shot 语音/歌声转换模型，若需本地部署查看[GitHub页面](https://github.com/Plachtaa/seed-vc)<br>"
@@ -345,7 +332,7 @@ if __name__ == "__main__":
     inputs = [
         gr.Audio(type="filepath", label="Source Audio / 源音频"),
         gr.Audio(type="filepath", label="Reference Audio / 参考音频"),
-        gr.Slider(minimum=1, maximum=200, value=25, step=1, label="Diffusion Steps / 扩散步数", info="25 by default, 50~100 for best quality / 默认为 25，50~100 为最佳质量"),
+        gr.Slider(minimum=1, maximum=200, value=10, step=1, label="Diffusion Steps / 扩散步数", info="10 by default, 50~100 for best quality / 默认为 10，50~100 为最佳质量"),
         gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label="Length Adjust / 长度调整", info="<1.0 for speed-up speech, >1.0 for slow-down speech / <1.0 加速语速，>1.0 减慢语速"),
         gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0.7, label="Inference CFG Rate", info="has subtle influence / 有微小影响"),
         gr.Checkbox(label="Use F0 conditioned model / 启用F0输入", value=False, info="Must set to true for singing voice conversion / 歌声转换时必须勾选"),
@@ -355,11 +342,11 @@ if __name__ == "__main__":
     ]
 
     examples = [["examples/source/yae_0.wav", "examples/reference/dingzhen_0.wav", 25, 1.0, 0.7, False, True, 0],
-                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, False, True, 0],
+                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, True, True, 0],
                 ["examples/source/Wiz Khalifa,Charlie Puth - See You Again [vocals]_[cut_28sec].wav",
-                 "examples/reference/kobe_0.wav", 50, 1.0, 0.7, True, False, -6],
+                 "examples/reference/teio_0.wav", 25, 1.0, 0.7, True, False, 0],
                 ["examples/source/TECHNOPOLIS - 2085 [vocals]_[cut_14sec].wav",
-                 "examples/reference/trump_0.wav", 50, 1.0, 0.7, True, False, -12],
+                 "examples/reference/trump_0.wav", 25, 1.0, 0.7, True, False, -12],
                 ]
 
     outputs = [gr.Audio(label="Stream Output Audio / 流式输出", streaming=True, format='mp3'),
@@ -372,4 +359,4 @@ if __name__ == "__main__":
                  title="Seed Voice Conversion",
                  examples=examples,
                  cache_examples=False,
-                 ).launch()
+                 ).launch()