Spaces:
Sleeping
Sleeping
File size: 18,690 Bytes
96ea114 c70fa17 96ea114 75afec6 af4a49e 96ea114 8b3f08b ba6c1db dd168eb 96ea114 c70fa17 7b8da10 c70fa17 ba6c1db c70fa17 ba6c1db c70fa17 96ea114 e407867 96ea114 41aa3b8 96ea114 c13f2b9 96ea114 6243a1f 5d8b016 96ea114 4589722 5e94b8c 75afec6 96ea114 dcaea67 96ea114 5e94b8c 96ea114 5e94b8c 96ea114 05dfc64 96ea114 05dfc64 96ea114 d0db4b3 96ea114 5e94b8c 787717e 96ea114 75afec6 96ea114 e20e705 96ea114 5e94b8c 96ea114 5e94b8c 96ea114 69a5bb6 f97a03d 4b51a1b 69a5bb6 96ea114 5e94b8c 96ea114 5a81c64 4b51a1b c70fa17 96ea114 75afec6 96ea114 75afec6 96ea114 75afec6 96ea114 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 |
import argparse
import gc
import hashlib
import json
import os
import shlex
import subprocess
from contextlib import suppress
from urllib.parse import urlparse, parse_qs
import gradio as gr
import librosa
import numpy as np
import soundfile as sf
import sox
import yt_dlp
from pedalboard import Pedalboard, Reverb, Compressor, HighpassFilter
from pedalboard.io import AudioFile
from pydub import AudioSegment
import pyloudnorm as pyln
from mdx import run_mdx
from rvc import Config, load_hubert, get_vc, rvc_infer
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
mdxnet_models_dir = os.path.join(BASE_DIR, 'mdxnet_models')
rvc_models_dir = os.path.join(BASE_DIR, 'rvc_models')
output_dir = os.path.join(BASE_DIR, 'song_output')
def get_youtube_video_id(url, ignore_playlist=True):
"""
Examples:
http://youtu.be/SA2iWivDJiE
http://www.youtube.com/watch?v=_oPAwA_Udwc&feature=feedu
http://www.youtube.com/embed/SA2iWivDJiE
http://www.youtube.com/v/SA2iWivDJiE?version=3&hl=en_US
"""
query = urlparse(url)
if query.hostname == 'youtu.be':
if query.path[1:] == 'watch':
return query.query[2:]
return query.path[1:]
if query.hostname in {'www.youtube.com', 'youtube.com', 'music.youtube.com'}:
if not ignore_playlist:
# use case: get playlist id not current video in playlist
with suppress(KeyError):
return parse_qs(query.query)['list'][0]
if query.path == '/watch':
return parse_qs(query.query)['v'][0]
if query.path[:7] == '/watch/':
return query.path.split('/')[1]
if query.path[:7] == '/embed/':
return query.path.split('/')[2]
if query.path[:3] == '/v/':
return query.path.split('/')[2]
# returns None for invalid YouTube url
return None
def yt_download(link):
ydl_opts = {
'format': 'bestaudio',
'outtmpl': '%(title)s',
'nocheckcertificate': True,
'ignoreerrors': True,
'no_warnings': True,
'quiet': True,
'extractaudio': True,
'postprocessors': [{'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3'}],
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
result = ydl.extract_info(link, download=True)
download_path = ydl.prepare_filename(result, outtmpl='%(title)s.mp3')
return download_path
def raise_exception(error_msg, is_webui):
if is_webui:
raise gr.Error(error_msg)
else:
raise Exception(error_msg)
def get_rvc_model(voice_model, is_webui):
rvc_model_filename, rvc_index_filename = None, None
model_dir = os.path.join(rvc_models_dir, voice_model)
#for file in os.listdir(model_dir):
# ext = os.path.splitext(file)[1]
# if ext == '.pth':
# rvc_model_filename = file
# if ext == '.index':
# rvc_index_filename = file
for root, dirs, files in os.walk(model_dir):
# Iterate over the files in the current directory
for file in files:
if file.endswith('.pth'):
# Append the full path of the file to the list
rvc_model_filename = os.path.join(root, file)
if file.endswith('.index'):
rvc_index_filename = file
if rvc_model_filename is None:
error_msg = f'No model file exists in {model_dir}.'
raise_exception(error_msg, is_webui)
return os.path.join(model_dir, rvc_model_filename), os.path.join(model_dir, rvc_index_filename) if rvc_index_filename else ''
def get_audio_paths(song_dir):
orig_song_path = None
instrumentals_path = None
main_vocals_dereverb_path = None
backup_vocals_path = None
for file in os.listdir(song_dir):
if file.endswith('_Instrumental.wav'):
instrumentals_path = os.path.join(song_dir, file)
orig_song_path = instrumentals_path.replace('_Instrumental', '')
elif file.endswith('_Vocals_Main_DeReverb.wav'):
main_vocals_dereverb_path = os.path.join(song_dir, file)
elif file.endswith('_Vocals_Backup.wav'):
backup_vocals_path = os.path.join(song_dir, file)
return orig_song_path, instrumentals_path, main_vocals_dereverb_path, backup_vocals_path
def convert_to_stereo(audio_path):
wave, sr = librosa.load(audio_path, mono=False, sr=44100)
# check if mono
if type(wave[0]) != np.ndarray:
print("QUI DENTRO STEREOOOOOOOOO!")
stereo_path = f'{os.path.splitext(audio_path)[0]}_stereo.mp3'
command = shlex.split(f'ffmpeg -y -loglevel error -i "{audio_path}" -ac 2 -f mp3 "{stereo_path}"')
subprocess.run(command)
return stereo_path
else:
return audio_path
def normalize_audio(input_audio_path, output_audio_path):
y1, sr1 = librosa.load(input_audio_path, mono = False)
meter = pyln.Meter(sr1)
lufs_left1 = meter.integrated_loudness(y1[0])
lufs_right1 = meter.integrated_loudness(y1[1])
y2, sr2 = librosa.load(output_audio_path, mono = False)
meter2 = pyln.Meter(sr2)
lufs_left2 = meter2.integrated_loudness(y2[0])
lufs_right2 = meter2.integrated_loudness(y2[1])
y2_normalized_left = pyln.normalize.loudness(y2[0], lufs_left2, lufs_left1)
y2_normalized_right = pyln.normalize.loudness(y2[1], lufs_right2, lufs_right1)
y2_normalized = np.stack((y2_normalized_left, y2_normalized_right), axis=0)
output_path = f"{output_audio_path}_normalized.mp3"
sf.write(output_path, y2_normalized.T, sr2)
return output_path
def pitch_shift(audio_path, pitch_change):
output_path = f'{os.path.splitext(audio_path)[0]}_p{pitch_change}.wav'
if not os.path.exists(output_path):
y, sr = sf.read(audio_path)
tfm = sox.Transformer()
tfm.pitch(pitch_change)
y_shifted = tfm.build_array(input_array=y, sample_rate_in=sr)
sf.write(output_path, y_shifted, sr)
return output_path
def get_hash(filepath):
with open(filepath, 'rb') as f:
file_hash = hashlib.blake2b()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()[:11]
def display_progress(message, percent, is_webui, progress=None):
if is_webui:
progress(percent, desc=message)
else:
print(message)
def preprocess_song(song_input, mdx_model_params, song_id, is_webui, input_type, progress=None):
keep_orig = False
if input_type == 'yt':
display_progress('[~] Downloading song...', 0, is_webui, progress)
song_link = song_input.split('&')[0]
orig_song_path = yt_download(song_link)
elif input_type == 'local':
orig_song_path = song_input
keep_orig = True
else:
orig_song_path = None
song_output_dir = os.path.join(output_dir, song_id)
orig_song_path = convert_to_stereo(orig_song_path)
display_progress('[~] Separating Vocals from Instrumental...', 0.1, is_webui, progress)
vocals_path, instrumentals_path = run_mdx(mdx_model_params, song_output_dir, os.path.join(mdxnet_models_dir, 'UVR-MDX-NET-Voc_FT.onnx'), orig_song_path, denoise=True, keep_orig=keep_orig)
display_progress('[~] Separating Main Vocals from Backup Vocals...', 0.2, is_webui, progress)
backup_vocals_path, main_vocals_path = run_mdx(mdx_model_params, song_output_dir, os.path.join(mdxnet_models_dir, 'UVR_MDXNET_KARA_2.onnx'), vocals_path, suffix='Backup', invert_suffix='Main', denoise=True)
display_progress('[~] Applying DeReverb to Vocals...', 0.3, is_webui, progress)
_, main_vocals_dereverb_path = run_mdx(mdx_model_params, song_output_dir, os.path.join(mdxnet_models_dir, 'Reverb_HQ_By_FoxJoy.onnx'), main_vocals_path, invert_suffix='DeReverb', exclude_main=True, denoise=True)
return orig_song_path, vocals_path, instrumentals_path, main_vocals_path, backup_vocals_path, main_vocals_dereverb_path
def voice_change(voice_model, vocals_path, output_path, pitch_change, f0_method, index_rate, filter_radius, rms_mix_rate, protect, crepe_hop_length, is_webui):
rvc_model_path, rvc_index_path = get_rvc_model(voice_model, is_webui)
device = 'cpu'
config = Config(device, False)
hubert_model = load_hubert(device, config.is_half, os.path.join(rvc_models_dir, 'hubert_base.pt'))
cpt, version, net_g, tgt_sr, vc = get_vc(device, False, config, rvc_model_path)
# convert main vocals
rvc_infer(rvc_index_path, index_rate, vocals_path, output_path, pitch_change, f0_method, cpt, version, net_g, filter_radius, tgt_sr, rms_mix_rate, protect, crepe_hop_length, vc, hubert_model)
del hubert_model, cpt
gc.collect()
def add_audio_effects(audio_path, reverb_rm_size, reverb_wet, reverb_dry, reverb_damping):
output_path = f'{os.path.splitext(audio_path)[0]}_mixed.wav'
# Initialize audio effects plugins
board = Pedalboard(
[
HighpassFilter(),
Compressor(ratio=4, threshold_db=-15),
Reverb(room_size=reverb_rm_size, dry_level=reverb_dry, wet_level=reverb_wet, damping=reverb_damping)
]
)
with AudioFile(audio_path) as f:
with AudioFile(output_path, 'w', f.samplerate, f.num_channels) as o:
# Read one second of audio at a time, until the file is empty:
while f.tell() < f.frames:
chunk = f.read(int(f.samplerate))
effected = board(chunk, f.samplerate, reset=False)
o.write(effected)
return output_path
def combine_audio(audio_paths, output_path, main_gain, backup_gain, inst_gain, output_format):
main_vocal_audio = AudioSegment.from_file(audio_paths[0]) - 4 + main_gain
backup_vocal_audio = AudioSegment.from_file(audio_paths[1]) - 6 + backup_gain
#instrumental_audio = AudioSegment.from_wav(audio_paths[2]) - 7 + inst_gain
main_vocal_audio.overlay(backup_vocal_audio).export(output_path, format=output_format)
# song_input is the main vocals that is going through the rvc processs
# backup vocals are the backup vocals to be added at the end
def song_cover_pipeline(main_vocals, backup_vocals, voice_model, pitch_change,
is_webui=0, main_gain=0, backup_gain=0, inst_gain=0, index_rate=0.5, filter_radius=3,
rms_mix_rate=1, f0_method='rmvpe', crepe_hop_length=128, protect=0.33, pitch_change_all=0,
reverb_rm_size=0.15, reverb_wet=0.2, reverb_dry=0.8, reverb_damping=0.7, output_format='mp3',
progress=gr.Progress()):
try:
print(f"DEBUG PRINT: main_vocals: {main_vocals}, backup_vocals: {backup_vocals}")
if not main_vocals or not voice_model:
raise_exception('Ensure that the song input field and voice model field is filled.', is_webui)
display_progress('[~] Starting AI vocals Cover Generation Pipeline...', 0, is_webui, progress)
with open(os.path.join(mdxnet_models_dir, 'model_data.json')) as infile:
mdx_model_params = json.load(infile)
# local audio file
input_type = 'local'
main_vocals = main_vocals.strip('\"')
if os.path.exists(main_vocals):
song_id = get_hash(main_vocals)
else:
error_msg = f'{main_vocals} does not exist.'
song_id = None
raise_exception(error_msg, is_webui)
orig_song_path = convert_to_stereo(main_vocals)
song_dir = os.path.join(output_dir, song_id)
print(f"DEBUG PRINT: song_dir {song_dir}")
if not os.path.exists(song_dir):
os.makedirs(song_dir)
pitch_change = pitch_change * 12 + pitch_change_all
ai_vocals_path = os.path.join(song_dir, f'{os.path.basename(os.path.splitext(main_vocals)[0])}_{os.path.basename(voice_model)}_p{pitch_change}_i{index_rate}_fr{filter_radius}_rms{rms_mix_rate}_pro{protect}_{f0_method}{"" if f0_method != "mangio-crepe" else f"_{crepe_hop_length}"}.wav')
ai_cover_path = os.path.join(song_dir, f'{os.path.basename(os.path.splitext(main_vocals)[0])}({os.path.basename(voice_model)} Ver)_cover.{output_format}')
print(f"DEBUG PRINT: ai_vocals_path: {ai_vocals_path}, ai_cover_path: {ai_cover_path}")
if not os.path.exists(ai_vocals_path):
display_progress('[~] Converting voice using RVC...', 0.5, is_webui, progress)
voice_change(voice_model, main_vocals, ai_vocals_path, pitch_change, f0_method, index_rate, filter_radius, rms_mix_rate, protect, crepe_hop_length, is_webui)
display_progress('[~] Applying audio effects to Vocals...', 0.8, is_webui, progress)
ai_vocals_mixed_path = add_audio_effects(ai_vocals_path, reverb_rm_size, reverb_wet, reverb_dry, reverb_damping)
if pitch_change_all != 0:
display_progress('[~] Applying overall pitch change', 0.85, is_webui, progress)
backup_vocals = pitch_shift(backup_vocals, pitch_change_all)
if backup_vocals == "":
print("####### ENTRATO QUA, BACKUP VOCALS NON DATI IN INPUT ########")
ai_vocals_mixed_path = convert_to_stereo(ai_vocals_mixed_path)
ai_vocals_mixed_path = normalize_audio(main_vocals, ai_vocals_mixed_path)
return ai_vocals_mixed_path
display_progress('[~] Combining AI Vocals and Instrumentals...', 0.9, is_webui, progress)
combine_audio([ai_vocals_mixed_path, backup_vocals], ai_cover_path, main_gain, backup_gain, inst_gain, output_format)
ai_cover_path = convert_to_stereo(ai_cover_path)
ai_cover_path = normalize_audio(main_vocals, ai_cover_path)
return ai_cover_path
except Exception as e:
raise_exception(str(e), is_webui)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Generate a AI cover song in the song_output/id directory.', add_help=True)
parser.add_argument('-main_v', '--main-v', type=str, default= "spongebob.wav", required=False, help='Link to a YouTube video or the filepath to a local mp3/wav file to create an AI cover of')
parser.add_argument('-backup_v', '--backup-v', type=str, default="", required=False,
help='Link to a YouTube video or the filepath to a local mp3/wav file to create an AI cover of')
parser.add_argument('-dir', '--rvc-dirname', type=str, default = "plankton", required=False, help='Name of the folder in the rvc_models directory containing the RVC model file and optional index file to use')
parser.add_argument('-p', '--pitch-change', type=int, required=False, default=0, help='Change the pitch of AI Vocals only. Generally, use 1 for male to female and -1 for vice-versa. (Octaves)')
parser.add_argument('-ir', '--index-rate', type=float, default=0.5, help='A decimal number e.g. 0.5, used to reduce/resolve the timbre leakage problem. If set to 1, more biased towards the timbre quality of the training dataset')
parser.add_argument('-fr', '--filter-radius', type=int, default=3, help='A number between 0 and 7. If >=3: apply median filtering to the harvested pitch results. The value represents the filter radius and can reduce breathiness.')
parser.add_argument('-rms', '--rms-mix-rate', type=float, default=0, help="A decimal number e.g. 0.25. Control how much to use the original vocal's loudness (0) or a fixed loudness (1).")
parser.add_argument('-palgo', '--pitch-detection-algo', type=str, default='rmvpe', help='Best option is rmvpe (clarity in vocals), then mangio-crepe (smoother vocals).')
parser.add_argument('-hop', '--crepe-hop-length', type=int, default=128, help='If pitch detection algo is mangio-crepe, controls how often it checks for pitch changes in milliseconds. The higher the value, the faster the conversion and less risk of voice cracks, but there is less pitch accuracy. Recommended: 128.')
parser.add_argument('-pro', '--protect', type=float, default=0.33, help='A decimal number e.g. 0.33. Protect voiceless consonants and breath sounds to prevent artifacts such as tearing in electronic music. Set to 0.5 to disable. Decrease the value to increase protection, but it may reduce indexing accuracy.')
parser.add_argument('-mv', '--main-vol', type=int, default=0, help='Volume change for AI main vocals in decibels. Use -3 to decrease by 3 decibels and 3 to increase by 3 decibels')
parser.add_argument('-bv', '--backup-vol', type=int, default=0, help='Volume change for backup vocals in decibels')
parser.add_argument('-iv', '--inst-vol', type=int, default=0, help='Volume change for instrumentals in decibels')
parser.add_argument('-pall', '--pitch-change-all', type=int, default=0, help='Change the pitch/key of vocals and instrumentals. Changing this slightly reduces sound quality')
parser.add_argument('-rsize', '--reverb-size', type=float, default=0.15, help='Reverb room size between 0 and 1')
parser.add_argument('-rwet', '--reverb-wetness', type=float, default=0.2, help='Reverb wet level between 0 and 1')
parser.add_argument('-rdry', '--reverb-dryness', type=float, default=0.8, help='Reverb dry level between 0 and 1')
parser.add_argument('-rdamp', '--reverb-damping', type=float, default=0.7, help='Reverb damping between 0 and 1')
parser.add_argument('-oformat', '--output-format', type=str, default='mp3', help='Output format of audio file. mp3 for smaller file size, wav for best quality')
args = parser.parse_args()
rvc_dirname = args.rvc_dirname
if not os.path.exists(os.path.join(rvc_models_dir, rvc_dirname)):
raise Exception(f'The folder {os.path.join(rvc_models_dir, rvc_dirname)} does not exist.')
cover_path = song_cover_pipeline(main_vocals = "spongebob.wav", backup_vocals=args.backup_v, voice_model = "plankton", pitch_change = args.pitch_change,
main_gain=args.main_vol, backup_gain=args.backup_vol, inst_gain=args.inst_vol,
index_rate=args.index_rate, filter_radius=args.filter_radius,
rms_mix_rate=args.rms_mix_rate, f0_method=args.pitch_detection_algo,
crepe_hop_length=args.crepe_hop_length, protect=args.protect,
pitch_change_all=args.pitch_change_all,
reverb_rm_size=args.reverb_size, reverb_wet=args.reverb_wetness,
reverb_dry=args.reverb_dryness, reverb_damping=args.reverb_damping,
output_format=args.output_format)
print(f'[+] Cover generated at {cover_path}')
|