Adorable-Qin
Change F0 extractor to Crepe
2d4dca5
# Copyright (c) 2023 Amphion.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import librosa
import numpy as np
import torch
import parselmouth
import torchcrepe
import pyworld as pw
def get_bin_index(f0, m, M, n_bins, use_log_scale):
"""
WARNING: to abandon!
Args:
raw_f0: tensor whose shpae is (N, frame_len)
Returns:
index: tensor whose shape is same to f0
"""
raw_f0 = f0.clone()
raw_m, raw_M = m, M
if use_log_scale:
f0[torch.where(f0 == 0)] = 1
f0 = torch.log(f0)
m, M = float(np.log(m)), float(np.log(M))
# Set normal index in [1, n_bins - 1]
width = (M + 1e-7 - m) / (n_bins - 1)
index = (f0 - m) // width + 1
# Set unvoiced frames as 0, Therefore, the vocabulary is [0, n_bins- 1], whose size is n_bins
index[torch.where(f0 == 0)] = 0
# TODO: Boundary check (special: to judge whether 0 for unvoiced)
if torch.any(raw_f0 > raw_M):
print("F0 Warning: too high f0: {}".format(raw_f0[torch.where(raw_f0 > raw_M)]))
index[torch.where(raw_f0 > raw_M)] = n_bins - 1
if torch.any(raw_f0 < raw_m):
print("F0 Warning: too low f0: {}".format(raw_f0[torch.where(f0 < m)]))
index[torch.where(f0 < m)] = 0
return torch.as_tensor(index, dtype=torch.long, device=f0.device)
def f0_to_coarse(f0, pitch_bin, pitch_min, pitch_max):
## TODO: Figure out the detail of this function
f0_mel_min = 1127 * np.log(1 + pitch_min / 700)
f0_mel_max = 1127 * np.log(1 + pitch_max / 700)
is_torch = isinstance(f0, torch.Tensor)
f0_mel = 1127 * (1 + f0 / 700).log() if is_torch else 1127 * np.log(1 + f0 / 700)
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (pitch_bin - 2) / (
f0_mel_max - f0_mel_min
) + 1
f0_mel[f0_mel <= 1] = 1
f0_mel[f0_mel > pitch_bin - 1] = pitch_bin - 1
f0_coarse = (f0_mel + 0.5).long() if is_torch else np.rint(f0_mel).astype(np.int32)
assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (
f0_coarse.max(),
f0_coarse.min(),
)
return f0_coarse
def interpolate(f0):
"""Interpolate the unvoiced part. Thus the f0 can be passed to a subtractive synthesizer.
Args:
f0: A numpy array of shape (seq_len,)
Returns:
f0: Interpolated f0 of shape (seq_len,)
uv: Unvoiced part of shape (seq_len,)
"""
uv = f0 == 0
if len(f0[~uv]) > 0:
# interpolate the unvoiced f0
f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
uv = uv.astype("float")
uv = np.min(np.array([uv[:-2], uv[1:-1], uv[2:]]), axis=0)
uv = np.pad(uv, (1, 1))
return f0, uv
def get_log_f0(f0):
f0[np.where(f0 == 0)] = 1
log_f0 = np.log(f0)
return log_f0
# ========== Methods ==========
def get_f0_features_using_pyin(audio, cfg):
"""Using pyin to extract the f0 feature.
Args:
audio
fs
win_length
hop_length
f0_min
f0_max
Returns:
f0: numpy array of shape (frame_len,)
"""
f0, voiced_flag, voiced_probs = librosa.pyin(
y=audio,
fmin=cfg.f0_min,
fmax=cfg.f0_max,
sr=cfg.sample_rate,
win_length=cfg.win_size,
hop_length=cfg.hop_size,
)
# Set nan to 0
f0[voiced_flag == False] = 0
return f0
def get_f0_features_using_parselmouth(audio, cfg, speed=1):
"""Using parselmouth to extract the f0 feature.
Args:
audio
mel_len
hop_length
fs
f0_min
f0_max
speed(default=1)
Returns:
f0: numpy array of shape (frame_len,)
pitch_coarse: numpy array of shape (frame_len,)
"""
hop_size = int(np.round(cfg.hop_size * speed))
# Calculate the time step for pitch extraction
time_step = hop_size / cfg.sample_rate * 1000
f0 = (
parselmouth.Sound(audio, cfg.sample_rate)
.to_pitch_ac(
time_step=time_step / 1000,
voicing_threshold=0.6,
pitch_floor=cfg.f0_min,
pitch_ceiling=cfg.f0_max,
)
.selected_array["frequency"]
)
# Pad the pitch to the mel_len
# pad_size = (int(len(audio) // hop_size) - len(f0) + 1) // 2
# f0 = np.pad(f0, [[pad_size, mel_len - len(f0) - pad_size]], mode="constant")
# Get the coarse part
pitch_coarse = f0_to_coarse(f0, cfg.pitch_bin, cfg.f0_min, cfg.f0_max)
return f0, pitch_coarse
def get_f0_features_using_dio(audio, cfg):
"""Using dio to extract the f0 feature.
Args:
audio
mel_len
fs
hop_length
f0_min
f0_max
Returns:
f0: numpy array of shape (frame_len,)
"""
# Get the raw f0
_f0, t = pw.dio(
audio.astype("double"),
cfg.sample_rate,
f0_floor=cfg.f0_min,
f0_ceil=cfg.f0_max,
channels_in_octave=2,
frame_period=(1000 * cfg.hop_size / cfg.sample_rate),
)
# Get the f0
f0 = pw.stonemask(audio.astype("double"), _f0, t, cfg.sample_rate)
return f0
def get_f0_features_using_harvest(audio, mel_len, fs, hop_length, f0_min, f0_max):
"""Using harvest to extract the f0 feature.
Args:
audio
mel_len
fs
hop_length
f0_min
f0_max
Returns:
f0: numpy array of shape (frame_len,)
"""
f0, _ = pw.harvest(
audio.astype("double"),
fs,
f0_floor=f0_min,
f0_ceil=f0_max,
frame_period=(1000 * hop_length / fs),
)
f0 = f0.astype("float")[:mel_len]
return f0
def get_f0_features_using_crepe_legacy(
audio, mel_len, fs, hop_length, hop_length_new, f0_min, f0_max, threshold=0.3
):
"""Using torchcrepe to extract the f0 feature.
Args:
audio
mel_len
fs
hop_length
hop_length_new
f0_min
f0_max
threshold(default=0.3)
Returns:
f0: numpy array of shape (frame_len,)
"""
# Currently, crepe only supports 16khz audio
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
audio_16k = librosa.resample(audio, orig_sr=fs, target_sr=16000)
audio_16k_torch = torch.FloatTensor(audio_16k).unsqueeze(0).to(device)
# Get the raw pitch
f0, pd = torchcrepe.predict(
audio_16k_torch,
16000,
hop_length_new,
f0_min,
f0_max,
pad=True,
model="full",
batch_size=1024,
device=device,
return_periodicity=True,
)
# Filter, de-silence, set up threshold for unvoiced part
pd = torchcrepe.filter.median(pd, 3)
pd = torchcrepe.threshold.Silence(-60.0)(pd, audio_16k_torch, 16000, hop_length_new)
f0 = torchcrepe.threshold.At(threshold)(f0, pd)
f0 = torchcrepe.filter.mean(f0, 3)
# Convert unvoiced part to 0hz
f0 = torch.where(torch.isnan(f0), torch.full_like(f0, 0), f0)
# Interpolate f0
nzindex = torch.nonzero(f0[0]).squeeze()
f0 = torch.index_select(f0[0], dim=0, index=nzindex).cpu().numpy()
time_org = 0.005 * nzindex.cpu().numpy()
time_frame = np.arange(mel_len) * hop_length / fs
f0 = np.interp(time_frame, time_org, f0, left=f0[0], right=f0[-1])
return f0
def get_f0_features_using_crepe(audio, cfg):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
audio_torch = torch.FloatTensor(audio).unsqueeze(0).to(device)
crepe_pitch, pd = torchcrepe.predict(audio_torch, cfg.sample_rate, cfg.hop_size, fmin=cfg.f0_min, fmax=cfg.f0_max, return_periodicity=True)
threshold = 0.3
# Filter, de-silence, set up threshold for unvoiced part
pd = torchcrepe.filter.median(pd, 3)
pd = torchcrepe.threshold.Silence(-60.0)(pd, audio_torch, cfg.sample_rate, 256)
crepe_pitch = torchcrepe.threshold.At(threshold)(crepe_pitch, pd)
crepe_pitch = torchcrepe.filter.mean(crepe_pitch, 3)
# Convert unvoiced part to 0hz
crepe_pitch = torch.where(torch.isnan(crepe_pitch), torch.full_like(crepe_pitch, 0), crepe_pitch)
return crepe_pitch[0].cpu().numpy()
def get_f0(audio, cfg):
if cfg.pitch_extractor == "dio":
f0 = get_f0_features_using_dio(audio, cfg)
elif cfg.pitch_extractor == "pyin":
f0 = get_f0_features_using_pyin(audio, cfg)
elif cfg.pitch_extractor == "parselmouth":
f0, _ = get_f0_features_using_parselmouth(audio, cfg)
elif cfg.pitch_extractor == "crepe":
f0 = get_f0_features_using_crepe(audio, cfg)
# elif cfg.data.f0_extractor == 'cwt': # todo
return f0
def get_cents(f0_hz):
"""
F_{cent} = 1200 * log2 (F/440)
Reference:
APSIPA'17, Perceptual Evaluation of Singing Quality
"""
voiced_f0 = f0_hz[f0_hz != 0]
return 1200 * np.log2(voiced_f0 / 440)
def get_pitch_derivatives(f0_hz):
"""
f0_hz: (,T)
"""
f0_cent = get_cents(f0_hz)
return f0_cent[1:] - f0_cent[:-1]
def get_pitch_sub_median(f0_hz):
"""
f0_hz: (,T)
"""
f0_cent = get_cents(f0_hz)
return f0_cent - np.median(f0_cent)