Kaggle competition: Pitch estimation and voicing detection
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<groupby>
def autocorr_method(frame, sfreq, threshold=0.46, fmin=50, fmax=400):

 frame = frame.astype(np.float)
 frame -= frame.mean()
amax = np.abs(frame ).max()
if amax > 0:
frame /= amax
else:
return 0
corr = correlate(frame, frame)
 corr = corr[len(corr)//2:]
dcorr = np.diff(corr)
 rmin = np.where(dcorr > 0)[0]
if len(rmin)> 0:
rmin1 = rmin[0]
else:
return 0
peak = np.argmax(corr[rmin1:])+ rmin1
rmax = corr[peak]/corr[0]
f0 = sfreq / peak
if rmax > threshold and f0 >= fmin and f0 <= fmax:
return f0
else:
return 0
country = "Vietnam"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()<save_model>
class Counters:
def __init__(self, gross_threshold=0.2):
self.num_voiced = 0
self.num_unvoiced = 0
self.num_voiced_unvoiced = 0
self.num_unvoiced_voiced = 0
self.num_voiced_voiced = 0
self.num_gross_errors = 0
self.fine_error = 0
self.e2 = 0
self.gross_threshold = gross_threshold
self.nfiles = 0
def add(self, other):
if other is not None:
self.num_voiced += other.num_voiced
self.num_unvoiced += other.num_unvoiced
self.num_voiced_unvoiced += other.num_voiced_unvoiced
self.num_unvoiced_voiced += other.num_unvoiced_voiced
self.num_voiced_voiced += other.num_voiced_voiced
self.num_gross_errors += other.num_gross_errors
self.fine_error += other.fine_error
self.e2 += other.e2
self.nfiles += 1
def __repr__(self):
nframes = self.num_voiced + self.num_unvoiced
if self.nfiles > 0:
self.fine_error /= self.nfiles
str = [
f"Num.frames:\t{self.num_unvoiced + self.num_voiced} = {self.num_unvoiced} unvoiced + {self.num_voiced} voiced",
f"Unvoiced frames as voiced:\t{self.num_unvoiced_voiced}/{self.num_unvoiced}({100*self.num_unvoiced_voiced/self.num_unvoiced:.2f}%)",
f"Voiced frames as unvoiced:\t{self.num_voiced_unvoiced}/{self.num_voiced}({100*self.num_voiced_unvoiced/self.num_voiced:.2f}%)",
f"Gross voiced errors(>{100*self.gross_threshold}%):\t{self.num_gross_errors}/{self.num_voiced_voiced}({100*self.num_gross_errors/self.num_voiced_voiced:.2f}%)",
f"MSE of fine errors:\t{100*self.fine_error:.2f}%",
f"RMSE:\t{np.sqrt(self.e2/nframes):.2f}"
]
return '
'.join(str )
animator.save('confirm_animation.gif', writer='imagemagick', fps=2)
 display(Image(url='confirm_animation.gif'))<feature_engineering>
def compare(fref, pitch):
vref = np.loadtxt(fref)
 vtest = np.array(pitch)
 diff_frames = len(vref)- len(vtest)
 if abs(diff_frames)> 5:
print(f"Error: number of frames in ref({len(vref)})!= number of frames in test({len(vtest)})")
 return None
elif diff_frames > 0:
vref = np.resize(vref, vtest.shape)
 elif diff_frames < 0:
vtest = np.resize(vtest, vref.shape)
 counters = Counters()
counters.num_voiced = np.count_nonzero(vref)
 counters.num_unvoiced = len(vref)- counters.num_voiced
counters.num_unvoiced_voiced = np.count_nonzero(np.logical_and(vref == 0, vtest != 0))
counters.num_voiced_unvoiced = np.count_nonzero(np.logical_and(vref != 0, vtest == 0))
voiced_voiced = np.logical_and(vref != 0, vtest != 0)
 counters.num_voiced_voiced = np.count_nonzero(voiced_voiced)
 f = np.absolute(vref[voiced_voiced] - vtest[voiced_voiced])/vref[voiced_voiced]
gross_errors = f > counters.gross_threshold
counters.num_gross_errors = np.count_nonzero(gross_errors)
 fine_errors = np.logical_not(gross_errors)
 counters.fine_error = np.sqrt(np.square(f[fine_errors] ).mean())if np.count_nonzero(fine_errors)> 0 else 0
counters.e2 = np.square(vref - vtest ).sum()
return counters
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<feature_engineering>
def wav2f0(options, gui):
fs = open(options.submission, 'w')if options.submission is not None else None
totalCounters = Counters()
with open(gui)as f:
if fs is not None:
print('id,frequency', file=fs)
 for line in f:
line = line.strip()
if len(line)== 0:
continue
filename = os.path.join(options.datadir, line + ".wav")
 f0ref_filename = os.path.join(options.datadir, line + ".f0ref")
 print("Processing:", filename)
 sfreq, data = wavfile.read(filename)
 nsamples = len(data)
 ns_windowlength = int(round(( options.windowlength * sfreq)/ 1000))
ns_frameshift = int(round(( options.frameshift * sfreq)/ 1000))
ns_left_padding = int(round(( options.left_padding * sfreq)/ 1000))
ns_right_padding = int(round(( options.right_padding * sfreq)/ 1000))
pitch = []
for ini in range(-ns_left_padding, nsamples - ns_windowlength + ns_right_padding + 1, ns_frameshift):
first_sample = max(0, ini)
 last_sample = min(nsamples, ini + ns_windowlength)
 frame = data[first_sample:last_sample]
f0 = autocorr_method(frame, sfreq)
 pitch.append(f0)
 if fs is not None:
for frame_id, f0 in enumerate(pitch):
print(line + '_' + str(frame_id)+ ',', f0, file=fs)
 if os.path.isfile(f0ref_filename):
counters = compare(f0ref_filename, pitch)
 totalCounters.add(counters)
 if totalCounters.num_voiced + totalCounters.num_unvoiced > 0:
print("
print(totalCounters)
 print("-------------------------------
" )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<compute_test_metric>
fda_ue_options = SimpleNamespace(
 windowlength=32, frameshift=15, left_padding=16, right_padding=16, datadir='.. /input', submission=None)
 wav2f0(fda_ue_options, '.. /input/fda_ue.gui' )
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
test_options = SimpleNamespace(
 windowlength=26.5, frameshift=10, left_padding=13.25, right_padding=7, datadir='.. /input/test', submission='submission.csv')
 wav2f0(test_options, '.. /input/test.gui' )