Kaggle competition: COVID19 Global Forecasting (Week 2)
def apk(actual, predicted, k=10):

 if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i,p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i+1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=10):

 return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)])
 def extract_features(files, path):
features = {}
cnt = 0
for f in tqdm(files):
features[f] = {}
fs, data = scipy.io.wavfile.read(os.path.join(path, f))
abs_data = np.abs(data)
 diff_data = np.diff(data)
 def calc_part_features(data, n=2, prefix=''):
f_i = 1
for i in range(0, len(data), len(data)//n):
features[f]['{}mean_{}_{}'.format(prefix, f_i, n)] = np.mean(data[i:i + len(data)//n])
 features[f]['{}std_{}_{}'.format(prefix, f_i, n)] = np.std(data[i:i + len(data)//n])
 features[f]['{}min_{}_{}'.format(prefix, f_i, n)] = np.min(data[i:i + len(data)//n])
 features[f]['{}max_{}_{}'.format(prefix, f_i, n)] = np.max(data[i:i + len(data)//n])
 features[f]['len'] = len(data)
 if features[f]['len'] > 0:
n = 1
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 n = 2
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 n = 3
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 cnt += 1
features = pd.DataFrame(features ).T.reset_index()
features.rename(columns={'index': 'fname'}, inplace=True)
 return features
path = os.path.join(data_path, 'audio_train')
 train_files = train.fname.values
train_features = extract_features(train_files, path)
 path = os.path.join(data_path, 'audio_test')
 test_files = ss.fname.values
test_features = extract_features(test_files, path )<merge>
from sklearn.tree import DecisionTreeRegressor
from sklearn import preprocessing
train_data = train_data.merge(train_features, on='fname', how='left')
 test_data = test_data.merge(test_features, on='fname', how='left')
 train_data.head()<prepare_x_and_y>
df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 df.head()
X = train_data.drop(['label', 'fname'], axis=1)
 feature_names = list(X.columns)
 X = X.values
labels = np.sort(np.unique(train_data.label.values))
num_class = len(labels)
 c2i = {}
i2c = {}
for i, c in enumerate(labels):
c2i[c] = i
i2c[i] = c
y = np.array([c2i[x] for x in train_data.label.values] )<train_model>
df['Date'] = pd.to_datetime(df['Date'])
 df['Date'] = pd.to_timedelta(df.Date ).dt.total_seconds().astype(int)
 df = df.replace(np.nan, '', regex=True)
 le_cr = preprocessing.LabelEncoder()
le_ps = preprocessing.LabelEncoder()
le_cr.fit(df['Country_Region'])
 le_ps.fit(df['Province_State'])
 df['Country_Region'] = le_cr.transform(df['Country_Region'])
 df['Province_State'] = le_ps.transform(df['Province_State'] )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=10, shuffle = True)
 clf = XGBClassifier(max_depth=5, learning_rate=0.05, n_estimators=3000,
n_jobs=-1, random_state=0, reg_alpha=0.2,
colsample_bylevel=0.9, colsample_bytree=0.9)
 clf.fit(X_train, y_train)
 print(accuracy_score(clf.predict(X_val), y_val))
def proba2labels(preds, i2c, k=3):
ans = []
ids = []
for p in preds:
idx = np.argsort(p)[::-1]
ids.append([i for i in idx[:k]])
 ans.append(' '.join([i2c[i] for i in idx[:k]]))
return ans, ids<save_to_csv>
regressor_cc = DecisionTreeRegressor(random_state = 0, max_features="auto")
 regressor_f = DecisionTreeRegressor(random_state = 0, max_features="auto")
 regressor_cc.fit(df[['Country_Region','Province_State','Date']], df[['ConfirmedCases']])
 regressor_f.fit(df[['Country_Region','Province_State','Date']], df[['Fatalities']] )
clf.fit(X, y)
 str_preds, _ = proba2labels(clf.predict_proba(test_data.drop(['label', 'fname'], axis = 1 ).values), i2c, k=3)
 subm = pd.DataFrame()
subm['fname'] = audio_test_files
subm['label'] = str_preds
subm.to_csv('submission.csv', index=False )<load_from_csv>
df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv' )
audio_train_files = os.listdir('.. /input/audio_train')
 audio_test_files = os.listdir('.. /input/audio_test')
 train = pd.read_csv('.. /input/train.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )<load_pretrained>
df_test['Date'] = pd.to_datetime(df_test['Date'])
 df_test['Date'] = pd.to_timedelta(df_test.Date ).dt.total_seconds().astype(int)
 df_test = df_test.replace(np.nan, '', regex=True)
 df_test['Country_Region'] = le_cr.transform(df_test['Country_Region'])
 df_test['Province_State'] = le_ps.transform(df_test['Province_State'] )
def clean_filename(fname, string):
file_name = fname.split('/')[1]
if file_name[:2] == '__':
file_name = string + file_name
return file_name
def get_mfcc(name, path):
b, _ = librosa.core.load(path + name, sr = SAMPLE_RATE)
 assert _ == SAMPLE_RATE
try:
ft1 = librosa.feature.mfcc(b, sr = SAMPLE_RATE, n_mfcc=20)
 ft2 = librosa.feature.zero_crossing_rate(b)[0]
ft3 = librosa.feature.spectral_rolloff(b)[0]
ft4 = librosa.feature.spectral_centroid(b)[0]
ft5 = librosa.feature.spectral_contrast(b)[0]
ft6 = librosa.feature.spectral_bandwidth(b)[0]
ft1_trunc = np.hstack(( np.mean(ft1, axis=1), np.std(ft1, axis=1), skew(ft1, axis = 1), np.max(ft1, axis = 1), np.min(ft1, axis = 1)))
 ft2_trunc = np.hstack(( np.mean(ft2), np.std(ft2), skew(ft2), np.max(ft2), np.min(ft2)))
 ft3_trunc = np.hstack(( np.mean(ft3), np.std(ft3), skew(ft3), np.max(ft3), np.min(ft3)))
 ft4_trunc = np.hstack(( np.mean(ft4), np.std(ft4), skew(ft4), np.max(ft4), np.min(ft4)))
 ft5_trunc = np.hstack(( np.mean(ft5), np.std(ft5), skew(ft5), np.max(ft5), np.min(ft5)))
 ft6_trunc = np.hstack(( np.mean(ft6), np.std(ft6), skew(ft6), np.max(ft6), np.m(ft6)))
 return pd.Series(np.hstack(( ft1_trunc, ft2_trunc, ft3_trunc, ft4_trunc, ft5_trunc, ft6_trunc)))
 except:
print('bad file')
 return pd.Series([0]*125 )<feature_engineering>
df_s = pd.DataFrame()
df_s['ForecastId'] = df_test['ForecastId']
df_s['ConfirmedCases'] = regressor_cc.predict(df_test[['Country_Region','Province_State','Date']])
 df_s['Fatalities'] = regressor_f.predict(df_test[['Country_Region','Province_State','Date']] )
train_data = pd.DataFrame()
train_data['fname'] = train['fname']
test_data = pd.DataFrame()
test_data['fname'] = audio_test_files
train_data = train_data['fname'].progress_apply(get_mfcc, path='.. /input/audio_train/')
 print('done loading train mfcc')
 test_data = test_data['fname'].progress_apply(get_mfcc, path='.. /input/audio_test/')
 print('done loading test mfcc')
 train_data['fname'] = train['fname']
test_data['fname'] = audio_test_files
train_data['label'] = train['label']
test_data['label'] = np.zeros(( len(audio_test_files)) )<compute_test_metric>
df_s.to_csv('submission.csv', index=False )