Kaggle competition: Freesound General-Purpose Audio Tagging Challenge
<SOS> metric: MAP@{K} Kaggle data source: freesound-audio-tagging<drop_column>
%matplotlib inline
tqdm_notebook().pandas(smoothing=0.7)
 mpl.rcParams['font.size'] = 14
alld = drop_columns(alld, to_drop )<compute_test_metric>
cache = False
run_full_notebook = False
train_root = '.. /input/audio_train/'
test_root = '.. /input/audio_test/'
if cache:
train_root_trimmed = '.. /input/audio_train_trimmed/'
test_root_trimmed = '.. /input/audio_test_trimmed/'
os.makedirs('.. /input/audio_train_trimmed', exist_ok=True)
 os.makedirs('.. /input/audio_test_trimmed', exist_ok=True)
 os.makedirs('.. /cache', exist_ok=True)
 os.makedirs('.. /output', exist_ok=True)
 else:
train_root_trimmed = train_root
test_root_trimmed = test_root
def macro_f1(y_pred, targ):
target = targ.cpu().numpy().argmax(axis=1)
 y_predict = y_pred.cpu().numpy().argmax(axis=1)
 return f1_score(target, y_predict, average='macro' )<data_type_conversions>
test_df = pd.read_csv(".. /input/sample_submission.csv")
 train_df = pd.read_csv(".. /input/train.csv")
 train_df.head()
<data_type_conversions>
n_test = test_df.shape[0]
n_training = train_df.shape[0]
n_categories = len(train_df.label.unique())
 print("Number of training examples: {}".format(n_training))
print("Number of testing examples: {}".format(n_test))
print("Number of unique categories: {}".format(n_categories))
<data_type_conversions>
random_int = random.randint(0, n_training)
 random_wavfile = train_df.fname.iloc[random_int]
print(train_df.label.iloc[random_int])
 play_audio(random_wavfile )
for v in cats: alld[v] = alld[v].astype('category' ).cat.as_ordered()<data_type_conversions>
def wavfile_stats(fname, root):
try:
data, fs = librosa.core.load(root + fname, sr=None)
 mean = np.mean(data)
 minimum = np.min(data)
 maximum = np.max(data)
 std = np.std(data)
 length = len(data)
 rms = np.sqrt(np.mean(data**2))
skewness = skew(data)
 kurt = kurtosis(data)
 return pd.Series([length, mean, minimum, maximum, std, rms, skewness, kurt])
 except ValueError:
print("Bad file at {}".format(fname))
return pd.Series([0, 0, 0, 0, 0, 0, 0, 0] )
alld['Target'] = alld['Target'].astype('int' )<drop_column>
if os.path.isfile('.. /cache/train_1.csv')and cache:
train_df = pd.read_csv('.. /cache/train_1.csv')
 test_df = pd.read_csv('.. /cache/test_1.csv')
 assert len(train_df.index)== n_training
assert len(test_df.index)== n_test
print("Files loaded from cache")
 else:
train_df[['length', 'data_mean', 'data_min', 'data_max', 'data_std', 'data_rms', 'skewness', 'kurtosis']] = \
train_df['fname'].progress_apply(wavfile_stats, root=train_root)
 test_df[['length', 'data_mean', 'data_min', 'data_max', 'data_std', 'data_rms', 'skewness', 'kurtosis']] = \
test_df['fname'].progress_apply(wavfile_stats, root=test_root)
 if cache:
train_df.to_csv('.. /cache/train_1.csv', index=False)
 test_df.to_csv('.. /cache/test_1.csv', index=False )
alld.drop('rez_esc', axis=1, inplace=True )<create_dataframe>
train_df['rms_std'] = train_df['data_rms'] / train_df['data_std']
test_df['rms_std'] = test_df['data_rms'] / test_df['data_std']
train_df['max_min'] = train_df['data_max'] / train_df['data_min']
test_df['max_min'] = test_df['data_max'] / test_df['data_min']
DataFrameSummary(alld ).summary()<define_variables>
test_df.fillna(0, inplace=True )
contin_vars = [c for c in alld.columns if c not in cats + ['Target', 'Id']]<data_type_conversions>
def data_split(train_df, test_df, shuffle=True, test_size=0.25, random_state=0, verbose=True):
X_train = train_df.drop(['fname', 'label', 'manually_verified'], axis=1 ).values
X_test = test_df.drop(['fname', 'label'], axis=1 ).values
feature_names = list(test_df.drop(['fname', 'label'], axis=1 ).columns.values)
 y_train = pd.get_dummies(train_df.label)
 labels = y_train.columns.values
y_train = y_train.values
y_train = [np.argmax(row)for row in y_train]
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=test_size,
random_state=random_state, shuffle=shuffle)
 if verbose:
print("Train X shape = {}
Train y shape = {}".format(X_train.shape, len(y_train)))
 print("
Valid X shape = {}
Valid y shape = {}".format(X_valid.shape, len(y_valid)))
 assert X_train.shape[1] == X_valid.shape[1] == X_test.shape[1]
assert len(y_train)== X_train.shape[0]
assert len(y_valid)== X_valid.shape[0]
return X_train, X_valid, y_train, y_valid, X_test, feature_names, labels
for v in contin_vars:
alld[v] = alld[v].fillna(0 ).astype('float32' )<prepare_x_and_y>
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = data_split(train_df, test_df )
_, _, nas, mapper = proc_df(alld, 'Target', do_scale=True )<filter>
def lgb_dset(X_train, X_valid, y_train, y_valid, feature_names):
d_train = lgb.Dataset(X_train, label=y_train, feature_name=feature_names)
 d_valid = lgb.Dataset(X_valid, label=y_valid, feature_name=feature_names)
 params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': 5,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'num_threads': os.cpu_count() ,
'lambda_l2': 1.0,
'min_gain_to_split': 0,
'num_class': n_categories,
}
return d_train, d_valid, params
traindf = alld.loc[set(ptrain_head.index.tolist())]
traindf.shape[0] == ptrain_head.shape[0]<filter>
d_train, d_valid, params = lgb_dset(X_train, X_valid, y_train, y_valid, feature_names)
 clf = lgb.train(params, d_train, num_boost_round=500, valid_sets=d_valid, verbose_eval=100, early_stopping_rounds=100 )
testdf = alld.loc[set(ptest_head.index.tolist())]
testdf.shape[0] == ptest_head.shape[0]<categorify>
def apk(actual, predicted, k=10):
if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i,p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i+1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=10):
return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)] )
df_test, _, nas, mapper = proc_df(testdf, 'Target', do_scale=True, mapper=mapper )<prepare_x_and_y>
p = clf.predict(X_valid, num_iteration=clf.best_iteration)
 predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
actual = [[i] for i in y_valid]
valid_score = mapk(actual, predictions, k=3)
 print("Score = {:.4f}".format(valid_score))
df, y, nas, mapper = proc_df(traindf, 'Target', do_scale=True, mapper=mapper )<create_dataframe>
def preds_to_labels(p, labels):
predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
prediction_labels = []
for pred in predictions:
label_list = []
for output in pred:
label_list.append(labels[output])
 prediction_labels.append(label_list)
 return prediction_labels
lab = preds_to_labels(p, labels)
 t = PrettyTable(['Truth', 'Prediction'])
 [t.add_row([labels[l[1][0]], l[0]])for l in zip(lab[:10], actual[:10])]
print(t )
DataFrameSummary(df ).summary()<define_variables>
preds = clf.predict(X_test, num_iteration=clf.best_iteration)
 lab = preds_to_labels(preds, labels )
samp_size = df.shape[0]
train_ratio = 0.8
train_size = int(samp_size * train_ratio)
 train_size<define_variables>
random_int = random.randint(0, n_test)
 random_wavfile = test_df.fname.iloc[random_int]
print(lab[random_int])
 play_audio(test_df.fname.iloc[random_int], dset='test' )
val_idx = random.sample(range(0, df.shape[0]), df.shape[0] - train_size )<sort_values>
def create_submission(predictions, name='submission.csv'):
predictions = ['{} {} {}'.format(x[0], x[1], x[2])for x in predictions]
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.label = predictions
submission.to_csv('{}'.format(name), index=False)
 print("Submission saved to '{}'".format(name))
if run_full_notebook:
create_submission(lab, 'submission-{:.4f}.csv'.format(valid_score))
val_idx = sorted(val_idx)
 len(val_idx )<define_variables>
def trim_silence(fname, root, window_length=0.5):
try:
trimmed_ends = 0
trimmed_int = 0
data, fs = librosa.core.load(root + fname, sr=None)
 length = len(data)
 data, _ = librosa.effects.trim(data, top_db=40)
 length_int = len(data)
 ratio_int = length_int/length
splits = librosa.effects.split(data, top_db=40)
 if len(splits)> 1:
data = np.concatenate([data[x[0]:x[1]] for x in splits])
 length_final = len(data)
 ratio_final = length_final/length_int
if cache:
librosa.output.write_wav('{}_trimmed/{}'.format(root[:-1], fname), data, fs)
 return pd.Series([length_int, length_final, ratio_int, ratio_final])
 except ValueError:
print("Bad file at {}".format(fname))
return pd.Series([0, 0, 0, 0] )
yl = np.zeros(( y.size, 4))<define_variables>
if os.path.isfile('.. /cache/train_2.csv')and cache:
train_df = pd.read_csv('.. /cache/train_2.csv')
 test_df = pd.read_csv('.. /cache/test_2.csv')
 assert len(train_df.index)== n_training
assert len(test_df.index)== n_test
print("Files loaded from cache")
 else:
train_df[['length_int', 'length_final', 'ratio_int', 'ratio_final']] = \
train_df['fname'].progress_apply(trim_silence, root=train_root)
 test_df[['length_int', 'length_final', 'ratio_int', 'ratio_final']] = \
test_df['fname'].progress_apply(trim_silence, root=test_root)
 if cache:
train_df.to_csv('.. /cache/train_2.csv', index=False)
 test_df.to_csv('.. /cache/test_2.csv', index=False )
yindex = np.array([t -1 for t in y]); yindex 
<feature_engineering>
if run_full_notebook:
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = data_split(train_df, test_df)
 d_train, d_valid, params = lgb_dset(X_train, X_valid, y_train, y_valid, feature_names)
 clf = lgb.train(params, d_train, num_boost_round=1000, valid_sets=d_valid, verbose_eval=100, early_stopping_rounds=100 )
yl[np.arange(y.size), yindex] = 1; yl<create_dataframe>
if run_full_notebook:
p = clf.predict(X_valid, num_iteration=clf.best_iteration)
 predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
actual = [[i] for i in y_valid]
valid_score = mapk(actual, predictions, k=3)
 print("Score = {:.4f}".format(valid_score))
md = ColumnarModelData.from_data_frame(PATH, val_idx, df, yl, cat_flds=cats, bs=64, is_reg=False, is_multi=True, test_df=df_test )<define_variables>
if os.path.isfile('.. /cache/train_spectral.csv')and cache:
train_df = pd.read_csv('.. /cache/train_spectral.csv')
 test_df = pd.read_csv('.. /cache/test_spectral.csv')
 assert len(train_df.index)== n_training
assert len(test_df.index)== n_test
print("Files loaded from cache")
 else:
feature_names = spectral_features(return_fnames=True)
 train_df[feature_names] = train_df['fname'].progress_apply(spectral_features, root=train_root_trimmed)
 test_df[feature_names] = test_df['fname'].progress_apply(spectral_features, root=test_root_trimmed)
 if cache:
train_df.to_csv('.. /cache/train_spectral.csv', index=False)
 test_df.to_csv('.. /cache/test_spectral.csv', index=False )
cat_sz = [(c, len(alld[c].cat.categories)+1)for c in cats]<define_variables>
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = data_split(train_df, test_df, verbose=False)
 d_train, d_valid, params = lgb_dset(X_train, X_valid, y_train, y_valid, feature_names)
 clf = lgb.train(params, d_train, num_boost_round=2000, valid_sets=d_valid, verbose_eval=200, early_stopping_rounds=100)
 p = clf.predict(X_valid, num_iteration=clf.best_iteration)
 predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
actual = [[i] for i in y_valid]
valid_score = mapk(actual, predictions, k=3)
 print("
Score = {:.4f}".format(valid_score))
emb_szs = [(c, min(50,(c+1)//2)) for _,c in cat_sz]<create_dataframe>
p = clf.predict(X_test, num_iteration=clf.best_iteration)
 lab = preds_to_labels(p, labels)
 create_submission(lab, 'submission-{:.4f}.csv'.format(valid_score))
DataFrameSummary(df )<choose_model_class>
if run_full_notebook:
params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': 5,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'num_threads': os.cpu_count() ,
'lambda_l2': 1.0,
'min_gain_to_split': 0,
'num_class': n_categories,
}
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = \
data_split(train_df, test_df, test_size=0)
 d_train = lgb.Dataset(X_train, label=y_train, feature_name=feature_names )
<train_model><EOS> 
if run_full_notebook:
print("Begin training...")
 clf = lgb.train(params, d_train, num_boost_round=1135)
 print("Begin test predictions...")
 p = clf.predict(X_test)
 lab = preds_to_labels(p, labels)
 create_submission(lab, 'submission-test.csv')
 print("Submission created.")