Kaggle competition: Freesound General-Purpose Audio Tagging Challenge
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age'] 
<define_variables>
audio_train_files = os.listdir('.. /input/audio_train')
 audio_test_files = os.listdir('.. /input/audio_test')
 train = pd.read_csv('.. /input/train.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
def clean_filename(fname, string):
file_name = fname.split('/')[1]
if file_name[:2] == '__':
file_name = string + file_name
return file_name
def get_mfcc(name, path):
b, _ = librosa.core.load(path + name, sr = SAMPLE_RATE)
 assert _ == SAMPLE_RATE
try:
ft1 = librosa.feature.mfcc(b, sr = SAMPLE_RATE, n_mfcc=20)
 ft2 = librosa.feature.zero_crossing_rate(b)[0]
ft3 = librosa.feature.spectral_rolloff(b)[0]
ft4 = librosa.feature.spectral_centroid(b)[0]
ft5 = librosa.feature.spectral_contrast(b)[0]
ft6 = librosa.feature.spectral_bandwidth(b)[0]
ft1_trunc = np.hstack(( np.mean(ft1, axis=1), np.std(ft1, axis=1), skew(ft1, axis = 1), np.max(ft1, axis = 1), np.min(ft1, axis = 1)))
 ft2_trunc = np.hstack(( np.mean(ft2), np.std(ft2), skew(ft2), np.max(ft2), np.min(ft2)))
 ft3_trunc = np.hstack(( np.mean(ft3), np.std(ft3), skew(ft3), np.max(ft3), np.min(ft3)))
 ft4_trunc = np.hstack(( np.mean(ft4), np.std(ft4), skew(ft4), np.max(ft4), np.min(ft4)))
 ft5_trunc = np.hstack(( np.mean(ft5), np.std(ft5), skew(ft5), np.max(ft5), np.min(ft5)))
 ft6_trunc = np.hstack(( np.mean(ft6), np.std(ft6), skew(ft6), np.max(ft6), np.m(ft6)))
 return pd.Series(np.hstack(( ft1_trunc, ft2_trunc, ft3_trunc, ft4_trunc, ft5_trunc, ft6_trunc)))
 except:
print('bad file')
 return pd.Series([0]*125 )
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
train_data = pd.DataFrame()
train_data['fname'] = train['fname']
test_data = pd.DataFrame()
test_data['fname'] = audio_test_files
train_data = train_data['fname'].progress_apply(get_mfcc, path='.. /input/audio_train/')
 print('done loading train mfcc')
 test_data = test_data['fname'].progress_apply(get_mfcc, path='.. /input/audio_test/')
 print('done loading test mfcc')
 train_data['fname'] = train['fname']
test_data['fname'] = audio_test_files
train_data['label'] = train['label']
test_data['label'] = np.zeros(( len(audio_test_files)) )
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )<data_type_conversions>
def apk(actual, predicted, k=10):

 if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i,p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i+1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=10):

 return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)])
 def extract_features(files, path):
features = {}
cnt = 0
for f in tqdm(files):
features[f] = {}
fs, data = scipy.io.wavfile.read(os.path.join(path, f))
abs_data = np.abs(data)
 diff_data = np.diff(data)
 def calc_part_features(data, n=2, prefix=''):
f_i = 1
for i in range(0, len(data), len(data)//n):
features[f]['{}mean_{}_{}'.format(prefix, f_i, n)] = np.mean(data[i:i + len(data)//n])
 features[f]['{}std_{}_{}'.format(prefix, f_i, n)] = np.std(data[i:i + len(data)//n])
 features[f]['{}min_{}_{}'.format(prefix, f_i, n)] = np.min(data[i:i + len(data)//n])
 features[f]['{}max_{}_{}'.format(prefix, f_i, n)] = np.max(data[i:i + len(data)//n])
 features[f]['len'] = len(data)
 if features[f]['len'] > 0:
n = 1
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 n = 2
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 n = 3
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 cnt += 1
features = pd.DataFrame(features ).T.reset_index()
features.rename(columns={'index': 'fname'}, inplace=True)
 return features
path = os.path.join(data_path, 'audio_train')
 train_files = train.fname.values
train_features = extract_features(train_files, path)
 path = os.path.join(data_path, 'audio_test')
 test_files = ss.fname.values
test_features = extract_features(test_files, path )
upper=corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop=[column for column in upper.columns if any(upper[column]>0.95)]
to_drop<filter>
train_data = train_data.merge(train_features, on='fname', how='left')
 test_data = test_data.merge(test_features, on='fname', how='left')
 train_data.head()
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9] 
<drop_column>
X = train_data.drop(['label', 'fname'], axis=1)
 feature_names = list(X.columns)
 X = X.values
labels = np.sort(np.unique(train_data.label.values))
num_class = len(labels)
 c2i = {}
i2c = {}
for i, c in enumerate(labels):
c2i[c] = i
i2c[i] = c
y = np.array([c2i[x] for x in train_data.label.values] )
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )<feature_engineering>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=10, shuffle = True)
 clf = XGBClassifier(max_depth=5, learning_rate=0.05, n_estimators=3000,
n_jobs=-1, random_state=0, reg_alpha=0.2,
colsample_bylevel=0.9, colsample_bytree=0.9)
 clf.fit(X_train, y_train)
 print(accuracy_score(clf.predict(X_val), y_val))
def proba2labels(preds, i2c, k=3):
ans = []
ids = []
for p in preds:
idx = np.argsort(p)[::-1]
ids.append([i for i in idx[:k]])
 ans.append(' '.join([i2c[i] for i in idx[:k]]))
return ans, ids
heads['hhsize-diff']=heads['tamviv']-heads['hhsize']<feature_engineering>
clf.fit(X, y)
 str_preds, _ = proba2labels(clf.predict_proba(test_data.drop(['label', 'fname'], axis = 1 ).values), i2c, k=3)
 subm = pd.DataFrame()
subm['fname'] = audio_test_files
subm['label'] = str_preds
subm.to_csv('submission.csv', index=False )