Kaggle competition: Freesound General-Purpose Audio Tagging Challenge
<SOS> metric: MAP@{K} Kaggle data source: freesound-audio-tagging<categorify>
COMPLETE_RUN = False
np.random.seed(1001)
 %matplotlib inline
matplotlib.style.use('ggplot' )
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
extract_features(train)
 extract_features(test )<drop_column>
train = pd.read_csv(".. /input/freesound-audio-tagging/train.csv")
 test = pd.read_csv(".. /input/freesound-audio-tagging/sample_submission.csv" )
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
print("Number of training examples=", train.shape[0], " Number of classes=", len(train.label.unique()))
def split_data(train, y, households, test_percentage=0.20, seed=None):
train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
return X_train, y_train, X_test, y_test<split>
print('Minimum samples per category = ', min(train.label.value_counts()))
print('Maximum samples per category = ', max(train.label.value_counts()))
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=42)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<define_variables>
fname = '.. /input/freesound-audio-tagging/audio_train/' + '00044347.wav'
ipd.Audio(fname )
extra_drop_features = [
'geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'rez_esc',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']<drop_column>
wav = wave.open(fname)
 print("Sampling(frame)rate = ", wav.getframerate())
 print("Total samples(frames)= ", wav.getnframes())
 print("Duration = ", wav.getnframes() /wav.getframerate() )
lgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<compute_train_metric>
rate, data = wavfile.read(fname)
 print("Sampling(frame)rate = ", rate)
 print("Total samples(frames)= ", data.shape)
 print(data )
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 16, 'subsample': 0.94, 'reg_lambda': 0.5, }
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 95, 'num_leaves': 25, 'subsample': 0.94, 'reg_lambda': 0.5, }
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":800,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
ModelCheckpoint, TensorBoard, ReduceLROnPlateau)
 GlobalMaxPool1D, Input, MaxPool1D, concatenate)
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=1972)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
class Config(object):
def __init__(self,
sampling_rate=16000, audio_duration=2, n_classes=41,
use_mfcc=False, n_folds=12, learning_rate=0.0001,
max_epochs=80, n_mfcc=20):
self.sampling_rate = sampling_rate
self.audio_duration = audio_duration
self.n_classes = n_classes
self.use_mfcc = use_mfcc
self.n_mfcc = n_mfcc
self.n_folds = n_folds
self.learning_rate = learning_rate
self.max_epochs = max_epochs
self.audio_length = self.sampling_rate * self.audio_duration
if self.use_mfcc:
self.dim =(self.n_mfcc, 1 + int(np.floor(self.audio_length/512)) , 1)
 else:
self.dim =(self.audio_length, 1 )
clfs = []
for i in range(30):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
class DataGenerator(Sequence):
def __init__(self, config, data_dir, list_IDs, labels=None,
batch_size=64, preprocessing_fn=lambda x: x):
self.config = config
self.data_dir = data_dir
self.list_IDs = list_IDs
self.labels = labels
self.batch_size = batch_size
self.preprocessing_fn = preprocessing_fn
self.on_epoch_end()
self.dim = self.config.dim
def __len__(self):
return int(np.ceil(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
return self.__data_generation(list_IDs_temp)
 def on_epoch_end(self):
self.indexes = np.arange(len(self.list_IDs))
def __data_generation(self, list_IDs_temp):
cur_batch_size = len(list_IDs_temp)
 X = np.empty(( cur_batch_size, *self.dim))
input_length = self.config.audio_length
for i, ID in enumerate(list_IDs_temp):
file_path = self.data_dir + ID
data, _ = librosa.core.load(file_path, sr=self.config.sampling_rate,
res_type='kaiser_fast')
 if len(data)> input_length:
max_offset = len(data)- input_length
offset = np.random.randint(max_offset)
 data = data[offset:(input_length+offset)]
else:
if input_length > len(data):
max_offset = input_length - len(data)
 offset = np.random.randint(max_offset)
 else:
offset = 0
data = np.pad(data,(offset, input_length - len(data)- offset), "constant")
 if self.config.use_mfcc:
data = librosa.feature.mfcc(data, sr=self.config.sampling_rate,
n_mfcc=self.config.n_mfcc)
 data = np.expand_dims(data, axis=-1)
 else:
data = self.preprocessing_fn(data)[:, np.newaxis]
X[i,] = data
if self.labels is not None:
y = np.empty(cur_batch_size, dtype=int)
 for i, ID in enumerate(list_IDs_temp):
y[i] = self.labels[ID]
return X, to_categorical(y, num_classes=self.config.n_classes)
 else:
return X
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_train_metric>
def audio_norm(data):
max_data = np.max(data)
 min_data = np.min(data)
 data =(data-min_data)/(max_data-min_data+1e-6)
 return data-0.5
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
def get_1d_dummy_model(config):
nclass = config.n_classes
input_length = config.audio_length
inp = Input(shape=(input_length,1))
x = GlobalMaxPool1D()(inp)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
def get_1d_conv_model(config):
nclass = config.n_classes
input_length = config.audio_length
inp = Input(shape=(input_length,1))
x = Convolution1D(32, 9, activation=relu, padding="same" )(inp)
 x = Convolution1D(32, 9, activation=relu, padding="same" )(x)
 x = MaxPool1D(16 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(64, 3, activation=relu, padding="same" )(x)
 x = Convolution1D(64, 3, activation=relu, padding="same" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(64, 3, activation=relu, padding="same" )(x)
 x = Convolution1D(64, 3, activation=relu, padding="same" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(128, 3, activation=relu, padding="same" )(x)
 x = Convolution1D(128, 3, activation=relu, padding="same" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(256, 3, activation=relu, padding="same" )(x)
 x = Convolution1D(256, 3, activation=relu, padding="same" )(x)
 x = GlobalMaxPool1D()(x)
 x = Dropout(rate=0.2 )(x)
 x = Dense(64, activation=relu )(x)
 x = Dense(1028, activation=relu )(x)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
ranked_features = feature_importance(clf_final, X_train )<create_dataframe>
LABELS = list(train.label.unique())
 label_idx = {label: i for i, label in enumerate(LABELS)}
train.set_index("fname", inplace=True)
 test.set_index("fname", inplace=True)
 train["label_idx"] = train.label.apply(lambda x: label_idx[x])
 if not COMPLETE_RUN:
train = train[:2000]
test = test[:2000]
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>
config = Config(sampling_rate=16000, audio_duration=2, n_folds=10, learning_rate=0.001)
 if not COMPLETE_RUN:
config = Config(sampling_rate=100, audio_duration=1, n_folds=2, max_epochs=1 )
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1<save_to_csv>
PREDICTION_FOLDER = "predictions_1d_conv"
if not os.path.exists(PREDICTION_FOLDER):
os.mkdir(PREDICTION_FOLDER)
 if os.path.exists('logs/' + PREDICTION_FOLDER):
shutil.rmtree('logs/' + PREDICTION_FOLDER)
 skf = StratifiedKFold(train.label_idx, n_folds=config.n_folds)
 for i,(train_split, val_split)in enumerate(skf):
train_set = train.iloc[train_split]
val_set = train.iloc[val_split]
checkpoint = ModelCheckpoint('best_%d.h5'%i, monitor='val_loss', verbose=1, save_best_only=True)
 early = EarlyStopping(monitor="val_loss", mode="min", patience=5)
 tb = TensorBoard(log_dir='./logs/' + PREDICTION_FOLDER + '/fold_%d'%i, write_graph=True)
 callbacks_list = [checkpoint, early, tb]
print("Fold: ", i)
 print("
if COMPLETE_RUN:
model = get_1d_conv_model(config)
 else:
model = get_1d_dummy_model(config)
 train_generator = DataGenerator(config, '.. /input/freesound-audio-tagging/audio_train/', train_set.index,
train_set.label_idx, batch_size=64,
preprocessing_fn=audio_norm)
 val_generator = DataGenerator(config, '.. /input/freesound-audio-tagging/audio_train/', val_set.index,
val_set.label_idx, batch_size=64,
preprocessing_fn=audio_norm)
 history = model.fit_generator(train_generator, callbacks=callbacks_list, validation_data=val_generator,
epochs=config.max_epochs, use_multiprocessing=True, workers=6, max_queue_size=20)
 model.load_weights('best_%d.h5'%i)
 train_generator = DataGenerator(config, '.. /input/freesound-audio-tagging/audio_train/', train.index, batch_size=128,
preprocessing_fn=audio_norm)
 predictions = model.predict_generator(train_generator, use_multiprocessing=True,
workers=6, max_queue_size=20, verbose=1)
 np.save(PREDICTION_FOLDER + "/train_predictions_%d.npy"%i, predictions)
 test_generator = DataGenerator(config, '.. /input/freesound-audio-tagging/audio_test/', test.index, batch_size=128,
preprocessing_fn=audio_norm)
 predictions = model.predict_generator(test_generator, use_multiprocessing=True,
workers=6, max_queue_size=20, verbose=1)
 np.save(PREDICTION_FOLDER + "/test_predictions_%d.npy"%i, predictions)
 top_3 = np.array(LABELS)[np.argsort(-predictions, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test['label'] = predicted_labels
test[['label']].to_csv(PREDICTION_FOLDER + "/predictions_%d.csv"%i )
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )<set_options>
SAMPLE_RATE = 44100
fname = '.. /input/freesound-audio-tagging/audio_train/' + '00044347.wav'
wav, _ = librosa.core.load(fname, sr=SAMPLE_RATE)
 wav = wav[:2*44100]
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k' 
<load_from_csv>
mfcc = librosa.feature.mfcc(wav, sr = SAMPLE_RATE, n_mfcc=40)
 mfcc.shape
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<data_type_conversions>
GlobalMaxPool2D, MaxPool2D, concatenate, Activation)
maps={'yes':1,'no':0}
for df in [train,test]:
df['dependency'] = df['dependency'].replace(maps ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(maps ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(maps ).astype(np.float64 )<concatenate>
def get_2d_dummy_model(config):
nclass = config.n_classes
inp = Input(shape=(config.dim[0],config.dim[1],1))
x = GlobalMaxPool2D()(inp)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
def get_2d_conv_model(config):
nclass = config.n_classes
inp = Input(shape=(config.dim[0],config.dim[1],1))
x = Convolution2D(32,(4,10), padding="same" )(inp)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Flatten()(x)
 x = Dense(64 )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
test['Target'] = np.nan
data = train.append(test, ignore_index = True )<count_unique_values>
config = Config(sampling_rate=44100, audio_duration=2, n_folds=10,
learning_rate=0.001, use_mfcc=True, n_mfcc=40)
 if not COMPLETE_RUN:
config = Config(sampling_rate=44100, audio_duration=2, n_folds=2,
max_epochs=1, use_mfcc=True, n_mfcc=40 )
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<groupby>
def prepare_data(df, config, data_dir):
X = np.empty(shape=(df.shape[0], config.dim[0], config.dim[1], 1))
input_length = config.audio_length
for i, fname in enumerate(df.index):
print(fname)
 file_path = data_dir + fname
data, _ = librosa.core.load(file_path, sr=config.sampling_rate, res_type="kaiser_fast")
 if len(data)> input_length:
max_offset = len(data)- input_length
offset = np.random.randint(max_offset)
 data = data[offset:(input_length+offset)]
else:
if input_length > len(data):
max_offset = input_length - len(data)
 offset = np.random.randint(max_offset)
 else:
offset = 0
data = np.pad(data,(offset, input_length - len(data)- offset), "constant")
 data = librosa.feature.mfcc(data, sr=config.sampling_rate, n_mfcc=config.n_mfcc)
 data = np.expand_dims(data, axis=-1)
 X[i,] = data
return X
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))<groupby>
X_train = prepare_data(train, config, '.. /input/freesound-audio-tagging/audio_train/')
 X_test = prepare_data(test, config, '.. /input/freesound-audio-tagging/audio_test/')
 y_train = to_categorical(train.label_idx, num_classes=config.n_classes )
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )<feature_engineering>
mean = np.mean(X_train, axis=0)
 std = np.std(X_train, axis=0)
 X_train =(X_train - mean)/std
X_test =(X_test - mean)/std
for household in not_equal.index:
true_target=int(train[(train['idhogar']==household)&(train['parentesco1']==1.0)]["Target"])
 train.loc[train["idhogar"]==household,"Target"]=true_target
all_equal=train.groupby("idhogar")["Target"].apply(lambda x:x.nunique() ==1)
 not_equal=all_equal[all_equal !=True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<feature_engineering>
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-file/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("2d_conv_ensembled_submission.csv", index=False )
missing=pd.DataFrame(data.isnull().sum())
 missing=missing.rename(columns={0:'total'})
 missing["percentage"]=missing['total']/len(data)
 missing.sort_values('percentage',ascending=False ).head(10 ).drop("Target" )<groupby>
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-data-2d-conv-reduced-lr/test_predictions_%d.npy"%i))
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-file/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("1d_2d_ensembled_submission.csv", index=False )