Kaggle competition: Freesound Audio Tagging 2019
<SOS> metric: weightedlabelrankingaverageprecision Kaggle data source: freesound-audio-tagging-2019<define_variables>
py.init_notebook_mode(connected=True)
 %matplotlib inline
t1 = time.time()
sz=64<create_dataframe>
train_audio_path = '.. /input/freesound-audio-tagging-2019/train_curated/'
train_files = os.listdir(train_audio_path)
 train_annot = pd.read_csv('.. /input/freesound-audio-tagging-2019/train_curated.csv')
 test_audio_path = '.. /input/freesound-audio-tagging-2019/test/'
test_files = np.sort(os.listdir(test_audio_path))
print(test_files[:5])
 USE_WEIGHTS = False
len(train_files), len(test_files)
data = get_data(sz )<normalization>
label_dict, classes, all_labels_set, first_labels_set = create_unique_labels(train_annot.labels)
 files = train_annot.fname
print(len(files), len(train_files))
data = data.resize(int(sz*1.3), TMP_PATH )<load_pretrained>
binarize = MultiLabelBinarizer(classes=classes)
 encode = LabelEncoder()
Y_split = encode.fit_transform(first_labels_set)
 Y = binarize.fit_transform(all_labels_set )
learn = ConvLearner.pretrained(f_model, data, metrics=metrics )<init_hyperparams>
X_raw = []
for f in tqdm_notebook(files):
sample_rate, sample_ = wavfile.read(str(train_audio_path)+ f)
 X_raw.append(sample_ )
lr = 0.2<train_model>
def pad_audio(samples):
if len(samples)>= L: return samples
else: return np.pad(samples, pad_width=(L - len(samples), 0), mode='constant', constant_values=(0, 0))
def chop_audio(samples):
beg = np.random.randint(0, len(samples)- L)
 return samples[beg: beg + L]
def log_specgram(audio,
sample_rate,
window_size=20,
step_size=10, eps=1e-10):
nperseg = int(round(window_size * sample_rate / 1e3))
noverlap = int(round(step_size * sample_rate / 1e3))
freqs, times, spec = signal.spectrogram(audio,
fs=sample_rate,
window='hann',
nperseg=nperseg,
noverlap=noverlap,
detrend=False)
 return freqs, times, np.log(spec.astype(np.float32)+ eps )
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )<define_search_space>
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, labels, batch_size=32, dim=(256,256,1), n_channels=1,
n_classes=80, shuffle=True):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
max_index = min(( index+1)*self.batch_size, len(self.list_IDs))
indexes = self.indexes[index*self.batch_size:max_index]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
X, y = self.__data_generation(list_IDs_temp)
 return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
X = []
y = []
t1 = time.time()
for i, ID in enumerate(list_IDs_temp):
xx = X_raw[ID].copy()
xx = pad_audio(xx)
 if len(xx)> L:
xx = chop_audio(xx)
 _, _, specgram = log_specgram(xx, sample_rate=SAMPLE_RATE, window_size=10, step_size=5)
 X.append(specgram)
 y.append(self.labels[ID, :])
 t2 = time.time()
y = np.array(y, dtype='float32')
 X = np.expand_dims(np.array(X), -1)
 return X, y
class TestDataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, test_files, test_base_path, batch_size=32, dim=(256,256,1), n_channels=1,
n_classes=80):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.test_base_path = test_base_path
self.test_files = test_files
self.n_channels = n_channels
self.n_classes = n_classes
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.test_files)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
max_index = min(( index+1)*self.batch_size, len(self.test_files))
indexes = self.indexes[index*self.batch_size:max_index]
list_IDs_temp = [self.test_files[k] for k in indexes]
X = self.__data_generation(list_IDs_temp)
 return X
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.test_files))
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
X = []
for i, ID in enumerate(list_IDs_temp):
sample_rate, xx = wavfile.read(str(self.test_base_path)+ ID)
 xx = pad_audio(xx)
 if len(xx)> L:
xx = chop_audio(xx)
 _, _, specgram = log_specgram(xx, sample_rate=SAMPLE_RATE, window_size=10, step_size=5)
 X.append(specgram)
 X = np.expand_dims(np.array(X), -1)
 return X
lrs = np.array([lr/9,lr/3,lr] )<train_model>
GlobalMaxPool2D, MaxPool2D, concatenate, Activation, Input, Dense, Dropout)
 def get_2d_conv_model(input_shape=(221, 298, 1), n_classes=80, learning_rate=0.001):
nclass = n_classes
inp = Input(shape=(input_shape[0],input_shape[1],1))
x = Convolution2D(96,(4,10), padding="same" )(inp)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(64,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(48,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Flatten()(x)
 x = Dropout(0.5 )(x)
 x = Dense(80 )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = Dense(80 )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 out = Dense(nclass, activation='softmax' )(x)
 model = Model(inputs=inp, outputs=out)
 opt = Adam(learning_rate)
 model.compile(optimizer=opt, loss=binary_crossentropy, metrics=['categorical_accuracy'])
 return model
learn.unfreeze()
learn.fit(lrs, 3, cycle_len=1, cycle_mult=2 )<save_model>
modelname = 'custom-v1-'
test_gen = TestDataGenerator(test_files, test_audio_path, batch_size=128 )
learn.save(f'{sz}' )<define_variables>
oof_y = np.zeros_like(Y, dtype='float32')
 test_Y = np.zeros(( len(test_files), 80), dtype='float32')
 kfold = StratifiedKFold(10, True, 6996)
 ifold = 0
for train_index, valid_index in kfold.split(X_raw, Y_split):
if ifold<=1:
print("--------------------------------------------")
 checkpoint = ModelCheckpoint(modelname + str(ifold)+ '.hdf5', monitor='val_categorical_accuracy', verbose=1,
save_best_only=True, save_weights_only=True, mode='auto', period=1)
 early = EarlyStopping(monitor='val_categorical_accuracy', min_delta=0.001, patience=12, verbose=1, mode='auto', restore_best_weights=True)
 reduce_lr = ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=6, verbose=1)
 model = get_2d_conv_model()
train_gen = DataGenerator(train_index, Y, batch_size=32, n_classes=80, shuffle=True)
 valid_gen = DataGenerator(valid_index, Y, batch_size=128, n_classes=80, shuffle=False)
 model.fit_generator(train_gen, epochs=60, verbose=0, callbacks=[checkpoint, early, reduce_lr], validation_data=valid_gen)
 res = model.predict_generator(valid_gen, verbose=0)
 res_Y = model.predict_generator(test_gen, verbose=0)
 for bag in range(9):
res_Y += model.predict_generator(test_gen, verbose=0)
 oof_y[valid_index, ] = res
test_Y = test_Y + res_Y / 10.
 ifold = ifold + 1
test_Y = test_Y / 5
sz=128<train_model>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
def calculate_overall_lwlrap_sklearn(truth, scores):

 sample_weight = np.sum(truth > 0, axis=1)
 nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)
 overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(
 truth[nonzero_weight_sample_indices, :] > 0,
scores[nonzero_weight_sample_indices, :],
sample_weight=sample_weight[nonzero_weight_sample_indices])
 return overall_lwlrap
class lwlrap_accumulator(object):

 def __init__(self):
self.num_classes = 0
self.total_num_samples = 0
def accumulate_samples(self, batch_truth, batch_scores):

 assert batch_scores.shape == batch_truth.shape
num_samples, num_classes = batch_truth.shape
if not self.num_classes:
self.num_classes = num_classes
self._per_class_cumulative_precision = np.zeros(self.num_classes)
 self._per_class_cumulative_count = np.zeros(self.num_classes,
dtype=np.int)
 assert num_classes == self.num_classes
for truth, scores in zip(batch_truth, batch_scores):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores, truth))
self._per_class_cumulative_precision[pos_class_indices] +=(
 precision_at_hits)
 self._per_class_cumulative_count[pos_class_indices] += 1
self.total_num_samples += num_samples
def per_class_lwlrap(self):

 return(self._per_class_cumulative_precision /
np.maximum(1, self._per_class_cumulative_count))
def per_class_weight(self):

 return(self._per_class_cumulative_count /
float(np.sum(self._per_class_cumulative_count)))
 def overall_lwlrap(self):

 return np.sum(self.per_class_lwlrap() * self.per_class_weight())
learn.set_data(get_data(sz))
learn.freeze()
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )<train_model>
truth = Y
scores = oof_y
print("lwlrap from sklearn.metrics =", calculate_overall_lwlrap_sklearn(truth, scores))
<define_variables><EOS> 
sort_idx = np.argsort(classes ).astype(int)
 sample_sub = pd.read_csv('.. /input/freesound-audio-tagging-2019/sample_submission.csv')
 test_Y_sort = test_Y[:, sort_idx]
sample_sub.iloc[:, 1:] = test_Y_sort
sample_sub.to_csv('submission.csv', index=False)
 t2 = time.time()
print('Total time: ',(t2-t1))