Kaggle competition: Freesound Audio Tagging 2019
train_xgb = xgb.DMatrix(new_train, label=target)
 test_xgb = xgb.DMatrix(new_test )<compute_train_metric>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
def lwlrap(scores, truth, **kwargs):
score, weight = calculate_per_class_lwlrap(to_np(truth), to_np(scores))
return torch.Tensor([(score * weight ).sum() ] )
params = {
'max_depth': 4,
'eta': 0.3,
'silent': 1,
'objective': 'multi:softprob',
'num_class': 39,
}
xg = xgb.cv(params, train_xgb, nfold=3, early_stopping_rounds=10, metrics='mlogloss', verbose_eval=True )<predict_on_test>
training_curated_df = pd.read_csv(".. /input/train_curated.csv")
 training_noisy_df = pd.read_csv(".. /input/train_noisy.csv")
 training_df = [training_curated_df, training_noisy_df]
testing_df = pd.read_csv('.. /input/sample_submission.csv' )
train_xgb = xgb.train(params, train_xgb, 10)
 pred_xgb = train_xgb.predict(test_xgb )<create_dataframe>
labels_curated = training_curated_df['labels'].unique()
print(labels_curated.shape)
 print('_'*40)
 print(labels_curated )
submission1 = pd.DataFrame(pred_xgb,columns=LB.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,index=new_test.index)
<define_variables>
labels_noisy = training_noisy_df['labels'].unique()
print(labels_noisy.shape)
 print('_'*40)
 print(labels_noisy )
ensemble = 0.9*predictions + 0.1*pred_xgb<save_to_csv>
class EasyDict(dict):
def __init__(self, d=None, **kwargs):
if d is None:
d = {}
if kwargs:
d.update(**kwargs)
 for k, v in d.items() :
setattr(self, k, v)
 for k in self.__class__.__dict__.keys() :
if not(k.startswith('__')and k.endswith('__')) and not k in('update', 'pop'):
setattr(self, k, getattr(self, k))
def __setattr__(self, name, value):
if isinstance(value,(list, tuple)) :
value = [self.__class__(x)
 if isinstance(x, dict)else x for x in value]
elif isinstance(value, dict)and not isinstance(value, self.__class__):
value = self.__class__(value)
 super(EasyDict, self ).__setattr__(name, value)
 super(EasyDict, self ).__setitem__(name, value)
 __setitem__ = __setattr__
def update(self, e=None, **f):
d = e or dict()
d.update(f)
 for k in d:
setattr(self, k, d[k])
 def pop(self, k, d=None):
delattr(self, k)
 return super(EasyDict, self ).pop(k, d )
sub = pd.DataFrame(ensemble,columns=LB.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,index=new_test.index)
 sub.to_csv('submission.csv', index_label='Id' )<set_options>
def read_audio(conf, pathname, trim_long_data):
y, sr = librosa.load(pathname, sr=conf.sampling_rate)
 if 0 < len(y):
y, _ = librosa.effects.trim(y)
 if len(y)> conf.samples:
if trim_long_data:
y = y[0:0+conf.samples]
else:
padding = conf.samples - len(y)
 offset = padding // 2
y = np.pad(y,(offset, conf.samples - len(y)- offset), 'constant')
 return y
def audio_to_melspectrogram(conf, audio):
spectrogram = librosa.feature.melspectrogram(audio,
sr=conf.sampling_rate,
n_mels=conf.n_mels,
hop_length=conf.hop_length,
n_fft=conf.n_fft,
fmin=conf.fmin,
fmax=conf.fmax)
 spectrogram = librosa.power_to_db(spectrogram)
 spectrogram = spectrogram.astype(np.float32)
 return spectrogram
def melspectrogram_to_delta(mels):
return librosa.feature.delta(mels)
 def show_melspectrogram(conf, mels, title='Log-frequency power spectrogram'):
librosa.display.specshow(mels, x_axis='time', y_axis='mel',
sr=conf.sampling_rate, hop_length=conf.hop_length,
fmin=conf.fmin, fmax=conf.fmax)
 plt.colorbar(format='%+2.0f dB')
 plt.title(title)
 plt.show()
def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):
x = read_audio(conf, pathname, trim_long_data)
 mels = audio_to_melspectrogram(conf, x)
 if debug_display:
delta = melspectrogram_to_delta(mels)
 delta_squared = melspectrogram_to_delta(delta)
 IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))
show_melspectrogram(conf, mels)
 show_melspectrogram(conf, delta)
 show_melspectrogram(conf, delta_squared)
 return mels
conf = EasyDict()
conf.sampling_rate = 44100
conf.duration = 2
conf.hop_length = 347 * conf.duration
conf.fmin = 20
conf.fmax = conf.sampling_rate // 2
conf.n_mels = 128
conf.n_fft = conf.n_mels * 20
conf.samples = conf.sampling_rate * conf.duration
pd.set_option("display.max_columns", None)
 %matplotlib inline 
<load_from_csv>
path = '.. /input/train_curated/0006ae4e.wav'
x = read_audio(conf, path, trim_long_data=False)
 print(x)
 print('_'*40)
 print(audio_to_melspectrogram(conf, x))
print(audio_to_melspectrogram(conf, x ).shape)
 x1 = read_as_melspectrogram(conf, path, trim_long_data=False, debug_display=True )
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
 train = pd.read_csv('/kaggle/input/sf-crime/train.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 test = pd.read_csv('/kaggle/input/sf-crime/test.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 print('done' )<count_duplicates>
training_curated_df, X_train_curated = convert_wav_to_image(training_curated_df, source=Path('.. /input/train_curated'), img_dest=Path('trn_curated'))
testing_df, X_test = convert_wav_to_image(testing_df, source=Path('.. /input/test'), img_dest=Path('test'))
print(f"Finished data conversion at {(time.time() -start_time)/3600} hours" )
print(train.duplicated().sum())
 train.drop_duplicates(inplace=True)
 assert train.duplicated().sum() ==0
print('done' )<predict_on_test>
bs = 64
size = 128
tfms = get_transforms(do_flip=False, max_rotate=0, max_lighting=0.1, max_zoom=0, max_warp=0.)
src =(ImageList.from_csv('/kaggle/working', '.. /input/train_curated.csv', folder='.. /input/train_curated')
.split_by_rand_pct(0.2 ).label_from_df(label_delim=','))
data = src.transform(tfms, size=size ).databunch(bs=bs ).normalize()
print(train.columns.difference(test.columns)) 
<count_values>
arch = models.resnet18
learn = cnn_learner(data, arch, pretrained=False, metrics=[lwlrap], wd = 0.1, ps = 0.5)
 learn.lr_find()
learn.recorder.plot()
print(train_gdf.loc[train_gdf.Y > 50].count() [0])
<categorify>
learn.fit_one_cycle(5, 1e-2)
 learn.save('first-attempt-128' )
train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='most_frequent')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 train_gdf = create_gdf(train)
 print('done' )<load_from_csv>
learn.unfreeze()
learn.fit_one_cycle(1 )
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
 train = pd.read_csv('/kaggle/input/sf-crime/train.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 test = pd.read_csv('/kaggle/input/sf-crime/test.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 print('done' )<categorify>
learn.fit(20, slice(2e-3, 2e-4))
learn.save('second-attempt-128' )
train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='most_frequent')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 print('done')
<prepare_x_and_y>
learn.fit(20, slice(1e-3, 1e-4))
learn.save('third-attempt-128' )
y_train = train['Category']
train_des = train['Descript']
train_res = train['Resolution']
train.drop(["Category", "Descript", "Resolution"], axis=1, inplace=True)
 print('done')
<drop_column>
size = 256
data = src.transform(tfms, size=size ).databunch(bs=bs ).normalize(imagenet_stats )
test_ID = test["Id"]
test.drop("Id", axis=1, inplace=True)
 print('done' )<categorify>
learn.data = data
learn.freeze()
learn.lr_find()
learn.recorder.plot()
le = LabelEncoder()
y_train = le.fit_transform(y_train)
 print(le.classes_)
 print('done' )<drop_column>
learn.fit(5, 3e-3)
 learn.save('first-attempt-256' )
<concatenate>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
num_train = train.shape[0]
all_data = pd.concat(( train, test), ignore_index=True)
 print('done' )<feature_engineering>
learn.fit(7, slice(1e-3, 1e-4))
learn.save('second-attempt-256' )
def feature_engineering(data):
date = pd.to_datetime(all_data['Dates'])
 data['year'] = date.dt.year
data['month'] = date.dt.month
data['day'] = date.dt.day
data['hour'] = date.dt.hour
data['minute'] = date.dt.minute
data['special_time'] = data['minute'].isin([0, 30] ).astype(int)
 data["n_days"] =(date - date.min() ).apply(lambda x: x.days)
 data.drop("Dates", axis=1, inplace=True)
 data['block'] = data["Address"].str.contains("block", case=False)
 data['ST'] = data['Address'].str.contains('ST', case=False)
 return data
all_data1 = feature_engineering(all_data)
 print('done' )<feature_engineering>
learn.fit_one_cycle(20, slice(5e-4, 5e-5), callbacks=[SaveModelCallback(learn, monitor='lwlrap', mode='max')] )
<feature_engineering><EOS> 
CUR_X_FILES, CUR_X = list(testing_df.fname.values), X_test
test = ImageList.from_csv(Path('/kaggle/working'), Path('.. /input/sample_submission.csv'), folder=Path('.. /input/test'))
learn = load_learner(Path('/kaggle/working'), test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 testing_df[learn.data.classes] = preds
testing_df.to_csv('submission.csv', index=False)
 testing_df.head()