Kaggle competition: Freesound Audio Tagging 2019
train = train[train.item_price > 0].reset_index(drop = True)
 train.loc[train.item_cnt_day < 1, "item_cnt_day"] = 0<feature_engineering>
DATA = Path('.. /input')
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
TRN_CURATED = DATA/'train_curated'
TRN_NOISY = DATA/'train_noisy'
TEST = DATA/'test'
WORK = Path('work')
 IMG_TRN_CURATED = WORK/'image/trn_curated'
IMG_TRN_NOISY = WORK/'image/train_noisy'
IMG_TEST = WORK/'image/test'
for folder in [WORK, IMG_TRN_CURATED, IMG_TRN_NOISY, IMG_TEST]:
Path(folder ).mkdir(exist_ok=True, parents=True)
 df = pd.read_csv(CSV_TRN_CURATED)
 test_df = pd.read_csv(CSV_SUBMISSION )
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<feature_engineering>
def read_audio(conf, pathname, trim_long_data):
y, sr = librosa.load(pathname, sr=conf.sampling_rate)
 if 0 < len(y):
y, _ = librosa.effects.trim(y)
 if len(y)> conf.samples:
if trim_long_data:
y = y[0:0+conf.samples]
else:
padding = conf.samples - len(y)
 offset = padding // 2
y = np.pad(y,(offset, conf.samples - len(y)- offset), 'constant')
 return y
def audio_to_melspectrogram(conf, audio):
spectrogram = librosa.feature.melspectrogram(audio,
sr=conf.sampling_rate,
n_mels=conf.n_mels,
hop_length=conf.hop_length,
n_fft=conf.n_fft,
fmin=conf.fmin,
fmax=conf.fmax)
 spectrogram = librosa.power_to_db(spectrogram)
 spectrogram = spectrogram.astype(np.float32)
 return spectrogram
def show_melspectrogram(conf, mels, title='Log-frequency power spectrogram'):
librosa.display.specshow(mels, x_axis='time', y_axis='mel',
sr=conf.sampling_rate, hop_length=conf.hop_length,
fmin=conf.fmin, fmax=conf.fmax)
 plt.colorbar(format='%+2.0f dB')
 plt.title(title)
 plt.show()
def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):
x = read_audio(conf, pathname, trim_long_data)
 mels = audio_to_melspectrogram(conf, x)
 if debug_display:
IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))
show_melspectrogram(conf, mels)
 return mels
class conf:
sampling_rate = 44100
duration = 2
hop_length = 347*duration
fmin = 20
fmax = sampling_rate // 2
n_mels = 128
n_fft = n_mels * 20
samples = sampling_rate * duration
x = read_as_melspectrogram(conf, TRN_CURATED/'0006ae4e.wav', trim_long_data=False, debug_display=True )
shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" " ).map(lambda x: x[0])
 shops["category"] = shops.shop_name.str.split(" " ).map(lambda x: x[1])
 shops.loc[shops.city == "!Якутск", "city"] = "Якутск"<categorify>
def mono_to_color(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):
X = np.stack([X, X, X], axis=-1)
 mean = mean or X.mean()
std = std or X.std()
Xstd =(X - mean)/(std + eps)
 _min, _max = Xstd.min() , Xstd.max()
norm_max = norm_max or _max
norm_min = norm_min or _min
if(_max - _min)> eps:
V = Xstd
V[V < norm_min] = norm_min
V[V > norm_max] = norm_max
V = 255 *(V - norm_min)/(norm_max - norm_min)
 V = V.astype(np.uint8)
 else:
V = np.zeros_like(Xstd, dtype=np.uint8)
 return V
def convert_wav_to_image(df, source, img_dest):
X = []
for i, row in tqdm_notebook(df.iterrows()):
x = read_as_melspectrogram(conf, source/str(row.fname), trim_long_data=False)
 x_color = mono_to_color(x)
 X.append(x_color)
 return X
X_train = convert_wav_to_image(df, source=TRN_CURATED, img_dest=IMG_TRN_CURATED)
 X_test = convert_wav_to_image(test_df, source=TEST, img_dest=IMG_TEST )
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city)
 shops = shops[["shop_id", "shop_category", "shop_city"]]<feature_engineering>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
def lwlrap(scores, truth, **kwargs):
score, weight = calculate_per_class_lwlrap(to_np(truth), to_np(scores))
return torch.Tensor([(score * weight ).sum() ])
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"<categorify>
data.show_batch(3 )
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
 cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats = cats[["item_category_id", "subtype_code", "type_code"]]<categorify>
learn = cnn_learner(data, models.resnet18, pretrained=False, metrics=[lwlrap])
 learn.unfreeze()
learn.lr_find() ; learn.recorder.plot()
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
learn.fit_one_cycle(5, 1e-1)
 learn.fit_one_cycle(10, 1e-2 )
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )<feature_engineering>
learn.fit_one_cycle(20, 3e-3 )
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"<groupby>
learn.fit_one_cycle(20, 1e-3 )
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 group_sum = group_sum.reset_index()
drop_cols = []
for cat in group_sum.type.unique() :
if group_sum.loc[(group_sum.type == cat), "item_id"].values[0] <40:
drop_cols.append(cat)
 items.name2 = items.name2.apply(lambda x: "other" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )<categorify>
learn.fit_one_cycle(50, slice(1e-3, 3e-3))
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()<data_type_conversions>
learn.fit_one_cycle(10, slice(1e-4, 1e-3))
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts<feature_engineering>
learn.save('fat2019_fastai_cnn2d_stage-2')
 learn.export()
train["revenue"] = train["item_cnt_day"] * train["item_price"]<merge>
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK/'image', Path('.. /.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK/'image', test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test )
ts = time.time()
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).astype(np.float16)
 time.time() - ts<data_type_conversions>
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()
<concatenate><EOS> 
CUR_X_FILES, CUR_X = list(df.fname.values), X_train
learn = cnn_learner(data, models.resnet18, pretrained=False, metrics=[lwlrap])
 learn.load('fat2019_fastai_cnn2d_stage-2');