Kaggle competition: Freesound Audio Tagging 2019
sales_train["item_cnt_day"] = sales_train["item_cnt_day"].clip(0,200)
 sales_train["item_price"] = sales_train["item_price"].clip(0,5000 )<merge>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
train_unique = sales_train.groupby(["shop_id", "item_id"] ).size()
test_unique = sales_test.groupby(["shop_id", "item_id"] ).size()
train_unique = pd.DataFrame({"in_train":True}, index=train_unique.index)
 test_unique = pd.DataFrame({"in_test":True}, index=test_unique.index)
 combined = pd.merge(train_unique, test_unique, on=["shop_id", "item_id"], how="outer" ).fillna(False)
 combined["in_both"] = combined["in_train"] & combined["in_test"]
num_in_both = sum(combined["in_both"] == True)
 num_in_train = sum(( combined["in_train"] == True)&(combined["in_test"] == False))
num_in_test = sum(( combined["in_test"] == True)&(combined["in_train"] == False))
pd.DataFrame({"in_both":[num_in_both],
"in_train": [num_in_train],
"in_test": [num_in_test]} ).T.plot.pie(subplots=True )<data_type_conversions>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
%%time
matrix = []
index_cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = sales_train[sales_train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype=np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns=index_cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(index_cols, inplace = True)
 matrix<merge>
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED = Path('.. /input/fat2019_prep_mels1')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_TRN_NOISY_BEST50S = PREPROCESSED/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
MELS_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'
MELS_TRN_NOISY = PREPROCESSED/'mels_train_noisy.pkl'
MELS_TRN_NOISY_BEST50S = PREPROCESSED/'mels_trn_noisy_best50s.pkl'
MELS_TEST = PREPROCESSED/'mels_test.pkl'
trn_curated_df = pd.read_csv(CSV_TRN_CURATED)
 trn_noisy_df = pd.read_csv(CSV_TRN_NOISY)
 trn_noisy50s_df = pd.read_csv(CSV_TRN_NOISY_BEST50S)
 test_df = pd.read_csv(CSV_SUBMISSION)
 df = pd.concat([trn_curated_df, trn_noisy50s_df], ignore_index=True, sort=True)
 test_df = pd.read_csv(CSV_SUBMISSION)
 X_train = pickle.load(open(MELS_TRN_CURATED, 'rb')) + pickle.load(open(MELS_TRN_NOISY_BEST50S, 'rb'))
item_counts = sales_train.groupby(index_cols ).agg({"item_cnt_day": ["sum"]} ).reset_index()
matrix = pd.merge(matrix, item_counts, on=index_cols, how="left")
 matrix.fillna(0, inplace=True)
 matrix.rename(columns={matrix.columns[-1]: "item_cnt_month"}, inplace=True)
 matrix["item_cnt_month"] = matrix["item_cnt_month"].clip(0,20 ).astype(np.float16)
 matrix<concatenate>
data.show_batch(3 )
sales_test["date_block_num"] = 34
matrix = pd.concat([matrix, sales_test.drop(["ID"], axis=1)], ignore_index=True, keys=index_cols)
 matrix.fillna(0, inplace=True)
 matrix<data_type_conversions>
def lwlrap(y_pred,y_true):
score, weight = calculate_per_class_lwlrap(y_true.cpu().numpy() , y_pred.cpu().numpy())
 lwlrap =(score * weight ).sum()
return torch.from_numpy(np.array(lwlrap))
matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.dtypes<merge>
class MixUpCallback(LearnerCallback):
"Callback that creates the mixed-up input and target."
def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):
super().__init__(learn)
 self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y
def on_train_begin(self, **kwargs):
if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)
 def on_batch_begin(self, last_input, last_target, train, **kwargs):
"Applies mixup to `last_input` and `last_target` if `train`."
if not train: return
lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))
lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1 ).max(1)
 lambd = last_input.new(lambd)
 shuffle = torch.randperm(last_target.size(0)).to(last_input.device)
 x1, y1 = last_input[shuffle], last_target[shuffle]
if self.stack_x:
new_input = [last_input, last_input[shuffle], lambd]
else:
new_input =(last_input * lambd.view(lambd.size(0),1,1,1)+ x1 *(1-lambd ).view(lambd.size(0),1,1,1))
if self.stack_y:
new_target = torch.cat([last_target[:,None].float() , y1[:,None].float() , lambd[:,None].float() ], 1)
 else:
if len(last_target.shape)== 2:
lambd = lambd.unsqueeze(1 ).float()
new_target = last_target.float() * lambd + y1.float() *(1-lambd)
 return {'last_input': new_input, 'last_target': new_target}
def on_train_end(self, **kwargs):
if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()
class MixUpLoss(nn.Module):
"Adapt the loss function `crit` to go with mixup."
def __init__(self, crit, reduction='mean'):
super().__init__()
if hasattr(crit, 'reduction'):
self.crit = crit
self.old_red = crit.reduction
setattr(self.crit, 'reduction', 'none')
 else:
self.crit = partial(crit, reduction='none')
 self.old_crit = crit
self.reduction = reduction
def forward(self, output, target):
if len(target.size())== 2:
loss1, loss2 = self.crit(output,target[:,0].long()), self.crit(output,target[:,1].long())
 d =(loss1 * target[:,2] + loss2 *(1-target[:,2])).mean()
else: d = self.crit(output, target)
 if self.reduction == 'mean': return d.mean()
elif self.reduction == 'sum': return d.sum()
return d
def get_old(self):
if hasattr(self, 'old_crit'): return self.old_crit
elif hasattr(self, 'old_red'):
setattr(self.crit, 'reduction', self.old_red)
 return self.crit
def mixup(learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True)-> Learner:
"Add mixup https://arxiv.org/abs/1710.09412 to `learn`."
learn.callback_fns.append(partial(MixUpCallback, alpha=alpha, stack_x=stack_x, stack_y=stack_y))
return learn
Learner.mixup = mixup
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id', col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
lag_dates = [1,2,3,6,12]
matrix = lag_feature(matrix, lag_dates, "item_cnt_month")
 matrix.fillna(0, inplace=True)
 feature = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': 'mean'})
 feature.index = feature.index+1
feature.columns = ["item_cnt_month_mean_lag_1"]
matrix = pd.merge(matrix, feature, on=["date_block_num"], how="left")
 matrix = pd.merge(matrix, items.drop(["item_name"], axis=1), how="left", on="item_id")
 feature = matrix.groupby(['date_block_num', "item_id"] ).agg({'item_cnt_month': 'mean'} ).reset_index()
feature
feature.rename({"item_cnt_month": "means_cnt_item_id_lag_1"}, axis=1, inplace=True)
 feature["date_block_num"] += 1
feature
matrix = pd.merge(matrix, feature, on=["date_block_num", "item_id"], how="left")
 feature = matrix.groupby(['date_block_num', "shop_id"] ).agg({'item_cnt_month': 'mean'} ).reset_index()
feature
feature.rename({"item_cnt_month": "means_cnt_shop_id_lag_1"}, axis=1, inplace=True)
 feature["date_block_num"] += 1
feature
matrix = pd.merge(matrix, feature, on=["date_block_num", "shop_id"], how="left")
 piv = sales_train.pivot_table(values="item_price", index=["shop_id", "item_id"] ,columns="date_block_num", aggfunc="mean")
 piv = piv.fillna(method="ffill", axis=1 ).fillna(method="bfill", axis=1)
 piv_diff = piv.diff(axis=1)
 piv_diff.columns += 1
piv_div = piv_diff.stack()
piv_div.name = "price_diff_lag_1"
matrix = pd.merge(matrix, piv_div, on=["shop_id", "item_id", "date_block_num"], how="left")
 matrix.fillna(0, inplace=True )<categorify>
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
class Classifier(nn.Module):
def __init__(self, num_classes=1000):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = self.avgpool(x)
 x = self.fc(x)
 return x
shops['city'] = shops['shop_name'].apply(lambda x: x.split() [0].lower())
 shops.loc[shops.city == '!якутск', 'city'] = 'якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 coords = dict()
coords['якутск'] =(62.028098, 129.732555, 4)
 coords['адыгея'] =(44.609764, 40.100516, 3)
 coords['балашиха'] =(55.8094500, 37.9580600, 1)
 coords['волжский'] =(53.4305800, 50.1190000, 3)
 coords['вологда'] =(59.2239000, 39.8839800, 2)
 coords['воронеж'] =(51.6720400, 39.1843000, 3)
 coords['выездная'] =(0, 0, 0)
 coords['жуковский'] =(55.5952800, 38.1202800, 1)
 coords['интернет-магазин'] =(0, 0, 0)
 coords['казань'] =(55.7887400, 49.1221400, 4)
 coords['калуга'] =(54.5293000, 36.2754200, 4)
 coords['коломна'] =(55.0794400, 38.7783300, 4)
 coords['красноярск'] =(56.0183900, 92.8671700, 4)
 coords['курск'] =(51.7373300, 36.1873500, 3)
 coords['москва'] =(55.7522200, 37.6155600, 1)
 coords['мытищи'] =(55.9116300, 37.7307600, 1)
 coords['н.новгород'] =(56.3286700, 44.0020500, 4)
 coords['новосибирск'] =(55.0415000, 82.9346000, 4)
 coords['омск'] =(54.9924400, 73.3685900, 4)
 coords['ростовнадону'] =(47.2313500, 39.7232800, 3)
 coords['спб'] =(59.9386300, 30.3141300, 2)
 coords['самара'] =(53.2000700, 50.1500000, 4)
 coords['сергиев'] =(56.3000000, 38.1333300, 4)
 coords['сургут'] =(61.2500000, 73.4166700, 4)
 coords['томск'] =(56.4977100, 84.9743700, 4)
 coords['тюмень'] =(57.1522200, 65.5272200, 4)
 coords['уфа'] =(54.7430600, 55.9677900, 4)
 coords['химки'] =(55.8970400, 37.4296900, 1)
 coords['цифровой'] =(0, 0, 0)
 coords['чехов'] =(55.1477000, 37.4772800, 4)
 coords['ярославль'] =(57.6298700, 39.8736800, 2)
 shops['city_coord_1'] = shops['city'].apply(lambda x: coords[x][0])
 shops['city_coord_2'] = shops['city'].apply(lambda x: coords[x][1])
 shops['country_part'] = shops['city'].apply(lambda x: coords[x][2])
 shops = shops[['shop_id', 'city_code', 'city_coord_1', 'city_coord_2', 'country_part']]
matrix = pd.merge(matrix, shops, on=["shop_id"], how="left" )<merge>
def borrowed_model(pretrained=False, **kwargs):
return Classifier(**kwargs)
 f_score = partial(fbeta, thresh=0.1)
 learn = cnn_learner(data, borrowed_model, pretrained=False, metrics=[lwlrap] ).mixup(stack_y=False)
 learn.unfreeze()
map_dict = {
'Чистые носители(штучные)': 'Чистые носители',
'Чистые носители(шпиль)' : 'Чистые носители',
'PC ': 'Аксессуары',
'Служебные': 'Служебные '
}
items = pd.merge(items, item_categories, on='item_category_id')
 items['item_category'] = items['item_category_name'].apply(lambda x: x.split('-')[0])
 items['item_category'] = items['item_category'].apply(lambda x: map_dict[x] if x in map_dict.keys() else x)
 items['item_category_common'] = LabelEncoder().fit_transform(items['item_category'])
 items['item_category_code'] = LabelEncoder().fit_transform(items['item_category_name'])
 items = items[['item_id', 'item_category_common', 'item_category_code']]
matrix = pd.merge(matrix, items, on=['item_id'], how='left' )<merge>
learn.fit_one_cycle(255, 1e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )
first_item_block = matrix.groupby(['item_id'])['date_block_num'].min().reset_index()
first_item_block['item_first_interaction'] = 1
first_shop_item_buy_block = matrix[matrix['date_block_num'] > 0].groupby(['shop_id', 'item_id'])['date_block_num'].min().reset_index()
first_shop_item_buy_block['first_date_block_num'] = first_shop_item_buy_block['date_block_num']
matrix = pd.merge(matrix, first_item_block[['item_id', 'date_block_num', 'item_first_interaction']], on=['item_id', 'date_block_num'], how='left')
 matrix = pd.merge(matrix, first_shop_item_buy_block[['item_id', 'shop_id', 'first_date_block_num']], on=['item_id', 'shop_id'], how='left')
 matrix['first_date_block_num'].fillna(100, inplace=True)
 matrix['shop_item_sold_before'] =(matrix['first_date_block_num'] < matrix['date_block_num'] ).astype('int8')
 matrix.drop(['first_date_block_num'], axis=1, inplace=True)
 matrix['item_first_interaction'].fillna(0, inplace=True)
 matrix['shop_item_sold_before'].fillna(0, inplace=True)
 matrix['item_first_interaction'] = matrix['item_first_interaction'].astype('int8')
 matrix['shop_item_sold_before'] = matrix['shop_item_sold_before'].astype('int8' )<prepare_x_and_y>
learn.lr_find()
learn.fit_one_cycle(50, 1e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )
matrix = matrix[matrix["date_block_num"] >= 12]
X_train = matrix[matrix["date_block_num"] < 33].drop(["item_cnt_month"], axis=1)
 X_val = matrix[matrix["date_block_num"] == 33].drop(["item_cnt_month"], axis=1)
 X_test = matrix[matrix["date_block_num"] == 34].drop(["item_cnt_month"], axis=1)
 y_train = matrix[matrix["date_block_num"] < 33]["item_cnt_month"]
y_val = matrix[matrix["date_block_num"] == 33]["item_cnt_month"]<train_model>
del X_train
X_test = pickle.load(open(MELS_TEST, 'rb'))
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test )
<train_model><EOS> 
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()