Kaggle competition: Freesound Audio Tagging 2019
<SOS> metric: weightedlabelrankingaverageprecision Kaggle data source: freesound-audio-tagging-2019<save_to_csv>
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms
start_time = time.time()
Y_test = rf.predict(X_test ).clip(0, 20)
 print(f"Predicting test set took {time.time() - start_time} s")
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('rf_y_test.csv', index=False)
 train_preds = pd.DataFrame({
"ID": X_train.index,
"item_cnt_month": Y_train_pred
})
 train_preds.to_csv('rf_y_train.csv', index=False)
 valid_preds = pd.DataFrame({
"ID": X_valid.index,
"item_cnt_month": Y_valid_pred
})
 valid_preds.to_csv('rf_y_valid.csv', index=False)
<set_options>
torch.cuda.is_available()
pd.options.display.max_rows = 50
plt.style.use("seaborn" )<data_type_conversions>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 520
seed_everything(SEED )
def compact_types(df):
for col in df.columns:
if df[col].dtype == np.float64:
df[col] = df[col].astype(np.float32)
 elif df[col].dtype == np.int64:
df[col] = df[col].astype(np.int32 )<load_from_csv>
N_JOBS = cpu_count()
os.environ['MKL_NUM_THREADS'] = str(N_JOBS)
 os.environ['OMP_NUM_THREADS'] = str(N_JOBS)
 DataLoader = partial(DataLoader, num_workers=N_JOBS )
shops = pd.read_csv(in_path + "shops.csv", header=0, names=["name", "shop"], index_col="shop")
 cats = pd.read_csv(in_path + "item_categories.csv", header=0, names=["name", "cat"], index_col="cat")
 items = pd.read_csv(in_path + "items.csv", header=0, names=["name", "item", "cat"], index_col="item")
 train = pd.read_csv(in_path + "sales_train.csv", header=0,
names=["date", "block", "shop", "item", "price", "count"],
usecols=[1, 2, 3, 4, 5] 
)
 test = pd.read_csv(in_path + "test.csv", header=0, names=["ID", "shop", "item"] )<categorify>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
shops_to_replace = { 0: 57, 1: 58, 11: 10, 23: 24, 40: 39 }
train["shop"].replace(shops_to_replace, inplace=True)
 test ["shop"].replace(shops_to_replace, inplace=True)
 shops.drop(index=shops_to_replace.keys() , inplace=True )<drop_column>
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )
print("Deleting shops:", shops_to_delete)
 train = train[ ~train["shop"].isin(shops_to_delete)]
shops.drop(index=shops_to_delete, inplace=True )<create_dataframe>
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}
cities = shops["name"].str.extract("^([^ ]+)")[0]
shops["city"], cities = cities.factorize()
cities = cities.to_frame(index=False, name="name")
 cities.index.name = "city"
print("Extracted", len(cities), "cities" )<feature_engineering>
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()
cats["is_digital"] = cats["name"].str.contains("Цифра")
 names = cats["name"].str.replace("(Цифра)", "", regex=False)
 subcats = names.str.extract("^([^-]+)-? (.*)$")
 majors = subcats[0].str.strip()
minors = subcats[1].str.strip()
del subcats
minors[ minors == "" ] = majors[ minors == "" ]
cats["major"], majors = majors.factorize()
cats["minor"], minors = minors.factorize()
majors = majors.to_frame(index=False, name="name")
 majors.index.name = "major"
minors = minors.to_frame(index=False, name="name")
 minors.index.name = "minor"
print("Extracted", len(majors), "major and", len(minors), "minor categories" )<filter>
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()
outlier_index =(train["price"] <= 0)|(train["price"] > 49000)
 train[outlier_index]<merge>
labels = test_df.columns[1:].tolist()
labels
index_names = ["block", "shop", "item"]
train.set_index(index_names, inplace=True)
 train = train.join(shops[["city"]])
 train = train.join(items[["cat" ]])
 train = train.join(cats [["is_digital", "major", "minor"]], on="cat")
 compact_types(train)
 train.sort_index(inplace=True)
 train<merge>
num_classes = len(labels)
 num_classes
def get_sales(col):
sales = train.groupby(col ).agg(
 count=("count", "sum"),
count_mean=("count", "mean"),
price=("price", "mean"), 
)
 sales = sales.add_prefix(col + "_")
 compact_types(sales)
 return sales
blocks = get_sales("block")
 shops = shops.join(get_sales("shop"))
cities = cities.join(get_sales("city"))
items = items.join(get_sales("item"))
cats = cats.join(get_sales("cat"))
compact_types(train )<feature_engineering>
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape
index_names = ["block", "shop", "item"]
sales_block = train.groupby(index_names ).agg(
 block_shop_item_count=("count", "sum"),
block_shop_item_count_mean=("count", "mean"),
block_shop_item_price=("price", "mean"), 
)
 sales_block["block_shop_item_count"] = sales_block["block_shop_item_count"].clip(0, 20 )<merge>
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )
sales_block =(sales_block.join(blocks)
.join(shops.drop(columns="name")) 
.join(cities.drop(columns="name"), on="city")
.join(items.drop(columns="name")) 
.join(cats.drop(columns="name"), on="cat")
)
 del blocks, shops, cities, items, cats, majors, minors<feature_engineering>
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - base_dim)
 image = image.crop([crop, 0, crop + base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname
feat_names = ["shop", "item", "city", "cat"]
for feat in tqdm(feat_names):
sales_feat = train.groupby(["block", feat] ).agg(
 count=("count", "sum"),
count_mean=("count", "mean"),
price=("price", "mean"), 
)
 sales_feat = sales_feat.add_prefix("block_" + feat + "_")
 sales_block = sales_block.join(sales_feat, on=["block", feat])
 metric = feat + "_price"
block_price = sales_block["block_" + metric]
price = sales_block[metric]
sales_block["block_" + metric + "_change"] =(block_price - price)/ price
del sales_feat, block_price, price
del train
compact_types(sales_block)
 sales_block.info()<drop_column>
transforms_dict = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}
sales_block.drop(columns=[
name for name in sales_block.columns
if "block_" in name and "_lag" not in name and name != "block_shop_item_count"
], inplace=True)
 sales_block.reset_index(inplace=True )<feature_engineering>
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
sales_block["year"] =(sales_block["block"] // 12 + 2013)
 sales_block["month"] =(sales_block["block"] % 12 + 1)
 dates = sales_block[["year", "month"]].copy()
dates["day"] = 1
sales_block["days"] = pd.to_datetime(dates ).dt.daysinmonth
compact_types(sales_block )<data_type_conversions>
class Classifier(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = torch.mean(x, dim=3)
 x, _ = torch.max(x, dim=2)
 x = self.fc(x)
 return x
sales_block.fillna(0, inplace=True)
 for name in tqdm(sales_block.columns):
if "count" in name and "mean" not in name and "change" not in name:
if sales_block[name].dtype != np.int32:
sales_block[name] = sales_block[name].astype(np.int32)
 sales_block.to_pickle("sales.pickle")
 sales_block.info()<load_from_csv>
Classifier(num_classes=num_classes )
sales_block = pd.read_pickle("sales.pickle")
 test = pd.read_csv(in_path + "test.csv", header=0, names=["ID", "shop", "item"])
 sales_block = sales_block[[
"block",
"shop",
"item",
"block_shop_item_count",
"city",
"cat",
"is_digital",
"major",
"minor",
"month",
"days",
"block_shop_item_count_lag1",
"block_shop_item_price_lag1",
"block_count_mean_lag1",
"block_shop_count_mean_lag1",
"block_shop_price_change_lag1",
"block_item_count_mean_lag1",
"block_item_price_change_lag1",
"block_city_count_mean_lag1",
"block_cat_count_mean_lag1",
"block_shop_item_count_lag2",
"block_shop_count_mean_lag2",
"block_item_count_mean_lag2",
"block_shop_item_count_lag3",
"block_shop_count_mean_lag3",
"block_item_count_mean_lag3",
"block_shop_item_count_lag6",
"block_shop_count_mean_lag6",
"block_item_count_mean_lag6",
"block_shop_item_count_lag12",
"block_shop_count_mean_lag12",
"block_item_count_mean_lag12",
]]
sales_block.info()<prepare_x_and_y>
def train_model(x_train, y_train, train_transforms):
num_epochs = 118
batch_size = 128
test_batch_size = 256
lr = 1e-3
eta_min = 1e-5
t_max = 5
num_classes = y_train.shape[1]
x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)
 train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)
 valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)
 model = Classifier(num_classes=num_classes ).cuda()
criterion = nn.BCEWithLogitsLoss().cuda()
optimizer = Adam(params=model.parameters() , lr=lr, amsgrad=False)
 scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)
 best_epoch = -1
best_lwlrap = 0.
 mb = master_bar(range(num_epochs))
for epoch in mb:
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in train_loader:
preds = model(x_batch.cuda())
 loss = criterion(preds, y_batch.cuda())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( len(x_val), num_classes))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
preds = model(x_batch.cuda() ).detach()
loss = criterion(preds, y_batch.cuda())
 preds = torch.sigmoid(preds)
 valid_preds[i * test_batch_size:(i+1)* test_batch_size] = preds.cpu().numpy()
avg_val_loss += loss.item() / len(valid_loader)
 score, weight = calculate_per_class_lwlrap(y_val, valid_preds)
 lwlrap =(score * weight ).sum()
scheduler.step()
if(epoch + 1)% 1 == 0:
elapsed = time.time() - start_time
mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')
 if lwlrap > best_lwlrap:
best_epoch = epoch + 1
best_lwlrap = lwlrap
torch.save(model.state_dict() , 'weight_best.pt')
 return {
'best_epoch': best_epoch,
'best_lwlrap': best_lwlrap,
}
test_block = sales_block["block"].max()
valid_block = test_block - 1
X_train = sales_block[ sales_block["block"] < valid_block ].drop(columns="block_shop_item_count")
 y_train = sales_block[ sales_block["block"] < valid_block ]["block_shop_item_count"]
print("X_train:", X_train.shape, "y_train:", y_train.shape)
 X_valid = sales_block[ sales_block["block"] == valid_block ].drop(columns="block_shop_item_count")
 y_valid = sales_block[ sales_block["block"] == valid_block ]["block_shop_item_count"]
print("X_valid:", X_valid.shape, "y_valid:", y_valid.shape)
 X_test = sales_block[ sales_block["block"] == test_block ].drop(columns="block_shop_item_count")
 print("X_test :", X_test.shape)
 del sales_block<compute_test_metric>
result = train_model(x_train, y_train, transforms_dict['train'] )
def print_rmse(which, y_true, y_pred):
print("RMSE", which, np.sqrt(np.square(y_true - y_pred ).mean()))<train_model>
test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes, tta=35 )
model_xgb = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42,
tree_method=tree_method,
n_jobs=-1, 
)
 model_xgb.fit(
 X_train, y_train,
eval_metric="rmse",
eval_set=[(X_valid, y_valid)],
verbose=True,
early_stopping_rounds=10
)<predict_on_test>
test_df[labels] = test_preds.values
test_df.to_csv('submission.csv', index=False)
 test_df.head()