Kaggle competition: Freesound Audio Tagging 2019
<SOS> metric: weightedlabelrankingaverageprecision Kaggle data source: freesound-audio-tagging-2019<compute_test_metric>
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau, MultiStepLR
from fastprogress import master_bar, progress_bar
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms
import torchvision.models as models
train_pred2,val_pred2,test_pred2 = post_processing(bst,'xgboost',xgb.DMatrix(train_x.values),xgb.DMatrix(val_x.values),xgb.DMatrix(test.values),train_y,val_y,test )<set_options>
torch.cuda.is_available()
%matplotlib inline<load_from_csv>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 1314
seed_everything(SEED )
train = pd.read_csv(".. /input/sf-crime/train.csv.zip", parse_dates=['Dates'])
 test = pd.read_csv(".. /input/sf-crime/test.csv.zip", parse_dates=['Dates'], index_col='Id')
 sampleSubmission = pd.read_csv(".. /input/sf-crime/sampleSubmission.csv.zip" )<concatenate>
N_JOBS = cpu_count()
used = int(4)
 print(N_JOBS, used)
 os.environ['MKL_NUM_THREADS'] = str(used)
 os.environ['OMP_NUM_THREADS'] = str(used)
 DataLoader = partial(DataLoader, num_workers=used )
fe_name = list(test)
 df_train = train[fe_name]
df = pd.concat(( df_train,test))<prepare_x_and_y>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
target = train['Category']<count_values>
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 print(os.listdir(dataset_dir))
preprocessed_dir = Path('.. /input/fat2019_prep_mels1')
 print(os.listdir(preprocessed_dir))
target.value_counts()<categorify>
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': dataset_dir / 'train_noisy.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_train_noisy.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}
LB = LabelEncoder()
target = LB.fit_transform(target)
 print(LB.classes_ )<feature_engineering>
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 train_df = pd.concat([train_curated], sort=True, ignore_index=True)
 train_df.head()
date = pd.to_datetime(df['Dates'])
 df['Date'] = date.dt.date
df['Year'] = date.dt.year
df['Month'] = date.dt.month
df['Day'] = date.dt.day
df['Hour'] = date.dt.hour<drop_column>
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()
df.drop("Dates", axis = 1, inplace = True )<groupby>
labels = test_df.columns[1:].tolist()
num_classes = len(labels)
 num_classes
date = df.groupby('Date' ).count().iloc[:, 0]<categorify>
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape
lb = LabelEncoder()
df['PdDistrict'] = lb.fit_transform(df["PdDistrict"] )<count_values>
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 print(len(x_train))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )
df["PdDistrict"].value_counts()<categorify>
class FATTrainDataset(Dataset):
def __init__(self, mels, labels, transforms):
super().__init__()
self.mels = mels
self.labels = labels
self.transforms = transforms
def __len__(self):
return len(self.mels)
 def __getitem__(self, idx):
image = Image.fromarray(self.mels[idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - 1*base_dim)
 image = image.crop([crop, 0, crop + 1*base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 label = self.labels[idx]
label = torch.from_numpy(label ).float()
return image, label
lb = LabelEncoder()
df['PdDistrict'] = lb.fit_transform(df["PdDistrict"] )<feature_engineering>
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - 1*base_dim)
 image = image.crop([crop, 0, crop + 1*base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname
df['DayOfWeek'] = lb.fit_transform(df["DayOfWeek"] )<count_values>
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters() :
param.requires_grad = False
df["Address"].value_counts().head(20 )<feature_engineering>
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=False):
model_ft = None
input_size = 0
if model_name == "resnet18":

 model_ft = models.resnet18(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
 input_size = 224
elif model_name == "resnet34":

 model_ft = models.resnet34(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
 input_size = 224
elif model_name == "alexnet":

 model_ft = models.alexnet(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
 input_size = 224
elif model_name == "vgg":

 model_ft = models.vgg11_bn(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
 input_size = 224
elif model_name == "squeezenet":

 model_ft = models.squeezenet1_0(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))
model_ft.num_classes = num_classes
input_size = 224
elif model_name == "inception":

 model_ft = models.inception_v3(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.AuxLogits.fc.in_features
model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs,num_classes)
 input_size = 299
else:
print("Invalid model name, exiting...")
 exit()
return model_ft, input_size
df['block'] = df['Address'].str.contains('block', case=False)
 df['ST'] = df['Address'].str.contains('ST', case=False )<categorify>
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
df['block'] = lb.fit_transform(df["block"])
 df['ST'] = lb.fit_transform(df["ST"] )<drop_column>
class Classifier(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.LeakyReLU(0.2),
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = torch.mean(x, dim=3)
 x, _ = torch.max(x, dim=2)
 x = self.fc(x)
 return x
df.drop("Address", axis = 1, inplace = True )<filter>
def mixup_data(x, y, alpha=1.0, use_cuda=True):

 if alpha > 0.:
lam = np.random.beta(alpha, alpha)
 else:
lam = 1.
 batch_size = x.size() [0]
if use_cuda:
index = torch.randperm(batch_size ).cuda()
else:
index = torch.randperm(batch_size)
 mixed_x = lam * x +(1 - lam)* x[index,:]
mixed_y = lam * y +(1 - lam)* y[index]
return mixed_x, mixed_y
def mixup_criterion(y_a, y_b, lam):
return lambda criterion, pred: lam * criterion(pred, y_a)+(1 - lam)* criterion(pred, y_b )
print(len(df.loc[df["X"] >= -120.5, "X"]))
print(len(df.loc[df["Y"] >= 90, "Y"]))<feature_engineering>
def weighted_BCEWithLogits(pos_weight):
def weighted_cross_entropy_with_logits(logits, targets):
return(targets * -logits.sigmoid().log() * pos_weight +(1 - targets)* -(1 - logits.sigmoid() ).log() ).mean()
return weighted_cross_entropy_with_logits
X_median = df[df["X"] < -120.5]["X"].median()
Y_median = df[df["Y"] < 90]["Y"].median()
df.loc[df["X"] >= -120.5, "X"] = X_median
df.loc[df["Y"] >= 90, "Y"] = Y_median<feature_engineering>
def train_model(x_train, y_train, train_transforms, model_path, num_epochs = 5, batch_size = 32, test_batch_size = 64):
lr = 1e-3
eta_min = 1e-7
t_max = 10
num_classes = y_train.shape[1]
trn_loss = []
val_loss = []
val_lwlrap = []
lr_log = []
x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)
 train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)
 valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)
 model = model_ft.cuda()
if model_path and False:
model.load_state_dict(torch.load(model_path))
criterion = nn.BCEWithLogitsLoss().cuda()
optimizer = Adam(params=model.parameters() , lr=lr, amsgrad=False)
 scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)
 best_epoch = -1
best_lwlrap = 0.
 mb = master_bar(range(num_epochs))
for epoch in mb:
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in progress_bar(train_loader, parent=mb):
if mixup_on:
x_batch, y_batch = mixup_data(x_batch, y_batch, alpha=1, use_cuda=True)
 if model_type != 'inception':
preds = model(x_batch.cuda())
 loss = criterion(preds, y_batch.cuda())
 else:
outputs, aux_outputs = model(x_batch.cuda())
 loss1 = criterion(outputs, y_batch.cuda())
 loss2 = criterion(aux_outputs, y_batch.cuda())
 loss = loss1 + 0.4*loss2
optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( len(x_val), num_classes))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
preds = model(x_batch.cuda() ).detach()
loss = criterion(preds, y_batch.cuda())
 preds = torch.sigmoid(preds)
 valid_preds[i * test_batch_size:(i+1)* test_batch_size] = preds.cpu().numpy()
avg_val_loss += loss.item() / len(valid_loader)
 score, weight = calculate_per_class_lwlrap(y_val, valid_preds)
 lwlrap =(score * weight ).sum()
scheduler.step()
for param_group in optimizer.param_groups:
lr_temp = param_group['lr']
lr_log.append(lr_temp)
 if(epoch + 1)% 1 == 0:
elapsed = time.time() - start_time
mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')
 trn_loss.append(avg_loss)
 val_loss.append(avg_val_loss)
 val_lwlrap.append(lwlrap)
 if lwlrap > best_lwlrap:
best_epoch = epoch + 1
best_lwlrap = lwlrap
best_train_loss = avg_loss
best_val_loss = avg_val_loss
torch.save(model.state_dict() , model_path)
 return {
'best_epoch': best_epoch,
'best_lwlrap': best_lwlrap,
'best_train_loss':best_train_loss,
'best_val_loss':best_val_loss,
'trn_loss':trn_loss,
'val_loss':val_loss,
'val_lwlrap':val_lwlrap,
'lr_log':lr_log
}
df["X+Y"] = df["X"] + df["Y"]
df["X-Y"] = df["X"] - df["Y"]<drop_column>
num_epochs = 400
batch_size = 64
test_batch_size = 64
lr_method = 'cosine'
aug_flip = 'flip'
mixup_on = True
model_type = 'inception'
model_ft, input_size = initialize_model(model_type, num_classes, False, use_pretrained=False)
 if aug_flip == 'flip':
transforms_dict = {
'train': transforms.Compose([
transforms.RandomResizedCrop(input_size),
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomResizedCrop(input_size),
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}
elif aug_flip == 'noflip':
transforms_dict = {
'train': transforms.Compose([
transforms.RandomResizedCrop(input_size),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomResizedCrop(input_size),
transforms.ToTensor() ,
]),
}
df.drop("Date", axis = 1, inplace = True )<split>
model_path = '.. /working/weight_best_2s_%(m)s_%(lr)s_%(aug)s.pt'% \
{'m':model_type, 'lr':lr_method, 'aug':aug_flip}
output_path = '.. /working/submission_2s_%(m)s_%(lr)s_%(aug)s.csv'%\
{'m':model_type, 'lr':lr_method, 'aug':aug_flip}
print(model_path )
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]<train_model>
train_data = lgb.Dataset(new_train, label=target, categorical_feature=["PdDistrict", "DayOfWeek"])
 params = {'boosting':'gbdt',
'objective':'multiclass',
'num_class':39,
'max_delta_step':0.9,
'min_data_in_leaf': 21,
'learning_rate': 0.4,
'max_bin': 465,
'num_leaves': 41,
'verbose' : 1}
bst = lgb.train(params, train_data, 120 )<predict_on_test>
model_path = '.. /input/outputmodels2/weight_best_2s_inception_cosine_flip.pt'
predictions = bst.predict(new_test )<create_dataframe>
start_time = time.time()
test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes )
submission = pd.DataFrame(predictions,columns=LB.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,index=new_test.index)
<prepare_x_and_y>
test_df[labels] = test_preds.values
test_df.to_csv(output_path, index=False)
 test_df.to_csv('submission.csv', index=False)
 elapsed = time.time() - start_time
print(f'Prediction time: {elapsed:.0f}s' )