Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<predict_on_test>
warnings.filterwarnings('ignore' )
test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes )<save_to_csv>
TRAINING = False
test_df[labels] = test_preds.values
test_df.to_csv('submission.csv', index=False)
 test_df.head()<set_options>
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )<categorify>
SEED_VALUE = 42
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=SEED_VALUE )
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
MDL_NAME = 'trans_learn_drop_zero_target'
MDL_DIR = '.. /input/pytorch-transfer-learn-dropping-zero-targets'
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED = Path('.. /input/fat2019_prep_mels1')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_TRN_NOISY_BEST50S = PREPROCESSED/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
MELS_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'
MELS_TRN_NOISY = PREPROCESSED/'mels_train_noisy.pkl'
MELS_TRN_NOISY_BEST50S = PREPROCESSED/'mels_trn_noisy_best50s.pkl'
MELS_TEST = PREPROCESSED/'mels_test.pkl'
trn_curated_df = pd.read_csv(CSV_TRN_CURATED)
 trn_noisy_df = pd.read_csv(CSV_TRN_NOISY)
 trn_noisy50s_df = pd.read_csv(CSV_TRN_NOISY_BEST50S)
 test_df = pd.read_csv(CSV_SUBMISSION)
 df = pd.concat([trn_curated_df, trn_noisy50s_df], ignore_index=True, sort=True)
 test_df = pd.read_csv(CSV_SUBMISSION)
 X_train = pickle.load(open(MELS_TRN_CURATED, 'rb')) + pickle.load(open(MELS_TRN_NOISY_BEST50S, 'rb'))<define_variables>
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
data.show_batch(3 )<compute_test_metric>
tmp = train_targets_nonscored.iloc[:, 1:].sum()
drop_cols = list(tmp[tmp==0].index)
 train_targets_nonscored.drop(drop_cols, axis=1, inplace=True)
 print(f'{len(drop_cols)} targets are dropped' )
def lwlrap(y_pred,y_true):
score, weight = calculate_per_class_lwlrap(y_true.cpu().numpy() , y_pred.cpu().numpy())
 lwlrap =(score * weight ).sum()
return torch.from_numpy(np.array(lwlrap))<train_model>
train_genes = train_features[GENES].copy()
train_cells = train_features[CELLS].copy()
test_genes = test_features[GENES].copy()
test_cells = test_features[CELLS].copy()
class MixUpCallback(LearnerCallback):
"Callback that creates the mixed-up input and target."
def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):
super().__init__(learn)
 self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y
def on_train_begin(self, **kwargs):
if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)
 def on_batch_begin(self, last_input, last_target, train, **kwargs):
"Applies mixup to `last_input` and `last_target` if `train`."
if not train: return
lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))
lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1 ).max(1)
 lambd = last_input.new(lambd)
 shuffle = torch.randperm(last_target.size(0)).to(last_input.device)
 x1, y1 = last_input[shuffle], last_target[shuffle]
if self.stack_x:
new_input = [last_input, last_input[shuffle], lambd]
else:
new_input =(last_input * lambd.view(lambd.size(0),1,1,1)+ x1 *(1-lambd ).view(lambd.size(0),1,1,1))
if self.stack_y:
new_target = torch.cat([last_target[:,None].float() , y1[:,None].float() , lambd[:,None].float() ], 1)
 else:
if len(last_target.shape)== 2:
lambd = lambd.unsqueeze(1 ).float()
new_target = last_target.float() * lambd + y1.float() *(1-lambd)
 return {'last_input': new_input, 'last_target': new_target}
def on_train_end(self, **kwargs):
if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()
class MixUpLoss(nn.Module):
"Adapt the loss function `crit` to go with mixup."
def __init__(self, crit, reduction='mean'):
super().__init__()
if hasattr(crit, 'reduction'):
self.crit = crit
self.old_red = crit.reduction
setattr(self.crit, 'reduction', 'none')
 else:
self.crit = partial(crit, reduction='none')
 self.old_crit = crit
self.reduction = reduction
def forward(self, output, target):
if len(target.size())== 2:
loss1, loss2 = self.crit(output,target[:,0].long()), self.crit(output,target[:,1].long())
 d =(loss1 * target[:,2] + loss2 *(1-target[:,2])).mean()
else: d = self.crit(output, target)
 if self.reduction == 'mean': return d.mean()
elif self.reduction == 'sum': return d.sum()
return d
def get_old(self):
if hasattr(self, 'old_crit'): return self.old_crit
elif hasattr(self, 'old_red'):
setattr(self.crit, 'reduction', self.old_red)
 return self.crit
def mixup(learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True)-> Learner:
"Add mixup https://arxiv.org/abs/1710.09412 to `learn`."
learn.callback_fns.append(partial(MixUpCallback, alpha=alpha, stack_x=stack_x, stack_y=stack_y))
return learn
Learner.mixup = mixup 
<define_search_model>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
class Classifier(nn.Module):
def __init__(self, num_classes=1000):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = self.avgpool(x)
 x = self.fc(x)
 return x<choose_model_class>
n_comp = 157
pca = PCA(n_components=n_comp, random_state=SEED_VALUE)
 train2 = pca.fit_transform(train_features[GENES])
 test2 = pca.transform(test_features[GENES])
 train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))
def borrowed_model(pretrained=False, **kwargs):
return Classifier(**kwargs)
 f_score = partial(fbeta, thresh=0.1)
 learn = cnn_learner(data, borrowed_model, pretrained=False, metrics=[lwlrap] ).mixup(stack_y=False)
 learn.unfreeze() 
<train_model>
n_comp = 7
pca = PCA(n_components=n_comp, random_state=SEED_VALUE)
 train2 = pca.fit_transform(train_features[CELLS])
 test2 = pca.transform(test_features[CELLS])
 train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))
learn.fit_one_cycle(255, 1e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )<train_model>
def fe_stats(train, test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
for df in train, test:
df['g_mean'] = df[GENES].mean(axis = 1)
 df['g_std'] = df[GENES].std(axis = 1)
 df['g_kurt'] = df[GENES].kurtosis(axis = 1)
 df['g_skew'] = df[GENES].skew(axis = 1)
 df['c_mean'] = df[CELLS].mean(axis = 1)
 df['c_std'] = df[CELLS].std(axis = 1)
 df['c_kurt'] = df[CELLS].kurtosis(axis = 1)
 df['c_skew'] = df[CELLS].skew(axis = 1)
 return train, test
train_features, test_features = fe_stats(train_features, test_features )
learn.lr_find()
learn.fit_one_cycle(50, 1e-2,callbacks=[SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best')] )<load_from_csv>
train = train_features.merge(train_targets_scored, on='sig_id', how='left')
 train = train.merge(train_targets_nonscored, on='sig_id', how='left')
 train = train.merge(train_drug, on='sig_id', how='left')
 train_genes = train_genes[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_cells = train_cells[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_genes = test_genes[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_cells = test_cells[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
del X_train
X_test = pickle.load(open(MELS_TEST, 'rb'))
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test )<save_to_csv>
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
class MoADataset:
def __init__(self, features, genes, cells, targets):
self.features = features
self.genes = genes
self.cells = cells
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x1' : torch.tensor(self.features[idx, :], dtype=torch.float),
'x2' : torch.tensor(self.genes[idx, :], dtype=torch.float),
'x3' : torch.tensor(self.cells[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features, genes, cells):
self.features = features
self.genes = genes
self.cells = cells
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x1' : torch.tensor(self.features[idx, :], dtype=torch.float),
'x2' : torch.tensor(self.genes[idx, :], dtype=torch.float),
'x3' : torch.tensor(self.cells[idx, :], dtype=torch.float)
 }
return dct
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms<set_options>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
x1, x2, x3, targets = data['x1'].to(device), data['x2'].to(device), \
data['x3'].to(device), data['y'].to(device)
 outputs = model(x1, x2, x3)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
x1, x2, x3, targets = data['x1'].to(device), data['x2'].to(device), \
data['x3'].to(device), data['y'].to(device)
 outputs = model(x1, x2, x3)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
x1, x2, x3 = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device)
 with torch.no_grad() :
outputs = model(x1, x2, x3)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
torch.cuda.is_available()<set_options>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 520
seed_everything(SEED )<feature_engineering>
class Model(nn.Module):
def __init__(self, num_features, num_genes, num_cells, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.layer_norm1_2 = nn.LayerNorm(num_genes)
 self.dense1_2 = nn.Linear(num_genes, self.hidden_size[0])
 self.layer_norm1_3 = nn.LayerNorm(num_cells)
 self.dense1_3 = nn.Linear(num_cells, self.hidden_size[0]//2)
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0]*2+self.hidden_size[0]//2)
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0]*2+self.hidden_size[0]//2, self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x1, x2, x3):
x1 = self.batch_norm1(x1)
 x1 = F.leaky_relu(self.dense1(x1))
x2 = self.layer_norm1_2(x2)
 x2 = F.leaky_relu(self.dense1_2(x2))
x3 = self.layer_norm1_3(x3)
 x3 = F.leaky_relu(self.dense1_3(x3))
x = torch.cat(( x1, x2, x3), 1)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
N_JOBS = cpu_count()
os.environ['MKL_NUM_THREADS'] = str(N_JOBS)
 os.environ['OMP_NUM_THREADS'] = str(N_JOBS)
 DataLoader = partial(DataLoader, num_workers=N_JOBS )<define_variables>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, len(GENES), len(CELLS), num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )<define_variables>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}<load_from_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()<load_from_csv>
model = Model(num_features, len(GENES), len(CELLS), num_all_targets)
 model
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()<data_type_conversions>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
labels = test_df.columns[1:].tolist()
labels<define_variables>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 train_genes_df = train_genes[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_genes_df = train_genes[train_[kfold_col] == fold_id].reset_index(drop=True)
 train_cells_df = train_cells[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_cells_df = train_cells[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols], train_df[target_cols_now]
x_valid, y_valid = valid_df[feature_cols], valid_df[target_cols_now]
train_dataset = MoADataset(x_train.values, train_genes_df.values, train_cells_df.values, y_train.values)
 valid_dataset = MoADataset(x_valid.values, valid_genes_df.values, valid_cells_df.values, y_valid.values)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 if TRAINING:
optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 best_loss = np.inf
best_epoch = None
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
best_epoch = epoch
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"./{MDL_NAME}_{tag_name}_SEED{seed_id}_FOLD{fold_id}_.pth")
 print(f'
else:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
print(f'
return oof
if TRAINING:
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, len(GENES), len(CELLS), num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, len(GENES), len(CELLS), num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"./{MDL_NAME}_ALL_TARGETS_SEED{seed_id}_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, len(GENES), len(CELLS), num_targets)
 model.load_state_dict(torch.load(f"./{MDL_NAME}_SCORED_ONLY_SEED{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 else:
model = Model(num_features, len(GENES), len(CELLS), num_targets)
 model.load_state_dict(torch.load(f"{MDL_DIR}/{MDL_NAME}_SCORED_ONLY_SEED{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 oof = train_model(model, 'SCORED_ONLY', target_cols)
 x_test = test_[feature_cols]
testdataset = TestDataset(x_test.values, test_genes.values, test_cells.values)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
num_classes = len(labels)
 num_classes<prepare_x_and_y>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
score = 0
for i in range(len(target_cols)) :
_score = log_loss(train.loc[:,target_cols[i]].values, oof[:,i])
 score += _score / len(target_cols)
 print(f"
return oof, predictions, score
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape<load_pretrained>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 scores = []
time_begin = time()
for seed_id in SEED:
oof_, predictions_, score_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 scores.append(score_)
 print('-'*80)
 for i in range(len(scores)) :
print(f'
score = 0
for i in range(len(target_cols)) :
_score = log_loss(train.loc[:,target_cols[i]].values, oof[:,i])
 score += _score / len(target_cols)
 print(f"
time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )<categorify>
str(timedelta(seconds=time_diff))
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - base_dim)
 image = image.crop([crop, 0, crop + base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname<categorify>
train[['sig_id']+target_cols].to_csv(f'oof_{MDL_NAME}.csv', index=False )
transforms_dict = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}<define_search_model>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels, downsample = None):
super().__init__()
self.downsample = downsample
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.conv2(out)
 out = F.avg_pool2d(out, 2)
 return out<choose_model_class>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv(f'submission.csv', index=False )