Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<load_from_csv>
KAGGLE_PATH = '.. '
train_data = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv' )<prepare_x_and_y>
DataLoader,
Dataset,
random_split, 
)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 for dirname, _, filenames in os.walk(f'{KAGGLE_PATH}/input/lish-moa'):
for filename in filenames:
print(os.path.join(dirname, filename))
X_train = train_data.drop(labels = ["label"],axis = 1)
 Y_train = train_data["label"]
Y_train = to_categorical(Y_train, num_classes = 10 )<split>
test_features = pd.read_csv(f'{KAGGLE_PATH}/input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv(f'{KAGGLE_PATH}/input/lish-moa/sample_submission.csv')
 train_features = pd.read_csv(f'{KAGGLE_PATH}/input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv(f'{KAGGLE_PATH}/input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(f'{KAGGLE_PATH}/input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv(f'{KAGGLE_PATH}/input/lish-moa/train_drug.csv' )
X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size = 0.2, random_state = 42 )<choose_model_class>
frequent_drugs = train_drug.groupby('drug_id' ).count().sort_values('sig_id', ascending=False ).loc[(train_drug.groupby('drug_id' ).count().sort_values('sig_id', ascending=False)>= 18 ).values].index.to_list()
train_drug['drug_id'] = train_drug['drug_id'].apply(lambda x: x if x in frequent_drugs else 'drug0' )
train_datagen = ImageDataGenerator(rescale = 1./255, shear_range = 0.2, zoom_range = 0.2 )<define_variables>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
test_datagen = ImageDataGenerator(rescale = 1./255 )<train_model>
class Config:
def __init__(self):
self.cat = []
self.ordinal = ['cp_time', 'cp_dose']
self.n_features = 838
self.n_classes = 206
self.n_classes_nonscored = 402
self.batch_size = 1024
self.num_epoch = 120
self.model_path = f'{KAGGLE_PATH}/working/'
self.drop_features = ['sig_id', 'cp_type', 'fold']
self.n_folds = 7
self.weight_decay = 1e-5
self.early_stopping_steps = 20
self.learning_rate = 4e-3
self.learning_rate_nonscored = 4e-3
self.hidden_size = 1024
self.seeds = [i for i in range(5)]
def seed_everything(seed_value=228):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
config = Config()
seed_everything()
train = train_datagen.flow(X_train, Y_train, batch_size = 128 )<compute_test_metric>
class MoaDataset(Dataset):
def __init__(self, features, targets=None, transforms=None):
self.features = torch.tensor(features.drop(columns=config.drop_features, errors='ignore' ).to_numpy().astype(np.float32)).squeeze()
if targets is not None:
self.targets = torch.tensor(targets.drop(columns=config.drop_features, errors='ignore' ).to_numpy().astype(np.float32)).squeeze()
else:
self.targets = None
self.ixs = features.sig_id.unique()
def __len__(self):
return len(self.features)
 def __getitem__(self, index):
ix = self.ixs[index]
if self.targets is None:
return {
'id': ix,
'features': self.features[index],
}
else:
return {
'id': ix,
'features': self.features[index],
'targets': self.targets[index],
}
def random_split(self, val_size):
num_val = int(val_size * len(self))
num_train = len(self)- num_val
return random_split(self, [num_train, num_val])
 def get_targets(self):
return self.targets
def get_features(self):
return self.features
test = test_datagen.flow(X_test, Y_test, batch_size = 128 )<choose_model_class>
class Layer(nn.Module):
def __init__(self, input_size, output_size, dropout, activation_f=F.leaky_relu):
super().__init__()
self.batch_norm = nn.BatchNorm1d(input_size)
 self.dropout = nn.Dropout(dropout)
 self.dense = nn.utils.weight_norm(nn.Linear(input_size, output_size))
self.activation_f = activation_f
def forward(self, x):
x = self.batch_norm(x)
 x = self.dropout(x)
 x = self.activation_f(self.dense(x))
return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=config.hidden_size):
super().__init__()
self.input_layer = Layer(num_features, hidden_size, 0.3)
 self.hidden_layer = Layer(hidden_size, hidden_size, 0.3)
 self.classifier = Layer(hidden_size, num_targets, 0.3, activation_f=torch.sigmoid)
 def forward(self, x):
x = self.input_layer(x)
 x = self.hidden_layer(x)
 x = self.classifier(x)
 return x
callback = EarlyStopping(monitor='loss', patience=8, restore_best_weights=True )<find_best_params>
class basic_encoder:
@staticmethod
def merge_samples(samples):
for i in range(len(samples)) :
samples[i]['sample'] = i
mega_sample = pd.concat(samples)
 return mega_sample
@staticmethod
def split_samples(samples, mega_sample):
for i in range(len(samples)) :
samples[i].drop(columns='sample', inplace=True)
 samples[i] = mega_sample[mega_sample['sample'] == i].drop(columns='sample' ).reset_index(drop=True)
 return samples
class encode_categorical(basic_encoder):
def __init__(self, cat_features=config.cat):
self.cat_features = cat_features
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 mega_sample = pd.get_dummies(mega_sample, columns=self.cat_features)
 return self.split_samples(samples, mega_sample)
 class encode_ordinal(basic_encoder):
def __init__(self, ordinal_features=config.ordinal):
self.ordinal_features = ordinal_features
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 for of in self.ordinal_features:
for i, u in enumerate(mega_sample[of].unique()):
mega_sample.loc[mega_sample[of] == u, of] = i
return self.split_samples(samples, mega_sample)
 class encode_PCA(basic_encoder):
def __init__(self, columns, n_pca, prefix='pca_g-', ignore_cols=False):
self.columns = columns
self.n_pca = n_pca
self.prefix = prefix
self.ignore_cols = ignore_cols
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 if self.ignore_cols:
self.columns = set(self.columns ).intersection(mega_sample.columns)
 pca = PCA(n_components=self.n_pca, random_state=42, svd_solver='full')
 pca_sample = pd.DataFrame(pca.fit_transform(mega_sample[self.columns]))
pca_sample.columns = [self.prefix + str(i)for i in range(1, pca_sample.shape[1] + 1)]
mega_sample = pd.concat([mega_sample.reset_index(drop=True), pca_sample], axis=1)
 return self.split_samples(samples, mega_sample)
 class normalize(basic_encoder):
def __init__(self,):
pass
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 cols = [c for c in mega_sample.columns if c not in config.drop_features + ['sample']]
mega_sample.loc[:, cols] =(mega_sample.loc[:, cols] - mega_sample.loc[:, cols].mean())/ mega_sample.loc[:, cols].std()
return self.split_samples(samples, mega_sample)
 class drop_low_varience(basic_encoder):
def __init__(self, threshold=0.7):
self.threshold=threshold
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 cols_to_drop = mega_sample.var() [mega_sample.var() <= self.threshold].index.to_list()
cols_to_drop.remove('sample')
 mega_sample.drop(columns=cols_to_drop, inplace=True)
 return self.split_samples(samples, mega_sample)
 class rank_gauss(basic_encoder):
def __init__(self, n_quantiles=100):
self.n_quantiles=n_quantiles
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 transformer = QuantileTransformer(n_quantiles=self.n_quantiles, random_state=0, output_distribution="normal")
 cols = [c for c in mega_sample.columns if c not in config.drop_features + ['sample']]
mega_sample.loc[:, cols] = transformer.fit_transform(mega_sample[cols])
 return self.split_samples(samples, mega_sample)
 class rank_gauss2(basic_encoder):
def __init__(self):
pass
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 transformer = GaussRankScaler()
cols = [c for c in mega_sample.columns if c not in config.drop_features + ['sample']]
mega_sample.loc[:, cols] = transformer.fit_transform(mega_sample[cols])
 return self.split_samples(samples, mega_sample)
 class add_statistics(basic_encoder):
def __init__(self, stats, columns, prefix, ignore_cols=False):
self.stats = stats
self.columns = columns
self.prefix = prefix
self.ignore_cols = ignore_cols
def __call__(self, samples):
mega_sample = self.merge_samples(samples)
 if self.ignore_cols:
self.columns = set(self.columns ).intersection(mega_sample.columns)
 for stat in self.stats:
mega_sample[self.prefix + stat] = getattr(mega_sample[self.columns], stat )(axis=1)
 return self.split_samples(samples, mega_sample)
 def preprocess(samples, transforms):
for transform in transforms:
samples = transform(samples)
 return samples
cnn = tf.keras.models.Sequential()<choose_model_class>
train = train_features[train_features.cp_type == 'trt_cp'].reset_index(drop=True)
 test = test_features[test_features.cp_type == 'trt_cp'].reset_index(drop=True)
 train_targets_scored_drop = train_targets_scored[train_features.cp_type == 'trt_cp'].reset_index(drop=True)
 train_targets_nonscored_drop = train_targets_nonscored[train_features.cp_type == 'trt_cp'].reset_index(drop=True)
 train, test = preprocess([train, test], [drop_low_varience() ,
encode_categorical() ,
encode_ordinal() ,
add_statistics(['sum', 'mean', 'std', 'kurt', 'skew'], GENES, 'g_', ignore_cols=True),
add_statistics(['sum', 'mean', 'std', 'kurt', 'skew'], CELLS, 'c_', ignore_cols=True),
drop_low_varience() ,
rank_gauss2() ])
cnn.add(tf.keras.layers.Conv2D(filters = 32, kernel_size = 5, padding = 'same', activation = 'relu', input_shape = [28, 28, 1]))<choose_model_class>
class SmoothCrossEntropyLoss(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0, logits=False):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
self.logits = logits
@staticmethod
def _smooth(targets, n_classes, smoothing=0.0):
assert 0 <= smoothing <= 1
with torch.no_grad() :
targets = targets *(1 - smoothing)+ torch.ones_like(targets ).to(device)* smoothing / n_classes
return targets
def forward(self, inputs, targets):
targets = SmoothCrossEntropyLoss()._smooth(targets, inputs.shape[1], self.smoothing)
 if self.weight is not None:
inputs = inputs * self.weight.unsqueeze(0)
 if self.logits:
loss = F.binary_cross_entropy_with_logits(inputs, targets)
 else:
loss = F.binary_cross_entropy(inputs, targets)
 return loss
cnn.add(tf.keras.layers.MaxPool2D(pool_size = 2, strides = 2, padding = 'valid'))<choose_model_class>
def train_val_fn(model, optimizer, scheduler, loss_fn, metric, dataloader, device, dataset_sizes, phase='train', epoch=0, cur_val_los=None):
gc.collect()
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
epoch_metric = 0
preds = []
for batch in dataloader:
x = batch['features'].to(device)
 targets = batch['targets'].to(device)
 if phase == 'train':
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
pred = model(x)
 if not pred.isnan().any() :
loss = loss_fn(pred, targets)
 else:
model.zero_grad()
continue
if phase == 'train':
loss.backward()
optimizer.step()
else:
preds.append(pred)
 with torch.no_grad() :
epoch_metric += metric(pred, targets ).item() * batch['targets'].size(0)
 running_loss += loss.item() * batch['targets'].size(0)
 if phase == 'val':
try:
preds = np.concatenate([p.cpu().numpy() for p in preds])
 except:
return float('inf'), None, float('inf')
 return running_loss / dataset_sizes[phase], preds, epoch_metric / dataset_sizes[phase]
@torch.no_grad()
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for batch in dataloader:
x = batch['features']
x = x.to(device)
 pred = model(x)
 preds.append(pred)
 preds = np.concatenate([p.cpu().numpy() for p in preds])
 return preds
cnn.add(tf.keras.layers.Conv2D(filters = 64, kernel_size = 3, padding = 'same'))
cnn.add(tf.keras.layers.MaxPool2D(pool_size = 2, strides = 2, padding='valid'))<define_search_model>
def train_fold(features, targets, test, fold, verbose=10, from_pretrain=False):
t_ix = features[features['fold'] != fold].index
v_ix = features[features['fold'] == fold].index
X = {
'train': features.loc[t_ix],
'val': features.loc[v_ix]
}
y = {
'train': targets.loc[t_ix],
'val': targets.loc[v_ix]
}
if from_pretrain:
model = Model(config.n_features, config.n_classes_nonscored)
 model.load_state_dict(torch.load(config.model_path + f'{fold}_model.pt'))
model.classifier = Layer(config.hidden_size, config.n_classes, 0.25, torch.sigmoid)
 model = model.to(device)
 else:
model = Model(config.n_features, config.n_classes ).to(device)
 datasets = {sample: MoaDataset(X[sample], y[sample])
 for sample in ['train', 'val']}
dataloaders = {
sample: DataLoader(
 datasets[sample],
batch_size=config.batch_size,
shuffle=(sample == 'train'),
num_workers=0 
)for sample in ['train', 'val']}
dataset_sizes = {sample: len(datasets[sample])for sample in ['train', 'val']}
loss_fn = SmoothCrossEntropyLoss(smoothing=0.001)
 metric = nn.BCELoss()
learning_rate = config.learning_rate
num_train_steps = int(dataset_sizes['train'] / config.batch_size)* config.num_epoch
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=config.weight_decay)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, factor=0.5, patience=5)
 best_loss = float('inf')
 epoch_loss = float('inf')
 early_step = 0
oos = np.zeros_like(targets.drop(columns=config.drop_features, errors='ignore'), dtype=np.float)
 best_epoch = config.num_epoch
print(f' FOLD {fold} MODEL TRAINING '.ljust(99, '=' ).rjust(190, '='))
stop = False
for epoch in range(config.num_epoch):
for phase in ['train', 'val']:
epoch_loss, preds, epoch_metric = train_val_fn(model, optimizer, scheduler, loss_fn, metric, dataloaders[phase], device, dataset_sizes, phase, epoch, cur_val_los=epoch_loss)
 if phase == 'val':
scheduler.step(epoch_metric)
 if phase == 'val' and epoch_metric < best_loss:
best_loss = epoch_metric
best_model_wts = copy.deepcopy(model.state_dict())
 oos[v_ix] = preds
best_epoch = epoch
early_step = 0
elif phase == 'val':
if preds is None:
model.load_state_dict(best_model_wts)
 early_step += 1
if early_step >= config.early_stopping_steps:
stop = True
if epoch % verbose == 0:
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 if stop:
break
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 print(f"Best epoch {best_epoch} validation loss: {best_loss}")
 torch.save(best_model_wts, config.model_path + f'{fold}_model.pt')
 model.load_state_dict(best_model_wts)
 dataset_test = MoaDataset(test)
 dataloader_test = DataLoader(
 dataset_test,
batch_size=config.batch_size,
shuffle=False,
num_workers=0)
 test_pred = inference_fn(model, dataloader_test, device)
 return oos, test_pred
def pretrain_fold(features, targets, fold, verbose=10):
t_ix = features[features['fold'] != fold].index
v_ix = features[features['fold'] == fold].index
X = {
'train': features.loc[t_ix],
'val': features.loc[v_ix]
}
y = {
'train': targets.loc[t_ix],
'val': targets.loc[v_ix]
}
model = Model(config.n_features, config.n_classes_nonscored ).to(device)
 datasets = {sample: MoaDataset(X[sample], y[sample])
 for sample in ['train', 'val']}
dataloaders = {
sample: DataLoader(
 datasets[sample],
batch_size=config.batch_size,
shuffle=(sample == 'train'),
num_workers=0 
)for sample in ['train', 'val']}
dataset_sizes = {sample: len(datasets[sample])for sample in ['train', 'val']}
loss_fn = SmoothCrossEntropyLoss(smoothing=0.1)
 metric = nn.BCELoss()
learning_rate = config.learning_rate_nonscored
num_train_steps = int(dataset_sizes['train'] / config.batch_size)* config.num_epoch
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=config.weight_decay)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, factor=0.5, patience=5)
 best_loss = float('inf')
 epoch_loss = float('inf')
 early_step = 0
best_epoch = config.num_epoch
print(f' FOLD {fold} MODEL PRETRAINING '.ljust(99, '=' ).rjust(190, '='))
stop = False
for epoch in range(config.num_epoch):
for phase in ['train', 'val']:
epoch_loss, preds, epoch_metric = train_val_fn(model, optimizer, scheduler, loss_fn, metric, dataloaders[phase], device, dataset_sizes, phase, epoch, cur_val_los=epoch_loss)
 if phase == 'val':
scheduler.step(epoch_metric)
 if phase == 'val' and epoch_metric < best_loss:
best_loss = epoch_metric
best_model_wts = copy.deepcopy(model.state_dict())
 best_epoch = epoch
early_step = 0
elif phase == 'val':
if preds is None:
model.load_state_dict(best_model_wts)
 early_step += 1
if early_step >= config.early_stopping_steps:
stop = True
if epoch % verbose == 0:
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 if stop:
break
print(f'Epoch {epoch} {phase} loss'.ljust(35), ':', epoch_loss, 'metric:', epoch_metric)
 print(f"Best epoch {best_epoch} validation loss: {best_loss}")
 torch.save(best_model_wts, config.model_path + f'{fold}_model.pt' )
cnn.add(tf.keras.layers.MaxPool2D(pool_size = 2, strides = 2, padding='valid'))
cnn.add(Droupout(0.5))<set_options>
def run_k_fold(NFOLDS, verbose=10, from_pretrain=False):
oos = np.zeros_like(train_targets_scored_drop.drop(columns=config.drop_features, errors='ignore'), dtype=np.float)
 predictions = np.zeros(( test.shape[0], config.n_classes), dtype=np.float)
 for fold in range(NFOLDS):
oos_, pred_ = train_fold(train, train_targets_scored_drop, test, fold, verbose, from_pretrain)
 predictions += pred_ / NFOLDS
oos += oos_
return oos, predictions
def run_k_fold_pretrain(NFOLDS, verbose=10):
for fold in range(NFOLDS):
pretrain_fold(train, train_targets_nonscored_drop, fold, verbose)
 def run_n_seeds(train_df, targets, targets_nonscored=None, SEEDS=config.seeds, verbose=10, pretrain=False, eval=False):
assert pretrain == False or pretrain == True and targets_nonscored is not None
oos = []
test_pred = []
for seed in SEEDS:
print(f' SEED {seed} '.ljust(90, '*' ).rjust(190, '*'))
seed_everything(seed)
 kf = MultilabelStratifiedKFold(config.n_folds, shuffle=True, random_state=seed)
 targets_to_strat = targets.merge(pd.get_dummies(train_drug, columns=['drug_id']), on='sig_id', how='left')
 if pretrain:
for f,(t_idx, v_idx)in enumerate(kf.split(X=train, y=targets_to_strat.drop(columns=config.drop_features, errors='ignore'))):
train.loc[v_idx, 'fold'] = f
train['fold'] = train['fold'].astype(int)
 run_k_fold_pretrain(config.n_folds)
 oos_seed, test_pred_seed = run_k_fold(config.n_folds, verbose=verbose, from_pretrain=True)
 else:
for f,(t_idx, v_idx)in enumerate(kf.split(X=train, y=targets_to_strat.drop(columns=config.drop_features, errors='ignore'))):
train.loc[v_idx, 'fold'] = f
train['fold'] = train['fold'].astype(int)
 oos_seed, test_pred_seed = run_k_fold(config.n_folds, verbose=verbose, from_pretrain=False)
 oos.append(oos_seed)
 test_pred.append(test_pred_seed)
 if eval:
print('*'*20)
 print(f'LOSS SEED {seed}:', nn.BCELoss()(torch.tensor(oos_seed, dtype=torch.float), torch.tensor(targets.drop(columns=config.drop_features, errors='ignore' ).to_numpy() , dtype=torch.float)))
 print('*'*20)
 return np.mean(oos, 0), np.mean(test_pred, 0 )
cnn.add(tf.keras.layers.Flatten() )<choose_model_class>
os.environ['CUDA_LAUNCH_BLOCKING'] = "1"
config.n_features = train.drop(columns=config.drop_features, errors='ignore' ).shape[1]
t0 = time()
date_time = datetime.now().strftime("%Y%m%d-%H%M")
 oos, test_pred = run_n_seeds(train, train_targets_scored_drop, train_targets_nonscored_drop, pretrain=True, eval=True)
 print(f'Done! Time {timedelta(seconds=time() - t0)}' )
cnn.add(tf.keras.layers.Dense(units=256, activation='relu'))<choose_model_class>
oos_df = pd.DataFrame(oos)
 oos_df.insert(0, 'sig_id', train.sig_id)
 oos_df.columns = train_targets_scored.columns
oos_df = train_targets_scored[['sig_id']].merge(oos_df, on='sig_id', how='left' ).fillna(0 )
cnn.add(tf.keras.layers.Dense(units=10, activation='softmax'))<choose_model_class>
print('loss without ctl: ', nn.BCELoss()(torch.tensor(oos, dtype=torch.float), torch.tensor(train_targets_scored_drop.drop(columns=config.drop_features,errors='ignore' ).to_numpy() , dtype=torch.float)).item())
 print('loss with ctl: ', nn.BCELoss()(torch.tensor(oos_df.drop(columns='sig_id' ).to_numpy() , dtype=torch.float), torch.tensor(train_targets_scored.drop(columns='sig_id' ).to_numpy() , dtype=torch.float)).item() )
cnn.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'] )<train_model>
test_pred_df = pd.DataFrame(test_pred)
 test_pred_df.insert(0, 'sig_id', test.sig_id)
 test_pred_df.columns = sample_submission.columns
test_pred_df = sample_submission[['sig_id']].merge(test_pred_df, on='sig_id', how='left' ).fillna(0 )
<compute_test_metric><EOS> 
test_pred_df.to_csv('submission.csv', index=False )