Kaggle competition: Mechanisms of Action (MoA) Prediction
class LoaderMetricCallback(Callback):
def __init__(self, input_key, output_key, metric_fn, prefix="metric"):
super().__init__(CallbackOrder.Metric)
 self.input_key = input_key
self.output_key = output_key
self.metric_fn = metric_fn
self.prefix = prefix
self.data = {k: [] for k in self.output_key+self.input_key}
def on_batch_end(self, state):
y_hat = state.output['logits'].detach().cpu().numpy()
y = state.input['targets'].detach().cpu().numpy()
for k in self.output_key:
self.data[k].append(state.output[k].detach().cpu())
 for k in self.input_key:
self.data[k].append(state.input[k].detach().cpu())
 def on_loader_end(self, state):
if state.is_valid_loader:
for k in self.output_key+self.input_key:
self.data[k] = torch.cat(self.data[k])
 state.loader_metrics[self.prefix] = self.metric_fn(**self.data)
 self.data = {k: [] for k in self.output_key+self.input_key}<choose_model_class>
df = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = df.loc[df['cp_type']!='ctl_vehicle'].index.to_list()
df = df.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 data_X_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 data_X_test = data_X_test.drop(['sig_id','cp_dose','cp_time'],axis=1)
 dada = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 dada = dada.drop('sig_id',axis=1)
 df_label = dada.values
df = df.iloc[non_ctl_idx]
df_label = df_label[non_ctl_idx]
class Mish(nn.Module):
def forward(self, x):
return x * torch.tanh(F.softplus(x))
class Conv1dStack(nn.Module):
def __init__(self, in_dim, out_dim, kernel_size=3, padding=1, dilation=1):
super(Conv1dStack, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv1d(in_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 self.res = nn.Sequential(
 nn.Conv1d(out_dim, out_dim, kernel_size=kernel_size, padding=padding, dilation=dilation, bias=False),
nn.BatchNorm1d(out_dim),
nn.Dropout(0.1),
nn.LeakyReLU() , 
)
 def forward(self, x):
x = self.conv(x)
 h = self.res(x)
 return x + h
class SeqEncoder(nn.Module):
def __init__(self, in_dim: int):
super(SeqEncoder, self ).__init__()
self.conv0 = Conv1dStack(in_dim, 128, 3, padding=1)
 self.conv1 = Conv1dStack(128, 64, 6, padding=5, dilation=2)
 self.conv2 = Conv1dStack(64, 32, 15, padding=7, dilation=1)
 self.conv3 = Conv1dStack(32, 32, 30, padding=29, dilation=2)
 def forward(self, x):
x1 = self.conv0(x)
 x2 = self.conv1(x1)
 x3 = self.conv2(x2)
 x4 = self.conv3(x3)
 x = torch.cat([x1, x2, x3, x4], dim=1)
 return x
class Model(torch.nn.Module):
def __init__(self):
super().__init__()
self.pretrain_mode = False
prenet_hidden = 128
seq_hidden_size = 1024
hidden_cls_dim = 128
feature_prenet_hidden = 32
n_feature = 4
self.sequence_embed = nn.Embedding(4, 8)
 self.predicted_loop_type_embed = nn.Embedding(7, 16)
 self.structure_embed = nn.Embedding(4, 8)
 self.feature_prenet = nn.Sequential(
 nn.Conv1d(n_feature, feature_prenet_hidden, kernel_size=1),
nn.BatchNorm1d(feature_prenet_hidden), 
)
 self.prenet = nn.Sequential(
 nn.Conv1d(( 8+16+8)* 2 + 1 + feature_prenet_hidden*2, prenet_hidden, kernel_size=1),
nn.BatchNorm1d(prenet_hidden), 
)
 self.seq_encoder = SeqEncoder(prenet_hidden)
 self.reccurent_layers = nn.ModuleList()
self.dense_layers = nn.ModuleList()
for idx in range(2):
self.reccurent_layers.append(nn.GRU(
 input_size= 256 if idx==0 else seq_hidden_size*2,
hidden_size=seq_hidden_size,
num_layers=1,
batch_first=True,
bidirectional=True, 
))
self.dense_layers.append(nn.Sequential(
 nn.Conv1d(seq_hidden_size*2, seq_hidden_size, kernel_size=1),
nn.BatchNorm1d(seq_hidden_size),
Mish() ,
nn.Dropout(0.3)
))
self.classifier = nn.Sequential(
 nn.Linear(seq_hidden_size*2, hidden_cls_dim), Mish() , nn.Dropout(p=0.3),
nn.Linear(hidden_cls_dim, hidden_cls_dim), Mish() , nn.Dropout(p=0.2),
nn.Linear(hidden_cls_dim, 5)
)
 self.pretrain = nn.Sequential(
 nn.Linear(seq_hidden_size*2, hidden_cls_dim), Mish() , nn.Dropout(p=0.3),
nn.Linear(hidden_cls_dim, hidden_cls_dim), Mish() , nn.Dropout(p=0.2),
nn.Linear(hidden_cls_dim, 4)
)
 def _attention_head(self, x, bbps):
return torch.bmm(bbps, x)
 def forward(self, sequence, predicted_loop_type, structure, bbps, features):
sequence = self.sequence_embed(sequence)
 predicted_loop_type = self.predicted_loop_type_embed(predicted_loop_type)
 structure = self.structure_embed(structure)
 features = self.feature_prenet(features ).permute(0,2,1)
 x = torch.cat([sequence, predicted_loop_type, structure, features], dim=-1)
 if self.pretrain_mode:
n_elements_to_mask = 3
for i in range(x.shape[0]):
mask = list(np.random.randint(x.shape[1], size=n_elements_to_mask))
x[i, mask]=0
x_sec = self._attention_head(x, bbps)
 attention_sum_f = bbps.sum(-1)[:,:,None]
x = torch.cat([x, x_sec, attention_sum_f], dim=-1)
 x = self.prenet(x.permute(0,2,1)).permute(0,2,1)
 x = self.seq_encoder(x.permute(0,2,1)).permute(0,2,1)
 for idx,(reccurent_layer, linear_layer)in enumerate(zip(self.reccurent_layers, self.dense_layers)) :
x, _ = reccurent_layer(x)
 x = linear_layer(x.permute(0,2,1)).permute(0,2,1)
 x = torch.cat([x, self._attention_head(x, bbps)], axis=-1)
 if self.pretrain_mode:
out = self.pretrain(x)
 else:
out = self.classifier(x)
 return out<drop_column>
cs = df.columns.str.contains('c-')
 gs = df.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
def remove_useless_checkpoints(path, usefull_checkpoint):
for el in listdir(path):
if el != usefull_checkpoint:
print(f'rm {path}/{el}')
 !rm '{path}/{el}'<load_pretrained>
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def dataloader_train_init(data, device='cuda'):
dataset = CompDataset(data, bbps_folder, device=device, train_mode=False)
 return DataLoader(dataset, 16, shuffle=True, drop_last=True, collate_fn=random_window_collate_fn)
 device = 'cuda'
dataloader_107 = dataloader_train_init(train.append(test[test['seq_length']==107]), device)
 dataloader_130 = dataloader_train_init(test[test['seq_length']==130], device)
 model = Model()
model.pretrain_mode = True
optimizer = Lookahead(RAdam(model.parameters() , lr=1e-3))
criterion = {"loss": PreTrainLoss() }
callbacks =[CriterionCallback(input_key='sequence', output_key='logits', prefix="loss", criterion_key="loss"),
OptimizerCallback(metric_key="loss", accumulation_steps=1),]
input_keys = ['sequence', 'predicted_loop_type', 'structure', 'bbps', 'features']
input_citerion_keys = ['sequence']
runner = SupervisedRunner(device=device, input_key=input_keys, input_target_key=input_citerion_keys )<train_model>
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, verbose=0,mode='min',
patience=3, min_lr=1E-7)
 early_st = tf.keras.callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=7, verbose=0, mode='min',
baseline=None, restore_best_weights=True )
for i in range(5):
runner.train(
 model=model,
optimizer=optimizer,
criterion=criterion,
loaders={'train': dataloader_107},
logdir='pretrain',
callbacks=callbacks,
num_epochs=1,
verbose= False,
main_metric="loss",
minimize_metric=True, 
)
 runner.train(
 model=model,
optimizer=optimizer,
criterion=criterion,
loaders={'train': dataloader_130},
logdir='pretrain',
callbacks=callbacks,
num_epochs=1,
verbose=False,
main_metric="loss",
minimize_metric=True, 
)
 remove_useless_checkpoints('pretrain/checkpoints', 'last.pth' )<load_pretrained>
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.0015),metrics=logloss 
)
 return model
device = 'cuda'
input_keys = ['sequence', 'predicted_loop_type', 'structure', 'bbps', 'features']
input_citerion_keys = ['targets', 'error', 'filter_1', 'filter_2']
batch_size = 16
def dataloader_train_init(idx):
dataset = CompDataset(train.iloc[idx], bbps_folder, device=device)
 return DataLoader(dataset, batch_size, shuffle=True, drop_last=True, collate_fn=random_window_collate_fn)
 def dataloader_valid_init(idx):
dataset = CompDataset(train.iloc[idx], bbps_folder, device=device)
 return DataLoader(dataset, batch_size, shuffle=False, drop_last=False)
 def model_init() :
model = Model()
model.load_state_dict(torch.load(f'pretrain/checkpoints/last.pth')['model_state_dict'])
 return model
optimizer_init = lambda model : Lookahead(RAdam(model.parameters() , lr=1e-3))
scheduler_init = lambda optimizer : torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.25, patience=4, cooldown=5, min_lr=1e-6)
 loss_init = lambda : CompLoss()
callbacks_init = lambda : [CriterionCallback(input_key=input_citerion_keys, output_key=['logits'], prefix="loss", criterion_key="loss"),
LoaderMetricCallback(input_key=input_citerion_keys, output_key=['logits'], prefix="metric", metric_fn=comp_metric_fn),
OptimizerCallback(metric_key="loss", accumulation_steps=1),
EarlyStoppingCallback(patience=15, metric='metric', minimize=True),
SchedulerCallback(mode='epoch')]<choose_model_class>
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.001),metrics=logloss 
)
 return model
for i,(train_idx, val_idx)in enumerate(folds):
model = model_init()
optimizer = optimizer_init(model)
 scheduler = scheduler_init(optimizer)
 criterion = {"loss": loss_init() }
loaders = {'train': dataloader_train_init(train_idx), 'valid': dataloader_valid_init(val_idx)}
callbacks = callbacks_init()
runner = SupervisedRunner(device=device, input_key=input_keys, input_target_key=input_citerion_keys)
 runner.train(
 model=model,
optimizer=optimizer,
scheduler=scheduler,
criterion=criterion,
loaders=loaders,
logdir=f'fold_{i}',
callbacks=callbacks,
num_epochs=9999,
verbose=True,
load_best_on_end=True,
main_metric="metric",
minimize_metric=True, 
)
 remove_useless_checkpoints(f'fold_{i}/checkpoints', 'best.pth' )<find_best_model_class>
X = 976
n_seeds = 6
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_labels = df_label.shape[1]
n_train = df.shape[0]
n_test = data_X_test.shape[0]
n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
print(fold)
 mlkf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for n,(train, test)in enumerate(mlkf.split(df,df_label)) :
tf.print('OOF score is ',oof)
 print(f'Fold {n}')
 train_x, validation_x = preprocessor(df.iloc[train].values,
df.iloc[test].values)
 _,data_test = preprocessor(df.iloc[train].values,
data_X_test.drop('cp_type',axis=1 ).values)
 train_y = df_label[train]
validation_y = df_label[test]
if seed%2 == 0:
model = create_model1(X)
 else:
model = create_model(X)
 model.compile(optimizer='adam',loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 hist = model.fit(train_x,train_y, batch_size=128, epochs=200,verbose=0,validation_data =(validation_x,validation_y),callbacks =[reduce_lr, early_st])
 hists.append(hist)
 model.save('_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(validation_x)
 oof += logloss(tf.constant(validation_y,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1
cv_scores = []
oof_prediction = torch.zeros(( train.shape[0], 107, 5))
for i,(train_idx, val_idx)in enumerate(folds):
best_model_path = f'fold_{i}/checkpoints/best.pth'
runner = SupervisedRunner(device='cuda', input_key=input_keys, input_target_key=input_citerion_keys)
 dataloader = dataloader_valid_init(val_idx)
 prediction = runner.predict_loader(loader=dataloader, model=Model() , resume=best_model_path)
 prediction = torch.cat([b['logits'] for b in prediction])
 fold_score = comp_metric_fn(prediction, *[dataloader.dataset.data[el] for el in input_citerion_keys])
 cv_scores.append(fold_score)
 oof_prediction[val_idx] = prediction.cpu()
cv_scores = np.array(cv_scores)
 train_data = CompDataset(train, bbps_folder ).data
off_score = comp_metric_fn(oof_prediction, *[train_data[el] for el in input_citerion_keys])
 np.save("oof_prediction", oof_prediction[:,:,[0,1,3,2,4]])
 final_log = f'Fold scores: {list(cv_scores)}
OFF score: {off_score}
Fold mean: {cv_scores.mean() }
Fold std : {cv_scores.std() }'
with open(f"final_log.txt", 'w')as file:
file.write(final_log)
 print(final_log )<create_dataframe>
final = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 final.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 final.iloc[data_X_test['cp_type'] == 'ctl_vehicle',1:] = 0
final.to_csv('submission.csv', index=False)
 final.head()