Kaggle competition: Mechanisms of Action (MoA) Prediction
measure_to_monitor = 'val_auc'
modality = 'max'
early_stopping = EarlyStopping(monitor=measure_to_monitor,
mode=modality,
patience=5,
verbose=0)
 model_checkpoint = ModelCheckpoint('best.model',
monitor=measure_to_monitor,
mode=modality,
save_best_only=True,
verbose=0)
 reduce_learning = ReduceLROnPlateau(monitor=measure_to_monitor, mode=modality,
factor=0.25, patience=2, min_lr=1e-6, verbose=0 )<init_hyperparams>
X, y, genes, cells, classnames, features, X_test, test_control, submission = load(onehot=False)
 splits = sorted(glob.glob(".. /input/moa-pipelines/c*.pkl"))
folds, pipes = list(zip(*[joblib.load(split)for split in splits]))
print(splits )
skf = StratifiedKFold(n_splits=FOLDS,
shuffle=True,
random_state=SEED)
 roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
best_iteration = list()
for fold,(train_idx, test_idx)in enumerate(skf.split(X, y)) :
tb = TabularTransformer(numeric = numeric_variables,
ordinal = [],
lowcat = [],
highcat = categorical_variables)
 tb.fit(X.iloc[train_idx])
 sizes = tb.shape(X.iloc[train_idx])
 categorical_levels = dict(zip(categorical_variables, sizes[1:]))
print(f"Input array sizes: {sizes}")
 print(f"Categorical levels: {categorical_levels}
")
 model = tabular_dnn(numeric_variables, categorical_variables,
categorical_levels,
feature_selection_dropout=0.1,
categorical_dropout=0.1,
first_dense = 256,
second_dense = 256,
dense_dropout = 0.1,
activation_type=gelu)
 model = compile_model(model, binary_crossentropy, [AUC(name='auc'), mAP], Adam(learning_rate=0.0001))
train_batch = DataGenerator(X.iloc[train_idx],
y[train_idx],
tabular_transformer=tb,
batch_size=BATCH_SIZE,
shuffle=True)
 history = model.fit_generator(train_batch,
validation_data=(tb.transform(X.iloc[test_idx]), y[test_idx]),
epochs=30,
callbacks=[model_checkpoint, early_stopping, reduce_learning],
class_weight=[1.0,(np.sum(y==0)/ np.sum(y==1)) ],
verbose=1)
 print("
FOLD %i" % fold)
 plot_keras_history(history, measures=['auc', 'loss'])
 best_iteration.append(np.argmax(history.history['val_auc'])+ 1)
 preds = model.predict(tb.transform(X.iloc[test_idx]),
verbose=1,
batch_size=1024 ).flatten()
oof[test_idx] = preds
roc_auc.append(roc_auc_score(y_true=y[test_idx], y_score=preds))
average_precision.append(average_precision_score(y_true=y[test_idx], y_score=preds))<compute_test_metric>
multi_input = resnet_multi_input(pipes[0])
 device =('cuda' if torch.cuda.is_available() else 'cpu')
 pseudo = 1
pseudo_optim =(torch.optim.SGD, dict(lr=1e-5, momentum=0.9, nesterov=True))
epochs =(30, 5)
 lr = 2e-3
wd = 1e-5
early_stopping = -1
top_bias = 0
hidden = {'cat': 128, 'genes': 1024, 'inter': 256, 'cells': 32, 'pca': 256, 'other': 64}
embed_size =(32, 16)
 lookahead = 10
smoothing = 3e-4
clipping = 1e-4
dropout = 0.2
batch_size = 128
df = 1000
ps = 0.15
max_lr = 0.01
activation = partial(nn.LeakyReLU, negative_slope=0.05)
 optim_params = {
"lr": lr,
"weight_decay": wd
}
sched_params = {
"epochs": epochs[0],
"pct_start": ps,
"div_factor": df,
"max_lr": max_lr
}
model_params = {
"n_genes": len(multi_input[2][0]),
"n_cells": len(multi_input[3][0]),
"n_pca" : len(multi_input[-2][0]),
"n_other": len(multi_input[-1][0]),
"dropout": dropout,
"embed_size": embed_size,
"hidden": hidden,
"n_clusters": [2, 2, 2, 2, 18, 11],
"device": device,
"activation": activation
}
fitter_params = dict(
 estimator=ResNet,
estimator_params=model_params,
optim_params=optim_params,
sched_params=sched_params,
lookahead=lookahead,
pseudo=pseudo,
pseudo_optim=pseudo_optim,
savepath="validation.pth",
device=device,
loss_fn=LabelSmoothingCrossEntropy(smoothing, softmax=True, clipping=clipping), 
)
 fitter = Fitter(**fitter_params)
 train_params = {
"epochs": epochs,
"batch_size": batch_size,
"early_stopping": early_stopping,
"multi_input": multi_input,
"dataset": MultiInputDataset,
"verbose": 100
}
postprocessor = None
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof))<train_on_grid>
%%time
seeds = np.arange(len(folds))
oof_train, oof_test = mlp_oof(X, y, X_test[~test_control], fitter, train_params, folds, seeds, pipes, postprocessor )
tb = TabularTransformer(numeric = numeric_variables,
ordinal = [],
lowcat = [],
highcat = categorical_variables)
 tb.fit(X)
 sizes = tb.shape(X)
 categorical_levels = dict(zip(categorical_variables, sizes[1:]))
print(f"Input array sizes: {sizes}")
 print(f"Categorical levels: {categorical_levels}
")
 model = tabular_dnn(numeric_variables, categorical_variables,
categorical_levels,
feature_selection_dropout=0.1,
categorical_dropout=0.1,
first_dense = 256,
second_dense = 256,
dense_dropout = 0.1,
activation_type=gelu)
 model = compile_model(model, binary_crossentropy, [AUC(name='auc'), mAP], Adam(learning_rate=0.0001))
train_batch = DataGenerator(X, y,
tabular_transformer=tb,
batch_size=BATCH_SIZE,
shuffle=True)
 history = model.fit_generator(train_batch,
epochs=int(np.median(best_iteration)) ,
class_weight=[1.0,(np.sum(y==0)/ np.sum(y==1)) ],
verbose=1 )<save_to_csv>
preds_clipping = 1e-5
oof_train = np.clip(oof_train, preds_clipping, 1 - preds_clipping)
 oof_test = np.clip(oof_test, preds_clipping, 1 - preds_clipping )
preds = model.predict(tb.transform(Xt[X.columns]),
verbose=1,
batch_size=1024 ).flatten()
submission = pd.DataFrame(Xt.id)
 submission['Action'] = preds
submission.to_csv("tabular_dnn_submission.csv", index=False)
 tabular_dnn_submission = submission.copy()<save_to_csv>
oof_result(y, oof_train, oof_test )
dnn_rank = rankdata(tabular_dnn_submission.Action, method='dense')/ len(Xt)
 cat_rank = rankdata(cat_boost_submission.Action, method='dense')/ len(Xt)
 submission = pd.DataFrame(Xt.id)
 submission['Action'] = 0.5 * dnn_rank + 0.5 * cat_rank
submission.to_csv("blended_submission.csv", index=False )<import_modules>
prediction = join_control_group(oof_test, test_control, submission, classnames)
 submit_preds(prediction, submission, test_control, classnames)
 pd.read_csv("submission.csv" ).iloc[:5, :5]