Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<choose_model_class>
%%capture
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
history = model.fit(X_train, Y_train,
batch_size=128, epochs=10,
verbose=1 )<compute_test_metric>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
score = model.evaluate(X_test, Y_test)
 print('Test accuracy:', score[1] )<predict_on_test>
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.9
decompo = "PCA"
ncompo_genes = 0.9
ncompo_cells = 0.9
encoding = "dummy"
use_categories = True
results = model.predict(test_data )<save_to_csv>
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )
results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )<predict_on_test>
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )
predicted_classes = model.predict_classes(X_test)
 correct_indices = np.nonzero(predicted_classes == y_test)[0]
incorrect_indices = np.nonzero(predicted_classes != y_test)[0]<install_modules>
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore')
<define_variables>
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass
BATCH_SIZE = 32 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 8<define_search_space>
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.7:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.7)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.8)
 else:
image = tf.image.central_crop(image, central_fraction=.9)
 elif p_crop >.4:
crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 return image, label<normalization>
categorical_cols = [cc for cc in data_all.columns if "cp_" in cc]
categorical_cols
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
return image
def center_crop(image):
image = tf.reshape(image, [600, 800, CHANNELS])
 h, w = image.shape[0], image.shape[1]
if h > w:
image = tf.image.crop_to_bounding_box(image,(h - w)// 2, 0, w, w)
 else:
image = tf.image.crop_to_bounding_box(image, 0,(w - h)// 2, h, h)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<train_model>
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
relation_cols = []
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1)
 relation_cols.append("g_" + stats)
 relation_cols.append("c_" + stats)
 relation_cols.append("gc_" + stats )
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<choose_model_class>
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed)
 pca_genes.fit(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed)
 pca_cells.fit(data_all[CELLS])
 g_pca = [f"pca_g-{i}" for i in range(pca_genes.n_components_)]
c_pca = [f"pca_c-{i}" for i in range(pca_cells.n_components_)]
pca_genes_df = pd.DataFrame(pca_genes.transform(data_all[GENES]), columns = g_pca)
 pca_cells_df = pd.DataFrame(pca_cells.transform(data_all[CELLS]), columns = c_pca)
 pca_cols = g_pca + c_pca
if use_categories:
cols_to_get = cols_numeric + relation_cols + categorical_cols
else:
cols_to_get = cols_numeric + relation_cols
data_all = pd.concat([pca_genes_df, pca_cells_df, data_all[cols_to_get]], axis = 1)
 else:
pass
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='input_image')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='softmax', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model.summary() 
<load_pretrained>
GENES_PCA = [g for g in data_all.columns if "pca_g-" in g]
CELLS_PCA = [c for c in data_all.columns if "pca_c-" in c]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["pca_g-" + stats] = getattr(data_all[GENES_PCA], stats )(axis = 1)
 data_all["pca_c-" + stats] = getattr(data_all[CELLS_PCA], stats )(axis = 1)
 data_all["pca_gc-" + stats] = getattr(data_all[GENES_PCA + CELLS_PCA], stats )(axis = 1 )
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]<save_to_csv>
if use_categories:
cat_idx = []
for col in categorical_cols:
cat_idx.append(data_all.columns.get_loc(col))
print(cat_idx)
 else:
cat_idx = []
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<define_variables>
with open("data_all.pickle", "wb")as f:
pickle.dump(data_all, f )
package_path = '.. /input/vision-transformer-pytorch/VisionTransformer-Pytorch'
sys.path.append(package_path)
<define_variables>
with open("data_all.pickle", "rb")as f:
data_all = pickle.load(f )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
sys.path.append(package_path )<import_modules>
features_to_drop = ["cp_type"]
try:
data_all.drop(features_to_drop, axis = 1, inplace = True)
 except:
pass
try:
targets.drop("sig_id", axis = 1, inplace = True)
 except:
pass
train_df = data_all[: train.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits)+ y_true * np.log(logits)
 return np.mean(-aux )
CFG = {
'fold_num': 10,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 384,
'epochs': 32,
'train_bs': 28,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=1e-5):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=1e-5):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
MAX_EPOCH = 300
tabnet_params = dict(
 n_d = 64,
n_a = 64,
n_steps = 1,
gamma = 1.2,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 3e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = 28749679,
verbose = 10,
cat_idxs = cat_idx 
)
 scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 5
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 976234598, shuffle = True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1)
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 30,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = SmoothBCEwLogits() 
)
 print(y_, '-' * 60)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(np.clip(1 /(1 + np.exp(-preds_test)) , 1e-5, 1-1e-5))
oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )
train.label.value_counts()<load_from_csv>
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )
<categorify><EOS> 
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()