Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<import_modules>
import os
import random
import time
import json
import warnings
import numpy as np
import pandas as pd
from sklearn.utils.class_weight import compute_class_weight
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.base import BaseEstimator
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, QuantileTransformer
from sklearn.metrics import fbeta_score
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import regularizers
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
import matplotlib.pyplot as plt
from xgboost import XGBClassifier
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
sys.path += ['/kaggle/input/iterstrat', '/kaggle/input/tabnet']
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
class EnsembleClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model1 = VisionTransformer.from_name('ViT-B_16', num_classes=5)
 self.model1.load_state_dict(torch.load('.. /input/vit-model-1/ViT-B_16.pt'))
self.model2 = CassvaImgClassifier(model_arch, n_class, pretrained)
 def forward(self, x):
x1 = self.model1(x)
 x2 = self.model2(x)
 return 0.6 * x1 + 0.4 * x2
def load(self, state_dict):
self.model2.load_state_dict(state_dict )<split>
def set_random_seeds(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 set_random_seeds(43 )
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = EnsembleClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
del model
torch.cuda.empty_cache()<feature_engineering>
X_full = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 Y_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 Y_nonscored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 drug = pd.read_csv('/kaggle/input/lish-moa/train_drug.csv')
 X_test_full = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sig_id = X_test_full['sig_id'].values.reshape(( -1, 1))
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
cat_cols = ['cp_type', 'cp_time', 'cp_dose']
numerical_cols = [c for c in X_full.columns if c not in ['sig_id'] + cat_cols]
label_cols = [c for c in Y_scored.columns if c != 'sig_id']
num_features = X_full.shape[1] - 1
num_labels = Y_scored.shape[1] - 1
num_nonscored_labels = Y_nonscored.shape[1] - 1
test.to_csv('submission.csv', index=False )<import_modules>
def logloss(Y_true, Y_preds, label_smoothing=0, remove_unpredicted=False):
if remove_unpredicted:
predicted =(Y_preds.sum(axis=1)!= 0)
 Y_true = Y_true.loc[predicted, :]
Y_preds = Y_preds.loc[predicted, :]
Y_true = Y_true.astype(np.float32)
 return tf.reduce_mean(keras.losses.binary_crossentropy(Y_true, Y_preds, label_smoothing=label_smoothing)).numpy()
def metrics(Y_true, Y_preds):
pr = tf.keras.metrics.Precision()
pr.update_state(Y_true, Y_preds)
 re = tf.keras.metrics.Recall()
re.update_state(Y_true, Y_preds)
 pr = pr.result().numpy()
re = re.result().numpy()
f1 = 2 *(pr * re)/(pr + re)
 auc = tf.keras.metrics.AUC(curve='PR', multi_label=True)
 auc.update_state(Y_true, Y_preds)
 return pr, re, f1, auc.result().numpy()
print("Tensorflow version " + tf.__version__)
<load_pretrained>
def add_outputs(Y, Y_preds, X=None, idx_list=None, col=None):
if X is not None:
if col is None:
Y.loc[X['cp_type'] == 'trt_cp', :] += Y_preds
else:
Y.loc[X['cp_type'] == 'trt_cp', Y.columns[col]] += Y_preds
elif idx_list is not None:
if col is None:
Y.iloc[idx_list, :] += Y_preds
else:
Y.iloc[idx_list, col] += Y_preds
else:
raise Exception("Must provide X or idx_list" )
dense201 = tf.keras.models.load_model('.. /input/train-model-cassava/densenet201.h5')
 inception = tf.keras.models.load_model('.. /input/train-model-cassava/inceptionv3.h5')
 efficient_net = tf.keras.models.load_model(
 '.. /input/train-model-cassava/efficient_netb3.h5',
compile=False,
custom_objects={'FixedDropout':FixedDropout}
)<define_variables>
def balance_class_weights(Y_true, class_weights=(0.7, 1.6)) :
n_labels = Y_true.shape[1]
weights = np.empty([n_labels, 2])
 for i in range(n_labels):
if class_weights == 'balanced':
total = len(Y_true[:, i])
 pos = np.sum(Y_true[:, i])
 neg = total - pos
weight_for_0 =(1 / neg)*(total)/2.0
weight_for_1 =(1 / pos)*(total)/2.0
weights[i] = [weight_for_0, weight_for_1]
else:
weights[i] = np.array(class_weights)
 return weights
def weighted_binary_crossentropy(weights, label_smoothing=0):
def weighted_loss(Y_true, Y_pred):
bce = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 return K.mean(
(weights[:, 0]**(1 - Y_true)) *(weights[:, 1]**(Y_true)) * bce(Y_true, Y_pred),
axis=-1 
)
 return weighted_loss
JPEG_PATH = ".. /input/cassava-leaf-disease-classification/test_images"
def load_image(jpeg_path, image_id):
img = cv2.imread(os.path.join(jpeg_path, image_id)) /255.0
img = cv2.resize(img,(512, 512)) [:, :, ::-1]
return img
def generator(filepath, paths, batch_size=32):
i=0
print(len(paths))
while i <= len(paths):
batch = []
for cpt in range(batch_size):
if i + cpt >= len(paths):
i += batch_size
break
batch.append(load_image(filepath, paths[i+cpt]))
i += batch_size
yield np.stack(batch )<load_from_csv>
def clear_session() :
curr_session = tf.compat.v1.get_default_session()
if curr_session is not None:
curr_session.close()
K.clear_session()
s = tf.compat.v1.InteractiveSession()
tf.compat.v1.keras.backend.set_session(s )
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv' )<compute_test_metric>
def preprocess(df, targets=None):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df = df.drop(columns='sig_id')
 if targets is not None:
targets = targets.copy()
targets = targets.drop(columns='sig_id')
 targets = targets[df['cp_type'] == 0].reset_index(drop=True)
 df = df[df['cp_type'] == 0].reset_index(drop=True)
 if targets is None:
return df
return df, targets
def vote_in_ensemble(v1, v2, v3):
if v1 == v2:
return v1
if v2 == v3:
return v2
if v1 == v3:
return v3
return v1<predict_on_test>
def complete_train_labels(Y, train_idx, test_idx):

 train_idx, test_idx = list(train_idx), list(test_idx)
 Y_train = Y.iloc[train_idx, :]
train_pos_cnts = Y_train.sum(axis=0)
 missing_labels = train_pos_cnts[train_pos_cnts == 0].index
removed_from_test = []
for l in missing_labels:
for i in test_idx:
if Y[l].iloc[i] == 1:
train_idx.append(i)
 removed_from_test.append(i)
 break
test_idx = [i for i in test_idx if i not in removed_from_test]
return train_idx, test_idx
def predict_for_pretrained(model):
ds_test = generator(JPEG_PATH,np.sort(submission.image_id.values))
preds = np.argmax(model.predict(ds_test, verbose=True), axis=-1)
 return preds
dense_preds = predict_for_pretrained(dense201)
 inception_preds = predict_for_pretrained(inception)
 efficient_net_preds = predict_for_pretrained(efficient_net)
 result = []
for idx, val in enumerate(dense_preds):
result.append(
 vote_in_ensemble(
 dense_preds[idx],
inception_preds[idx],
efficient_net_preds[idx] 
)
 )<save_to_csv>
X_train, Y_train = preprocess(X_full, Y_scored)
 X_test = preprocess(X_test_full)
 Y_train_nonscored = Y_nonscored.drop(columns='sig_id')[X_full['cp_type'] == 'trt_cp'].reset_index(drop=True)
 initial_bias = -np.log(Y_train.mean(axis=0 ).values)
 nonscored_initial_bias = -np.log(Y_train_nonscored.mean(axis=0 ).values + 1e-6 )
submission["label"] = result
submission.to_csv("submission.csv", index=False )<install_modules>
def multilabel_split_by_drugs(scored, n_folds, seed):
scored = scored.copy().merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc <= 18].index
vc2 = vc.loc[vc > 18].index
dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
 tmp = scored.groupby('drug_id')[label_cols].mean().loc[vc1]
for fold,(train_idx, test_idx)in enumerate(skf.split(tmp, tmp[label_cols])) :
dd = {k:fold for k in tmp.index[test_idx].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(train_idx, test_idx)in enumerate(skf.split(tmp, tmp[label_cols])) :
dd = {k:fold for k in tmp.sig_id[test_idx].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() , 'fold'] = scored.loc[scored.fold.isna() , 'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 test_idx = [np.where(scored['fold'] == fold)[0].tolist() for fold in range(n_folds)]
train_idx = [np.where(scored['fold'] != fold)[0].tolist() for fold in range(n_folds)]
return zip(train_idx, test_idx )
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
def select_features(X):
return X.iloc[:, [
1, 2, 3, 4, 5, 6, 7, 9, 11, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31,
32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46,
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 58, 59, 60,
61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
74, 75, 76, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88,
89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101,
102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 120, 121, 122, 123, 124, 125, 126, 127, 128,
129, 130, 131, 132, 133, 136, 137, 138, 139, 140, 141, 142, 143,
144, 145, 146, 147, 149, 150, 151, 152, 153, 154, 155, 156, 157,
158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,
171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183,
184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 197,
198, 199, 200, 202, 203, 204, 205, 206, 208, 209, 210, 211, 212,
213, 214, 215, 216, 217, 218, 219, 220, 221, 223, 224, 225, 226,
227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
240, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253,
254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266,
267, 268, 269, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280,
281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294,
295, 296, 298, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309,
310, 311, 312, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336,
337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 367, 368, 369, 370, 371, 374, 375, 376, 377,
378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 390, 391,
392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404,
405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 416, 417, 418,
419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431,
432, 434, 435, 436, 437, 438, 439, 440, 442, 443, 444, 445, 446,
447, 448, 449, 450, 453, 454, 456, 457, 458, 459, 460, 461, 462,
463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475,
476, 477, 478, 479, 481, 482, 483, 484, 485, 486, 487, 488, 489,
490, 491, 492, 493, 494, 495, 496, 498, 500, 501, 502, 503, 505,
506, 507, 509, 510, 511, 512, 513, 514, 515, 518, 519, 520, 521,
522, 523, 524, 525, 526, 527, 528, 530, 531, 532, 534, 535, 536,
538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 549, 550, 551,
552, 554, 557, 559, 560, 561, 562, 565, 566, 567, 568, 569, 570,
571, 572, 573, 574, 575, 577, 578, 580, 581, 582, 583, 584, 585,
586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 599,
600, 601, 602, 606, 607, 608, 609, 611, 612, 613, 615, 616, 617,
618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630,
631, 632, 633, 634, 635, 636, 637, 638, 639, 641, 642, 643, 644,
645, 646, 647, 648, 649, 650, 651, 652, 654, 655, 656, 658, 659,
660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672,
673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685,
686, 687, 688, 689, 691, 692, 693, 694, 695, 696, 697, 699, 700,
701, 702, 704, 705, 707, 708, 709, 710, 711, 713, 714, 716, 717,
718, 720, 721, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732,
733, 734, 735, 737, 738, 739, 740, 742, 743, 744, 745, 746, 747,
748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 759, 760, 761,
762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774,
775, 776, 777, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788,
789, 790, 792, 793, 794, 795, 796, 797, 798, 800, 801, 802, 803,
804, 805, 806, 808, 809, 811, 813, 814, 815, 816, 817, 818, 819,
821, 822, 823, 825, 826, 827, 828, 829, 830, 831, 832, 834, 835,
837, 838, 839, 840, 841, 842, 845, 846, 847, 848, 850, 851, 852,
854, 855, 856, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869,
870, 871, 872, 873, 874
]].values
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 21
seed_everything(seed)
 warnings.filterwarnings('ignore' )<define_variables>
gene_cols = [c for c in X_train.columns if c.startswith('g-')]
cell_cols = [c for c in X_train.columns if c.startswith('c-')]
BATCH_SIZE = 32 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 8<define_search_space>
qtransform = joblib.load('/kaggle/input/qtransform/qtransform.joblib')
 X_train_selected = select_features(X_train)
 X_test_selected = select_features(X_test)
 X_train_qtrans = np.concatenate([X_train_selected[:, :2], qtransform.transform(X_train_selected[:, 2:])], axis=1)
 X_test_qtrans = np.concatenate([X_test_selected[:, :2], qtransform.transform(X_test_selected[:, 2:])], axis=1)
 pca_qtrans_2 = joblib.load('/kaggle/input/qtransform/pca_qtrans.joblib' )
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.7:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.7)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.8)
 else:
image = tf.image.central_crop(image, central_fraction=.9)
 elif p_crop >.4:
crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 return image, label<normalization>
std_scaler_2 = joblib.load('/kaggle/input/qtransform/std_scaler_2.joblib')
 gene_pca_2 = joblib.load('/kaggle/input/qtransform/gene_pca_2.joblib')
 cell_pca_2 = joblib.load('/kaggle/input/qtransform/cell_pca_2.joblib' )
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
return image
def center_crop(image):
image = tf.reshape(image, [600, 800, CHANNELS])
 h, w = image.shape[0], image.shape[1]
if h > w:
image = tf.image.crop_to_bounding_box(image,(h - w)// 2, 0, w, w)
 else:
image = tf.image.crop_to_bounding_box(image, 0,(w - h)// 2, h, h)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<train_model>
kmeans_g = joblib.load('/kaggle/input/qtransform/kmeans_g.joblib')
 kmeans_c = joblib.load('/kaggle/input/qtransform/kmeans_c.joblib' )
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<choose_model_class>
LOAD_PRETRAINED_NN = True
REUSE_SPLITS_NN = True
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='input_image')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='softmax', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model.summary()<predict_on_test>
def preprocess_nn(X):
X_pca = pca_qtrans_2.transform(X)
 return np.concatenate([X, X_pca], axis=1 )
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]<save_to_csv>
X_train_nn = preprocess_nn(X_train_qtrans)
 X_test_nn = preprocess_nn(X_test_qtrans )
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<import_modules>
class MultiLabelNN(BaseEstimator):
def __init__(self,
n_input_features=num_features,
n_output_labels=num_labels,
n_hidden_units=2048):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.n_hidden_units = n_hidden_units
self._model_nn = None
def _create_nn_model(self, loss_fn=None, loss_fn_aux=None):
output_bias = keras.initializers.Constant(initial_bias)
 aux_output_bias = keras.initializers.Constant(nonscored_initial_bias)
 inp = keras.layers.Input(self.n_input_features)
 gauss_noise = keras.layers.GaussianNoise(0.001 )(inp)
 norm_1 = keras.layers.BatchNormalization()(gauss_noise)
 dense_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(norm_1)
 norm_2 = keras.layers.BatchNormalization()(dense_1)
 dropout_1 = keras.layers.Dropout(0.4 )(norm_2)
 dense_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units,
activation='elu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_1)
 norm_3 = keras.layers.BatchNormalization()(dense_2)
 dropout_2 = keras.layers.Dropout(0.4 )(norm_3)
 dropout_3 = keras.layers.Dropout(0.5 )(norm_2)
 dense_3 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_hidden_units // 2,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)
))(dropout_3)
 norm_4 = keras.layers.BatchNormalization()(dense_3)
 dropout_4 = keras.layers.Dropout(0.4 )(norm_4)
 output_1 = tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
), name='main' )(dropout_4)
 output_2 = tfa.layers.WeightNormalization(keras.layers.Dense(
 Y_train_nonscored.shape[1],
activation='sigmoid',
bias_initializer=aux_output_bias 
), name='aux' )(dropout_2)
 model_nn = keras.models.Model(inputs=inp, outputs=[output_1, output_2])
 model_nn.compile(
 loss=[
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn is None else loss_fn,
keras.losses.BinaryCrossentropy(label_smoothing=2e-4)if loss_fn_aux is None else loss_fn_aux
],
optimizer=keras.optimizers.Nadam(learning_rate=0.01),
metrics=['binary_crossentropy'],
loss_weights=[0.9, 0.1] 
)
 return model_nn
def fit(self, X, Y, X_valid, Y_valid, Y_aux, Y_aux_valid, max_epochs=100, save_weights=True):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 Y_aux = tf.cast(Y_aux, tf.float32)
 Y_aux_valid = tf.cast(Y_aux_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.7, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=1e-3)
 class_weights_aux = balance_class_weights(Y_aux.numpy() , class_weights=(0.7, 1.4))
loss_fn_aux = weighted_binary_crossentropy(class_weights_aux, label_smoothing=1e-3)
 self._model_nn = self._create_nn_model(loss_fn=loss_fn, loss_fn_aux=loss_fn_aux)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=10,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_nn/weights_{repeat}_{fold}.h5',
monitor='val_main_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self._model_nn.fit(
 X, [Y, Y_aux],
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, [Y_valid, Y_aux_valid]),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self._model_nn.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)[0]
return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self._model_nn.save_weights('model_nn/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self._model_nn is None:
self._model_nn = self._create_nn_model()
if pretrained:
base_path = '/kaggle/input/model-nn/'
else:
base_path = './working/'
self._model_nn.load_weights(base_path + 'model_nn/weights_{}_{}.h5'.format(repeat, fold))
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
<set_options>
n_repeats = 5
n_folds = 7
kfold_seeds = [74, 1156, 415, 608, 1993] if REUSE_SPLITS_NN else np.random.randint(42, 1337, n_repeats)
 kfold_seeds = kfold_seeds[:n_repeats]
assert len(kfold_seeds)== n_repeats
print(kfold_seeds )
CFG = {
'vit_img_size': 384,
'tta': 3,
'valid_bs': 16,
'device': 'cuda' if torch.cuda.is_available() else 'cpu',
'vit_models': ['model_5.pt', 'model_6.pt', 'model_7.pt', 'model_8.pt']
}<categorify>
val_losses = []
histories = []
Y_train_preds_nn = pd.DataFrame(np.zeros(( X_train.shape[0], num_labels)) , columns=label_cols)
 Y_test_preds_nn = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 set_random_seeds(42)
 for repeat, kf_seed in enumerate(kfold_seeds):
kf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=kf_seed, shuffle=True)
 for fold,(train_idx, test_idx)in enumerate(kf.split(X_train, Y_train)) :
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
train_idx, test_idx = complete_train_labels(Y_train, train_idx, test_idx)
 train_idx = tf.random.shuffle(train_idx)
 X_train_fold = tf.gather(X_train_nn, train_idx, axis=0)
 Y_train_fold = tf.gather(Y_train, train_idx, axis=0)
 Y_train_aux_fold = tf.gather(Y_train_nonscored, train_idx, axis=0)
 X_valid_fold = tf.gather(X_train_nn, test_idx, axis=0)
 Y_valid_fold = tf.gather(Y_train, test_idx, axis=0)
 Y_valid_aux_fold = tf.gather(Y_train_nonscored, test_idx, axis=0)
 model_nn = MultiLabelNN(n_input_features=X_train_nn.shape[1])
 if LOAD_PRETRAINED_NN:
model_nn.load_weights(repeat, fold)
 val_loss = model_nn.score(X_valid_fold, Y_valid_fold.numpy())
 else:
history = model_nn.fit(
 X_train_fold, Y_train_fold,
X_valid_fold, Y_valid_fold,
Y_train_aux_fold, Y_valid_aux_fold 
)
 histories.append(history)
 val_loss = min(history.history['val_main_binary_crossentropy'])
 model_nn.load_weights(repeat, fold, pretrained=False)
 val_preds = model_nn.predict(X_valid_fold)[0]
add_outputs(Y_train_preds_nn, val_preds, idx_list=test_idx)
 test_preds = model_nn.predict(X_test_nn)[0]
add_outputs(Y_test_preds_nn, test_preds, X=X_test_full)
 val_losses.append(val_loss)
 print('Val loss: {}'.format(val_loss))
del model_nn
print('----------')
 print('Repeat avg val loss: {}'.format(np.mean(val_losses[repeat*n_folds:(( repeat + 1)*n_folds)])))
 print('Repeat OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_nn /(repeat + 1), remove_unpredicted=True)))
 print('----------')
 Y_train_preds_nn /= n_repeats
Y_test_preds_nn.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats)
 print('==========')
 print('Overall avg val loss: {}'.format(np.mean(val_losses)))
 print('Overall OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_nn, remove_unpredicted=True)) )
class DiseaseDatasetInference(torch.utils.data.Dataset):
def __init__(self, df, transform=None, opt_label=True):
self.df = df.reset_index(drop=True ).copy()
self.transform = transform
self.opt_label = opt_label
if self.opt_label:
self.data = [(row['image_id'], row['label'])for _, row in self.df.iterrows() ]
else:
self.data = [(row['image_id'])for _, row in self.df.iterrows() ]
self.data = np.asarray(self.data)
 def __len__(self):
return len(self.data)
 def __getitem__(self, index):
if self.opt_label:
image_path, label = self.data[index]
else:
image_path = self.data[index]
image = cv2.imread(image_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform is not None:
image = self.transform(image=image)['image']
if self.opt_label == True:
return(image, int(label))
else:
return image<normalization>
LOAD_PRETRAINED_TABNET = True
REUSE_SPLITS_TABNET = True
def get_inference_transforms(img_size = 512):
return Compose([
CenterCrop(img_size, img_size, p=0.5),
Resize(img_size, img_size),
Transpose(p=0.5),
RandomRotate90(p=0.25),
ShiftScaleRotate(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<load_from_csv>
def preprocess_tabnet(X, X_qtrans):
X_pca = pca_qtrans_2.transform(X_qtrans)
 g_std = X[gene_cols].std(axis=1 ).values.reshape(-1, 1)
 g_kurt = X[gene_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 g_skew = X[gene_cols].skew(axis=1 ).values.reshape(-1, 1)
 c_std = X[cell_cols].std(axis=1 ).values.reshape(-1, 1)
 c_kurt = X[cell_cols].kurtosis(axis=1 ).values.reshape(-1, 1)
 c_skew = X[cell_cols].skew(axis=1 ).values.reshape(-1, 1)
 X_std = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 g_cluster = kmeans_g.predict(X_std[gene_cols] ).reshape(-1, 1)
 c_cluster = kmeans_c.predict(X_std[cell_cols] ).reshape(-1, 1)
 return np.concatenate([
X_qtrans, X_pca,
g_std, g_kurt, g_skew,
c_std, c_kurt, c_skew,
g_cluster, c_cluster
], axis=1 )
df = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/sample_submission.csv')
 PATH = '/kaggle/input/cassava-leaf-disease-classification/test_images/'<feature_engineering>
X_train_tabnet = preprocess_tabnet(X_train, X_train_qtrans)
 X_test_tabnet = preprocess_tabnet(X_test, X_test_qtrans )
test_csv = df.copy()
test_csv['image_id'] = PATH + test_csv['image_id']<predict_on_test>
class MultiLabelTabNet(BaseEstimator):
def __init__(self, n_input_features, n_output_labels):
self.n_input_features = n_input_features
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, loss_fn=None, output_bias=None):
if output_bias is not None:
output_bias = keras.initializers.Constant(output_bias)
 clf = StackedTabNetClassifier(
 feature_columns=None,
num_classes=1024,
num_layers=1,
feature_dim=2048,
output_dim=1024,
num_features=self.n_input_features,
num_decision_steps=1,
relaxation_factor=1.5,
sparsity_coefficient=1e-5,
num_groups=-1,
output_activation='relu' 
)
 model = keras.models.Sequential([
keras.layers.InputLayer(self.n_input_features),
keras.layers.Dropout(0.5),
keras.layers.GaussianNoise(0.001),
clf,
keras.layers.Dropout(0.7),
tfa.layers.WeightNormalization(keras.layers.Dense(
 self.n_output_labels,
activation='sigmoid',
bias_initializer=output_bias 
))
])
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(weight_decay=1e-5),
metrics=['binary_crossentropy'] 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=200):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 15.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(loss_fn=loss_fn, output_bias=initial_bias)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss', min_delta=1e-5,
patience=3, verbose=1, factor=0.1, min_lr=1e-7 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=1e-5, patience=10, restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_tabnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=[early_stop_cb, learning_rate_cb, checkpoint_cb],
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_tabnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model()
if pretrained:
base_path = '/kaggle/input/model-tabnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + f'model_tabnet/weights_{repeat}_{fold}.h5' )
def inference(model, data_loader, device):
preds = []
model.eval()
test_tqdm = tq.tqdm(data_loader, total=len(data_loader), desc="Testing", position=0, leave=True)
 for images in test_tqdm:
images = images.to(device)
 preds.extend(model(images ).detach().cpu().numpy())
 return preds<choose_model_class>
n_repeats = 5
n_folds = 7
kfold_seeds = [1217, 231, 902, 3893, 111] if REUSE_SPLITS_TABNET else np.random.randint(42, 1337, n_repeats)
 kfold_seeds = kfold_seeds[:n_repeats]
assert len(kfold_seeds)== n_repeats
print(kfold_seeds )
class CassavaImageClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.head.in_features
self.model.head = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<load_from_csv>
val_losses = []
histories = []
Y_train_preds_tabnet = pd.DataFrame(np.zeros(( X_train.shape[0], num_labels)) , columns=label_cols)
 Y_test_preds_tabnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 set_random_seeds(888)
 for repeat, kf_seed in enumerate(kfold_seeds):
kf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=kf_seed, shuffle=True)
 for fold,(train_idx, test_idx)in enumerate(kf.split(X_train_tabnet, Y_train)) :
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
train_idx, test_idx = complete_train_labels(Y_train, train_idx, test_idx)
 train_idx = tf.random.shuffle(train_idx)
 X_train_fold = tf.gather(X_train_tabnet, train_idx, axis=0)
 Y_train_fold = tf.gather(Y_train, train_idx, axis=0)
 X_valid_fold = tf.gather(X_train_tabnet, test_idx, axis=0)
 Y_valid_fold = tf.gather(Y_train, test_idx, axis=0)
 model_tabnet = MultiLabelTabNet(
 n_input_features=X_train_tabnet.shape[1],
n_output_labels=Y_train.shape[1] 
)
 if LOAD_PRETRAINED_TABNET:
model_tabnet.load_weights(repeat, fold)
 val_loss = model_tabnet.score(X_valid_fold, Y_valid_fold.numpy())
 else:
history = model_tabnet.fit(
 X_train_fold, Y_train_fold,
X_valid_fold, Y_valid_fold,
repeat, fold 
)
 histories.append(history)
 val_loss = min(history.history['val_binary_crossentropy'])
 model_tabnet.load_weights(repeat, fold, pretrained=False)
 val_preds = model_tabnet.predict(X_valid_fold)
 add_outputs(Y_train_preds_tabnet, val_preds, idx_list=test_idx)
 test_preds = model_tabnet.predict(X_test_tabnet)
 add_outputs(Y_test_preds_tabnet, test_preds, X=X_test_full)
 val_losses.append(val_loss)
 print('Val loss: {}'.format(val_loss))
del model_tabnet
print('----------')
 print('Repeat avg val loss: {}'.format(np.mean(val_losses[repeat*n_folds:(( repeat + 1)*n_folds)])))
 print('Repeat OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_tabnet /(repeat + 1), remove_unpredicted=True)))
 print('----------')
 Y_train_preds_tabnet /= n_repeats
Y_test_preds_tabnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats)
 print('==========')
 print('Overall avg val loss: {}'.format(np.mean(val_losses)))
 print('Overall OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_tabnet, remove_unpredicted=True)) )
vit_test_ds = DiseaseDatasetInference(test_csv, transform=get_inference_transforms(img_size=CFG['vit_img_size']), opt_label=False)
 vit_test_loader = torch.utils.data.DataLoader(vit_test_ds, batch_size=CFG['valid_bs'], shuffle=False, pin_memory=False )<feature_engineering>
LOAD_PRETRAINED_RESNET = True
REUSE_SPLITS_RESNET = True
vit_preds = []
for vit_model_name in CFG['vit_models']:
print("Model: ", vit_model_name)
 vit_model = torch.load('/kaggle/input/vit-cassava/'+vit_model_name, map_location=torch.device(CFG['device']))
with torch.no_grad() :
for i in range(CFG['tta']):
vit_preds += [inference(vit_model, vit_test_loader, CFG['device'])]
vit_preds = np.mean(vit_preds, axis=0 )<concatenate>
def preprocess_resnet(X):
X_scaled = pd.DataFrame(std_scaler_2.transform(X[numerical_cols]), columns=numerical_cols)
 X_genes = gene_pca_2.transform(X_scaled[gene_cols])
 X_cells = cell_pca_2.transform(X_scaled[cell_cols])
 return select_features(X), np.concatenate([X_genes, X_cells], axis=1 )
vit_outcomes = pd.concat([df['image_id'], pd.DataFrame(vit_preds)], axis=1 ).sort_values(['image_id'] )<data_type_conversions>
X_train_resnet_1, X_train_resnet_2 = preprocess_resnet(X_train)
 X_test_resnet_1, X_test_resnet_2 = preprocess_resnet(X_test )
final_preds = vit_outcomes.drop('image_id', axis=1 ).to_numpy().argmax(1 )<save_to_csv>
class MultiLabelResNet(BaseEstimator):
def __init__(self, n_input_1, n_input_2, n_output_labels):
self.n_input_1 = n_input_1
self.n_input_2 = n_input_2
self.n_output_labels = n_output_labels
self.estimator_ = None
def _create_model(self, initial_bias, loss_fn=None):
output_bias = keras.initializers.Constant(initial_bias)
 inp_1 = keras.layers.Input(self.n_input_1, name='raw_inp')
 inp_2 = keras.layers.Input(self.n_input_2, name='pca_inp')
 gauss_2 = keras.layers.GaussianNoise(1e-4 )(inp_2)
 head_1 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(keras.layers.Dense(512, activation='elu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='elu'))
], name='head1')
 seq_1 = head_1(inp_1)
 seq_1_inp_concat = keras.layers.Concatenate()([gauss_2, seq_1])
 head_2 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(512, "elu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "relu")) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, "elu"))
], name='head2')
 seq_2 = head_2(seq_1_inp_concat)
 seq_1_seq_2_avg = keras.layers.Average()([seq_1, seq_2])
 head_3 = keras.models.Sequential([
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(256, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(206, activation='relu')) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(keras.layers.Dense(self.n_output_labels, bias_initializer=output_bias, activation='sigmoid'))
], name='head3')
 output = head_3(seq_1_seq_2_avg)
 model = keras.models.Model(inputs=[inp_1, inp_2], outputs=output)
 model.compile(
 loss=keras.losses.BinaryCrossentropy(label_smoothing=2e-3)if loss_fn is None else loss_fn,
optimizer=tfa.optimizers.AdamW(learning_rate=0.02, weight_decay=1e-5),
metrics=['binary_crossentropy'], 
)
 return model
def fit(self, X, Y, X_valid, Y_valid, repeat, fold, max_epochs=100, save_weights=True, initial_bias=initial_bias, transfer_model=None):
Y = tf.cast(Y, tf.float32)
 Y_valid = tf.cast(Y_valid, tf.float32)
 class_weights = balance_class_weights(Y.numpy() , class_weights=(0.5, 2.0))
loss_fn = weighted_binary_crossentropy(class_weights, label_smoothing=2e-3)
 self.estimator_ = self._create_model(initial_bias, loss_fn=loss_fn)
 if transfer_model is not None:
self.copy_weights(transfer_model)
 learning_rate_cb = keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
patience=3, verbose=1, factor=0.1, min_lr=1e-8 
)
 early_stop_cb = keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0.0001, patience=15,
restore_best_weights=False 
)
 checkpoint_cb = keras.callbacks.ModelCheckpoint(
 f'model_resnet/weights_{repeat}_{fold}.h5',
monitor='val_binary_crossentropy',
save_best_only=True,
save_weights_only=True 
)
 callbacks = [early_stop_cb, learning_rate_cb, checkpoint_cb] if save_weights else [early_stop_cb, learning_rate_cb]
history = self.estimator_.fit(
 X, Y,
batch_size=128,
epochs=max_epochs,
validation_data=(X_valid, Y_valid),
callbacks=callbacks,
shuffle=True,
verbose=0 
)
 return history
def predict(self, X):
return self.estimator_.predict(X)
 def score(self, X, Y_true):
Y_preds = self.predict(X)
 return logloss(Y_true, Y_preds)
 def save_weights(self, repeat, fold):
self.estimator_.save_weights('model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def load_weights(self, repeat, fold, pretrained=True):
if self.estimator_ is None:
self.estimator_ = self._create_model(initial_bias)
 if pretrained:
base_path = '/kaggle/input/model-resnet/'
else:
base_path = './'
self.estimator_.load_weights(base_path + 'model_resnet/weights_{}_{}.h5'.format(repeat, fold))
def copy_weights(self, from_model):
for to_layer, from_layer in zip(self.estimator_.layers[:-1], from_model.estimator_.layers[:-1]):
to_layer.set_weights(from_layer.get_weights())
 for to_layer, from_layer in zip(self.estimator_.layers[-1].layers[:-1], from_model.estimator_.layers[-1].layers[:-1]):
to_layer.set_weights(from_layer.get_weights() )
submit = pd.DataFrame({'image_id': df['image_id'].values, 'label': final_preds})
 submit.to_csv('submission.csv', index=False )<load_from_csv>
n_repeats = 7
n_folds = 7
kfold_seeds = [188, 526, 1045, 453, 200, 8684, 87352] if REUSE_SPLITS_RESNET else np.random.randint(42, 1337, n_repeats)
 kfold_seeds = kfold_seeds[:n_repeats]
assert len(kfold_seeds)== n_repeats
print(kfold_seeds )
test_df = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 print(test_df)
 AUTOTUNE = tf.data.experimental.AUTOTUNE
GCS_PATH = '.. /input/cassava-leaf-disease-classification'
BATCH_SIZE = 16*8
IMAGE_SIZE = [512,512]
CLASSES = ["1", "2", "3", "4", "5"]
def dataset_sizes(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return(np.sum(n))
TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH+"/test_tfrecords/ld_test*.tfrec")
 NUM_TEST_IMAGES = dataset_sizes(TEST_FILENAMES )<categorify>
val_losses = []
histories = []
Y_train_preds_resnet = pd.DataFrame(np.zeros(( X_train.shape[0], num_labels)) , columns=label_cols)
 Y_test_preds_resnet = pd.DataFrame(np.zeros(( X_test_full.shape[0], num_labels)) , columns=label_cols)
 set_random_seeds(123)
 for repeat, kf_seed in enumerate(kfold_seeds):
kf = MultilabelStratifiedKFold(n_splits=n_folds, random_state=kf_seed, shuffle=True)
 for fold,(train_idx, test_idx)in enumerate(kf.split(X_train, Y_train)) :
print('** Repeat {}/{}.Fold {}/{}'.format(repeat + 1, n_repeats, fold + 1, n_folds))
K.clear_session()
train_idx, test_idx = complete_train_labels(Y_train, train_idx, test_idx)
 train_idx = tf.random.shuffle(train_idx)
 X_train_1_fold = tf.gather(X_train_resnet_1, train_idx, axis=0)
 X_train_2_fold = tf.gather(X_train_resnet_2, train_idx, axis=0)
 Y_train_fold = tf.gather(Y_train, train_idx, axis=0)
 X_valid_1_fold = tf.gather(X_train_resnet_1, test_idx, axis=0)
 X_valid_2_fold = tf.gather(X_train_resnet_2, test_idx, axis=0)
 Y_valid_fold = tf.gather(Y_train, test_idx, axis=0)
 Y_nonscored_train_fold = tf.gather(Y_train_nonscored, train_idx, axis=0)
 Y_nonscored_valid_fold = tf.gather(Y_train_nonscored, test_idx, axis=0)
 model_resnet = MultiLabelResNet(
 n_input_1=X_train_resnet_1.shape[1],
n_input_2=X_train_resnet_2.shape[1],
n_output_labels=Y_train.shape[1] 
)
 if LOAD_PRETRAINED_RESNET:
model_resnet.load_weights(repeat, fold)
 val_loss = model_resnet.score([X_valid_1_fold, X_valid_2_fold], Y_valid_fold.numpy())
 else:
print('Pretrain model')
 transfer_model = MultiLabelResNet(
 n_input_1=X_train_resnet_1.shape[1],
n_input_2=X_train_resnet_2.shape[1],
n_output_labels=Y_train_nonscored.shape[1] 
)
 transfer_model.fit(
 [X_train_1_fold, X_train_2_fold], Y_nonscored_train_fold,
[X_valid_1_fold, X_valid_2_fold], Y_nonscored_valid_fold,
repeat, fold,
max_epochs=10,
save_weights=False,
initial_bias=nonscored_initial_bias 
)
 print('Train model')
 history = model_resnet.fit(
 [X_train_1_fold, X_train_2_fold], Y_train_fold,
[X_valid_1_fold, X_valid_2_fold], Y_valid_fold,
repeat, fold,
transfer_model=transfer_model 
)
 histories.append(history)
 val_loss = min(history.history['val_binary_crossentropy'])
 model_resnet.load_weights(repeat, fold, pretrained=False)
 del transfer_model
val_preds = model_resnet.predict([X_valid_1_fold, X_valid_2_fold])
 add_outputs(Y_train_preds_resnet, val_preds, idx_list=test_idx)
 test_preds = model_resnet.predict([X_test_resnet_1, X_test_resnet_2])
 add_outputs(Y_test_preds_resnet, test_preds, X=X_test_full)
 val_losses.append(val_loss)
 print('Val loss: {}'.format(val_loss))
del model_resnet
print('----------')
 print('Repeat avg val loss: {}'.format(np.mean(val_losses[repeat*n_folds:(( repeat + 1)*n_folds)])))
 print('Repeat OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_resnet /(repeat + 1), remove_unpredicted=True)))
 print('----------')
 Y_train_preds_resnet /= n_repeats
Y_test_preds_resnet.loc[X_test_full['cp_type'] == 'trt_cp', :] /=(n_folds * n_repeats)
 print('==========')
 print('Overall avg val loss: {}'.format(np.mean(val_losses)))
 print('Overall OOF val loss: {}'.format(logloss(Y_train, Y_train_preds_resnet, remove_unpredicted=True)) )
def to_float32(image, label):
return tf.cast(image, tf.float32), label
def decode_img(img):
img = tf.io.decode_jpeg(img, channels = 3)
 img = tf.cast(img, tf.float32)/255.0
img = tf.reshape(img, [*IMAGE_SIZE, 3])
 return img
def read_tfrecord(example, labeled):
if labeled:
TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 }
else:
TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, TFREC_FORMAT)
 img = decode_img(example["image"])
 if labeled:
label = tf.cast(example["target"], tf.int32)
 return(img, label)
 else:
idNum = example["image_name"]
return(img, idNum)
 def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTOTUNE)
 dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled = labeled), num_parallel_calls = AUTOTUNE)
 return dataset
def get_test_data(ordered = False):
dataset = load_dataset(filenames = TEST_FILENAMES, labeled = False, ordered = ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset<predict_on_test>
test_ds = get_test_data(ordered=True)
 test_ds = test_ds.map(to_float32)
 testing_dataset = get_test_data()
testing_dataset = testing_dataset.unbatch().batch(1)
 print('Computing predictions...')
 test_images_ds = testing_dataset
test_images_ds = test_ds.map(lambda image, idnum: image)
 prob1 = model_15.predict(test_images_ds)
 prob2 = model_17.predict(test_images_ds)
 prob6 = model_21.predict(test_images_ds)
 prob10 = model_28_1.predict(test_images_ds)
 probabilities = 0.25*(prob1+prob2+prob6+prob10)
 predictions = np.argmax(probabilities, axis = -1)
 print(predictions )<save_to_csv>
weights = [0.22, 0.43, 0.35]
print(sum(weights))
assert(abs(sum(weights)- 1)< 1e-6)
 all_train_preds =(
 Y_train_preds_nn * weights[0] +
Y_train_preds_tabnet * weights[1] +
Y_train_preds_resnet * weights[2] 
)
 print(logloss(Y_train, all_train_preds, remove_unpredicted=True))
print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='image_id,label', comments='' )<load_from_csv>
final_preds =(
 Y_test_preds_nn * weights[0] +
Y_test_preds_tabnet * weights[1] +
Y_test_preds_resnet * weights[2] 
)
 final_preds.loc[X_test_full['cp_type'] != 'trt_cp', :] = 0
<prepare_x_and_y><EOS> 
final_preds = np.concatenate([sig_id, final_preds], axis=1)
 final_preds = pd.DataFrame(final_preds, columns=Y_scored.columns)
 final_preds.to_csv('submission.csv', index=False )