Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<train_model>
import pandas as pd
import numpy as np
import cv2
from glob import glob
import sklearn
from sklearn.model_selection import GroupKFold, StratifiedKFold
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
from sklearn.metrics import log_loss
from skimage import io
import os
from datetime import datetime
import time
import random
import torchvision
from torchvision import transforms
from tqdm import tqdm
import matplotlib.pyplot as plt
import torch
from torch import nn
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import warnings
import joblib
from scipy.ndimage.interpolation import zoom
mpnn.fit({'adj_input' : in_edges_train[:train_size], 'nod_input': nodes_train[:train_size]}, y = out_labels[:train_size], batch_size = batch_size, epochs = epochs,
callbacks = [lrate, stop_early], use_multiprocessing = True, initial_epoch = 0, verbose = 2,
validation_data =({'adj_input' : in_edges_train[train_size:], 'nod_input': nodes_train[train_size:]},out_labels[train_size:]))<predict_on_test>
CFG = {
'fold_num': 12,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 384,
'epochs': 120,
'train_bs': 28,
'valid_bs': 32,
'lr': 1e-2,
'num_workers': 5,
'accum_iter': 1,
'verbose_step': 2,
'device': 'cuda:0',
'tta': 10,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}
preds = mpnn.predict({'adj_input' : in_edges_test, 'nod_input': nodes_test} )<save_model>
def all_seed(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
np.save("preds_kernel.npy" , preds )<load_from_csv>
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
train = pd.read_csv(datadir + "champs-scalar-coupling/train.csv")
 test = pd.read_csv(datadir + "champs-scalar-coupling/test.csv")
 test_group = test.groupby('molecule_name')
 scale_min = train['scalar_coupling_constant'].min()
scale_max = train['scalar_coupling_constant'].max()
scale_mid =(scale_max + scale_min)/2
scale_norm = scale_max - scale_mid<concatenate>
img = get_img('.. /input/cassava-leaf-disease-classification/train_images/1000015157.jpg')
 plt.imshow(img)
 plt.show()
def make_outs(test_group, preds):
i = 0
x = np.array([])
 for test_gp, preds in zip(test_group, preds):
if(not i%1000):
print(i)
 gp = test_gp[1]
x = np.append(x,(preds[gp['atom_index_0'].values, gp['atom_index_1'].values] + preds[gp['atom_index_1'].values, gp['atom_index_0'].values])/2.0)
 i = i+1
return x<normalization>
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv' )
out_unscaled = make_outs(test_group, preds )<save_to_csv>
train.label.value_counts()
test['scalar_coupling_constant'] = out_unscaled
test['scalar_coupling_constant'] = test['scalar_coupling_constant']*scale_norm + scale_mid
test[['id','scalar_coupling_constant']].to_csv('submission.csv', index=False )<load_from_csv>
sample_submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 sample_submission.head()
sub1 = pd.read_csv('.. /input/lgb-public-kernels-plus-more-features/sub_lgb_model_individual.csv')
 sub2 = pd.read_csv('.. /input/staking-and-stealing-like-a-molecule/submission.csv')
 sample = pd.read_csv('.. /input/champs-scalar-coupling/sample_submission.csv' )<save_to_csv>
class CassavaDataset(Dataset):
def __init__(self, df, data_root, transforms = None, output_label = True):

 super().__init__()
self.df = df.reset_index(drop = True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):

 return self.df.shape[0]
def __getitem__(self, index: int):

 path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image = img)['image']
if self.output_label:
target = self.df.iloc[index]['label']
if self.output_label:
return img, target
else:
return img
sample['scalar_coupling_constant'] =(0.6*sub2['scalar_coupling_constant'] + 0.4*sub1['scalar_coupling_constant'])
 sample.to_csv('stackers_blend.csv', index=False )<load_from_csv>
HorizontalFlip, VerticalFlip, Transpose, ShiftScaleRotate,
HueSaturationValue,RandomResizedCrop, RandomBrightnessContrast,
Compose, Normalize, Cutout, CoarseDropout, CenterCrop, Resize 
)
pd.options.display.float_format = '{:,}'.format
train_df = pd.read_csv(".. /input/champs-scalar-coupling/train.csv")
 test_df = pd.read_csv(".. /input/champs-scalar-coupling/test.csv")
 structures = pd.read_csv(".. /input/champs-scalar-coupling/structures.csv")
 yuk = pd.read_csv(".. /input/submolecularyukawapotential/structures_yukawa.csv",
usecols=[
'dist_O_0', 'dist_O_1',
'dist_F_0','dist_F_1',
'dist_C_0','dist_C_1', 'dist_C_2', 'dist_C_3',
'dist_H_0','dist_H_1', 'dist_H_2', 'dist_H_3'
] 
)
 yuk['molecule_name'] = structures.molecule_name
yuk['atom_index'] = structures.atom_index
yuk.to_csv("yuk.csv", index=False)
 del yuk
target = train_df['scalar_coupling_constant']
train_size = train_df.shape[0]
train_df = train_df\ 
.merge(structures, left_on=['molecule_name', 'atom_index_0'], right_on=['molecule_name', 'atom_index'], how='left')\ 
.drop(['atom_index'], axis=1)\ 
.merge(structures, left_on=['molecule_name', 'atom_index_1'], right_on=['molecule_name', 'atom_index'], how='left',
suffixes=('_0', '_1')) \ 
.drop(['atom_index'], axis=1)
 test_df = test_df\ 
.merge(structures, left_on=['molecule_name', 'atom_index_0'], right_on=['molecule_name', 'atom_index'], how='left')\ 
.drop(['atom_index'], axis=1)\ 
.merge(structures, left_on=['molecule_name', 'atom_index_1'], right_on=['molecule_name', 'atom_index'], how='left',
suffixes=('_0', '_1')) \ 
.drop(['atom_index'], axis=1)
 del structures
gc.collect()<save_to_csv>
def get_train_transforms() :

 return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
target = train_df.pop('scalar_coupling_constant')
 features = [col for col in train_df.columns if col != 'id' and col != 'scalar_coupling_constant']
all_data = pd.concat([train_df[features], test_df[features]], axis=0, sort=False)
 all_data[['molecule_name', 'atom_index_0', 'atom_index_1']].to_csv("all_data_index.csv", index=False)
 test_df = test_df[['id']]
del train_df
gc.collect()
all_data['distance'] =(
(all_data['x_0'] - all_data['x_1'] ).pow(2)+(all_data['y_0'] - all_data['y_1'] ).pow(2)+(
 all_data['z_0'] - all_data['z_1'] ).pow(2)
 ).pow(0.5 )<load_from_csv>
def get_inference_transforms() :

 return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
structures = pd.read_csv(".. /input/champs-scalar-coupling/structures.csv")
 structures_df = structures.merge(structures, how='left', on= ['molecule_name'], suffixes =('_0', '_1'))
del structures
gc.collect()
structures_df['distance'] =(
(structures_df['x_0'] - structures_df['x_1'] ).pow(2)+(structures_df['y_0'] - structures_df['y_1'] ).pow(2)+(
 structures_df['z_0'] - structures_df['z_1'] ).pow(2)
 ).pow(0.5)
 structures_df = structures_df[structures_df.atom_1.isin(['F', 'O'])]
gc.collect()<data_type_conversions>
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
sys.path.append(package_path)
for atom in ['O', 'F']:
rank_col = '{0}_rank'.format(atom)
 structures_df.loc[structures_df.atom_1 == atom, rank_col] = structures_df[structures_df.atom_1 == atom].groupby(
 ['molecule_name', 'atom_index_0'])['distance'].rank(method='first')
 atom_distance_ranks = structures_df[structures_df[rank_col] <= 2][['molecule_name', 'atom_index_0', 'distance', rank_col]]
atom_distance_ranks[rank_col] = atom_distance_ranks[rank_col].astype(int ).astype(str)
 atom_distance_ranks = atom_distance_ranks.pivot_table(
 index=['molecule_name', 'atom_index_0'], columns=[rank_col], values=['distance'])
 atom_distance_ranks.columns = [atom + '_' + '_'.join(col)for col in atom_distance_ranks.columns]
atom_distance_ranks = atom_distance_ranks.reset_index()
all_data = all_data.merge(atom_distance_ranks, on=['molecule_name', 'atom_index_0'], how='left')
 del atom_distance_ranks
gc.collect()
del structures_df
gc.collect()
for atom in ['H', 'C', 'N']:
rank_col = '{0}_rank'.format(atom)
 all_data.loc[all_data.atom_1 == atom, rank_col] = all_data[all_data.atom_1 == atom].groupby(
 ['molecule_name', 'atom_index_0'])['distance'].rank(method='first')
 atom_distance_ranks = all_data[all_data[rank_col] <= 4]
atom_distance_ranks[rank_col] = atom_distance_ranks[rank_col].astype(int ).astype(str)
 atom_distance_ranks = atom_distance_ranks.pivot_table(
 index=['molecule_name', 'atom_index_0'], columns=[rank_col], values=['distance'])
 atom_distance_ranks.columns = [atom + '_' + '_'.join(col)for col in atom_distance_ranks.columns]
atom_distance_ranks = atom_distance_ranks.reset_index()
all_data = all_data.merge(atom_distance_ranks, on=['molecule_name', 'atom_index_0'], how='left')
 del atom_distance_ranks
gc.collect()
all_data.drop(columns=['C_rank', 'H_rank', 'N_rank'], inplace=True)
 gc.collect()
num_features = 0
prev_features = 0
num_features = all_data.shape[1]
all_data.iloc[:5,prev_features:num_features].head()<merge>
class CassavaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):

 super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
grps = [['molecule_name', 'atom_index_0'], ['molecule_name', 'atom_index_1']]
grp_columns = ['distance']
gc.collect()
for grp in grps:
for column in grp_columns:
stats = all_data.groupby(grp ).agg(
 {column: ["mean", "min", "std", "max"]} 
)
 stats.columns = ['_'.join(col)for col in stats.columns]
stats.columns = ['_'.join(grp)+ '_' + col for col in stats.columns]
all_data = all_data.join(stats, on=grp, how='left')
 del stats
gc.collect()
columns_prefix = '_'.join(grp)+ '_' + column
all_data[columns_prefix + '_mean_ratio'] = all_data[column] / all_data[columns_prefix + '_mean']
all_data[columns_prefix + '_mean_diff'] = all_data[column] - all_data[columns_prefix + '_mean']
all_data[columns_prefix + '_mean_diff_std_ratio'] = all_data[columns_prefix + '_mean_diff'] \
/ all_data[columns_prefix + '_std']
prev_features = num_features
num_features = all_data.shape[1]
all_data.iloc[:5,prev_features:num_features].head()<load_from_csv>
package_path = '.. /input/visiontransformer-pytorch/VisionTransformer-Pytorch-main'
sys.path.append(package_path)
bond_data = pd.read_csv(".. /input/submolecular-bond-data/structures_bond.csv", usecols = [
'molecule_name', 'atom_index', 'n_bonds', 'bond_lengths_mean', 'bond_lengths_std'])
 all_data = all_data.merge(bond_data, left_on=['molecule_name', 'atom_index_0'],
right_on=['molecule_name', 'atom_index'], how='left')\
all_data.drop(columns=['atom_index'], inplace=True)
 all_data = all_data.merge(bond_data, left_on=['molecule_name', 'atom_index_1'],
right_on=['molecule_name', 'atom_index'], how='left', suffixes=('_0', '_1')) \
del bond_data
gc.collect()
all_data.drop(columns=['atom_index'], inplace=True)
 prev_features = num_features
num_features = all_data.shape[1]
all_data.iloc[:5,prev_features:num_features].head()
structures = pd.read_csv(".. /input/champs-scalar-coupling/structures.csv")
 atom_counts = structures.pivot_table(
 index=['molecule_name'], columns=['atom'], values='atom_index', aggfunc='count')\ 
.reset_index()
atom_counts.columns = [col + '_structure_count' if col != 'molecule_name' else col for col in atom_counts.columns ]
del structures
gc.collect()
atoms_unique = all_data.pivot_table(
 index='molecule_name', columns=['atom_1'], values='atom_index_1', aggfunc='nunique')\ 
.reset_index()
atoms_unique.columns = [col + '_atoms_count' if col != 'molecule_name' else col for col in atoms_unique.columns ]
all_data = all_data.merge(atom_counts, on=['molecule_name'], how='left')
 all_data = all_data.merge(atoms_unique, on=['molecule_name'], how='left')
 del atoms_unique, atom_counts<define_search_space>
class EnsembleClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):

 super().__init__()
self.model1 = VisionTransformer.from_name('ViT-B_16', num_classes=5)
 self.model1.load_state_dict(torch.load('.. /input/vit-model-1/ViT-B_16.pt'))
self.model2 = CassavaImgClassifier(model_arch, n_class, pretrained)
 def forward(self, x):
x1 = self.model1(x)
 x2 = self.model2(x)
 return 0.6 * x1 + 0.4 * x2
def load(self, state_dict):
self.model2.load_state_dict(state_dict )
atomic_radius = {'H':0.38, 'C':0.77, 'N':0.75, 'O':0.73, 'F':0.71}
fudge_factor = 0.05
atomic_radius = {k:v + fudge_factor for k,v in atomic_radius.items() }
print(atomic_radius)
 electronegativity = {'H':2.2, 'C':2.55, 'N':3.04, 'O':3.44, 'F':3.98}
for idx in [0,1]:
atom = 'atom_{0}'.format(idx)
 atoms = all_data[atom].values
atoms_EN = [electronegativity[x] for x in atoms]
atoms_rad = [atomic_radius[x] for x in atoms]
all_data['rad_{0}'.format(idx)] = atoms_rad
all_data['EN_{0}'.format(idx)] = atoms_EN
all_data['inverse_square'] = 1 / all_data['distance'].pow(2)
 all_data['inverse_cube'] = 1 / all_data['distance'].pow(3)
 all_data['inv_distance_minus_R'] = 1 /(
 all_data['distance'] - all_data['rad_0'] - all_data['rad_1'] ).pow(2)
 all_data['inv_distance_EN'] = 1 /(
 all_data['distance']*(0.5*all_data['EN_0'] + 0.5*(all_data['EN_1'])) ).pow(2)
 all_data['inv_distance_sum_atom_0'] = all_data.groupby(
 ['molecule_name', 'atom_index_0'])['inv_distance_minus_R'].transform('sum')
 all_data['inv_distance_sum_atom_1'] = all_data.groupby(
 ['molecule_name', 'atom_index_1'])['inv_distance_minus_R'].transform('sum')
 all_data['inv_distance_EN_atom_0'] = all_data.groupby(
 ['molecule_name', 'atom_index_0'])['inv_distance_EN'].transform('sum')
 all_data['inv_distance_EN_atom_1'] = all_data.groupby(
 ['molecule_name', 'atom_index_1'])['inv_distance_EN'].transform('sum')
 all_data['inv_distance_harmonic'] = all_data['inv_distance_sum_atom_0']*all_data['inv_distance_sum_atom_1']\
/(all_data['inv_distance_sum_atom_0'] + all_data['inv_distance_sum_atom_1'])
 all_data['inv_distance_EN_harmonic'] = all_data['inv_distance_EN_atom_0']*all_data['inv_distance_EN_atom_1']\
/(all_data['inv_distance_EN_atom_0'] + all_data['inv_distance_EN_atom_1'])
 all_data.drop(columns=['rad_0', 'EN_0', 'rad_1', 'EN_1'
], inplace=True)
 del atoms_EN, atoms_rad, atoms
gc.collect()
prev_features = num_features
num_features = all_data.shape[1]
all_data.iloc[:5,prev_features:num_features].head()<feature_engineering>
if __name__ == '__main__':
all_seed(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/',
transforms = get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/',
transforms = get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size = CFG['valid_bs'],
num_workers = CFG['num_workers'],
shuffle = False,
pin_memory = False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size = CFG['valid_bs'],
num_workers = CFG['num_workers'],
shuffle = False,
pin_memory = False, 
)
 device = torch.device(CFG['device'])
 model = EnsembleClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
del model
torch.cuda.empty_cache()
all_data['dot_product'] = all_data['x_0']*all_data['x_1'] + all_data['y_0']*all_data['y_1'] + all_data['z_0']*all_data['z_1']
all_data['norm_atom_0'] = all_data['x_0'].pow(2)+ all_data['y_0'].pow(2)+ all_data['z_0'].pow(2)
 all_data['norm_atom_1'] = all_data['x_1'].pow(2)+ all_data['y_1'].pow(2)+ all_data['z_1'].pow(2)
 all_data['cosine'] = all_data['dot_product'] /(all_data['norm_atom_0']*all_data['norm_atom_1'])
 all_data.drop(columns=['dot_product', 'norm_atom_0', 'norm_atom_1'], inplace=True)
 gc.collect()<create_dataframe>
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()
<merge><EOS> 
test.to_csv('submission.csv', index = False )