Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<rename_columns>
! pip install.. /input/mlcollection/ml_collections-0.1.0-py3-none-any.whl
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )<feature_engineering>
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
from fastai.vision.all import *
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X<feature_engineering>
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'resnext101_ibn_a',
'model_arch_eff':'tf_efficientnet_b4_ns',
'img_size': 512,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda' if torch.cuda.is_available() else 'cpu',
'tta': 4,
}
ckpt_paths = ['.. /input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_4_0.89299',
'.. /input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_6_0.89065',
'.. /input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_8_0.89533',
'.. /input/tf-efficientnet-b4-ns-fold-0-8-089533/tf_efficientnet_b4_ns_fold_0_9_0.88995'
]
ckpt_paths = [
'.. /input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_6_0.88435',
'.. /input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_7_0.89299',
'.. /input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_8_0.88995',
'.. /input/resnext101-ibn-a-01-22-10-05-bit-ls015-896/resnext101_ibn_a_fold_0_9_0.89603']
ckpt_weights = [1,1,1,1]
ckpt_dir = '.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds'
ckpt_paths = [os.path.join(ckpt_dir, sub_path)for sub_path in os.listdir(ckpt_dir)if not ckpt_dir.endswith('.txt')]
ckpt_paths = ['.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_0_9_0.89439',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_0_19_0.89393',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_1_9_0.89603',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_1_8_0.89486',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_2_9_0.88759',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_2_17_0.88619',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_3_16_0.89928',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_3_19_0.89904',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_4_18_0.89857',
'.. /input/resnext101-ibn-a-01-26-01-27-bit-ema-5folds/resnext101_ibn_a_fold_4_9_0.89787',
]
ckpt_paths = [
'.. /input/vit-convert-tta-seed719/vit__1920-fold0_checkpoint-17.pth',
'.. /input/vit-convert-tta-seed719/vit__1920-fold1_checkpoint-14.pth',
'.. /input/vit-convert-tta-seed719/vit__1920-fold2_checkpoint-16.pth',
'.. /input/vit-convert-tta-seed719/vit__1920-fold3_checkpoint-8.pth',
'.. /input/vit-convert-tta-seed719/vit__1920-fold4_checkpoint-9.pth',
'.. /input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_0_11_0.89603',
'.. /input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_1_10_0.89369',
'.. /input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_2_13_0.88899',
'.. /input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_3_12_0.89834',
'.. /input/alldata-ema-resnext101-ibn-a/resnext101_ibn_a_fold_4_11_0.89741',
'.. /input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_0_12_0.89369',
'.. /input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_1_12_0.89416',
'.. /input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_2_8_0.89203',
'.. /input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_3_12_0.90115',
'.. /input/alldata-ema-resnext101-32x4d-swsl-adam-02-05-17-28/resnext101_32x4d_swsl_fold_4_14_0.89530',
]
model_names = [
'vit',
'vit',
'vit',
'vit',
'vit',
'resnext101_ibn_a',
'resnext101_ibn_a',
'resnext101_ibn_a',
'resnext101_ibn_a',
'resnext101_ibn_a',
'resnext101',
'resnext101',
'resnext101',
'resnext101',
'resnext101',
]
assert len(model_names)== len(ckpt_paths)
 print(len(ckpt_paths))
ckpt_weights = [1 for i in range(len(ckpt_paths)) ]
print(ckpt_paths)
 model_wts={}
model_wts['vit']=0.6582084173812914
model_wts['resnext101']=0.16946139705016672
model_wts['resnext101_ibn_a']=0.6465801475470692
create_time_features(X_train)
 create_time_features(X_test )<drop_column>
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()
X_train.drop("date", axis=1, inplace=True)
 X_test.drop("date", axis=1, inplace=True )<categorify>
train.label.value_counts()
X_train = pd.concat([X_train,pd.get_dummies(X_train['Province/State'], prefix='ps')],axis=1)
 X_train.drop(['Province/State'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Province/State'], prefix='ps')],axis=1)
 X_test.drop(['Province/State'],axis=1, inplace=True)
 X_train = pd.concat([X_train,pd.get_dummies(X_train['Country/Region'], prefix='cr')],axis=1)
 X_train.drop(['Country/Region'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Country/Region'], prefix='cr')],axis=1)
 X_test.drop(['Country/Region'],axis=1, inplace=True )<prepare_x_and_y>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()
Y1= train["ConfirmedCases"]<prepare_x_and_y>
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img
Y2 = train["Fatalities"]<train_model>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.2),
RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
model = RandomForestClassifier(bootstrap=True,max_depth=None, max_features='auto', max_leaf_nodes=None,
n_estimators=150, random_state=None, n_jobs=1, verbose=0)
 model.fit(X_train,Y1)
 pred1 = model.predict(X_test)
 pred1 = pd.DataFrame(pred1)
 pred1.columns = ["ConfirmedCases_prediction"]<train_model>
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
model = RandomForestClassifier(bootstrap=True,max_depth=None, max_features='auto', max_leaf_nodes=None,
n_estimators=150, random_state=None, n_jobs=1, verbose=0)
 model.fit(X_train,Y2)
 pred2 = model.predict(X_test)
 pred2 = pd.DataFrame(pred2)
 pred2.columns = ["Death_prediction"]<load_from_csv>
class IBNResnextCassava(nn.Module):
def __init__(self, arch='resnext101_ibn_a', n_class=5, pre=False):
super().__init__()
m = resnext101_ibn_a()
self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(2048, n_class)
)
 def forward(self, x):
x = self.enc(x)
 x = self.head(x)
 return x
data_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/submission.csv")
 data_submission.columns
sub_new = data_submission[["ForecastId"]]<concatenate>
class MishFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x)
 return x * torch.tanh(F.softplus(x))
@staticmethod
def backward(ctx, grad_output):
x = ctx.saved_variables[0]
sigmoid = torch.sigmoid(x)
 tanh_sp = torch.tanh(F.softplus(x))
return grad_output *(tanh_sp + x * sigmoid *(1 - tanh_sp * tanh_sp))
class Mish(nn.Module):
def forward(self, x):
return MishFunction.apply(x)
 def to_Mish(model):
for child_name, child in model.named_children() :
if isinstance(child, nn.ReLU):
setattr(model, child_name, Mish())
 else:
to_Mish(child)
 def _resnext(url, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 return model
concat = pd.concat([pred1,pred2,sub_new],axis=1)
 concat.head()
concat.columns = ['ConfirmedCases', 'Fatalities', 'ForecastId']
concat = concat[['ForecastId','ConfirmedCases', 'Fatalities']]<data_type_conversions>
semi_weakly_supervised_model_urls = {
'resnet18': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnet18-118f1556.pth',
'resnet50': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnet50-16a12f1b.pth',
'resnext50_32x4d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext50_32x4-72679e44.pth',
'resnext101_32x4d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x4-3f87e46b.pth',
'resnext101_32x8d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x8-b4712904.pth',
'resnext101_32x16d':'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x16-f3559a9c.pth',
}
class ResnextCassava(nn.Module):
def __init__(self, arch='resnext101_32x4d', n_class=5, pretrained=False):
super().__init__()
m = _resnext(semi_weakly_supervised_model_urls[arch], Bottleneck, [3, 4, 23, 3], False,
progress=False,groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.max_pool = nn.AdaptiveMaxPool2d(1)
 self.logit = nn.Linear(2048,n_class)
 def forward(self, x):
x = self.enc(x)
 x =(self.avg_pool(x)+ self.max_pool(x)).squeeze(-1 ).squeeze(-1)
 x = self.logit(x)
 return x
concat["ConfirmedCases"] = concat["ConfirmedCases"].astype(int)
 concat["Fatalities"] = concat["Fatalities"].astype(int )<save_to_csv>
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)

def forward(self, x):
x = self.model(x)
 return x
class IBN_CUSTOM1(nn.Module):
def __init__(self, arch='resnext101_ibn_a', n_class=5, pretrained=True):
super().__init__()
m = se_resnet101_ibn_a(pretrained=pretrained)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 self.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(2048, n_class)
)
 def forward(self, x):
x = self.enc(x)
 x = self.head(x)
 return x
def get_model(model_name, device='cuda'):
if 'efficientnet' in model_name:
model = CassvaImgClassifier(model_name, 5, pretrained=False ).to(device)
 elif model_name == 'ibn50a':
model = resnet50_ibn_a(pretrained=False ).to(device)
 model.avgpool = nn.AdaptiveAvgPool2d(1)
 model.fc = nn.Linear(2048, train.label.nunique())
 elif model_name == 'ibnse50a':
model = se_resnet50_ibn_a(pretrained=False ).to(device)
 model.avgpool = nn.AdaptiveAvgPool2d(1)
 model.fc = nn.Linear(2048, train.label.nunique())
 elif model_name == 'resnext101_ibn_a':
model = IBNResnextCassava(arch=model_name)
 elif model_name == 'se50_softlabel':
model = se_resnext50_32x4d(pretrained=None ).to(device)
 elif model_name == 'resnext101':
model = ResnextCassava(pretrained=False ).to(device)
 elif model_name == 'b4':
b4_model = timm.create_model('tf_efficientnet_b4_ns', pretrained=False)
 model=timm_class(b4_model,n=5 ).to(device)
 elif model_name == 'vit':
CONFIGS = {
'ViT-B_16': configs.get_b16_config() ,
'ViT-B_32': configs.get_b32_config() ,
'ViT-L_16': configs.get_l16_config() ,
'ViT-L_32': configs.get_l32_config() ,
'ViT-H_14': configs.get_h14_config() ,
'testing': configs.get_testing() ,
}
config = CONFIGS["ViT-B_16"]
model = VisionTransformer(config, CFG['img_size'], zero_head=True, num_classes=5 ).to(device)
 elif model_name == 'se_resnet101_ibn_a':
model = IBN_CUSTOM1(pretrained=False ).to(device)
 model = model.to(device)
 return model
concat.to_csv("submission.csv",index=False )<load_from_csv>
! pip install.. /input/mlcollection/ml_collections-0.1.0-py3-none-any.whl
train_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv")
 submission_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/submission.csv")
 test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv' )<data_type_conversions>
class AdaptiveConcatPool2d(nn.Module):
"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`"
def __init__(self, size=None):
super().__init__()
self.size = size or 1
self.ap = nn.AdaptiveAvgPool2d(self.size)
 self.mp = nn.AdaptiveMaxPool2d(self.size)
 def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1 )
train_df["Date"] = train_df["Date"].apply(lambda x: datetime.strptime(x,'%Y-%m-%d'))
train_df["Date"] = train_df["Date"].apply(lambda x: x.timestamp())
 train_df["Date"] = train_df["Date"].astype(int )<count_missing_values>
if __name__ == '__main__':
VALID = False
test_num = len(os.listdir('.. /input/cassava-leaf-disease-classification/test_images'))
print('test_num:', test_num)
 seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 val_preds = []
tst_preds = []
for i in range(len(ckpt_paths)) :
ckpt_path = ckpt_paths[i]
model = get_model(model_names[i], device)
 if model_names[i] == 'se_resnet101_ibn_a':
model.load_state_dict(torch.load(ckpt_path)['state_dict'])
 else:
model.load_state_dict(torch.load(ckpt_path,map_location=device))
with torch.no_grad() :
for _ in range(CFG['tta']):
if VALID:
if test_num == 1:
val_preds += [ckpt_weights[i]/sum(ckpt_weights)/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [
inference_one_epoch(model, tst_loader, device)*model_wts[model_names[i]]]
del model
torch.cuda.empty_cache()
if VALID:
if test_num == 1:
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)*3
if VALID:
if test_num == 1:
print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
cm = confusion_matrix(valid_.label.values, np.argmax(val_preds, axis=1), labels=[0, 1, 2, 3, 4])
 print('cm')
 print(cm)
 sns.set()
f, ax=plt.subplots()
sns.heatmap(cm,annot=True,ax=ax)
 ax.set_title('confusion matrix')
 ax.set_xlabel('predict')
 ax.set_ylabel('true')
train_df.isnull().sum()<drop_column>
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()
<count_missing_values><EOS> 
test.to_csv('submission.csv', index=False )