Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<feature_engineering>
package_paths = [
'.. /input/pytorch-image-models/pytorch-image-models-master',
'.. /input/adamp-optimizer/AdamP-master/adamp'
]
for pth in package_paths:
sys.path.append(pth )
for col in all_data_index.columns:
if col not in all_data.columns:
all_data[col] = all_data_index[col]<set_options>
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
from torch.nn.modules.loss import _WeightedLoss
import torch.nn.functional as F
import timm
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
from scipy.ndimage.interpolation import zoom
del all_data_index
gc.collect()<categorify>
CFG = {
'valid': False,
'fold_num': 5,
'seed': 719,
'model_arch1': 'tf_efficientnet_b4_ns',
'model_arch2': 'tf_efficientnet_b4_ns',
'model_arch3' : 'regnety_040',
'model_arch4' : 'regnety_040',
'model_arch5': 'tf_efficientnet_b4_ns',
'model_arch6': 'regnety_040',
'ckpt_path2': 'regnety4noresetadamp',
'ckpt_path3': 'regnety4nocv',
'weight' : [1/6 for _ in range(6)],
'img_size1': 512,
'img_size2': 512,
'img_size3': 512,
'img_size4': 512,
'epochs': 10,
'tta_num' : 3,
'train_bs': 64,
'valid_bs': 64,
'T_0': 10,
'lr': 1e-4,
'min_lr': 1e-6,
'weight_decay':1e-6,
'num_workers': 4,
'accum_iter': 2,
'verbose_step': 1,
'device': 'cuda:0',
'used_epochs': [5, 6, 7, 8, 9]
}
types = all_data.dtypes
cat_columns = [t[0] for t in types.iteritems() if(( t[1] not in ['int64', 'float64'])) ]
print('Label encoding categorical columns:', cat_columns)
 encoders = {}
for col in cat_columns:
lbl = preprocessing.LabelEncoder()
all_data[col] = lbl.fit_transform(all_data[col].astype(str))
encoders[col] = lbl<compute_train_metric>
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()
def group_mean_log_mae(y_true, y_pred, groups, floor=1e-9):
maes =(y_true-y_pred ).abs().groupby(groups ).mean()
print(( y_true-y_pred ).abs().groupby(groups))
return np.log(maes.map(lambda x: max(x, floor)) ).mean()<split>
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()
features = [col for col in all_data.columns if col != 'fc' and col != 'id' and col not in ['atom_index_0', 'atom_index_1', 'scalar_coupling_constant', 'molecule_name']]
def lgbm_model_oof(train_X, train_y, test_X, n_folds, lgbm_params):
folds = KFold(n_splits=n_folds, shuffle=False, random_state=7557)
 oof = np.zeros(len(train_X))
predictions = np.zeros(len(test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_X, train_y)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(
 train_X.iloc[trn_idx][features], label=train_y.iloc[trn_idx], free_raw_data=True)
 val_data = lgb.Dataset(
 train_X.iloc[val_idx][features], label=train_y.iloc[val_idx], free_raw_data=True)
 num_round = 1000
clf = lgb.train(lgbm_params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000,
early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train_X.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_X, num_iteration=clf.best_iteration)/ folds.n_splits
del trn_data, val_data
gc.collect()
return predictions, oof, feature_importance_df<init_hyperparams>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
params_dict = {}
params_dict['1JHC'] = param = {
'num_leaves': 150,
'objective': 'huber',
'colsample_bytree': 0.5,
'subsample': 0.9,
'eta': 0.05,
'n_estimators': 20000,
'metric': 'mae'
}
params_dict['2JHC'] = param = {
'num_leaves': 200,
'objective': 'huber',
'colsample_bytree': 0.5,
'subsample': 0.9,
'eta': 0.05,
'n_estimators': 10000,
'metric': 'mae'
}
params_dict['1JHN'] = {
'num_leaves': 200,
'objective': 'huber',
'colsample_bytree': 0.5,
'subsample': 0.9,
'eta': 0.05,
'n_estimators': 10000,
'metric': 'mae'
}<categorify>
def rand_bbox(size, lam):
W = size[0]
H = size[1]
cut_rat = np.sqrt(1.- lam)
 cut_w = np.int(W * cut_rat)
 cut_h = np.int(H * cut_rat)
 cx = np.random.randint(W)
 cy = np.random.randint(H)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2
class CassavaDataset(Dataset):
def __init__(self, df, data_root,
transforms=None,
output_label=True, 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
if self.output_label == True:
self.labels = self.df['label'].values
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.labels[index]
img = get_img("{}/{}".format(self.data_root, self.df.loc[index]['image_id']))
if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img
predictions_type = np.zeros(len(test_df))
feature_importance_dfs = {}
oof_type = np.zeros(train_size)
 for typ in all_data.type.unique() :
gc.collect()
print('Predicting type:', typ)
 print(encoders['type'].classes_[typ])
 if encoders['type'].classes_[typ] in params_dict.keys() :
param = params_dict[encoders['type'].classes_[typ]]
else:
param = {
'num_leaves': 200,
'min_child_samples': 79,
'objective': 'huber',
'colsample_bytree': 0.75,
'subsample': 0.8,
'eta': 0.3,
'n_estimators': 10000,
'reg_alpha': 0.1,
'reg_lambda': 0.3,
'metric': 'mae'
}
print(param)
 type_df_train = all_data[:train_size][all_data.type == typ]
type_df_train_indices = type_df_train[type_df_train.type == typ].index
type_target = target.iloc[type_df_train_indices]
type_df_test = all_data[train_size:].reset_index().drop(columns=['index'])
 type_df_test = type_df_test[type_df_test.type == typ]
type_df_test_indices = type_df_test[type_df_test.type == typ].index
predictions, oof, type_feature_importance_df = lgbm_model_oof(
 type_df_train[features], type_target, type_df_test[features], 3, param)
 predictions_type[type_df_test_indices] += predictions
oof_type[type_df_train_indices] += oof
feature_importance_dfs[typ] = type_feature_importance_df
print(
 type_feature_importance_df.groupby(['Feature'])[['importance']].mean().sort_values(
 "importance", ascending=False ).head(10)
)
 del type_df_train, type_target, type_df_test
gc.collect()<compute_test_metric>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize, Rotate,
ShiftScaleRotate, CenterCrop, Resize, Rotate, RandomShadow, RandomSizedBBoxSafeCrop,
ChannelShuffle, MotionBlur 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size1'], CFG['img_size1']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.) ,
Resize(CFG['img_size1'], CFG['img_size1']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms1() :
return Compose([
OneOf([
Resize(CFG['img_size1'], CFG['img_size1'], p=1.) ,
CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.) ,
RandomResizedCrop(CFG['img_size1'], CFG['img_size1'], p=1.)
], p=1.) ,
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
Resize(CFG['img_size1'], CFG['img_size1']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms2() :
return Compose([
OneOf([
Resize(CFG['img_size2'], CFG['img_size2'], p=1.) ,
CenterCrop(CFG['img_size2'], CFG['img_size2'], p=1.) ,
RandomResizedCrop(CFG['img_size2'], CFG['img_size2'], p=1.)
], p=1.) ,
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
Resize(CFG['img_size2'], CFG['img_size2']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
types = encoders['type'].inverse_transform(all_data[:train_size]['type'])
 maes =(target-oof_type ).abs().groupby(types ).mean()
floor=1e-9
print('Log of Mean absolute errors on VAL:')
 print(np.log(maes.map(lambda x: max(x, floor))))
print('')
 print('Overal Log Mae:')
 print(np.log(maes.map(lambda x: max(x, floor)) ).mean() )<save_to_csv>
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 if model_arch == 'regnety_040':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(1088, n_class)
)
 elif model_arch == 'regnety_320':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(3712, n_class)
)
 elif model_arch == 'regnety_080':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(2016, n_class)
)
 elif model_arch == 'regnety_160':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(3024, n_class)
)
 else:
n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
test_df['scalar_coupling_constant'] = predictions_type
test_df[['id', 'scalar_coupling_constant']].to_csv('submission.csv', index=False )<save_to_csv>
class CassvaImgClassifier_ViT(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 self.model.head = nn.Linear(self.model.head.in_features, n_class)
 for module in self.model.modules() :
if isinstance(module, nn.BatchNorm2d):
if hasattr(module, 'weight'):
module.weight.requires_grad_(False)
 if hasattr(module, 'bias'):
module.bias.requires_grad_(False)
 def forward(self, x):
x = self.model(x)
 return x
sub1['scalar_coupling_constant'] = 0.25*sub1['scalar_coupling_constant'] + 0.2*sub2['scalar_coupling_constant'] + 0.3*sub3['scalar_coupling_constant'] + 0.25*sub4['scalar_coupling_constant']
sub1.to_csv('submission.csv', index=False )<import_modules>
def prepare_dataloader(df, trn_idx, val_idx, data_root='.. /input/cassava-leaf-disease-classification/train_images/'):
train_ = df.loc[trn_idx,:].reset_index(drop=True)
 valid_ = df.loc[val_idx,:].reset_index(drop=True)
 train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms() , output_label=True)
 valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms() , output_label=True)
 train_loader = torch.utils.data.DataLoader(
 train_ds,
batch_size=CFG['train_bs'],
pin_memory=False,
drop_last=False,
shuffle=True,
num_workers=CFG['num_workers'], 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 return train_loader, val_loader
def train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):
model.train()
t = time.time()
running_loss = None
for step,(imgs, image_labels)in enumerate(train_loader):
imgs = imgs.to(device ).float()
image_labels = image_labels.to(device ).long()
with autocast() :
image_preds = model(imgs)
 loss = loss_fn(image_preds, image_labels)
 scaler.scale(loss ).backward()
if running_loss is None:
running_loss = loss.item()
else:
running_loss = running_loss *.99 + loss.item() *.01
if(( step + 1)% CFG['accum_iter'] == 0)or(( step + 1)== len(train_loader)) :
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
if scheduler is not None and schd_batch_update:
scheduler.step()
if scheduler is not None and not schd_batch_update:
scheduler.step()
def valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):
model.eval()
t = time.time()
loss_sum = 0
sample_num = 0
image_preds_all = []
image_targets_all = []
for step,(imgs, image_labels)in enumerate(val_loader):
imgs = imgs.to(device ).float()
image_labels = image_labels.to(device ).long()
image_preds = model(imgs)
 image_preds_all += [torch.argmax(image_preds, 1 ).detach().cpu().numpy() ]
image_targets_all += [image_labels.detach().cpu().numpy() ]
loss = loss_fn(image_preds, image_labels)
 loss_sum += loss.item() *image_labels.shape[0]
sample_num += image_labels.shape[0]
image_preds_all = np.concatenate(image_preds_all)
 image_targets_all = np.concatenate(image_targets_all)
 print('epoch = {}'.format(epoch+1), 'validation multi-class accuracy = {:.4f}'.format(( image_preds_all==image_targets_all ).mean()))
if scheduler is not None:
if schd_loss_update:
scheduler.step(loss_sum/sample_num)
 else:
scheduler.step()
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
with torch.no_grad() :
for step,(imgs)in enumerate(data_loader):
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [torch.softmax(image_preds, 1 ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all
import shutil
from allennlp.common.params import Params
from allennlp.common.util import prepare_environment, dump_metrics
from allennlp.data.iterators import BasicIterator, BucketIterator
from allennlp.data.token_indexers import SingleIdTokenIndexer
from allennlp.data.token_indexers import PretrainedBertIndexer
from allennlp.data.tokenizers import WordTokenizer
from allennlp.data.tokenizers.word_splitter import BertBasicWordSplitter
from allennlp.data.vocabulary import Vocabulary
from allennlp.modules.seq2seq_encoders import PytorchSeq2SeqWrapper
from allennlp.modules.seq2vec_encoders import BagOfEmbeddingsEncoder
from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.training import Trainer
from pytorch_pretrained_bert import convert_tf_checkpoint_to_pytorch
from pytorch_pretrained_bert.optimization import BertAdam
from sklearn.model_selection import train_test_split
import torch
import torch.optim as optim
from swemencoder import SWEMEncoder
from toxiccommentclassificationreader import ToxicCommentClassificationReader
from toxiccommentpredictor import ToxicCommentPredictor
from toxicbaseclassifier import ToxicBaseClassifier
from toxicbertclassifier import ToxicBertClassifier<set_options>
def freeze_batchnorm_stats(net):
try:
for m in net.modules() :
if isinstance(m,nn.BatchNorm2d)or isinstance(m,nn.LayerNorm):
m.eval()
except ValuError:
print('error with batchnorm2d or layernorm')
 return
def unfreeze_batchnorm_stats(net):
try:
for m in net.modules() :
if isinstance(m,nn.BatchNorm2d)or isinstance(m,nn.LayerNorm):
m.train()
except ValuError:
print('error with batchnorm2d or layernorm')
 return
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss<init_hyperparams>
class LabelSmoothingCrossEntropy(nn.Module):

 def __init__(self, smoothing=0.1):

 super(LabelSmoothingCrossEntropy, self ).__init__()
assert smoothing < 1.0
self.smoothing = smoothing
self.confidence = 1.- smoothing
def forward(self, x, target):
logprobs = torch.nn.functional.log_softmax(x, dim=-1)
 nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))
nll_loss = nll_loss.squeeze(1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
class Lookahead(Optimizer):
def __init__(self, base_optimizer,alpha=0.5, k=6):
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
 if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
 self.optimizer = base_optimizer
self.param_groups = self.optimizer.param_groups
self.alpha = alpha
self.k = k
for group in self.param_groups:
group["step_counter"] = 0
self.slow_weights = [[p.clone().detach() for p in group['params']]
for group in self.param_groups]
for w in it.chain(*self.slow_weights):
w.requires_grad = False
self.state = base_optimizer.state
def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
loss = self.optimizer.step()
for group,slow_weights in zip(self.param_groups,self.slow_weights):
group['step_counter'] += 1
if group['step_counter'] % self.k != 0:
continue
for p,q in zip(group['params'],slow_weights):
if p.grad is None:
continue
q.data.add_(self.alpha,p.data - q.data)
 p.data.copy_(q.data)
 self.state = self.optimizer.state
return loss<load_from_csv>
if __name__ == '__main__':
seed_everything(CFG['seed'])
 oof_preds = np.zeros(len(train))
print('Model 1 Start')
 sub1 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model1 = CassvaImgClassifier(CFG['model_arch1'], train.label.nunique() ).to(device)
 tst_preds = []
model1.load_state_dict(torch.load('.. /input/leaf-weight-v9-2/model9_2/swa_{}_fold_{}_{}'.format(CFG['model_arch1'], fold, '9')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model1, tst_loader2, device)]
sub1 += [np.mean(tst_preds, axis=0)]
del model1;
torch.cuda.empty_cache()
print('Model 2 Start')
 sub2 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model2 = CassvaImgClassifier(CFG['model_arch2'], train.label.nunique() ).to(device)
 tst_preds = []
model2.load_state_dict(torch.load('.. /input/905-training-efficientnetb4-merged-bs32/swa_{}_fold_{}_{}'.format(CFG['model_arch2'], fold, '9')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model2, tst_loader2, device)]
sub2 += [np.mean(tst_preds, axis=0)]
del model2;
torch.cuda.empty_cache()
print('Model 3 Start')
 sub3 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model3 = CassvaImgClassifier(CFG['model_arch3'], train.label.nunique() ).to(device)
 tst_preds = []
model3.load_state_dict(torch.load('.. /input/regnety4noresetadamp/swa_{}_fold_{}_{}'.format(CFG['model_arch3'], fold, '19')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model3, tst_loader2, device)]
sub3 += [np.mean(tst_preds, axis=0)]
del model3;
torch.cuda.empty_cache()
print('Model 4 Start')
 sub4 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model4 = CassvaImgClassifier(CFG['model_arch4'], train.label.nunique() ).to(device)
 tst_preds = []
model4.load_state_dict(torch.load('.. /input/0214v1-hwkim-regnet-40-reset-swalr-swastep-ep24/swa_{}_fold_{}_{}'.format(CFG['model_arch4'], fold, '23')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model4, tst_loader2, device)]
sub4 += [np.mean(tst_preds, axis=0)]
del model4;
torch.cuda.empty_cache()
print('Model 5 Start')
 sub5 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model5 = CassvaImgClassifier(CFG['model_arch5'], train.label.nunique() ).to(device)
 tst_preds = []
model5.load_state_dict(torch.load('.. /input/905-training-efficientnetb4-seed720/swa_{}_fold_{}_{}'.format(CFG['model_arch5'], fold, '9')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model5, tst_loader2, device)]
sub5 += [np.mean(tst_preds, axis=0)]
del model5;
torch.cuda.empty_cache()
print('Model 6 Start')
 sub6 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model6 = CassvaImgClassifier(CFG['model_arch6'], train.label.nunique() ).to(device)
 tst_preds = []
model6.load_state_dict(torch.load('.. /input/reg-distill/swa_{}_fold_{}_{}'.format(CFG['model_arch6'], fold, '19')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model6, tst_loader2, device)]
sub6 += [np.mean(tst_preds, axis=0)]
del model6;
torch.cuda.empty_cache()
sub1 = [e * CFG['weight'][0] for e in sub1]
sub2 = [e * CFG['weight'][1] for e in sub2]
sub3 = [e * CFG['weight'][2] for e in sub3]
sub4 = [e * CFG['weight'][3] for e in sub4]
sub5 = [e * CFG['weight'][4] for e in sub5]
sub6 = [e * CFG['weight'][5] for e in sub6]
sub = [e1 + e2 + e3 + e4 + e5 + e6 for(e1, e2, e3, e4, e5, e6)in zip(sub1,sub2,sub3, sub4, sub5, sub6)]
reader = ToxicCommentClassificationReader(token_indexers={
"tokens1": SingleIdTokenIndexer() ,
"tokens2": SingleIdTokenIndexer() ,
})
 all_dataset = reader.read('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 train_dataset, validation_dataset = train_test_split(all_dataset, test_size=0.2, random_state=11)
 vocab = Vocabulary.from_instances(train_dataset, min_count={'tokens': 3})
 iterator = BucketIterator(
 batch_size=512,
sorting_keys=[("tokens", "num_tokens")], 
)
 iterator.index_with(vocab )<categorify>
test['label'] = np.argmax(np.mean(sub, axis=0), axis=1)
 test.head()
glove_params = Params({
'pretrained_file': '.. /input/glove-stanford/glove.twitter.27B.200d.txt',
'embedding_dim': 200,
'trainable': False
})
 fasttext_params = Params({
'pretrained_file': '.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec',
'embedding_dim': 300,
'trainable': False
})
 glove_embedding = Embedding.from_params(vocab, glove_params)
 fasttext_embedding = Embedding.from_params(vocab, fasttext_params)
 word_embeddings = BasicTextFieldEmbedder({"tokens1": glove_embedding, "tokens2": fasttext_embedding})
 seq2vec_encoder = BagOfEmbeddingsEncoder(embedding_dim=word_embeddings.get_output_dim() )<choose_model_class>
test.to_csv('submission.csv', index=False )