Kaggle competition: Cassava Leaf Disease Classification
train = pd.read_csv(f'{PATH}train.csv')
 test = pd.read_csv(f'{PATH}test.csv' )<drop_column>
!nvidia-smi
%%time
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id', 'bin_0'], axis=1, inplace=True)
 test.drop(['id','bin_0'], axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<categorify>
!pip install.. /input/timmmodels/dist/timm-0.3.4.tar
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4
if not os.path.exists('/root/.cache/torch/hub/checkpoints/'):
os.makedirs('/root/.cache/torch/hub/checkpoints/')
 !cp.. /input/timm-pretrained-efficientnet/efficientnet/tf_efficientnet_b4_ns-d6313a46.pth /root/.cache/torch/hub/checkpoints/
%%time
traintest = pd.concat([train, test])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 train_ohe = dummies.iloc[:train.shape[0], :]
test_ohe = dummies.iloc[train.shape[0]:, :]
print(train_ohe.shape)
 print(test_ohe.shape )<categorify>
import cv2
import copy
import time
import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
import torchvision
from torchvision import models
from torch.utils.data import DataLoader, Dataset
from torch.cuda import amp
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.utils import class_weight
from tqdm.notebook import tqdm
from collections import defaultdict
import albumentations as A
from albumentations.pytorch import ToTensorV2
import timm
import pretrainedmodels
%%time
train_ohe = train_ohe.sparse.to_coo().tocsr()
test_ohe = test_ohe.sparse.to_coo().tocsr()<train_model>
ROOT_DIR = ".. /input/cassava-leaf-disease-classification"
TRAIN_DIR = ".. /input/cassava-leaf-disease-classification/train_images"
TEST_DIR = ".. /input/cassava-leaf-disease-classification/test_images"
print('
print('StratifiedKFold training...')
 folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)
 model = LogisticRegression(solver='lbfgs', max_iter=200, C=0.095)
 score = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_ohe, target, groups=target)) :
print('Fold:',fold_+1)
 tr_x, tr_y = train_ohe[trn_idx,:], target[trn_idx]
vl_x, v_y = train_ohe[val_idx,:], target[val_idx]
%time model.fit(tr_x, tr_y)
 y_pred = model.predict(vl_x)
 score.append(roc_auc_score(v_y, y_pred))
print(f'AUC score : {roc_auc_score(v_y, y_pred)}')
 print('Average AUC score', np.mean(score))
print('<train_model>
class CFG:
model_name = 'tf_efficientnet_b4_ns'
img_size = 512
scheduler = 'CosineAnnealingWarmRestarts'
T_max = 10
T_0 = 10
lr = 1e-4
min_lr = 1e-6
batch_size = 16
weight_decay = 1e-6
seed = 42
num_classes = 5
num_epochs = 14
n_fold = 5
smoothing = 0.2
t1 = 0.6
t2 = 1.6
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 savemodel = True
savemodelpath = '/kaggle/working/bitemp-32.pth'
dosubmission = True
crop_size = 512
crop_p = 1.0
%time model.fit(train_ohe, target )<predict_on_test>
def set_seed(seed = 42):

 np.random.seed(seed)
 random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
os.environ['PYTHONHASHSEED'] = str(seed)
 set_seed(CFG.seed )
pred_test = model.predict_proba(test_ohe)[:,0]<save_to_csv>
df = pd.read_csv(f"{ROOT_DIR}/train.csv" )
sub = pd.read_csv(f'{PATH}sample_submission.csv')
 sub['target'] = pred_test
sub.to_csv('log_reg_0.1.csv', index=None, header=True )<load_from_csv>
skf = StratifiedKFold(n_splits=CFG.n_fold)
 for fold,(_, val_)in enumerate(skf.split(X=df, y=df.label)) :
df.loc[val_ , "kfold"] = int(fold)
 df['kfold'] = df['kfold'].astype(int )
sub = simple.Blend(['.. /input/why-not-logistic-regression/submission.csv', 'log_reg_0.1.csv'], 'id', 'target', 'blend01.csv' )<load_from_csv>
def log_t(u, t):

 if t==1.0:
return u.log()
else:
return(u.pow(1.0 - t)- 1.0)/(1.0 - t)
 def exp_t(u, t):

 if t==1:
return u.exp()
else:
return(1.0 +(1.0-t)*u ).relu().pow(1.0 /(1.0 - t))
def compute_normalization_fixed_point(activations, t, num_iters):

 mu, _ = torch.max(activations, -1, keepdim=True)
 normalized_activations_step_0 = activations - mu
normalized_activations = normalized_activations_step_0
for _ in range(num_iters):
logt_partition = torch.sum(
 exp_t(normalized_activations, t), -1, keepdim=True)
 normalized_activations = normalized_activations_step_0 * \
logt_partition.pow(1.0-t)
 logt_partition = torch.sum(
 exp_t(normalized_activations, t), -1, keepdim=True)
 normalization_constants = - log_t(1.0 / logt_partition, t)+ mu
return normalization_constants
def compute_normalization_binary_search(activations, t, num_iters):

 mu, _ = torch.max(activations, -1, keepdim=True)
 normalized_activations = activations - mu
effective_dim = \
torch.sum(
(normalized_activations > -1.0 /(1.0-t)).to(torch.int32),
dim=-1, keepdim=True ).to(activations.dtype)
 shape_partition = activations.shape[:-1] +(1,)
 lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)
 upper = -log_t(1.0/effective_dim, t)* torch.ones_like(lower)
 for _ in range(num_iters):
logt_partition =(upper + lower)/2.0
sum_probs = torch.sum(
 exp_t(normalized_activations - logt_partition, t),
dim=-1, keepdim=True)
 update =(sum_probs < 1.0 ).to(activations.dtype)
 lower = torch.reshape(
 lower * update +(1.0-update)* logt_partition,
shape_partition)
 upper = torch.reshape(
 upper *(1.0 - update)+ update * logt_partition,
shape_partition)
 logt_partition =(upper + lower)/2.0
return logt_partition + mu
class ComputeNormalization(torch.autograd.Function):

 @staticmethod
def forward(ctx, activations, t, num_iters):
if t < 1.0:
normalization_constants = compute_normalization_binary_search(activations, t, num_iters)
 else:
normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)
 ctx.save_for_backward(activations, normalization_constants)
 ctx.t=t
return normalization_constants
@staticmethod
def backward(ctx, grad_output):
activations, normalization_constants = ctx.saved_tensors
t = ctx.t
normalized_activations = activations - normalization_constants
probabilities = exp_t(normalized_activations, t)
 escorts = probabilities.pow(t)
 escorts = escorts / escorts.sum(dim=-1, keepdim=True)
 grad_input = escorts * grad_output
return grad_input, None, None
def compute_normalization(activations, t, num_iters=5):

 return ComputeNormalization.apply(activations, t, num_iters)
 def tempered_sigmoid(activations, t, num_iters = 5):

 internal_activations = torch.stack([activations,
torch.zeros_like(activations)],
dim=-1)
 internal_probabilities = tempered_softmax(internal_activations, t, num_iters)
 return internal_probabilities[..., 0]
def tempered_softmax(activations, t, num_iters=5):

 if t == 1.0:
return activations.softmax(dim=-1)
 normalization_constants = compute_normalization(activations, t, num_iters)
 return exp_t(activations - normalization_constants, t)
 def bi_tempered_binary_logistic_loss(activations,
labels,
t1,
t2,
label_smoothing = 0.0,
num_iters=5,
reduction='mean'):

 internal_activations = torch.stack([activations,
torch.zeros_like(activations)],
dim=-1)
 internal_labels = torch.stack([labels.to(activations.dtype),
1.0 - labels.to(activations.dtype)],
dim=-1)
 return bi_tempered_logistic_loss(internal_activations,
internal_labels,
t1,
t2,
label_smoothing = label_smoothing,
num_iters = num_iters,
reduction = reduction)
 def bi_tempered_logistic_loss(activations,
labels,
t1,
t2,
label_smoothing=0.0,
num_iters=5,
reduction = 'mean'):

 if len(labels.shape)<len(activations.shape):
labels_onehot = torch.zeros_like(activations)
 labels_onehot.scatter_(1, labels[..., None], 1)
 else:
labels_onehot = labels
if label_smoothing > 0:
num_classes = labels_onehot.shape[-1]
labels_onehot =(1 - label_smoothing * num_classes /(num_classes - 1)) \
* labels_onehot + \
label_smoothing /(num_classes - 1)
 probabilities = tempered_softmax(activations, t2, num_iters)
 loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1)\
- labels_onehot * log_t(probabilities, t1)\
- labels_onehot.pow(2.0 - t1)/(2.0 - t1)\
+ probabilities.pow(2.0 - t1)/(2.0 - t1)
 loss_values = loss_values.sum(dim = -1)
 if reduction == 'none':
return loss_values
if reduction == 'sum':
return loss_values.sum()
if reduction == 'mean':
return loss_values.mean()
sub = simple.Blend(['.. /input/entity-embeddings-to-handle-categories/submission.csv', 'blend01.csv'], 'id', 'target', 'blend02.csv' )<load_from_csv>
def train_model(model, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold):
start = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
history = defaultdict(list)
 scaler = amp.GradScaler()
for epoch in range(1,num_epochs+1):
print('Epoch {}/{}'.format(epoch, num_epochs))
print('-' * 10)
 for phase in ['train','valid']:
if(phase == 'train'):
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0.0
for inputs,labels in tqdm(dataloaders[phase]):
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
with amp.autocast() :
outputs = model(inputs)
 _, preds = torch.max(outputs,1)
 loss = bi_tempered_logistic_loss(outputs, labels, t1=CFG.t1, t2=CFG.t2, label_smoothing=CFG.smoothing)
 if phase == 'train':
scaler.scale(loss ).backward()
scaler.step(optimizer)
 scaler.update()
running_loss += loss.item() *inputs.size(0)
 running_corrects += torch.sum(preds == labels.data ).double().item()
epoch_loss = running_loss/dataset_sizes[phase]
epoch_acc = running_corrects/dataset_sizes[phase]
history[phase + ' loss'].append(epoch_loss)
 history[phase + ' acc'].append(epoch_acc)
 if phase == 'train' and scheduler != None:
scheduler.step()
print('{} Loss: {:.4f} Acc: {:.4f}'.format(
 phase, epoch_loss, epoch_acc))
if phase=='valid' and epoch_acc >= best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
 PATH = f"Fold{fold}_{best_acc}_epoch{epoch}.bin"
print()
end = time.time()
time_elapsed = end - start
print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(
 time_elapsed // 3600,(time_elapsed % 3600)// 60,(time_elapsed % 3600)% 60))
print("Best Accuracy ",best_acc)
 model.load_state_dict(best_model_wts)
 return model, history
sub = simple.Blend(['.. /input/r-glmnet/submission_glmnet.csv', 'blend02.csv'], 'id', 'target', 'blend03.csv' )<import_modules>
def run_fold(model, optimizer, scheduler, device, fold, num_epochs=10):
valid_df = df[df.kfold == fold]
train_df = df[df.kfold != fold]
train_data = CassavaLeafDataset(TRAIN_DIR, train_df, transforms=data_transforms["train"])
 valid_data = CassavaLeafDataset(TRAIN_DIR, valid_df, transforms=data_transforms["valid"])
 dataset_sizes = {
'train' : len(train_data),
'valid' : len(valid_data)
 }
train_loader = DataLoader(dataset=train_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=True)
 valid_loader = DataLoader(dataset=valid_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=False)
 dataloaders = {
'train' : train_loader,
'valid' : valid_loader
}
model, history = train_model(model, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold)
 return model, history
import pandas as pd
import category_encoders as ce
from sklearn import linear_model
from sklearn.model_selection import StratifiedKFold<load_from_csv>
model = timm.create_model(CFG.model_name, pretrained=True )
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 train.sort_index(inplace=True)
 train_y = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True )<categorify>
num_features = model.classifier.in_features
model.classifier = nn.Linear(num_features, CFG.num_classes)
 model.to(CFG.device);
cat_feat_to_encode = train.columns.tolist()
smoothing=50.0
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(
 n_splits=5, random_state=1, shuffle=True ).split(
 train, train_y):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, train_y)
 train = oof.sort_index()
test = ce_target_encoder.transform(test )<train_model>
optimizer = optim.Adam(model.parameters() , lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False )
glm = linear_model.LogisticRegression(
 random_state=1, solver='lbfgs', max_iter=5000, fit_intercept=True,
penalty='none', verbose=0)
 glm.fit(train, train_y )<save_to_csv>
def fetch_scheduler(optimizer):
if CFG.scheduler == 'CosineAnnealingLR':
scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr)
 elif CFG.scheduler == 'CosineAnnealingWarmRestarts':
scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr)
 elif CFG.scheduler == None:
return None
return scheduler
pd.DataFrame({'id': test_id, 'target': glm.predict_proba(test)[:,1]} ).to_csv(
 'sub_' + str(datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) + '.csv',
index=False )<import_modules>
scheduler = fetch_scheduler(optimizer )
import numpy as np
import pandas as pd 
<load_from_csv>
model, history = run_fold(model, optimizer, scheduler, device=CFG.device, fold=0, num_epochs=CFG.num_epochs )
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape)
 train['ord_5_a'] = train['ord_5'].str[0]
train['ord_5_b'] = train['ord_5'].str[1]
test['ord_5_a'] = test['ord_5'].str[0]
test['ord_5_b'] = test['ord_5'].str[1]
train.drop(['ord_5'],inplace=True,axis=1)
 test.drop(['ord_5'],inplace=True,axis=1)
 test.head()<import_modules>
if(CFG.savemodel):
torch.save(model, CFG.savemodelpath )
<data_type_conversions><EOS> 
if(CFG.dosubmission):
t_df = pd.read_csv(f"{ROOT_DIR}/sample_submission.csv")
 T_DIR = TEST_DIR
t_data = CassavaLeafDataset(T_DIR, t_df, transforms=data_transforms["valid"])
 t_loader = DataLoader(dataset=t_data, batch_size=1, num_workers=4, pin_memory=True, shuffle=False)
 submit_df = pd.DataFrame(t_df ).copy(deep=True)
 for i,(inputs, label)in enumerate(t_loader):
inputs = inputs.to(CFG.device)
 outputs = model(inputs ).detach().cpu().numpy()
pred_label = np.argmax(outputs)
 submit_df.iloc[i] = [t_df.iloc[i]['image_id'], pred_label]
submit_df.to_csv("/kaggle/working/submission.csv", index=False )