Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<train_model>
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import torch.nn.functional as F
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss
FOLDS = 5
X = train_df[feature_columns].values
y = np.array(train_df['target'].values)
 splits = list(StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=42 ).split(X, y))
models = []
for idx,(train_idx, val_idx)in enumerate(splits):
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = build_model()
model.fit(train_X, train_y, eval_set=(val_X,val_y))
models.append(model)
<predict_on_test>
CFG = {
'fold_num': 5,
'seed': 324,
'model_arch': 'SE-Net',
'img_size': 28,
'epochs': 10,
'train_bs': 16,
'valid_bs': 32,
'T_0': 10,
'lr': 1e-2,
'min_lr': 1e-6,
'weight_decay':1e-6,
'num_workers': 2,
'accum_iter': 4,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 15,
'used_epochs': [8],
'weights': [1,1,1,1,1]
}
preds_test = []
X = test_df[feature_columns].values
for i in range(FOLDS):
pred = models[i].predict_proba(X)
 preds_test.append(pred)
<prepare_output>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission.head()
preds = np.mean(np.array(preds_test),axis=0 )<prepare_output>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def show_batch_imgs(examples, df, transforms=None):
imgs = df.sample(frac=1)[:examples]
plt.figure(figsize=(8,8))
for i in range(examples):
img = imgs.iloc[i][1:].values.astype(np.uint8 ).reshape(( 28,28))
if transforms:
img = transforms(image=img)['image']
img = img.numpy().reshape(( 28,28))
plt.subplot(1, examples, i%examples+1)
 plt.axis('off')
 plt.imshow(img )
submission_df = pd.DataFrame(data={'ID_code': list(test_df['ID_code'].values), 'target': list(preds)})
 submission_df['target'] = submission_df['target'].apply(lambda x: float(x[0])).astype('float')
 submission_df[submission_df['target'] > 0.5].head(10)
<save_to_csv>
class KMnistDataset(Dataset):
def __init__(self, data, transforms=None, do_fmix=None, fmix_params=None, do_cutmix=None, cutmix_params={'alpha':1}):
self.data = data
self.transforms = transforms
self.do_fmix = do_fmix
self.fmix_params = fmix_params
self.do_cutmix = do_cutmix
self.cutmix_params = cutmix_params
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
img = self.data.iloc[index, 1:].values.astype(np.float32 ).reshape(( 28,28))
if self.transforms:
img = self.transforms(image=img)['image']
return img
submission_df.to_csv('submission.csv', index=False )<load_from_csv>
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=30, p=0.5),
CoarseDropout(p=0.1),
Cutout(p=0.1),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)
%matplotlib inline
print(os.listdir(".. /input"))
h2o.init()
train_df = pd.read_csv(".. /input/train.csv")
 valid_rate =.15
train_df, valid_df, tr_y, va_y = train_test_split(train_df, train_df['target'], stratify = train_df['target'], test_size=valid_rate, random_state = 42)
 train = h2o.H2OFrame(train_df)
 valid = h2o.H2OFrame(valid_df)
 test = h2o.import_file(".. /input/test.csv")
 del train_df, valid_df, tr_y, va_y
gc.collect()<feature_engineering>
class Sq_Ex_Block(nn.Module):
def __init__(self, in_ch, r):
super(Sq_Ex_Block, self ).__init__()
self.se = nn.Sequential(
 GlobalAvgPool() ,
nn.Linear(in_ch, in_ch//r),
nn.ReLU(inplace=True),
nn.Linear(in_ch//r, in_ch),
nn.Sigmoid() 
)
 def forward(self, x):
se_weight = self.se(x ).unsqueeze(-1 ).unsqueeze(-1)
 return x.mul(se_weight)
 class GlobalAvgPool(nn.Module):
def __init__(self):
super(GlobalAvgPool, self ).__init__()
def forward(self, x):
return x.view(*(x.shape[:-2]),-1 ).mean(-1)
 class SE_Net(nn.Module):
def __init__(self,in_channels):
super(SE_Net,self ).__init__()
self.c1 = nn.Conv2d(in_channels=in_channels, out_channels=64,kernel_size=3,stride=1,padding=0)
 self.bn1 = nn.BatchNorm2d(num_features=64,eps=1e-3,momentum=0.01)
 self.c2 = nn.Conv2d(64,64,3,1,0)
 self.bn2 = nn.BatchNorm2d(64,1e-3,0.01)
 self.c3 = nn.Conv2d(64,64,5,1,2)
 self.bn3 = nn.BatchNorm2d(64,1e-3,0.01)
 self.m1 = nn.MaxPool2d(2)
 self.d1 = nn.Dropout(0.4)
 self.c4 = nn.Conv2d(64,128,3,1,0)
 self.bn4 = nn.BatchNorm2d(128,1e-3,0.01)
 self.c5 = nn.Conv2d(128,128,3,1,0)
 self.bn5 = nn.BatchNorm2d(128,1e-3,0.01)
 self.c6 = nn.Conv2d(128,128,5,1,2)
 self.bn6 = nn.BatchNorm2d(128,1e-3,0.01)
 self.m2 = nn.MaxPool2d(2)
 self.d2 = nn.Dropout(0.4)
 self.c7 = nn.Conv2d(128,256,3,1,0)
 self.bn7 = nn.BatchNorm2d(256,1e-3,0.01)
 self.se3 = Sq_Ex_Block(in_ch=256,r=8)
 self.m3 = nn.MaxPool2d(2)
 self.d3 = nn.Dropout(0.4)
 self.fc1 = nn.Linear(256*1*1,256)
 self.bn8 = nn.BatchNorm1d(256,1e-3,0.01)
 self.out = nn.Linear(256,10)
 self.init_linear_weights()
def forward(self,x):
x = self.bn1(F.leaky_relu(self.c1(x),0.1))
x = self.bn2(F.leaky_relu(self.c2(x),0.1))
x = self.bn3(F.leaky_relu(self.c3(x),0.1))
x = self.d1(self.m1(x))
x = self.bn4(F.leaky_relu(self.c4(x),0.1))
x = self.bn5(F.leaky_relu(self.c5(x),0.1))
x = self.bn6(F.leaky_relu(self.c6(x),0.1))
x = self.d2(self.m2(x))
x = self.bn7(F.leaky_relu(self.c7(x),0.1))
x = self.se3(x)
 x = self.d3(self.m3(x))
x = x.view(-1, 256*1*1)
 x = self.bn8(F.leaky_relu(self.fc1(x),0.1))
return self.out(x)
 def init_linear_weights(self):
nn.init.kaiming_normal_(self.fc1.weight, mode='fan_in')
 nn.init.kaiming_normal_(self.out.weight, mode='fan_in' )
y = 'target'
x = train.columns[2:]
train[y] = train[y].asfactor() 
<choose_model_class>
seed_everything(CFG['seed'])
 test_data = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 test_ds = KMnistDataset(test_data, transforms=get_inference_transforms())
 tst_preds = []
device = torch.device(CFG['device'] )
n_models = 8
grid_params = {
'max_depth': [2, 3],
'col_sample_rate': [.6,.7],
'learn_rate': [.09,.1],
'learn_rate_annealing': [1],
'min_rows': [110, 90],
'sample_rate': [.7]
}
gbm_grid = H2OGridSearch(model=h2o.estimators.H2OGradientBoostingEstimator,
grid_id='gbm_grid',
hyper_params=grid_params,
search_criteria={'strategy': 'RandomDiscrete', 'max_models': n_models})
<train_model>
model_paths = ['.. /input/eesm5720/SE-Net_fold_0_9',
'.. /input/eesm5720/SE-Net_fold_1_9',
'.. /input/eesm5720/SE-Net_fold_2_9',
'.. /input/eesm5720/SE-Net_fold_3_9',
'.. /input/eesm5720/SE-Net_fold_4_9',]
gbm_grid.train(x=x, y=y, training_frame=train, validation_frame=valid,
distribution='bernoulli',
ntrees=2500,
score_tree_interval = 20,
stopping_rounds = 4,
stopping_metric = "AUC",
stopping_tolerance = 1e-4,
seed = 1 )<find_best_params>
for o, model_path in enumerate(model_paths):
model = SE_Net(in_channels=1 ).to(device)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 print("using the model from {}".format(model_path))
model.load_state_dict(torch.load(model_path))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [1/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0 )
gridperf = gbm_grid.get_grid(sort_by='auc', decreasing=True)
 best_model = gridperf.models[0]
for par in grid_params:
if par in best_model.params:
print('par: ' + par); print(best_model.params[par])
<create_dataframe>
test_data['label'] = np.argmax(tst_preds, axis=1 )
history = pd.DataFrame(best_model.scoring_history())
 history.plot(x='number_of_trees', y = ['validation_auc', 'validation_pr_auc'])
<save_to_csv>
test_data = pd.DataFrame(test_data, columns=['id','label'] )
<set_options><EOS> 
test_data.to_csv('submission.csv', index=False )