Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
import torch
import torchvision
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
import copy
import time
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<choose_model_class>
torch.backends.cudnn.enabled
class IBNResnextCassava(nn.Module):
def __init__(self, arch='resnext101_ibn_a', n_class=5, pre=False):
super().__init__()
m = resnext101_ibn_a()
self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(2048, n_class)
)
 def forward(self, x):
x = self.enc(x)
 x = self.head(x)
 return x<feature_engineering>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device )
class MishFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x)
 return x * torch.tanh(F.softplus(x))
@staticmethod
def backward(ctx, grad_output):
x = ctx.saved_variables[0]
sigmoid = torch.sigmoid(x)
 tanh_sp = torch.tanh(F.softplus(x))
return grad_output *(tanh_sp + x * sigmoid *(1 - tanh_sp * tanh_sp))
class Mish(nn.Module):
def forward(self, x):
return MishFunction.apply(x)
 def to_Mish(model):
for child_name, child in model.named_children() :
if isinstance(child, nn.ReLU):
setattr(model, child_name, Mish())
 else:
to_Mish(child)
 def _resnext(url, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 return model 
<define_variables>
Num_CNN = 12
n_epochs = 25
batch_size_train = 64
batch_size_test = 1000
learning_rate = 0.01
momentum = 0.5
log_interval = 100
random_seed = 121
torch.manual_seed(random_seed)
 kaggle_input_data = "/kaggle/input/digit-recognizer/train.csv"
kaggle_input_validation = "/kaggle/input/digit-recognizer/test.csv"
kaggle_model_path = "/kaggle/working/model_id_%d.pth"
kaggle_optimizer_path = "/kaggle/working/optimizer_id_%d.pth"
prediction_path = "/kaggle/working/predictions_essemble.csv"
semi_weakly_supervised_model_urls = {
'resnet18': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnet18-118f1556.pth',
'resnet50': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnet50-16a12f1b.pth',
'resnext50_32x4d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext50_32x4-72679e44.pth',
'resnext101_32x4d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x4-3f87e46b.pth',
'resnext101_32x8d': 'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x8-b4712904.pth',
'resnext101_32x16d':'https://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_weakly_supervised_resnext101_32x16-f3559a9c.pth',
}
class ResnextCassava(nn.Module):
def __init__(self, arch='resnext101_32x4d', n_class=5, pretrained=False):
super().__init__()
m = _resnext(semi_weakly_supervised_model_urls[arch], Bottleneck, [3, 4, 23, 3], False,
progress=False,groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.max_pool = nn.AdaptiveMaxPool2d(1)
 self.logit = nn.Linear(2048,n_class)
 def forward(self, x):
x = self.enc(x)
 x =(self.avg_pool(x)+ self.max_pool(x)).squeeze(-1 ).squeeze(-1)
 x = self.logit(x)
 return x 
<choose_model_class>
class Dataset(torch.utils.data.Dataset):
def __init__(self, dataframe):
self.labels = dataframe["label"].to_numpy()
self.dataframe = dataframe.loc[:,dataframe.columns != "label"]
def __len__(self):
return self.dataframe.shape[0]
def __getitem__(self, index):
X = torch.from_numpy(self.dataframe.iloc[index].values.reshape(1,28,28)).float()
y = self.labels[index]
return X,y
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)

def forward(self, x):
x = self.model(x)
 return x
class IBN_CUSTOM1(nn.Module):
def __init__(self, arch='resnext101_ibn_a', n_class=5, pretrained=True):
super().__init__()
m = se_resnet101_ibn_a(pretrained=pretrained)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 self.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(2048, n_class)
)
 def forward(self, x):
x = self.enc(x)
 x = self.head(x)
 return x
def get_model(model_name, device='cuda'):
if 'efficientnet' in model_name:
model = CassvaImgClassifier(model_name, 5, pretrained=False ).to(device)
 elif model_name == 'ibn50a':
model = resnet50_ibn_a(pretrained=False ).to(device)
 model.avgpool = nn.AdaptiveAvgPool2d(1)
 model.fc = nn.Linear(2048, train.label.nunique())
 elif model_name == 'ibnse50a':
model = se_resnet50_ibn_a(pretrained=False ).to(device)
 model.avgpool = nn.AdaptiveAvgPool2d(1)
 model.fc = nn.Linear(2048, train.label.nunique())
 elif model_name == 'resnext101_ibn_a':
model = IBNResnextCassava(arch=model_name)
 elif model_name == 'se50_softlabel':
model = se_resnext50_32x4d(pretrained=None ).to(device)
 elif model_name == 'resnext101':
model = ResnextCassava(pretrained=False ).to(device)
 elif model_name == 'b4':
b4_model = timm.create_model('tf_efficientnet_b4_ns', pretrained=False)
 model=timm_class(b4_model,n=5 ).to(device)
 elif model_name == 'vit':
CONFIGS = {
'ViT-B_16': configs.get_b16_config() ,
'ViT-B_32': configs.get_b32_config() ,
'ViT-L_16': configs.get_l16_config() ,
'ViT-L_32': configs.get_l32_config() ,
'ViT-H_14': configs.get_h14_config() ,
'testing': configs.get_testing() ,
}
config = CONFIGS["ViT-B_16"]
model = VisionTransformer(config, CFG['img_size'], zero_head=True, num_classes=5 ).to(device)
 elif model_name == 'se_resnet101_ibn_a':
model = IBN_CUSTOM1(pretrained=False ).to(device)
 model = model.to(device)
 return model 
<install_modules>
df_input = pd.read_csv(kaggle_input_data)
 df_validation = pd.read_csv(kaggle_input_validation)
 df_train, df_test = train_test_split(df_input, test_size = 0.01)
 training_set = Dataset(df_train)
 training_generator = torch.utils.data.DataLoader(training_set, batch_size = batch_size_train,
shuffle = True)
 test_set = Dataset(df_test)
 test_generator = torch.utils.data.DataLoader(test_set, batch_size = batch_size_test,
shuffle = True)
 kaggle_validation_set = torch.from_numpy(df_validation.to_numpy() ).view(
 df_validation.shape[0],-1,28,28 ).float()
! pip install.. /input/mlcollection/ml_collections-0.1.0-py3-none-any.whl<feature_engineering>
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
class AdaptiveConcatPool2d(nn.Module):
"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`"
def __init__(self, size=None):
super().__init__()
self.size = size or 1
self.ap = nn.AdaptiveAvgPool2d(self.size)
 self.mp = nn.AdaptiveMaxPool2d(self.size)
 def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1 )<init_hyperparams>
class Net(nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.conv1 = nn.Conv2d(1, 32, kernel_size=3)
 self.batchnorm1 = nn.BatchNorm2d(32)
 self.conv2 = nn.Conv2d(32, 32, kernel_size=3)
 self.batchnorm2 = nn.BatchNorm2d(32)
 self.conv3 = nn.Conv2d(32, 32, kernel_size=2, stride = 2)
 self.batchnorm3 = nn.BatchNorm2d(32)
 self.conv4 = nn.Conv2d(32, 64, kernel_size=5)
 self.batchnorm4 = nn.BatchNorm2d(64)
 self.conv5 = nn.Conv2d(64, 64, kernel_size=2, stride = 2)
 self.batchnorm5 = nn.BatchNorm2d(64)
 self.conv5_drop = nn.Dropout2d()
self.fc1 = nn.Linear(1024, 128)
 self.fc2 = nn.Linear(128, 10)
 def forward(self, x):
x = self.batchnorm1(F.relu(self.conv1(x)))
 x = self.batchnorm2(F.relu(self.conv2(x)))
 x = self.batchnorm3(F.relu(self.conv3(x)))
 x = self.batchnorm4(F.relu(self.conv4(x)))
 x = self.batchnorm5(F.relu(self.conv5(x)))
 x = self.conv5_drop(x)
 x = x.view(-1, 1024)
 x = F.relu(self.fc1(x))
x = F.log_softmax(self.fc2(x), dim=1)
 return x
if __name__ == '__main__':
VALID = False
test_num = len(os.listdir('.. /input/cassava-leaf-disease-classification/test_images'))
print('test_num:', test_num)
 seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 val_preds = []
tst_preds = []
for i in range(len(ckpt_paths)) :
ckpt_path = ckpt_paths[i]
model = get_model(model_names[i], device)
 if model_names[i] == 'se_resnet101_ibn_a':
model.load_state_dict(torch.load(ckpt_path)['state_dict'])
 else:
model.load_state_dict(torch.load(ckpt_path,map_location=device))
with torch.no_grad() :
for _ in range(CFG['tta']):
if VALID:
if test_num == 1:
val_preds += [ckpt_weights[i]/sum(ckpt_weights)/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [
inference_one_epoch(model, tst_loader, device)*model_wts[model_names[i]]]
del model
torch.cuda.empty_cache()
if VALID:
if test_num == 1:
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)*3
if VALID:
if test_num == 1:
print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
cm = confusion_matrix(valid_.label.values, np.argmax(val_preds, axis=1), labels=[0, 1, 2, 3, 4])
 print('cm')
 print(cm)
 sns.set()
f, ax=plt.subplots()
sns.heatmap(cm,annot=True,ax=ax)
 ax.set_title('confusion matrix')
 ax.set_xlabel('predict')
 ax.set_ylabel('true')
<feature_engineering>
train_losses = []
train_counter = []
test_losses = []
test_counter = [i*len(training_generator.dataset)for i in range(n_epochs + 1)]
log_interval = len(training_generator)
 network_list = []
prediction_tensor = torch.zeros(kaggle_validation_set.shape[0],10 )
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
def train(epoch, network, scheduler, network_id, device):
start_time = time.time()
network.train()
for batch_idx,(data, target)in enumerate(training_generator):
data = data.to(device)
 target = target.to(device)
 optimizer.zero_grad()
output = network(data)
 loss = F.nll_loss(output, target)
 loss.backward()
optimizer.step()
print('CNN : {},\t Train Epoch: {} \tLoss: {:.6f},\t runtime: {:.2f}'.format(
 network_id,
epoch, loss.item() , time.time() -start_time))
train_losses.append(loss.item())
 train_counter.append(epoch)
 torch.save(network.state_dict() , kaggle_model_path%network_id)
 torch.save(optimizer.state_dict() ,kaggle_optimizer_path%network_id)
 scheduler.step()
def test(network):
network.eval()
test_loss = 0
correct = 0
with torch.no_grad() :
for data, target in test_generator:
data = data.to(device)
 target = target.to(device)
 output = network(data)
 test_loss += F.nll_loss(output, target, reduction="sum" ).item()
pred = output.data.max(1, keepdim=True)[1]
correct += pred.eq(target.data.view_as(pred)).sum()
test_loss /= len(test_generator.dataset)
 test_losses.append(test_loss)
 print('
Test set: Avg.loss: {:.4f}, Accuracy: {}/{}({:.5f}%)
 '.format(
 test_loss, correct, len(test_generator.dataset),
100.* correct / len(test_generator.dataset)))
 def validation(network, validation_tensor, prediction_tensor):
with torch.no_grad() :
output = network(validation_tensor)
 prediction_tensor = prediction_tensor + output
return prediction_tensor
def initialise_network(device):
network = Net().to(device)
 optimizer = optim.SGD(network.parameters() , lr=learning_rate, momentum = momentum)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.95)
 return network, optimizer, exp_lr_scheduler
test.to_csv('submission.csv', index=False )<install_modules>
validation_tensor = kaggle_validation_set.to(device)
 prediction_tensor = prediction_tensor.to(device)
 for id_net in range(1, Num_CNN+1):
network, optimizer, scheduler = initialise_network(device)
 for epoch in range(1, n_epochs+1):
train(epoch, network, scheduler, id_net, device)
 test(network)
 prediction_tensor = validation(network, validation_tensor, prediction_tensor)
 network_list.append(copy.deepcopy(network))
!pip install -q '/kaggle/input/birdcall-identification-submission-custom/Keras_Applications-1.0.8-py3-none-any.whl'
!pip install -q '/kaggle/input/birdcall-identification-submission-custom/efficientnet-1.1.0-py3-none-any.whl'<import_modules>
output_df = prediction_tensor.max(1)[1].to("cpu")
 output_df = pd.DataFrame(output_df.numpy() , columns = ["Label"])
 output_df.index.name = "ImageId"
output_df.index = output_df.index + 1
import numpy as np
import pandas as pd
import tensorflow as tf
import efficientnet.tfkeras as efn
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm<define_search_space>
output_df.to_csv(prediction_path )