Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<save_to_csv>
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import time
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader, Dataset
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
data1 = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 data2 = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv' )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import torch
import torchtext
import torch.nn as nn
import torch.nn.functional as F
from torch.optim.optimizer import Optimizer<set_options>
data = pd.concat([data1, data2], axis = 0 )
seed = 0
torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed_all(seed )<load_from_csv>
target = data.pop('label')
 data = data/255.0
data['label'] = target
del data1
del data2
del target
data = pd.read_csv(".. /input/train.csv")
 data_train, data_val = train_test_split(data, test_size=0.12, random_state=6)
 data_test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
class CNNUnit(nn.Module):
def __init__(self, in_channels, out_channels):
super(CNNUnit, self ).__init__()
self.conv = nn.Conv2d(in_channels=in_channels,
out_channels=out_channels,
kernel_size = 3, stride = 1, padding = 1)
 self.bn = nn.BatchNorm2d(num_features=out_channels, momentum=0.9)
 self.LR = nn.LeakyReLU(inplace=True, negative_slope=0.1)
 def forward(self,x):
output = self.LR(self.bn(self.conv(x)))
 return output
print(np.sum(data_train.target)/ len(data_train))
print(np.sum(data_val.target)/ len(data_val))<define_variables>
class FCUnit(nn.Module):
def __init__(self, input_dim, output_dim, initrange):
super(FCUnit, self ).__init__()
self.fc = nn.Linear(in_features=input_dim,
out_features = output_dim)
 nn.init.uniform_(a = -initrange, b = initrange, tensor=self.fc.weight)
 nn.init.zeros_(self.fc.bias)
 self.LR = nn.LeakyReLU(inplace=True)
 self.bn = nn.BatchNorm1d(num_features=output_dim)
 def forward(self, x):
output = self.bn(self.LR(self.fc(x)))
 return output
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
char_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
def clean_text(x):
for dic in [contraction_mapping, mispell_dict, char_mapping]:
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
pd.set_option('mode.chained_assignment', None)
 data_train['question_text'] = data_train['question_text'].fillna("" ).apply(lambda x: clean_text(x))
data_val['question_text'] = data_val['question_text'].fillna("" ).apply(lambda x: clean_text(x))
data_test['question_text'] = data_test['question_text'].fillna("" ).apply(lambda x: clean_text(x))<save_to_csv>
class CNN(nn.Module):
def __init__(self):
super(CNN,self ).__init__()
self.unit1 = CNNUnit(in_channels=1, out_channels=32)
 self.unit2 = CNNUnit(in_channels=32, out_channels=32)
 self.unit3 = CNNUnit(in_channels=32, out_channels=32)
 self.pool1 = nn.MaxPool2d(kernel_size=2, stride = 2)
 self.unit4 = CNNUnit(in_channels=32, out_channels=64)
 self.unit5 = CNNUnit(in_channels=64, out_channels=64)
 self.unit6 = CNNUnit(in_channels=64, out_channels=64)
 self.unit7 = CNNUnit(in_channels=64, out_channels=64)
 self.pool2 = nn.MaxPool2d(kernel_size=2, stride = 2)
 self.unit8 = CNNUnit(in_channels=64, out_channels=128)
 self.unit9 = CNNUnit(in_channels=128, out_channels=128)
 self.pool3 = nn.MaxPool2d(kernel_size=2, stride = 2, padding = 1)
 self.unit10 = CNNUnit(in_channels=128, out_channels=256)
 self.avg = nn.AvgPool2d(kernel_size=2, stride = 2)
 self.fc1 = FCUnit(input_dim=2*2*256, output_dim=64, initrange=0.5)
 self.fc2 = nn.Linear(64,10)
 self.net = nn.Sequential(self.unit1,self.unit2,self.unit3,self.pool1,
self.unit4,self.unit5,self.unit6,self.unit7,self.pool2,
self.unit8,self.unit9,self.pool3,
self.unit10,self.avg)
 self.localization = nn.Sequential(
 nn.Conv2d(1, 8, kernel_size=7),
nn.MaxPool2d(2, stride=2),
nn.ReLU(True),
nn.Conv2d(8, 10, kernel_size=5),
nn.MaxPool2d(2, stride=2),
nn.ReLU(True))
self.fc_loc = nn.Sequential(
 nn.Linear(10 * 3 * 3, 32),
nn.ReLU(True),
nn.Linear(32, 3 * 2))
self.fc_loc[2].weight.data.zero_()
self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
def stn(self, x):
xs = self.localization(x)
 xs = xs.view(-1, 10 * 3 * 3)
 theta = self.fc_loc(xs)
 theta = theta.view(-1, 2, 3)
 grid = F.affine_grid(theta, x.size())
 x = F.grid_sample(x, grid)
 return x
def forward(self, image):
output = self.stn(image)
 output = self.net(output)
 output = output.view(output.size(0),-1)
 output = F.dropout(( self.fc1(output)) , p = 1-KEPT_PROB, training = False)
 output = F.log_softmax(self.fc2(output))
return output
!rm -rf split_data
!mkdir split_data
data_train.to_csv('split_data/train.csv', index=False)
 data_val.to_csv('split_data/val.csv', index=False)
 data_test.to_csv('split_data/test.csv', index=False)
 del data, data_train, data_val, data_test<load_from_csv>
class oversampdata(Dataset):
def __init__(self, data):
self.data = torch.FloatTensor(data.values.astype('float'))
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
target = self.data[index][-1]
data_val = self.data[index][:-1]
return data_val, target
text = torchtext.data.Field(lower=True, batch_first=True, tokenize='spacy', include_lengths=True)
 target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 qid = torchtext.data.Field()
train = torchtext.data.TabularDataset(path='split_data/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 val = torchtext.data.TabularDataset(path='split_data/val.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 test = torchtext.data.TabularDataset(path='split_data/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text',text)} )<compute_test_metric>
def train_func(sub_train_):
train_loss = []
train_acc = []
data = DataLoader(sub_train_, batch_size=BATCH_SIZE, shuffle=True)
 for i,(image, label)in enumerate(data):
optimizer.zero_grad()
image, label = image.to(device), label.to(device)
 image = image.view(-1,1,28,28)
 label = label.long()
output = model(image)
 loss = criterion(output, label)
 train_loss.append(float(loss.item()))
loss.backward()
optimizer.step()
del(image)
 output = output.data.max(dim=1)[1]
train_acc.append(((output.data == label.data ).float().mean() ).item())
 del(label)
 torch.cuda.empty_cache()
if i % 200 == 0:
print("Batch = {},\t loss = {:2.4f},\t accuracy = {}%".format(i, train_loss[-1], train_acc[-1]*100))
def test(data_):
loss = 0
acc = 0
data = DataLoader(data_, batch_size=BATCH_SIZE)
 for image, label in data:
image, label = image.to(device), label.to(device)
 image = image.view(-1,1,28,28)
 label = label.long()
with torch.no_grad() :
output = model(image)
 loss = criterion(output, label)
 loss += float(loss.item())
 del(image)
 output = output.data.max(dim=1)[1]
acc += float(( output.data == label.data ).sum().item() / len(data_))
del(label)
 torch.cuda.empty_cache()
return loss / len(data_), acc
text.build_vocab(train, val, test, min_freq=3)
 qid.build_vocab(test )<load_pretrained>
BATCH_SIZE = 32
KEPT_PROB = 0.5
N_EPOCHS = 10
LEARNING_RATE = 0.0001
text.vocab.load_vectors(torchtext.vocab.Vectors(".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"))<count_unique_values>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = CNN().to(device)
 criterion = nn.CrossEntropyLoss().to(device)
 optimizer = optim.Adam(model.parameters() , lr = LEARNING_RATE )
print(text.vocab.vectors.shape)
 print(f"Unique tokens in text vocabulary: {len(text.vocab)}" )<compute_test_metric>
train_data, test_data = train_test_split(data, test_size = 0.05, random_state = 128 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def get_metrics(outs, y):
outs = sigmoid(outs.cpu().data.numpy())
 y = y.cpu().data.numpy()
y_pred =(outs >= 0.5 ).astype(int)
 acc = np.sum(y_pred == y)/ len(y)
 tp = np.sum(( y_pred == y)&(y_pred == 1))
fp = np.sum(( y_pred != y)&(y_pred == 1))
fn = np.sum(( y_pred != y)&(y_pred == 0))
return acc, tp, fp, fn<define_variables>
sub_train = oversampdata(train_data)
 sub_test = oversampdata(test_data )
batch_size = 512
train_iter = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
shuffle=True,
sort_within_batch=True)
 val_iter = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
train=False,
sort_within_batch=True)
 test_iter = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
sort_within_batch=True )<categorify>
for epoch in range(N_EPOCHS):
start_time = time.time()
train_func(sub_train)
 valid_loss, valid_acc = test(sub_test)
 secs = int(time.time() - start_time)
 mins = secs / 60
secs = secs % 60
print('Epoch: %d' %(epoch + 1), " | time in %d minutes, %d seconds" %(mins, secs))
print('\tLoss: {valid_loss:.4f}(test)\t|\tAcc: {valid_acc:.1f}%(test)\t|'.
 format(valid_loss = valid_loss, valid_acc = valid_acc * 100))
class Attention(nn.Module):
def __init__(self, hidden_size, batch_first=False):
super(Attention, self ).__init__()
self.hidden_size = hidden_size
self.batch_first = batch_first
self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)
 stdv = 1.0 / np.sqrt(self.hidden_size)
 for weight in self.att_weights:
nn.init.uniform_(weight, -stdv, stdv)
 def get_mask(self):
pass
def forward(self, inputs, lengths):
if self.batch_first:
batch_size, max_len = inputs.size() [:2]
else:
max_len, batch_size = inputs.size() [:2]
weights = torch.bmm(inputs,
self.att_weights 
.permute(1, 0)
.unsqueeze(0)
.repeat(batch_size, 1, 1)
)
 attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)
 mask = torch.ones(attentions.size() , requires_grad=True ).cuda()
for i, l in enumerate(lengths):
if l < max_len:
mask[i, l:] = 0
masked = attentions * mask
_sums = masked.sum(-1 ).unsqueeze(-1)
 attentions = masked.div(_sums)
 weighted = torch.mul(inputs, attentions.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations, attentions<choose_model_class>
test_data = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_data = test_data.drop(['id'], axis = 1)
 test_data = test_data/255.0
test_data = torch.FloatTensor(test_data.values.astype('float'))
test_data = test_data.view(-1,1,28,28 )
class MyLSTM(nn.Module):
def __init__(self, pretrained_lm, hidden_dim=128, lstm_layer=2, dropout=0.2):
super(MyLSTM, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 self.embedding = nn.Embedding.from_pretrained(pretrained_lm)
 self.embedding.weight.requires_grad = False
self.lstm1 = nn.LSTM(input_size=self.embedding.embedding_dim,
hidden_size=hidden_dim,
num_layers=1,
bidirectional=True)
 self.atten1 = Attention(hidden_dim*2, batch_first=True)
 self.lstm2 = nn.LSTM(input_size=hidden_dim*2,
hidden_size=hidden_dim,
num_layers=1,
bidirectional=True)
 self.atten2 = Attention(hidden_dim*2, batch_first=True)
 self.fc1 = nn.Sequential(nn.Linear(hidden_dim*lstm_layer*2, hidden_dim*lstm_layer*2),
nn.BatchNorm1d(hidden_dim*lstm_layer*2),
nn.ReLU())
 self.fc2 = nn.Linear(hidden_dim*lstm_layer*2, 1)
 def forward(self, x, x_len):
x = self.embedding(x)
 x = self.dropout(x)
 x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)
 out1,(h_n, c_n)= self.lstm1(x)
 x, lengths = nn.utils.rnn.pad_packed_sequence(out1, batch_first=True)
 x, _ = self.atten1(x, lengths)
 out2,(h_n, c_n)= self.lstm2(out1)
 y, lengths = nn.utils.rnn.pad_packed_sequence(out2, batch_first=True)
 y, _ = self.atten2(y, lengths)
 z = torch.cat([x, y], dim=1)
 z = self.fc1(self.dropout(z))
z = self.fc2(self.dropout(z))
return z<choose_model_class>
def predict(data):
model.eval()
predictions = []
for image in data:
image = image.view(1,1,28,28)
 image = image.to(device)
 output = model(image)
 del image
predictions.append(int(output.argmax(1)))
 return predictions
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs<find_best_params>
predictions = predict(test_data )
<choose_model_class><EOS> 
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission = submission.drop(['label'], axis = 1)
 submission['label'] = predictions
submission.to_csv('submission.csv', index=False )