Kaggle competition: Kannada MNIST
texts = df_train.text.values
labels = df_train.target.values<load_pretrained>
train=pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test=pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')
 submission_set = pd.read_csv(".. /input/Kannada-MNIST/test.csv" ).iloc[:,1:]
train_data=train.drop('label',axis=1)
 train_targets=train['label']
test_images=test.drop('label',axis=1)
 test_labels=test['label']
train_images, val_images, train_labels, val_labels = train_test_split(train_data,
train_targets,
test_size=0.2)
 train_images.reset_index(drop=True, inplace=True)
 train_labels.reset_index(drop=True, inplace=True)
 val_images.reset_index(drop=True, inplace=True)
 val_labels.reset_index(drop=True, inplace=True)
 test_images.reset_index(drop=True, inplace=True)
 test_labels.reset_index(drop=True, inplace=True)
 print("Train Set")
 print(train_images.shape)
 print(train_labels.shape)
 print("Validation Set")
 print(val_images.shape)
 print(val_labels.shape)
 print("Validation 2")
 print(test_images.shape)
 print(test_labels.shape)
 print("Submission")
 print(submission_set.shape )
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 model = BertForSequenceClassification.from_pretrained('bert-base-uncased',num_labels=2)
 model.cuda()<split>
print("Train Distribution")
 print(train_labels.value_counts(normalize = True))
print("
Submission Distribution")
 print(test_labels.value_counts(normalize = True))
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )<data_type_conversions>
IMGSIZE = 28
train_trans = transforms.Compose(( [
transforms.ToPILImage() ,
transforms.RandomCrop(IMGSIZE),
transforms.RandomAffine(degrees=5, translate=(0.1, 0.1)) ,
transforms.ToTensor() ,
]))
val_trans = transforms.Compose(( [
transforms.ToPILImage() ,
transforms.ToTensor() ,
]))
class KannadaDataSet(torch.utils.data.Dataset):
def __init__(self, images, labels,transforms = None):
self.X = images
self.y = labels
self.transforms = transforms
def __len__(self):
return(len(self.X))
def __getitem__(self, i):
data = self.X.iloc[i,:]
data = np.array(data ).astype(np.uint8 ).reshape(IMGSIZE,IMGSIZE,1)
 if self.transforms:
data = self.transforms(data)
 if self.y is not None:
return(data, self.y[i])
 else:
return data
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )<load_pretrained>
batch_size = 128
train_data = KannadaDataSet(train_images, train_labels, train_trans)
 val_data = KannadaDataSet(val_images, val_labels, val_trans)
 test_data = KannadaDataSet(test_images, test_labels, val_trans)
 submission_data = KannadaDataSet(submission_set, None, val_trans)
 train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
shuffle=True)
 val_loader = torch.utils.data.DataLoader(val_data,
batch_size=batch_size,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(test_data,
batch_size=batch_size,
shuffle=False)
 submission_loader = torch.utils.data.DataLoader(submission_data,
batch_size=batch_size,
shuffle=False)
 classes =('0', '1', '2', '3', '4', '5', '6', '7', '8', '9' )
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )<categorify>
class Net(nn.Module):
def __init__(self, dropout = 0.40):
super(Net, self ).__init__()
self.dropout = dropout
self.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=5, stride=1, padding=2)
 self.conv1_bn = nn.BatchNorm2d(num_features=64)
 self.conv1_1 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=1, padding=2)
 self.conv1_1_bn = nn.BatchNorm2d(num_features=64)
 self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
 self.d2_1 = nn.Dropout2d(p=self.dropout)
 self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)
 self.conv2_bn = nn.BatchNorm2d(num_features=128)
 self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
 self.d2_2 = nn.Dropout2d(p=self.dropout)
 self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)
 self.conv3_bn = nn.BatchNorm2d(num_features=256)
 self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)
 self.d2_3 = nn.Dropout2d(p=self.dropout)
 self.fc1 = nn.Linear(256 * 3 * 3, 512)
 self.d1_1 = nn.Dropout(p=self.dropout)
 self.fc2 = nn.Linear(in_features=512, out_features=256)
 self.d1_2 = nn.Dropout(p=self.dropout)
 self.fc3 = nn.Linear(in_features=256, out_features=128)
 self.d1_3 = nn.Dropout(p=self.dropout)
 self.out = nn.Linear(in_features=128, out_features=10)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv1_bn(x)
 x = F.relu(x)
 x = self.conv1_1(x)
 x = self.conv1_1_bn(x)
 x = F.relu(x)
 x = self.d2_1(x)
 x = self.pool1(x)
 x = self.conv2(x)
 x = self.conv2_bn(x)
 x = F.relu(x)
 x = self.d2_2(x)
 x = self.pool2(x)
 x = self.conv3(x)
 x = self.conv3_bn(x)
 x = F.relu(x)
 x = self.d2_3(x)
 x = self.pool3(x)
 x = x.view(-1, 256 * 3 * 3)
 x = F.relu(self.fc1(x))
x = self.d1_1(x)
 x = F.relu(self.fc2(x))
x = self.d1_2(x)
 x = F.relu(self.fc3(x))
x = self.d1_3(x)
 x = self.out(x)
 return F.log_softmax(x, dim=-1)
 net = Net().to(device)
 net
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat)
 def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<define_variables>
def find_lr(trn_loader, init_value = 1e-8, final_value=10., beta = 0.98):
num = len(trn_loader)-1
mult =(final_value / init_value)**(1/num)
 lr = init_value
optimizer.param_groups[0]['lr'] = lr
avg_loss = 0.
 best_loss = 0.
 batch_num = 0
losses = []
log_lrs = []
for data in trn_loader:
batch_num += 1
inputs = data[0].to(device)
 labels = data[1].to(device)
 optimizer.zero_grad()
outputs = net(inputs)
 loss = criterion(outputs, labels)
 avg_loss = beta * avg_loss +(1-beta)*loss.item()
smoothed_loss = avg_loss /(1 - beta**batch_num)
 if batch_num > 1 and smoothed_loss > 4 * best_loss:
return log_lrs, losses
if smoothed_loss < best_loss or batch_num==1:
best_loss = smoothed_loss
losses.append(smoothed_loss)
 log_lrs.append(math.log10(lr))
loss.backward()
optimizer.step()
lr *= mult
optimizer.param_groups[0]['lr'] = lr
return log_lrs, losses
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )<train_model>
net = Net().to(device)
 criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters() , lr=1e-1)
 logs,losses = find_lr(trn_loader = train_loader)
 plt.plot(logs[10:-5],losses[10:-5] )
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
preds=[]
true=[]
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
preds.append(logits)
 true.append(label_ids)
 tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
<define_variables>
net = Net().to(device)
 EPOCHS = 30
nn_output = []
optimizer = optim.Adam(net.parameters() , lr=4e-3)
 criterion = nn.CrossEntropyLoss()
def get_num_correct(preds, labels):
return preds.argmax(dim=1 ).eq(labels ).sum().item()
for epoch in range(EPOCHS):
epoch_loss = 0
epoch_correct = 0
net.train()
for data in train_loader:
X = data[0].to(device)
 y = data[1].to(device)
 net.zero_grad()
output = net(X)
 tloss = criterion(output, y)
 tloss.backward()
optimizer.step()
epoch_loss += tloss.item()
epoch_correct += get_num_correct(output, y)
 net.eval()
val_loss = 0
val_correct = 0
test_loss = 0
test_correct = 0
with torch.no_grad() :
for data in val_loader:
X = data[0].to(device)
 y = data[1].to(device)
 preds = net(X)
 vloss = criterion(preds, y)
 val_correct += get_num_correct(preds, y)
 val_loss += vloss.item()
for data in test_loader:
X = data[0].to(device)
 y = data[1].to(device)
 preds = net(X)
 tstloss = criterion(preds, y)
 test_correct += get_num_correct(preds, y)
 test_loss += tstloss.item()
tmp_nn_output = [epoch + 1,EPOCHS,
epoch_loss/len(train_loader.dataset),epoch_correct/len(train_loader.dataset)*100,
val_loss/len(val_loader.dataset), val_correct/len(val_loader.dataset)*100,
test_loss/len(test_loader.dataset), test_correct/len(test_loader.dataset)*100
]
nn_output.append(tmp_nn_output)
 print('Epoch [{}/{}] train loss: {:.6f} acc: {:.3f} - valid loss: {:.6f} acc: {:.3f} - Test loss: {:.6f} acc: {:.3f}' 
.format(*tmp_nn_output))
flat_predictions = [item for sublist in preds for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
flat_true_labels = [item for sublist in true for item in sublist]<compute_test_metric>
pd_results = pd.DataFrame(nn_output,
columns = ['epoch','total_epochs','train_loss','train_acc','valid_loss','valid_acc','test_loss','test_acc'] 
)
 display(pd_results)
 print("Best Epoch: {}".format(pd_results.loc[pd_results.valid_acc.idxmax() ]['epoch']))
print(classification_report(flat_predictions,flat_true_labels))<define_variables>
num_classes = len(classes)
 net.eval()
predictions = torch.LongTensor().to(device)
 for images, _ in val_loader:
images = images.to(device)
 preds = net(images)
 predictions = torch.cat(( predictions, preds.argmax(dim=1)) , dim=0)
 cmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)
 for i in range(len(val_labels)) :
cmt[val_labels[i], predictions[i]] += 1
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size )<predict_on_test>
net.eval()
predictions = torch.LongTensor().to(device)
 for images in submission_loader:
images = images.to(device)
 preds = net(images)
 predictions = torch.cat(( predictions, preds.argmax(dim=1)) , dim=0 )
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()<save_to_csv>
submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv")
 submission['label'] = predictions.cpu().numpy()
submission.to_csv("predictions.csv", index=False )