Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, sampler
from torchvision import transforms
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
from random import shuffle, randint
from PIL import Image
import math
N_EPOCHS = EPOCHS
steps_per_epoch=len(train_df)// BATCH_SIZE
total_training_steps = steps_per_epoch * N_EPOCHS
warmup_steps = total_training_steps // 5
warmup_steps, total_training_steps<choose_model_class>
class DigitDataset(Dataset):

 def __init__(self, csv_file, root_dir, train=False, transform=None):
self.digit_df = pd.read_csv(root_dir + csv_file)
 self.transform = transform
self.train = train
def __len__(self):
return len(self.digit_df)
 def __getitem__(self, item):
if self.train:
digit = self.digit_df.iloc[item, 1:].values
digit = digit.astype('float' ).reshape(( 28, 28))
label = self.digit_df.iloc[item, 0]
else:
digit = self.digit_df.iloc[item, :].values
digit = digit.astype('float' ).reshape(( 28, 28))
label = 0
sample = [digit, label]
if self.transform:
sample[0] = self.transform(sample[0])
 return sample
optimizer = AdamW(model.parameters() , lr=2e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=warmup_steps,
num_training_steps=total_training_steps
)<train_model>
class Regularize(object):

 def __init__(self, max_pixel=255):
self.max_pixel = max_pixel
def __call__(self, digit):
assert isinstance(digit, np.ndarray)
 digit = digit / self.max_pixel
return digit
class ToTensor(object):

 def __call__(self, digit):
assert isinstance(digit, np.ndarray)
 digit = digit.reshape(( 1, 28, 28))
digit = torch.from_numpy(digit)
 digit = digit.float()
return digit
def train() :
model.train()
total_loss, total_accuracy = 0, 0
avg_loss = 0
total_preds=[]
for step,batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
print(' Batch {:>5,} of {:>5,}.'.format(step, len(train_dataloader)))
 input_ids = batch["input_ids"].to(device)
 attention_mask = batch["attention_mask"].to(device)
 labels = batch["labels"].to(device)
 model.zero_grad()
loss, outputs = model(input_ids, attention_mask, labels)
 total_loss = total_loss + loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
outputs=outputs.detach().cpu().numpy()
total_preds.append(outputs)
 avg_loss = total_loss / len(train_dataloader)
 print(f"{step}: {avg_loss}")
 total_preds = np.concatenate(total_preds, axis=0)
 return avg_loss, total_preds<import_modules>
data_np = DigitDataset('train.csv', '.. /input/', train=True)
 print("Number of Training Images: ", len(data_np))
plt.imshow(data_np[5][0], cmap='gray')
 plt.show()
print("Label for the Image: ", data_np[5][1] )
<train_model>
composed_transform = transforms.Compose([Regularize() , ToTensor() ])
 data_torch = DigitDataset('train.csv', '.. /input/', train=True, transform=composed_transform)
 dataloader = DataLoader(data_torch,
batch_size=4,
shuffle=True,
num_workers=4)
 for i, data in enumerate(dataloader, 0):
digits, labels = data
print("Type of Digits: ", type(digits))
print("Dimension of the Tensor: ", digits.shape)
 print("Type of Labels: ", type(labels))
print("Dimension of the Tensor: ", labels.shape)
 if i == 0:
break
def evaluate() :
print("
Evaluating...")
 model.eval()
total_loss, total_accuracy = 0, 0
total_preds = []
total_labels = []
for step,batch in enumerate(val_dataloader):
if step % 50 == 0 and not step == 0:
print(' Batch {:>5,} of {:>5,}.'.format(step, len(val_dataloader)))
 input_ids = batch["input_ids"].to(device)
 attention_mask = batch["attention_mask"].to(device)
 labels = batch["labels"].to(device)
 with torch.no_grad() :
loss, outputs = model(input_ids, attention_mask, labels)
 total_loss = total_loss + loss.item()
outputs = outputs.detach().cpu().numpy()
labels = labels.detach().cpu().numpy()
total_preds.append(outputs)
 total_labels.append(labels)
 avg_loss = total_loss / len(val_dataloader)
 print(f"{step}: {avg_loss}")
 total_preds = np.concatenate(total_preds, axis=0)
 total_labels = np.concatenate(total_labels, axis=0)
 true = np.array(total_labels)
 pred = np.array(total_preds>0.5)
 for i, name in enumerate(LABEL_COLUMNS):
try:
print(f"{name} roc_auc {roc_auc_score(true[:, i], pred[:, i])}")
 except Exception as e:
print(e)
 pass
print(f"Evaluate loss {total_loss / len(val_dataloader)}")
 return avg_loss, total_preds, total_labels<train_model>
def digits_per_class(digit_df, indices):
assert isinstance(digit_df, pd.DataFrame)
 assert isinstance(indices, list)
 digit_num = [0 for num in range(10)]
for idx in indices:
label = digit_df.iloc[idx, 0]
digit_num[label] += 1
return digit_num
%%time
best_valid_loss = float('inf')
 train_losses=[]
valid_losses=[]
EPOCHS = 2
for epoch in range(EPOCHS):
print('
Epoch {:} / {:}'.format(epoch + 1, EPOCHS))
train_loss, _ = train()
valid_loss, _, _ = evaluate()
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict() , 'saved_weights.pt')
 train_losses.append(train_loss)
 valid_losses.append(valid_loss)
 print(f'
Training Loss: {train_loss:.3f}')
 print(f'Validation Loss: {valid_loss:.3f}' )<train_model>
digit_class_num = digits_per_class(data_torch.digit_df,
[num for num in range(len(data_torch)) ])
 for i, num in enumerate(digit_class_num, 0):
print("Number of Images for Digit ", i, ": ", num)
 print("Overall Images: ", sum(digit_class_num))
def test() :
print("
Testing...")
 model.eval()
total_loss, total_accuracy = 0, 0
total_preds = []
_ids = []
for step,batch in enumerate(test_dataloader):
if step % 50 == 0 and not step == 0:
print(' Batch {:>5,} of {:>5,}.'.format(step, len(test_dataloader)))
 _id = batch["_id"]
input_ids = batch["input_ids"].to(device)
 attention_mask = batch["attention_mask"].to(device)
 with torch.no_grad() :
loss, outputs = model(input_ids, attention_mask)
 total_loss = total_loss + loss
outputs = outputs
_ids.append(_id)
 total_preds.append(outputs)
 avg_loss = total_loss / len(test_dataloader)
 _ids = np.concatenate(_ids, axis=0)
 total_preds = torch.cat(total_preds, axis=0)
 results = dict(id=_ids,
predictions = total_preds 
)
 return avg_loss, total_preds, results<compute_train_metric>
def train_validate_split(digit_df, test_ratio=0.2):
assert isinstance(digit_df, pd.DataFrame)
 digit_num = len(digit_df)
 overall_indices = [num for num in range(digit_num)]
overall_class_num = digits_per_class(digit_df, overall_indices)
 test_class_num = [int(num*test_ratio)for num in overall_class_num]
tmp_test_class_num = [0 for num in range(10)]
shuffle(overall_indices)
 train_indices = []
val_indices = []
for idx in overall_indices:
tmp_label = digit_df.iloc[idx, 0]
if tmp_test_class_num[tmp_label] < test_class_num[tmp_label]:
val_indices.append(idx)
 tmp_test_class_num[tmp_label] += 1
else:
train_indices.append(idx)
 return train_indices, val_indices
def evaluate_roc(probs, y_true):

 preds = probs
fpr, tpr, threshold = roc_curve(y_true, preds)
 roc_auc = auc(fpr, tpr)
 print(f'AUC: {roc_auc:.4f}')
 y_pred = np.where(preds >= 0.5, 1, 0)
 accuracy = accuracy_score(y_true, y_pred)
 print(f'Accuracy: {accuracy*100:.2f}%')
 plt.title('Receiver Operating Characteristic')
 plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
 plt.legend(loc = 'lower right')
 plt.plot([0, 1], [0, 1],'r--')
 plt.xlim([0, 1])
 plt.ylim([0, 1])
 plt.ylabel('True Positive Rate')
 plt.xlabel('False Positive Rate')
 plt.show()<compute_train_metric>
train_data, val_data = train_validate_split(data_torch.digit_df)
 train_class_num = digits_per_class(data_torch.digit_df, train_data)
 val_class_num = digits_per_class(data_torch.digit_df, val_data)
 for i, num in enumerate(train_class_num, 0):
print("Number of Images for Digit ", i, "- Train: ", num, "Validate: ", val_class_num[i])
 print("Train Images: ", sum(train_class_num), "Validate Images: ", sum(val_class_num))
avg_loss, total_preds, total_labels = evaluate()<compute_test_metric>
train_sampler = sampler.SubsetRandomSampler(train_data)
 train_dataloader = DataLoader(data_torch,
batch_size=4,
shuffle=False,
sampler=train_sampler,
num_workers=4 )
for i, name in enumerate(LABEL_COLUMNS):
print(f"label: {name}")
 evaluate_roc(total_preds[:,i]>0.5, total_labels[:,i] )<compute_test_metric>
class BasicLeNet(nn.Module):

 def __init__(self):
super(BasicLeNet, self ).__init__()
self.features = nn.Sequential(
 nn.Conv2d(1, 6, 5),
nn.ReLU(inplace=True),
nn.MaxPool2d(2),
nn.Conv2d(6, 16, 5),
nn.ReLU(inplace=True),
nn.MaxPool2d(2)
)
 self.classifier = nn.Sequential(
 nn.Linear(16*4*4, 120),
nn.ReLU(inplace=True),
nn.Linear(120, 84),
nn.ReLU(inplace=True),
nn.Linear(84, 10)
)
 def forward(self, x):

 x = self.features(x)
 x = x.view(x.size(0), 16*4*4)
 x = self.classifier(x)
 return x
avg_test_loss, total_test_preds, sub = test()<create_dataframe>
def training(network, criterion, optimizer, epoch_num, test=True):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("Start Training with", device, epoch_num, "overall epoch")
 network.to(device)

composed_transform = transforms.Compose([Regularize() , ToTensor() ])
 digit_dataset = DigitDataset('train.csv', '.. /input/', train=True, transform=composed_transform)
 if test:
train_indices, val_indices = train_validate_split(digit_dataset.digit_df)
 train_sampler = sampler.SubsetRandomSampler(train_indices)
 val_sampler = sampler.SubsetRandomSampler(val_indices)
 train_dataloader = DataLoader(
 digit_dataset,
batch_size=32,
shuffle=False,
sampler=train_sampler,
num_workers=4,
pin_memory=True 
)
 val_dataloader = DataLoader(
 digit_dataset,
batch_size=32,
shuffle=False,
sampler=val_sampler,
num_workers=4,
pin_memory=True 
)
 print("Training with validation, ", "Overall Data:", len(train_indices)+len(val_indices))
print("Training Data:", len(train_indices), "Validate Data:", len(val_indices))
else:
train_dataloader = DataLoader(
 digit_dataset,
batch_size=32,
shuffle=True,
num_workers=4,
pin_memory=True 
)
 val_dataloader = None
print("Training all data, ", "Overall Data:", len(digit_dataset))

 batch_num = 0
ita = []
loss_avg = []
val_acc = []
for epoch in range(epoch_num):
running_loss = 0.0
for i, data in enumerate(train_dataloader, 0):
digits, labels = data
digits, labels = digits.to(device), labels.to(device)
 optimizer.zero_grad()
outputs = network(digits)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
batch_num += 1
if test == True and i % 500 == 499:
ita.append(batch_num)
 loss_avg.append(running_loss/500.)
val_acc.append(validating(network, val_dataloader))
running_loss = 0.
 if test:
train_accuracy = validating(network, train_dataloader)
 val_accuracy = validating(network, val_dataloader)
 print('Training accuracy: %.5f' %(train_accuracy))
print('Validation accuracy: %.5f' %(val_accuracy))
return network, ita, loss_avg, val_acc
D = pd.DataFrame()
D['id'] = sub['id']
D<data_type_conversions>
def validating(network, loader):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 correct_num = 0
total_num = 0
for i, data in enumerate(loader, 0):
digits, labels = data
total_num += labels.size(0)
 digits, labels = digits.to(device), labels.to(device)
 outputs = network(digits)
 _, predicted = torch.max(outputs, 1)
 correct_num +=(( predicted == labels ).sum().to("cpu")).item()
accuracy = correct_num / total_num
return accuracy
D[LABEL_COLUMNS] =(sub['predictions'].cpu().numpy())
 D<save_to_csv>
lenet = BasicLeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters())
 lenet, batch_ita, loss_list, val_acc_list = training(lenet, criterion, optimizer, 30 )
D.to_csv("submission.csv", index=False )<import_modules>
class DigitDataset(Dataset):

 def __init__(self, csv_file, root_dir, train=False, argument=True, transform=None):
self.digit_df = pd.read_csv(root_dir + csv_file)
 self.transform = transform
self.train = train
self.argument = argument
def __len__(self):
if self.argument:
return 2 * len(self.digit_df)
 else:
return len(self.digit_df)
 def __getitem__(self, item):
if item < len(self.digit_df):
if self.train:
digit = self.digit_df.iloc[item, 1:].values
digit = digit.astype('float' ).reshape(( 28, 28))
label = self.digit_df.iloc[item, 0]
else:
digit = self.digit_df.iloc[item, :].values
digit = digit.astype('float' ).reshape(( 28, 28))
label = 0
else:
assert self.argument and self.train
digit = self.digit_df.iloc[item % len(self.digit_df), 1:].values
digit = digit.astype('float' ).reshape(( 28, 28))
rand_theta =(randint(-20, 20)/ 180)* math.pi
rand_x = randint(-2, 2)
 rand_y = randint(-2, 2)
 rand_scale = randint(9, 11)* 0.1
digit = digit_argument(digit, rand_theta, [rand_x, rand_y], rand_scale)
 label = self.digit_df.iloc[item % len(self.digit_df), 0]
sample = [digit, label]
if self.transform:
sample[0] = self.transform(sample[0])
 return sample
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.linear_model import LogisticRegression
from tensorflow import keras
from keras import layers
from keras.callbacks import Callback
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from textblob import TextBlob
from textblob import Word
import re
import string 
<load_from_disk>
def train_validate_split(digit_df, test_ratio=0.2, argument=True):
assert isinstance(digit_df, pd.DataFrame)
 digit_num = len(digit_df)
 overall_indices = [num for num in range(digit_num)]
overall_class_num = digits_per_class(digit_df, overall_indices)
 test_class_num = [int(num*test_ratio)for num in overall_class_num]
tmp_test_class_num = [0 for num in range(10)]
shuffle(overall_indices)
 train_indices = []
val_indices = []
for idx in overall_indices:
tmp_label = digit_df.iloc[idx, 0]
if tmp_test_class_num[tmp_label] < test_class_num[tmp_label]:
val_indices.append(idx)
 tmp_test_class_num[tmp_label] += 1
else:
train_indices.append(idx)
 if argument:
train_indices.append(idx + digit_num)
 return train_indices, val_indices
!unzip -q "/kaggle/input/jigsaw-toxic-comment-classification-challenge/*.zip"
!dir<load_from_csv>
class EnhancedLeNet(nn.Module):
def __init__(self):
super(EnhancedLeNet, self ).__init__()
self.features = nn.Sequential(
 nn.Conv2d(1, 64, 5, padding=2),
nn.ReLU(inplace=True),
nn.Conv2d(64, 64, 5, padding=2),
nn.ReLU(inplace=True),
nn.BatchNorm2d(64),
nn.MaxPool2d(2),
nn.Conv2d(64, 128, 5, padding=2),
nn.ReLU(inplace=True),
nn.Conv2d(128, 128, 5, padding=2),
nn.ReLU(inplace=True),
nn.BatchNorm2d(128),
nn.MaxPool2d(2), 
)
 self.classifier = nn.Sequential(
 nn.Linear(128*7*7, 512),
nn.ReLU(inplace=True),
nn.BatchNorm1d(512),
nn.Linear(512, 10)
)
 def forward(self, x):
x = self.features(x)
 x = x.view(x.size(0), 128*7*7)
 x = self.classifier(x)
 return x
train_data_file = "train.csv"
test_data_file = "test.csv"
submission_file = "sample_submission.csv"
train_data = pd.read_csv(train_data_file)
 test_data = pd.read_csv(test_data_file)
 submission_result = pd.read_csv(submission_file )<define_variables>
lenet = EnhancedLeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters())
 lenet, batch_ita, loss_list, val_acc_list = training(lenet, criterion, optimizer, 30 )
max_len = 120
embedding_dim = 300
vocabulary_size = 20000
num_tokens = vocabulary_size+1<categorify>
def testing(network):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 composed_transform = transforms.Compose([Regularize() , ToTensor() ])
 digit_dataset = DigitDataset('test.csv', '.. /input/', train=False, argument=False, transform=composed_transform)
 test_dataloader = DataLoader(
 digit_dataset,
batch_size=128,
shuffle=False,
num_workers=4,
pin_memory=True 
)
 test_results = []
for i, data in enumerate(test_dataloader, 0):
digits, label = data
digits = digits.to(device)
 outputs = network(digits)
 _, predicted = torch.max(outputs, 1)
 test_results += np.int_(predicted.to("cpu" ).numpy().squeeze() ).tolist()

 test_df = pd.read_csv(".. /input/sample_submission.csv")
 assert(len(test_df)== len(test_results))
test_df.loc[:, 'Label'] = test_results
test_df.to_csv('test_results.csv', index=False)
 print("Test Results for Kaggle Generated..." )
def preprocess(corpus):
printable = set(string.printable)
 corpus = ''.join(filter(lambda x: x in printable, corpus))
corpus = corpus.lower()
corpus = re.sub(r"won't", "will not", corpus)
 corpus = re.sub(r"can't", "can not", corpus)
 corpus = re.sub(r"ain't","is not", corpus)
 corpus = re.sub(r"shan't", "shall not", corpus)
 corputs = re.sub(r"let's", "let us", corpus)
 corpus = re.sub(r"n't", " not", corpus)
 corpus = re.sub(r"'re", " are", corpus)
 corpus = re.sub(r"'s", " is", corpus)
 corpus = re.sub(r"'d", " would", corpus)
 corpus = re.sub(r"'ll", " will", corpus)
 corpus = re.sub(r"'t", " not", corpus)
 corpus = re.sub(r"'ve", " have", corpus)
 corpus = re.sub(r"'m", " am", corpus)
 corpus = re.sub(r"'", " ", corpus)
 correction_list = {"youfuck": "you fuck", \
"fucksex": "fuck sex",\
"bitchbot": "bitch bot",\
"offfuck": "fuck off",\
"donkeysex": "donkey sex",\
"securityfuck": "security fuck",\
"ancestryfuck": "ancestry fuck",\
"turkeyfuck": "turkey fuck",\
"faggotgay": "faggot gay",\
"fuckbot": "fuck bot",\
"assfuckers": "ass fucker",\
"ckckck": "cock",\
"fuckfuck": "fuck",\
"lolol": "lol",\
"pussyfuck": "fuck",\
"gaygay": "gay",\
"haha": "ha",\
"sucksuck": "suck"
}
for old,new in correction_list.items() :
corpus = corpus.replace(old,new)
 return corpus<prepare_x_and_y>
lenet = EnhancedLeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lenet.parameters())
 lenet, batch_ita, loss_list, val_acc_list = training(lenet, criterion, optimizer, 50, test=False)
 testing(lenet )