Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
import numpy as np
import pandas as pd
import os
import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
from torch.optim import lr_scheduler
import time
import random
import copy
import matplotlib.pyplot as plt
from typing import Union, List, Dict, Any, cast
from PIL import Image
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+' )<string_transform>
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.enabled = True
seed_torch()
def wordCount(text):
try:
if text == 'No description yet':
return 0
else:
text = text.lower()
words = [w for w in text.split(" ")]
return len(words)
 except:
return 0<string_transform>
BATCH = 32
EPOCHS = 100
LR = 0.001
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 device = DEVICE
def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1 and x not in stopwords])
<data_type_conversions>
all_df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv", dtype=np.float32)
 test_df = pd.read_csv("/kaggle/input/digit-recognizer/test.csv", dtype=np.float32)
 sample_sub = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
def normalize_dataset_text(dataset):
dataset['item_description'] = dataset['item_description'].apply(lambda x: normalize_text(x))
dataset['brand_name'] = dataset['brand_name'].apply(lambda x: normalize_text(x))<feature_engineering>
train_df, validate_df = train_test_split(all_df, train_size=0.8, test_size=0.2 )
def delete_unseen(dataset):
dataset.loc[~dataset['brand_name'].isin(all_brand), 'brand_name'] = 'missing'
dataset.loc[~dataset['general_cat'].isin(all_general_cat), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(all_subcat_1), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(all_subcat_2), 'subcat_2'] = 'missing'<feature_engineering>
original_transforms = transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor()
])
 sub = [original_transforms]
train_transforms = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees=25, translate=(0.1, 0.1), scale=(0.8, 1.2)) ,
transforms.ToTensor()
] )
def text_length_feature(dataset, train = True):
if train:
dataset['desc_len'] = dataset['item_description'].apply(lambda x: wordCount(x))
dataset['name_len'] = dataset['name'].apply(lambda x: wordCount(x))
dataset[['desc_len', 'name_len']] = desc_normalizer.fit_transform(dataset[['desc_len', 'name_len']])
 else:
dataset['desc_len'] = dataset['item_description'].apply(lambda x: wordCount(x))
dataset['name_len'] = dataset['name'].apply(lambda x: wordCount(x))
dataset[['desc_len', 'name_len']] = desc_normalizer.transform(dataset[['desc_len', 'name_len']] )<data_type_conversions>
class GetData(Dataset):
def __init__(self, X_Train, Y_Train, Transform):
self.X = X_Train
self.transform = Transform
self.Y = Y_Train
def __len__(self):
return len(self.X)
 def __getitem__(self, index):
return self.transform(self.X[index] ).contiguous() , self.Y[index]
start_time = time.time()
print(strftime("%Y-%m-%d %H:%M:%S", gmtime()))<split>
def getSet(data_df, Transform):
data_df.head()
Y = data_df.label.values
X0 = data_df.loc[:, data_df.columns != 'label'].values/255
Y = torch.from_numpy(Y ).type(torch.LongTensor)
 X = X0.reshape(( -1, 1, 28, 28))
X = torch.from_numpy(X)
 return GetData(X, Y, Transform )
train_df, dev_df = train_test_split(train_df, random_state=200, train_size=0.70 )<prepare_x_and_y>
trainset = getSet(train_df, train_transforms)
 validateset = getSet(validate_df, original_transforms)
 dataloaders = { 'train': DataLoader(trainset, batch_size=BATCH, shuffle=True, num_workers=4),
'val': DataLoader(validateset, batch_size=BATCH, shuffle=True, num_workers=4)}
train_df = train_df.drop(train_df[(train_df.price < 1.0)].index)
 train_y = np.log1p(train_df["price"])
 dev_y = np.log1p(dev_df["price"] )<feature_engineering>
dataset_sizes = { 'train': len(trainset), 'val': len(validateset)}
dataset_sizes
train_df['general_cat'], train_df['subcat_1'], train_df['subcat_2'] = \
zip(*train_df['category_name'].apply(lambda x: split_cat(x)))
 train_df.drop('category_name', axis=1, inplace=True)
 print('[{}] Split categories completed.'.format(time.time() - start_time))
handle_missing_inplace(train_df)
 print('[{}] Handle missing completed.'.format(time.time() - start_time))
cutting(train_df)
 print('[{}] Cut completed.'.format(time.time() - start_time))
to_categorical(train_df)
 print('[{}] Convert categorical completed'.format(time.time() - start_time)) 
<categorify>
class VGG(nn.Module):
def __init__(
 self,
features: nn.Module,
num_classes: int = 1000,
init_weights: bool = True 
)-> None:
super(VGG, self ).__init__()
self.features = features
self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.classifier = nn.Sequential(
 nn.Linear(256, num_classes), 
)
 if init_weights:
self._initialize_weights()
def forward(self, x: torch.Tensor)-> torch.Tensor:
x = self.features(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.classifier(x)
 return x
def _initialize_weights(self)-> None:
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 if m.bias is not None:
nn.init.constant_(m.bias, 0)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 elif isinstance(m, nn.Linear):
nn.init.normal_(m.weight, 0, 0.01)
 nn.init.constant_(m.bias, 0)
 cfgs: Dict[str, List[Union[str, int]]] = {
'A': [64, 'M', 128, 128, 'M', 256, 256, 256, 'M'],
'B': [16, 'M', 256, 'M'],
'firstPadding': 2
}
def make_layers(cfg: List[Union[str, int]], batch_norm: bool = False)-> nn.Sequential:
layers: List[nn.Module] = []
in_channels = 1
in_padding = 5
i = 0
for v in cfg:
if v == 'M':
layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
else:
v = cast(int, v)
 in_padding = 1
if i == 5:
in_padding = 2
conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=in_padding)
 if batch_norm:
layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
else:
layers += [conv2d, nn.ReLU(inplace=True)]
in_channels = v
i = i + 1
return nn.Sequential(*layers)
 def _vgg(arch: str, cfg: str, batch_norm: bool,num_classes: int, **kwargs: Any)-> VGG:
model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), num_classes, **kwargs)
 return model
def selfDefineVgg(num_classes: int, **kwargs: Any)-> VGG:
return _vgg('vgg', 'A', True, num_classes, **kwargs)
 model = selfDefineVgg(10)
 model = model.to(DEVICE )
desc_normalizer = Normalizer()
name_normalizer = Normalizer()
text_length_feature(train_df)
 print('[{}] Calculate length features'.format(time.time() - start_time))
normalize_dataset_text(train_df)
 print('[{}] Normalization text'.format(time.time() - start_time))<categorify>
criterion = nn.CrossEntropyLoss()
all_brand = set(train_df["brand_name"].values)
 all_general_cat = set(train_df["general_cat"].values)
 all_subcat_1 = set(train_df["subcat_1"].values)
 all_subcat_2 = set(train_df["subcat_2"].values)
 le_brand = LabelEncoder()
le_general_cat = LabelEncoder()
le_subcat_1 = LabelEncoder()
le_subcat_2 = LabelEncoder()
le_brand.fit(train_df['brand_name'])
 train_df['encoded_brand_name'] = le_brand.transform(train_df['brand_name'])
 le_general_cat.fit(train_df['general_cat'])
 train_df['encoded_general_cat'] = le_general_cat.transform(train_df['general_cat'])
 le_subcat_1.fit(train_df['subcat_1'])
 train_df['encoded_subcat_1'] = le_subcat_1.transform(train_df['subcat_1'])
 le_subcat_2.fit(train_df['subcat_2'])
 train_df['encoded_subcat_2'] = le_subcat_2.transform(train_df['subcat_2'] )<feature_engineering>
optimizer = torch.optim.Adam(model.parameters() , lr=LR, betas=(0.9, 0.999), eps=1e-9)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.7, patience=3, verbose=True )
print("Tokenizing item description")
 tok_desc = Tokenizer()
tok_desc.fit_on_texts(train_df["item_description"].values)
 print("Tokenizing name")
 tok_name = Tokenizer()
tok_name.fit_on_texts(train_df["name"].values)
 print("Transforming text to sequences...")
 train_df['seq_item_description'] = tok_desc.texts_to_sequences(train_df["item_description"].values)
 train_df['seq_name'] = tok_name.texts_to_sequences(train_df["name"].values)
<define_variables>
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 if phase == 'train':
acc = 100.* running_corrects.double() / dataset_sizes[phase]
scheduler.step(acc)
 epoch_loss = running_loss / dataset_sizes[phase]
epoch_acc = running_corrects.double() / dataset_sizes[phase]
if phase == 'train':
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
 print('{} Loss: {:.4f} Acc: {:.4f}'.format(
 phase, epoch_loss, epoch_acc))
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
model.load_state_dict(best_model_wts)
 return model
MAX_NAME_SEQ = 15
MAX_ITEM_DESC_SEQ = 50
MAX_DESC_TEXT = len(tok_desc.word_index)+ 1
MAX_NAME_TEXT = len(tok_name.word_index)+ 1
MAX_BRAND = len(le_brand.classes_)
 MAX_GENCAT = len(le_general_cat.classes_)
 MAX_SUBCAT_1 = len(le_subcat_1.classes_)
 MAX_SUBCAT_2 = len(le_subcat_2.classes_)
 MAX_CONDITION = max(train_df.item_condition_id)+ 1<categorify>
model = train_model(model, criterion, optimizer, scheduler,
num_epochs=EPOCHS )
def get_rnn_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(dataset.encoded_brand_name),
'item_condition': np.array(dataset.item_condition_id),
'num_vars': np.array(dataset[["shipping"]]),
'desc_len': np.array(dataset[["desc_len"]]),
'name_len': np.array(dataset[["name_len"]]),
'general_cat': np.array(dataset.encoded_general_cat),
'subcat_1': np.array(dataset.encoded_subcat_1),
'subcat_2': np.array(dataset.encoded_subcat_2),
}
return X<prepare_x_and_y>
y_val = validate_df.label.values
x_val = validate_df.loc[:, validate_df.columns != 'label'].values/255
x_val = x_val.reshape(( -1, 1, 28, 28))
print(x_val[0].shape)
 with torch.no_grad() :
model.eval()
preds = model(torch.from_numpy(x_val ).to(DEVICE)).cpu().argmax(dim=1)
train_X = get_rnn_data(train_df )<define_variables>
with torch.no_grad() :
model.eval()
sample_sub['Label'] = model(torch.from_numpy(Xt ).to(DEVICE)).cpu().argmax(dim=1 )
<init_hyperparams><EOS> 
sample_sub.to_csv("submission.csv", index=False )