Kaggle competition: Digit Recognizer
del sales, categories, shops, shops_cats, temp, temp2, test, dupes, item_map,
df.head()<feature_engineering>
train_path = "/kaggle/input/digit-recognizer/train.csv"
test_path = "/kaggle/input/digit-recognizer/test.csv"
sample_submission_path = "/kaggle/input/digit-recognizer/sample_submission.csv"
df['item_cnt_unclipped'] = df['item_cnt']
df['item_cnt'] = df['item_cnt'].clip(0, 20 )<data_type_conversions>
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path)
 sample_submission = pd.read_csv(sample_submission_path )
def downcast(df):
float_cols = [c for c in df if df[c].dtype in ["float64"]]
int_cols = [c for c in df if df[c].dtype in ['int64']]
df[float_cols] = df[float_cols].astype('float32')
 df[int_cols] = df[int_cols].astype('int16')
 return df
df = downcast(df )<data_type_conversions>
import torch
from torchvision.transforms import Normalize
from torch.utils.data import Dataset, DataLoader, random_split
from tqdm.notebook import tqdm
df['item_age'] =(df['date_block_num'] - df.groupby('item_id')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first4_age'] =(df['date_block_num'] - df.groupby('item_name_first4')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first6_age'] =(df['date_block_num'] - df.groupby('item_name_first6')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first11_age'] =(df['date_block_num'] - df.groupby('item_name_first11')['date_block_num'].transform('min')).astype('int8')
 df['category_age'] =(df['date_block_num'] - df.groupby('category_id')['date_block_num'].transform('min')).astype('int8')
 df['group_age'] =(df['date_block_num'] - df.groupby('group_id')['date_block_num'].transform('min')).astype('int8')
 df['shop_age'] =(df['date_block_num'] - df.groupby('shop_id')['date_block_num'].transform('min')).astype('int8' )<merge>
class LoadImagesLabels(Dataset):
def __init__(self, data_frame, stats = None, kind="train"):
self.stats = stats
self.kind = kind
self.imgs, self.labels = self.get_data(data_frame)
 def get_data(self, data_frame):
imgs = np.zeros(shape=(data_frame.shape[0], 1, 28, 28), dtype=np.float32)
 labels = None
if self.kind != "test":
labels = np.zeros(shape=(data_frame.shape[0]), dtype=np.int64)
 for idx, row in tqdm(data_frame.iterrows() , total=data_frame.shape[0],
desc="Extracting images and labels"):
list_row = row.tolist()
if self.kind != "test":
labels[idx] = int(list_row[0])
 imgs[idx, :] = np.asarray(list_row[1:], dtype=np.float32 ).reshape(1, 28, 28)
 else:
imgs[idx, :] = np.asarray(list_row, dtype=np.float32 ).reshape(1, 28, 28)
 if self.stats is None:
self.stats = {"mean":(imgs.reshape(-1, 1, 28**2)/255.0 ).mean(2 ).sum(0)/imgs.shape[0],
"std":(imgs.reshape(-1, 1, 28**2)/255.0 ).std(2 ).sum(0)/imgs.shape[0]}
self.create_stats_dataset()
else:
self.normalize = Normalize(mean=self.stats["mean"], std=self.stats["std"])
 return imgs, labels
def create_stats_dataset(self):
self.normalize = Normalize(mean=self.stats["mean"], std=self.stats["std"])
 self.unormalize = Normalize(mean=-self.stats["mean"]/self.stats["std"], std=1/self.stats["std"])
 def __len__(self):
return len(self.imgs)
 def __getitem__(self, idx):
if self.labels is None:
return self.normalize(torch.from_numpy(self.imgs[idx]/255.0))
else:
return self.normalize(torch.from_numpy(self.imgs[idx]/255.0)) , torch.tensor(self.labels[idx], dtype=torch.int64 )
temp = df.query('item_cnt > 0' ).groupby(['item_id','shop_id'] ).agg({'date_block_num':'min'} ).reset_index()
temp.columns = ['item_id', 'shop_id', 'item_shop_first_sale']
df = pd.merge(df, temp, on=['item_id','shop_id'], how='left')
 df['item_shop_first_sale'] = df['item_shop_first_sale'].fillna(50)
 df['item_age_if_shop_sale'] =(df['date_block_num'] > df['item_shop_first_sale'])* df['item_age']
df['item_age_without_shop_sale'] =(df['date_block_num'] <= df['item_shop_first_sale'])* df['item_age']
del df['item_shop_first_sale']<merge>
dataset = LoadImagesLabels(train)
 train_dataset, val_dataset = random_split(dataset,
[int(len(dataset)*0.7), len(dataset)-int(len(dataset)*0.7)])
 test_dataset = LoadImagesLabels(test, dataset.stats, "test" )
def agg_cnt_col(df, merging_cols, new_col,aggregation):
temp = df.groupby(merging_cols ).agg(aggregation ).reset_index()
temp.columns = merging_cols + [new_col]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_cnt_col(df, ['date_block_num','item_id'],'item_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'item_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','group_id','shop_id'],'group_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','group_id'],'group_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_id'],'shop_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_city'],'city_cnt',{'item_cnt':'mean'} )<merge>
import matplotlib.pyplot as plt
import random
def new_item_sales(df, merging_cols, new_col):
temp =(
 df 
.query('item_age==0')
.groupby(merging_cols)['item_cnt'] 
.mean() 
.reset_index() 
.rename(columns={'item_cnt': new_col})
)
 df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = new_item_sales(df, ['date_block_num','category_id','shop_id'], 'new_items_in_cat')
 df = new_item_sales(df, ['date_block_num','category_id'], 'new_items_in_cat_all_shops' )<merge>
train_dataloader = DataLoader(train_dataset, num_workers=multiprocessing.cpu_count() , shuffle=True, batch_size=2**6)
 val_dataloader = DataLoader(val_dataset, num_workers=multiprocessing.cpu_count() , shuffle=False, batch_size=8 )
def agg_price_col(df, merging_cols, new_col):
temp = df.groupby(merging_cols ).agg({'revenue':'sum','item_cnt_unclipped':'sum'} ).reset_index()
temp[new_col] = temp['revenue']/temp['item_cnt_unclipped']
temp = temp[merging_cols + [new_col]]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_price_col(df,['date_block_num','item_id'],'item_price')
 df = agg_price_col(df,['date_block_num','category_id'],'category_price')
 df = agg_price_col(df,['date_block_num'],'block_price' )<categorify>
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import pytorch_lightning as pl
from torchvision import models
import pytorch_lightning.callbacks as pl_callbacks
df = downcast(df )<merge>
class ClassificationModel(pl.LightningModule):
def __init__(self, learning_rate, weights=None):
super(ClassificationModel, self ).__init__()
self.learning_rate = learning_rate
self.loss = nn.CrossEntropyLoss(weight=weights)
 pretrained_resnet = models.resnet18(pretrained=True)
 self.encoder = nn.Sequential(
 nn.Upsample(size=(224, 224), mode='bilinear', align_corners=True),
nn.Conv2d(in_channels=1, out_channels=3, kernel_size=1),
nn.BatchNorm2d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False),
pretrained_resnet 
)
 self.decoder = nn.Sequential(
 nn.ReLU(inplace=True),
nn.Linear(in_features=1000, out_features=10)
)
 def configure_optimizers(self):
return optim.Adam(self.parameters() , lr=(self.learning_rate))
def forward(self, x):
x = self.encoder(x)
 x = self.decoder(x)
 return x
def training_step(self, batch, batch_idx):
x, y = batch
y_pred = self(x)
 loss = self.loss(y_pred, y)
 self.log("train_loss", loss, on_epoch=True,
prog_bar=True, logger=False)
 return loss
def validation_step(self, batch, batch_idx):
x, y = batch
y_pred = self(x)
 loss = self.loss(y_pred, y)
 self.log("val_loss", loss, on_epoch=True,
prog_bar=True, logger=False )
def lag_feature(df, lag, col, merge_cols):
temp = df[merge_cols + [col]]
temp = temp.groupby(merge_cols ).agg({f'{col}':'first'} ).reset_index()
temp.columns = merge_cols + [f'{col}_lag{lag}']
temp['date_block_num'] += lag
df = pd.merge(df, temp, on=merge_cols, how='left')
 df[f'{col}_lag{lag}'] = df[f'{col}_lag{lag}'].fillna(0 ).astype('float32')
 return df<define_variables>
trainer.fit(model, train_dataloader, val_dataloader )
lag12_cols = {
'item_cnt':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops':['date_block_num', 'item_id'],
'category_cnt':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops':['date_block_num', 'category_id'],
'group_cnt':['date_block_num', 'shop_id', 'group_id'],
'group_cnt_all_shops':['date_block_num', 'group_id'],
'shop_cnt':['date_block_num', 'shop_id'],
'city_cnt':['date_block_num', 'shop_city'],
'new_items_in_cat':['date_block_num', 'shop_id', 'category_id'],
'new_items_in_cat_all_shops':['date_block_num', 'category_id']
}
for col,merge_cols in lag12_cols.items() :
df[f'{col}_lag1to12'] = 0
for i in range(1,13):
df = lag_feature(df, i, col, merge_cols)
 df[f'{col}_lag1to12'] += df[f'{col}_lag{i}']
if i > 2:
del df[f'{col}_lag{i}']
if col == 'item_cnt':
del df[f'{col}_lag1']
del df[f'{col}_lag2']
else:
del df[col]<drop_column>
train_accuracy = pl.metrics.Accuracy(compute_on_step=False)
 valid_accuracy = pl.metrics.Accuracy(compute_on_step=False)
 train_predictions, train_real = [], []
valid_predictions, valid_real = [], []
model.eval()
model.cuda()
for x, y in tqdm(train_dataloader):
with torch.no_grad() :
y_pred = model(x.cuda())
 train_accuracy(y_pred.cpu() , y)
 train_predictions +=(list(y_pred.argmax(dim=1 ).cpu()))
train_real += list(y)
 for x, y in tqdm(val_dataloader):
with torch.no_grad() :
y_pred = model(x.cuda())
 valid_accuracy(y_pred.cpu() , y)
 valid_predictions +=(list(y_pred.argmax(dim=1 ).cpu()))
valid_real += list(y)
 total_train_accuracy = train_accuracy.compute()
total_valid_accuracy = valid_accuracy.compute()
lag2_cols = {
'item_cnt_unclipped':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops_median':['date_block_num', 'item_id'],
'category_cnt_median':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops_median':['date_block_num', 'category_id']
}
for col in lag2_cols:
df = lag_feature(df, 1, col, merge_cols)
 df = lag_feature(df, 2, col, merge_cols)
 if col!='item_cnt_unclipped':
del df[col]<feature_engineering>
print("training accuracy", round(total_train_accuracy.item() * 100, 3), "%")
 print("validation accuracy", round(total_valid_accuracy.item() * 100, 3), "%" )
df['item_cnt_diff'] = df['item_cnt_unclipped_lag1']/df['item_cnt_lag1to12']
df['item_cnt_all_shops_diff'] = df['item_cnt_all_shops_lag1']/df['item_cnt_all_shops_lag1to12']
df['category_cnt_diff'] = df['category_cnt_lag1']/df['category_cnt_lag1to12']
df['category_cnt_all_shops_diff'] = df['category_cnt_all_shops_lag1']/df['category_cnt_all_shops_lag1to12']<drop_column>
valid_matrix = confusion_matrix(valid_real, valid_predictions, labels=range(10))
train_matrix = confusion_matrix(train_real, train_predictions, labels=range(10))
df = lag_feature(df, 1, 'category_price',['date_block_num', 'category_id'])
 df = lag_feature(df, 1, 'block_price',['date_block_num'])
 del df['category_price'], df['block_price']<feature_engineering>
fail_images = []
real_labels, fail_labels = [], []
for x, y in tqdm(val_dataloader):
with torch.no_grad() :
y_pred = model(x.cuda())
 y_pred = y_pred.argmax(dim=1 ).cpu()
mask =(y_pred == y ).numpy()
idxs = np.where(mask == False)[0]
if idxs.shape[0] > 0:
for idx in idxs:
fail_images.append(x[idx])
 real_labels.append(y[idx].item())
 fail_labels.append(y_pred[idx].item())
 fail_images = dataset.unormalize(torch.stack(fail_images, dim=0))
df.loc[(df['item_age']>0)&(df['item_cnt_lag1to12'].isna()), 'item_cnt_lag1to12'] = 0
df.loc[(df['category_age']>0)&(df['category_cnt_lag1to12'].isna()), 'category_cnt_lag1to12'] = 0
df.loc[(df['group_age']>0)&(df['group_cnt_lag1to12'].isna()), 'group_cnt_lag1to12'] = 0<feature_engineering>
model.to_torchscript("/kaggle/working/mnist_digit.torch.pt" )
df['item_cnt_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],df['shop_age'],[12]*len(df)) ]
df['item_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],[12]*len(df)) ]
df['category_cnt_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['category_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
df['group_cnt_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],df['shop_age'],[12]*len(df)) ]
df['group_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],[12]*len(df)) ]
df['city_cnt_lag1to12'] /= [min(idx)for idx in zip(df['date_block_num'],[12]*len(df)) ]
df['shop_cnt_lag1to12'] /= [min(idx)for idx in zip(df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]<categorify>
submission = sample_submission.copy()
df = downcast(df )<merge>
model.eval()
predicted_labels = []
for idx, img in tqdm(enumerate(test_dataset), total=len(test_dataset)) :
with torch.no_grad() :
y_predict = model(img.unsqueeze(0 ).cuda())
 predicted_labels.append(y_predict.argmax(dim=1 ).cpu().item() )
def past_information(df, merging_cols, new_col, aggregation):
temp = []
for i in range(1,35):
block = df.query(f'date_block_num < {i}' ).groupby(merging_cols ).agg(aggregation ).reset_index()
block.columns = merging_cols + [new_col]
block['date_block_num'] = i
block = block[block[new_col]>0]
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=['date_block_num']+merging_cols, how='left')
 return df
df = past_information(df, ['item_id'],'last_item_price',{'item_price':'last'})
 df = past_information(df, ['shop_id','item_id'],'item_cnt_sum_alltime',{'item_cnt':'sum'})
 df = past_information(df, ['item_id'],'item_cnt_sum_alltime_allshops',{'item_cnt':'sum'})
 del df['revenue'], df['item_cnt_unclipped'], df['item_price']<feature_engineering>
submission["Label"] = predicted_labels
submission.to_csv('/kaggle/working/submission.csv',index=False )
<data_type_conversions><EOS> 
json.dump(stats, open('/kaggle/working/stats.json', 'w'))