Kaggle competition: Digit Recognizer
sales =(sales 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day':'sum',
'revenue':'sum',
'first_sale_day':'first'
})
.reset_index() 
.rename(columns={'item_cnt_day':'item_cnt'})
)
 sales.sample(5 )<drop_column>
from __future__ import print_function, division
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
from torchvision import models, transforms
import matplotlib.pyplot as plt
import time
import copy
import pandas as pd
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader, Dataset
test['date_block_num'] = 34
del test['ID']<concatenate>
class MyDataset(Dataset):
def __init__(self, feature, target=None, transform=None):
self.X = feature
self.Y = target
self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
if self.transform is not None:
return self.transform(self.X[idx]), self.Y[idx]
elif self.Y is None:
return [self.X[idx]]
return self.X[idx], self.Y[idx]
df = pd.concat([df,test] ).fillna(0)
 df = df.reset_index()
del df['index']<merge>
train_data_dir = ".. /input/digit-recognizer/train.csv"
test_data_dir = ".. /input/digit-recognizer/test.csv"
train = pd.read_csv(train_data_dir)
 test = pd.read_csv(test_data_dir)
 raw_train_labels = train["label"]
raw_train_imgs = train.drop(labels = ["label"], axis = 1)
 normalized_train = raw_train_imgs/255.0
normalized_test = test/255.0
train_split, validation_split, train_labels_split, validation_labels_split = train_test_split(normalized_train, raw_train_labels, random_state=0)
 test_data = torch.from_numpy(normalized_test.values.reshape(( -1,1,28,28)))
 train_data = torch.from_numpy(train_split.values.reshape(( -1,1,28,28)))
 train_labels_data = torch.from_numpy(train_labels_split.values)
 validation_data = torch.from_numpy(validation_split.values.reshape(( -1,1,28,28)))
 validation_labels_data = torch.from_numpy(validation_labels_split.values)
 train_set = MyDataset(train_data.float() , train_labels_data)
 valid_set = MyDataset(validation_data.float() , validation_labels_data)
 test_set = MyDataset(test_data.float())
 batch_size = 128
train_loader = torch.utils.data.DataLoader(train_set, batch_size = batch_size, shuffle = True)
 valid_loader = torch.utils.data.DataLoader(valid_set, batch_size = batch_size, shuffle = False)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size = batch_size, shuffle = False )
df = pd.merge(df, sales, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 df = pd.merge(df, dates, on=['date_block_num','shop_id'], how='left')
 df = pd.merge(df, items.drop(columns=['item_name','group_name','category_name']), on='item_id', how='left' )<feature_engineering>
data = next(iter(train_loader))
mean = data[0].mean()
std = data[0].std()
transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees = 30),
transforms.RandomPerspective() ,
transforms.ToTensor() ,
transforms.Normalize(mean, std)])
 augmented_train_set = MyDataset(train_data.float() , train_labels_data, transform=transform)
 datasets = []
datasets.append(train_set)
 datasets.append(augmented_train_set)
 train_set = torch.utils.data.ConcatDataset(datasets)
 train_loader = torch.utils.data.DataLoader(train_set, batch_size = batch_size, shuffle = True )
shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops_cats = pd.DataFrame(
 np.array(list(product(*[df['shop_id'].unique() , df['category_id'].unique() ]))),
columns =['shop_id', 'category_id'] 
)
 temp = df.groupby(['category_id', 'shop_id'] ).agg({'item_cnt':'sum'} ).reset_index()
temp2 = temp.groupby('shop_id' ).agg({'item_cnt':'sum'} ).rename(columns={'item_cnt':'shop_total'})
 temp = temp.join(temp2, on='shop_id')
 temp['category_proportion'] = temp['item_cnt']/temp['shop_total']
temp = temp[['shop_id', 'category_id', 'category_proportion']]
shops_cats = pd.merge(shops_cats, temp, on=['shop_id','category_id'], how='left')
 shops_cats = shops_cats.fillna(0)
 shops_cats = shops_cats.pivot(index='shop_id', columns=['category_id'])
 kmeans = KMeans(n_clusters=7, random_state=0 ).fit(shops_cats)
 shops_cats['shop_cluster'] = kmeans.labels_.astype('int8')
 shops = shops.join(shops_cats['shop_cluster'], on='shop_id' )<feature_engineering>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 dataloaders = {'train': train_loader, 'val': valid_loader}
dataset_sizes= {'train': len(train_set), 'val': len(valid_set)}
def train_model(model, criterion, optimizer, scheduler, dataloaders, dataset_sizes, num_epochs=25):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 if phase == 'train':
scheduler.step()
epoch_loss = running_loss / dataset_sizes[phase]
epoch_acc = running_corrects.double() / dataset_sizes[phase]
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
model.load_state_dict(best_model_wts)
 return model
shops.dropna(inplace=True)
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_type'] = 'regular'
shops.loc[shops['shop_name'].str.contains(r'tc'), 'shop_type'] = 'tc'
shops.loc[shops['shop_name'].str.contains(r'mall|center|mega'), 'shop_type'] = 'mall'
shops.loc[shops['shop_id'].isin([9,20]), 'shop_type'] = 'special'
shops.loc[shops['shop_id'].isin([12,55]), 'shop_type'] = 'online'
shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
shops.shop_city = le.fit_transform(shops.shop_city.values)
 shops.shop_type = le.fit_transform(shops.shop_type.values)
 shops.head()<merge>
epochs = 20
model_ft = models.resnet18(pretrained=False)
 model_ft.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 10)
 model_ft = model_ft.to(device)
 criterion = nn.CrossEntropyLoss()
optimizer_ft = optim.Adam(model_ft.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=5, gamma=0.1)
 model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, dataloaders, dataset_sizes,
num_epochs=epochs )
df = pd.merge(df, shops.drop(columns='shop_name'), on='shop_id', how='left')
 df.head()<feature_engineering>
test_pred = torch.LongTensor()
with torch.no_grad() :
for images in test_loader:
images = torch.autograd.Variable(images[0])
 if torch.cuda.is_available() :
images = images.to(device)
 outputs = model_ft(images)
 predicted = outputs.cpu().data.max(1, keepdim=True)[1]
test_pred = torch.cat(( test_pred, predicted), dim=0 )
df['first_sale_day'] = df.groupby('item_id')['first_sale_day'].transform('max' ).astype('int16')
 df.loc[df['first_sale_day']==0, 'first_sale_day'] = 1035
df['prev_days_on_sale'] = [max(idx)for idx in zip(df['first_day_of_month']-df['first_sale_day'],[0]*len(df)) ]
del df['first_day_of_month']<drop_column>
out_df = pd.DataFrame(np.c_[np.arange(1, len(test_set)+1)[:,None], test_pred.numpy() ],
columns=['ImageId', 'Label'])
 out_df.to_csv('submission.csv', index=False )