Kaggle competition: Digit Recognizer
data = X_train.groupby('date_block_num' ).agg({'lgb_pred':'mean','target':'mean','sq_err':'mean'} ).reset_index()
data['new_item_rmse'] = np.sqrt(X_train.query('item_age<=1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data['old_item_rmse'] = np.sqrt(X_train.query('item_age>1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data = data.append([
{'date_block_num':33,
'target':X_val.target.mean() ,
'lgb_pred':X_val.lgb_pred.mean() ,
'sq_err':np.sqrt(X_val.sq_err.mean()),
'old_item_rmse':np.sqrt(X_val.query('item_age>1' ).sq_err.mean()),
'new_item_rmse':np.sqrt(X_val.query('item_age<=1' ).sq_err.mean())},
{'date_block_num':34,
'target':0,
'lgb_pred':X_test.lgb_pred.mean() ,
'sq_err':0,
'old_item_rmse':0,
'new_item_rmse':0}
],
ignore_index=True 
)
 data['date'] = [x[:7] for x in pd.date_range(start='2013-03',end='2015-09',freq='MS' ).astype('str')]+['Validation','Test']<merge>
def set_seed(seed=0):
np.random.seed(seed)
 random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
os.environ['PYTHONHASHSEED'] = str(seed)
 set_seed(0 )
df = pd.read_pickle('.. /input/files-top-scoring-notebook-output-exploration/df_complete.pkl')
(
 df
[df['category_id'].isin(X_test.category_id.unique())] 
.query('item_cnt>0')
.groupby('category_id')
.agg({
'category_age':'max',
'shop_id':['nunique','unique'],
'item_cnt':'sum'
})
.join(categories['category_name'])
.join(
 X_test 
.groupby('category_id')
.agg({'item_id':'nunique'})
.rename(columns={'item_id':'test_set_items'})
)
.sort_values(( 'shop_id', 'nunique')) 
.head(20)
 )<merge>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device )
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('shop_id')
.agg({
'sq_err':'mean',
'target':'mean',
'lgb_pred':'mean'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('shop_id')
.agg({'test_pred':'mean'})
)
.join(shops['shop_name'])
.sort_values('sq_err', ascending=False)
.head(20)
 )<merge>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 sample_sub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('category_id')
.agg({
'sq_err':['sum','mean'],
'target':'mean',
'lgb_pred':['sum','mean'],
'item_id':'nunique'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred','item_id':'test_items'})
.groupby('category_id')
.agg({
'test_pred':['sum','mean'],
'test_items':'nunique'
}),
on='category_id' 
)
.join(categories)
.sort_values(( 'sq_err', 'mean'), ascending=False)
.head(20)
 )<merge>
X = train.drop(columns='label' ).values.reshape(-1, 28, 28, 1)
 y = train.label.values
train_X, valid_X, train_y, valid_y = train_test_split(X,
y,
test_size=0.2
)
CATEGORY = 20 
(
 items[
items['item_id'].isin(X_val.item_id.unique())&
items['item_id'].isin(X_test.item_id.unique())
 ]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('item_id')
.agg({'test_pred':'mean'}),
on='item_id' 
)
.join(
 X_val 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'target':'mean',
'sq_err':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={
'lgb_pred':'val_pred',
'target':'val_target',
}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('sq_err',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )<merge>
class MNISTDataset(Dataset):
def __init__(self, X, y=None, is_test=False, transforms=None):
self.X = X
self.y = y
self.is_test = is_test
self.transforms = transforms
def __len__(self):
return len(self.X)
 def __getitem__(self, index):
image = self.X[index]
if self.transforms:
image = self.transforms(image=image)['image']
if self.is_test:
return image
else:
return image, self.y[index]
CATEGORY = 20 
(
 items[items['item_id'].isin(X_test.item_id.unique())]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={'lgb_pred':'test_pred'}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('test_pred',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'item_category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )<save_to_csv>
train_transforms = Compose([ToFloat(max_value=255),
ShiftScaleRotate(shift_limit=0.1,
scale_limit=0.1,
rotate_limit=10,
border_mode=cv2.BORDER_CONSTANT,
value=0,
p=1),
ToTensorV2()
])
 valid_transforms = Compose([ToFloat(max_value=255),
ToTensorV2()
] )
submission_1 = pd.read_csv('.. /input/blend-boosting-best-score-on-predict-future-sales/submission_blend_1.csv')
 submission_2 = pd.read_csv('initial_lgb_submission.csv')
 submission_1['item_cnt_month'] = np.median(np.concatenate([np.expand_dims(submission_1['item_cnt_month'], axis = 1), np.expand_dims(submission_2['item_cnt_month'], axis = 1)], axis = 1), axis = 1)
 submission_1[['ID', 'item_cnt_month']].to_csv('final_sub.csv', index=False )<install_modules>
example_transforms = Compose([ToFloat(max_value=255),
ShiftScaleRotate(shift_limit=0.3,
scale_limit=0.3,
rotate_limit=30,
p=1),
ToTensorV2()
] )
!pip install -q ".. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl"<import_modules>
class CNN(nn.Module):
def __init__(self):
super(CNN, self ).__init__()
self.dropout = nn.Dropout2d(0.5)
 self.bn1_1 = nn.BatchNorm2d(48)
 self.bn1_2 = nn.BatchNorm2d(48)
 self.bn2 = nn.BatchNorm1d(256)
 self.conv1 = nn.Conv2d(in_channels=1, out_channels=48, kernel_size=3, stride=1, padding=1)
 self.conv2 = nn.Conv2d(in_channels=48, out_channels=48, kernel_size=3, stride=2, padding=1)
 self.fc1 = nn.Linear(in_features=48 * 14 * 14, out_features=256)
 self.fc2 = nn.Linear(in_features=256, out_features=10)
 def forward(self, x):
x = F.relu(self.conv1(x))
x = self.bn1_1(x)
 x = F.relu(self.conv2(x))
x = self.bn1_2(x)
 x = self.dropout(x)
 x = torch.flatten(x, 1)
 x = F.relu(self.fc1(x))
x = self.bn2(x)
 x = self.dropout(x)
 x = self.fc2(x)
 return x
import pandas as pd
import numpy as np
import os
import tqdm
import cv2
import matplotlib.pyplot as plt
import seaborn as sns
import gc
import base64
from pycocotools import _mask as coco_mask
import typing as t
import zlib
from torch.utils.data import Dataset, DataLoader
import albumentations as A
from albumentations.pytorch.transforms import ToTensorV2
import torch
import torch.nn as nn
from collections import OrderedDict
import ttach as tta<load_from_csv>
class EarlyStopping:
def __init__(self, mode, path, patience=3, delta=0):
if mode not in {'min', 'max'}:
raise ValueError("Argument mode must be one of 'min' or 'max'.")
 if patience <= 0:
raise ValueError("Argument patience must be a postive integer.")
 if delta < 0:
raise ValueError("Argument delta must not be a negative number.")
 self.mode = mode
self.patience = patience
self.delta = delta
self.path = path
self.best_score = np.inf if mode == 'min' else -np.inf
self.counter = 0
def _is_improvement(self, val_score):

 if self.mode == 'max' and val_score > self.best_score + self.delta:
return True
elif self.mode == 'min' and val_score < self.best_score - self.delta:
return True
return False
def __call__(self, val_score, model):

 if self._is_improvement(val_score):
self.best_score = val_score
self.counter = 0
torch.save(model.state_dict() , self.path)
 print('Val loss improved.Saved model.')
 return False
else:
self.counter += 1
print(f'Early stopping counter: {self.counter}/{self.patience}')
 if self.counter >= self.patience:
print(f'Stopped early.Best val loss: {self.best_score:.4f}')
 return True
df = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv' )<define_variables>
def train_one_epoch(model, train_loader, optimizer, device, criterion):

 model.train()
running_loss_train = 0
for inputs, labels in train_loader:
inputs, labels = inputs.to(device), labels.to(device)
 optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss_train += loss.item()
train_loss = running_loss_train / len(train_loader.dataset)
 return train_loss
ROOT = '.. /input/hpa-single-cell-image-classification/'
train_or_test = 'test'<prepare_x_and_y>
def validate(model, valid_loader, device, criterion):

 model.eval()
correct = 0
running_loss_val = 0
with torch.no_grad() :
for inputs, labels in valid_loader:
inputs, labels = inputs.to(device), labels.to(device)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 pred = outputs.argmax(dim=1)
 correct += pred.eq(labels ).sum().item()
running_loss_val += loss.item()
val_acc = correct / len(valid_loader.dataset)
 val_loss = running_loss_val / len(valid_loader.dataset)
 return val_acc, val_loss
def get_cropped_cell(img, msk):
bmask = msk.astype(int)[...,None]
masked_img = img * bmask
true_points = np.argwhere(bmask)
 top_left = true_points.min(axis=0)
 bottom_right = true_points.max(axis=0)
 cropped_arr = masked_img[top_left[0]:bottom_right[0]+1,top_left[1]:bottom_right[1]+1]
return cropped_arr<compute_test_metric>
def fit(model, train_loader, valid_loader, learning_rate, num_epochs):
criterion = nn.CrossEntropyLoss(reduction='sum')
 optimizer = optim.Adam(model.parameters() , lr=learning_rate)
 es = EarlyStopping(mode='min', path='model.pth', patience=5)
 model = model.to(device)
 scheduler = ExponentialLR(optimizer, gamma=0.9)
 for epoch in range(1, num_epochs + 1):
train_loss = train_one_epoch(model, train_loader, optimizer, device, criterion)
 val_acc, val_loss = validate(model, valid_loader, device, criterion)
 scheduler.step()
print(f'Epoch {epoch:2}/{num_epochs}',
f'train loss: {train_loss:.4f}',
f'val loss: {val_loss:.4f}',
f'val acc: {val_acc:.2%}',
sep=' | ' 
)
 if es(val_loss, model):
break
def get_stats(cropped_cell):
x =(cropped_cell/255.0 ).reshape(-1,3 ).mean(0)
 x2 =(( cropped_cell/255.0)**2 ).reshape(-1,3 ).mean(0)
 return x, x2<data_type_conversions>
TRAIN_BATCH_SIZE = 64
VALID_BATCH_SIZE = 512
NUM_EPOCHS = 50
LEARNING_RATE = 1e-3
NUM_WORKERS = 0
mnist_train = MNISTDataset(train_X, train_y, is_test=False, transforms=train_transforms)
 mnist_valid = MNISTDataset(valid_X, valid_y, is_test=False, transforms=valid_transforms)
 train_loader = DataLoader(mnist_train, batch_size=TRAIN_BATCH_SIZE, shuffle=True, drop_last=True)
 valid_loader = DataLoader(mnist_valid, batch_size=VALID_BATCH_SIZE, shuffle=False)
 model = CNN()
start = time.time()
fit(model, train_loader, valid_loader, learning_rate=LEARNING_RATE, num_epochs=NUM_EPOCHS)
 print(f'Total training time: {time.time() - start}')
 model.load_state_dict(torch.load('model.pth'))
def read_img(image_id, color, train_or_test='test', image_size=None):
filename = f'{ROOT}/{train_or_test}/{image_id}_{color}.png'
assert os.path.exists(filename), f'not found {filename}'
img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)
 if image_size is not None:
img = cv2.resize(img,(image_size, image_size))
if img.max() > 255:
img_max = img.max()
img =(img/255 ).astype('uint8')
 return img<categorify>
TEST_BATCH_SIZE = 512
test_transforms = Compose([ToFloat(max_value=255),
ToTensorV2()
])
 test_X = test.values.reshape(-1, 28, 28, 1)
 mnist_test = MNISTDataset(test_X, is_test=True, transforms=test_transforms)
 test_loader = DataLoader(mnist_test, batch_size=TEST_BATCH_SIZE, shuffle=False )
def encode_binary_mask(mask: np.ndarray)-> t.Text:

 if mask.dtype != np.bool:
raise ValueError(
 "encode_binary_mask expects a binary mask, received dtype == %s" %
mask.dtype)
 mask = np.squeeze(mask)
 if len(mask.shape)!= 2:
raise ValueError(
 "encode_binary_mask expects a 2d mask, received shape == %s" %
mask.shape)
 mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)
 mask_to_encode = mask_to_encode.astype(np.uint8)
 mask_to_encode = np.asfortranarray(mask_to_encode)
 encoded_mask = coco_mask.encode(mask_to_encode)[0]["counts"]
binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)
 base64_str = base64.b64encode(binary_str)
 return base64_str.decode('ascii' )<choose_model_class>
def predict(model, test_loader, device):

 model.eval()
predictions = sample_sub['Label'].values
with torch.no_grad() :
for i, inputs in enumerate(test_loader):
inputs = inputs.to(device)
 outputs = model(inputs)
 pred = outputs.argmax(dim=1 ).to('cpu' ).numpy()
predictions[i*TEST_BATCH_SIZE:i*TEST_BATCH_SIZE+len(inputs)] = pred
output = sample_sub.copy()
output['Label'] = predictions
output.to_csv('submission.csv', index=False)
 return output
output = predict(model, test_loader, device)
 output