Kaggle competition: Predict Future Sales
subm.to_csv("submission3.csv", index=False )<load_from_csv>
PATH = '.. /input/competitive-data-science-predict-future-sales/'
train_data = pd.read_csv(PATH+'sales_train.csv')
 test_data = pd.read_csv(PATH+'test.csv' ).set_index('ID')
 item_data = pd.read_csv(PATH+'items.csv')
 item_category_data = pd.read_csv(PATH+'item_categories.csv')
 shop_data = pd.read_csv(PATH+'shops.csv' )
sub1 = pd.read_csv('.. /input/average-efficientnet/submission.csv')
 sub1.head()<load_from_csv>
train_data[train_data.item_price < 0]
sub2 = pd.read_csv('.. /input/classification-densenet201-efficientnetb7/submission.csv')
 sub2.head()<load_from_csv>
train_data = train_data[train_data.item_cnt_day < 1001]
train_data = train_data[train_data.item_price < 100000]
median = train_data[(train_data.shop_id==32)&(train_data.item_id==2973)&(train_data.date_block_num==4)& 
(train_data.item_price>0)].item_price.median()
train_data.loc[train_data.item_price < 0, 'item_price'] = median
sub3 = pd.read_csv('.. /input/tf-zoo-models-on-tpu/submission.csv')
 sub3.head()<load_from_csv>
train_data.loc[train_data.shop_id == 0, 'shop_id'] = 57
test_data.loc[test_data.shop_id == 0, 'shop_id'] = 57
train_data.loc[train_data.shop_id == 1, 'shop_id'] = 58
test_data.loc[test_data.shop_id == 1, 'shop_id'] = 58
train_data.loc[train_data.shop_id == 10, 'shop_id'] = 11
test_data.loc[test_data.shop_id == 10, 'shop_id'] = 11
sub4 = pd.read_csv('.. /input/fork-of-plant-2020-tpu-915e9c/submission.csv')
 sub4.head()<load_from_csv>
shop_data.loc[shop_data.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shop_data['city'] = shop_data['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shop_data.loc[shop_data.city == '!Якутск', 'city'] = 'Якутск'
shop_data['city_code'] = LabelEncoder().fit_transform(shop_data['city'])
 shop_data = shop_data[['shop_id','city_code']]
item_category_data['split'] = item_category_data['item_category_name'].str.split('-')
 item_category_data['type'] = item_category_data['split'].map(lambda x: x[0].strip())
 item_category_data['type_code'] = LabelEncoder().fit_transform(item_category_data['type'])
 item_category_data['subtype'] = item_category_data['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 item_category_data['subtype_code'] = LabelEncoder().fit_transform(item_category_data['subtype'])
 item_category_data = item_category_data[['item_category_id','type_code', 'subtype_code']]
item_data.drop(['item_name'], axis=1, inplace=True )
sub = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub.head()<feature_engineering>
%%time
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train_data[train_data.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 matrix.info()
sub.healthy =(sub1.healthy + sub2.healthy + sub3.healthy + sub4.healthy)/4<define_variables>
train_data['revenue'] = train_data['item_price'] * train_data['item_cnt_day']
sub.multiple_diseases =(sub1.multiple_diseases + sub2.multiple_diseases + sub3.multiple_diseases + sub4.multiple_diseases)/4<feature_engineering>
%%time
group = train_data.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
sub.rust =(sub1.rust + sub2.rust + sub3.rust + sub4.rust)/4<save_to_csv>
%%time
numerical_features = ['item_price','revenue']
transform_data = train_data
transform_data[numerical_features].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )
sub.to_csv('submission.csv', index=False )<save_to_csv>
%%time
transform_data[numerical_features].apply(lambda x: kurt(x.dropna())).sort_values(ascending=False )
sub.to_csv('submission.csv', index=False )<import_modules>
%%time
skewed_features = {feature: skew(transform_data[feature].dropna() -transform_data[feature].min())for feature in numerical_features if skew(transform_data[feature].dropna() -transform_data[feature].min())>=.5}
print(skewed_features)
 boxcox1p_skews = {}
log1p_skews = {}
exp_skews = {}
power_skews = {}
best_skews = {}
methods = {}
tmp = pd.DataFrame()
for feature in skewed_features.keys() :
tmp[feature] = boxcox1p(transform_data[feature]-transform_data[feature].min() , boxcox_normmax(transform_data[feature]-transform_data[feature].min() + 1))
boxcox1p_skews[feature] = skew(tmp[feature].dropna())
 tmp[feature] = np.log1p(transform_data[feature]-transform_data[feature].min())
 log1p_skews[feature] = skew(tmp[feature].dropna())
 tmp[feature] = np.exp(transform_data[feature]-transform_data[feature].min())
 exp_skews[feature] = skew(tmp[feature].dropna())
 tmp[feature] =(transform_data[feature]-transform_data[feature].min())**.5
power_skews[feature] = skew(tmp[feature].dropna())
 best_skews[feature] = min(boxcox1p_skews[feature],log1p_skews[feature],exp_skews[feature],power_skews[feature])
 methods[feature] = 'boxcox' if best_skews[feature]==boxcox1p_skews[feature] else('log' if best_skews[feature]==log1p_skews[feature] else('exp' if best_skews[feature]==exp_skews[feature] else('power' if best_skews[feature]==power_skews[feature] else 'original')))
 print(methods)
 df_skew = pd.DataFrame(index=skewed_features.keys() , columns=['Skew', 'Skew after boxcox1p'])
 df_skew['Skew Original'] = skewed_features.values()
df_skew['Skew after boxcox1p'] = boxcox1p_skews.values()
df_skew['Skew after log1p'] = log1p_skews.values()
df_skew['Skew after exp'] = exp_skews.values()
df_skew['Skew after power'] = power_skews.values()
df_skew['Skew after compare'] = best_skews.values()
fig = plt.figure(figsize=(22, 6))
sns.pointplot(x=df_skew.index, y='Skew Original', data=df_skew, markers=['o'], linestyles=['-'], label='original')
 sns.pointplot(x=df_skew.index, y='Skew after boxcox1p', data=df_skew, markers=['x'], linestyles=['--'], color='dodgerblue', label='boxcox1p')
 sns.pointplot(x=df_skew.index, y='Skew after log1p', data=df_skew, markers=['s'], linestyles=['--'], color='peru', label='log1p')
 sns.pointplot(x=df_skew.index, y='Skew after exp', data=df_skew, markers=['*'], linestyles=['--'], color='darkorchid', label='exp')
 sns.pointplot(x=df_skew.index, y='Skew after power', data=df_skew, markers=['+'], linestyles=['--'], color='yellow', label='power0.5')
 sns.pointplot(x=df_skew.index, y='Skew after compare', data=df_skew, markers=['v'], linestyles=['--'], color='lawngreen', label='best')
 plt.xlabel('Skewed Features', size=20, labelpad=12.5)
 plt.ylabel('Skewness', size=20, labelpad=12.5)
 plt.tick_params(axis='x', labelsize=11)
 plt.tick_params(axis='y', labelsize=15)
 plt.xticks(rotation=70)
 plt.title('Skewed Features Before and After Several Transformation Method', size=20)
 plt.legend(loc='best', shadow=True)
 plt.show()
import numpy as np
import pandas as pd
import random
import torch
import torch.nn.functional as F
import torchvision
import cv2
import albumentations
import albumentations.pytorch
import os
import PIL
import copy
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns<install_modules>
%%time
for feature in numerical_features:
method = methods[feature]
if method=='boxcox':
transform_data[feature] = boxcox1p(transform_data[feature]-transform_data[feature].min() , boxcox_normmax(transform_data[feature]-transform_data[feature].min() + 1))
elif method=='log':
transform_data[feature] = np.log1p(transform_data[feature]-transform_data[feature].min())
 elif method=='exp':
transform_data[feature] = np.exp(transform_data[feature]-transform_data[feature].min())
 elif method=='power':
transform_data[feature] =(transform_data[feature]-transform_data[feature].min())**.5
else:
pass
!pip install --upgrade git+git://github.com/carloalbertobarbano/pytorch-train-utils<import_modules>
%%time
numerical_features = ['item_cnt_month']
transform_data = matrix
transform_data[numerical_features].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )
from pytorchtrainutils import trainer
from pytorchtrainutils import metrics
from pytorchtrainutils import utils<init_hyperparams>
%%time
transform_data[numerical_features].apply(lambda x: kurt(x.dropna())).sort_values(ascending=False )
seed = 42
utils.set_seed(seed)
 device = torch.device('cuda')
 lr = 1e-2
batch_size = 8
n_epochs_224 = 30
n_epochs_448 = 20
arch = 'resnet18'
mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]<load_from_csv>
print('After log1p transform, skew = ', skew(np.log1p(transform_data[numerical_features] ).dropna()))
dataset_path = '/kaggle/input/plantpathology2020fgvc7pickles/plant-pathology-2020-fgvc7-pickles'
train_df = pd.read_csv(os.path.join(dataset_path, 'train.csv'))
test_df = pd.read_csv(os.path.join(dataset_path, 'test.csv'))<feature_engineering>
transform_data[numerical_features] = np.log1p(transform_data[numerical_features] )
def preprocess_df(df):
df['label'] = df.multiple_diseases * 2 + \
df.rust * 3 + \
df.scab * 4 + \
df.healthy
df.label -= 1
return df<categorify>
test_data['date_block_num'] = 34
test_data['date_block_num'] = test_data['date_block_num'].astype(np.int8)
 test_data['shop_id'] = test_data['shop_id'].astype(np.int8)
 test_data['item_id'] = test_data['item_id'].astype(np.int16 )
train_df = preprocess_df(train_df)
 train_df.head()<categorify>
matrix = pd.concat([matrix, test_data], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True )
class PlantDataset(torch.utils.data.dataset.Dataset):
def __init__(self, df, path, transform):
super().__init__()
self.df = df
self.path = os.path.join(path, 'images')
 self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
entry = self.df.iloc[index]
fname = entry.image_id + '.npy'
fname = os.path.join(self.path, fname)
 img = np.load(fname)
 img = self.transform(img)
 return img, entry.values[1:5].astype('float64' )<split>
%%time
matrix = pd.merge(matrix, shop_data, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, item_data, on=['item_id'], how='left')
 matrix = pd.merge(matrix, item_category_data, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8 )
train_df, val_df = train_test_split(train_df, test_size=0.3, random_state=seed, stratify=train_df.label )<define_variables>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
train_weight = val_weight = torch.tensor([
[1., 1.],
[1.5, 2.],
[1., 1.],
[1., 1.]
] ).to(device )<define_search_space>
%%time
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True )
class_weights = torch.tensor([1., 1.3, 1.1, 1.])
 sampler_weights = class_weights[train_df.label.values]<categorify>
%%time
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True )
def stack_image(image, **kwargs):
to_tensor = albumentations.pytorch.ToTensor()
vflip = albumentations.VerticalFlip(always_apply=True)
 hflip = albumentations.HorizontalFlip(always_apply=True)
 return torch.stack([
to_tensor(image=vflip(image=image)['image'])['image'],
to_tensor(image=hflip(image=image)['image'])['image'],
to_tensor(image=vflip(image=hflip(image=image)['image'])['image'])['image'],
to_tensor(image=image)['image']
])
 def get_transform(img_size, crop_size):
train_transform = albumentations.Compose([
albumentations.Resize(img_size, img_size, always_apply=True),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.ShiftScaleRotate(rotate_limit=30.0, scale_limit=0.2, shift_limit=0.15, p=0.7),
albumentations.CenterCrop(crop_size, crop_size, always_apply=True),
albumentations.Normalize(mean, std),
albumentations.pytorch.ToTensor()
])
 transform = albumentations.Compose([
albumentations.Resize(img_size, img_size, always_apply=True),
albumentations.CenterCrop(crop_size, crop_size, always_apply=True),
albumentations.Normalize(mean, std),
albumentations.pytorch.ToTensor()
])
 tta_transform = albumentations.Compose([
albumentations.Resize(img_size, img_size, always_apply=True),
albumentations.CenterCrop(crop_size, crop_size, always_apply=True),
albumentations.Normalize(mean, std),
albumentations.Lambda(stack_image, always_apply=True)
 ])
 lambda_train = lambda image: train_transform(image=image)['image']
lambda_valid = lambda image: transform(image=image)['image']
lambda_tta = lambda image: tta_transform(image=image)['image']
return lambda_train, lambda_valid, lambda_tta<choose_model_class>
%%time
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True )
model = torchvision.models.resnet18(pretrained=True)
 num_ft = model.fc.in_features
model.fc = torch.nn.Linear(in_features=num_ft, out_features=4, bias=True)
 model = model.to(device )<normalization>
%%time
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True )
class Softmaxer(torch.nn.Module):
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
x = self.model(x)
 return F.softmax(x, dim=1 )<compute_train_metric>
%%time
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True )
def bce(preds, targets, weight=None):
loss = F.binary_cross_entropy_with_logits(preds, targets.type(preds.dtype), reduction='none')
 if weight is not None:
weight = weight[:, targets.T.long() ]
idx = np.diag_indices(weight.shape[0])
 weight = weight[idx[0], idx[1], :]
loss *= weight.T
return loss.mean()<train_model>
%%time
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True )
img_size = 250
crop_size = 224
name = f'{arch}-{crop_size}'
print(f'{name} image size: {img_size}, crop size: {crop_size}' )<create_dataframe>
%%time
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True )
train_transform, valid_transform, tta_transform = get_transform(img_size=img_size, crop_size=crop_size)
 train_dataset = PlantDataset(train_df, dataset_path, train_transform)
 train_sampler = torch.utils.data.sampler.WeightedRandomSampler(sampler_weights, len(train_df))
train_loader = torch.utils.data.DataLoader(
 train_dataset, batch_size=batch_size, num_workers=4,
sampler=train_sampler, shuffle=False 
)
 val_dataset = PlantDataset(val_df, dataset_path, valid_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False )<choose_model_class>
%%time
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True )
criterion = bce
optimizer = torch.optim.SGD(model.parameters() , lr=lr, weight_decay=1e-5)
 lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True )<train_model>
%%time
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True )
tracked_metrics = [
metrics.MultilabelAccuracy() ,
metrics.MultilabelRocAuc() ,
metrics.MultilabelConfusionMatrix()
]
best_model = trainer.fit(
 model, train_dataloader=train_loader, val_dataloader=val_loader,
test_dataloader=None, test_every=0, criterion=criterion,
optimizer=optimizer, scheduler=lr_scheduler, metrics=tracked_metrics, n_epochs=n_epochs_224,
metric_choice='col-auc', mode='max',
name=name, device=device, weight={'train': train_weight, 'val': val_weight}
)<train_on_grid>
%%time
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True )
tracked_metrics = [
metrics.MultilabelAccuracy(metric='top1-acc', apply_sigmoid=False),
metrics.MultilabelRocAuc(apply_sigmoid=False),
metrics.MultilabelConfusionMatrix()
]
softmaxer = Softmaxer(model)
 softmaxer_best = Softmaxer(best_model)
 val_dataset = PlantDataset(val_df, dataset_path, tta_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False)
 val_logs = trainer.test(softmaxer, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True)
 best_val_logs = trainer.test(softmaxer_best, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True )<train_model>
%%time
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True )
print(f'Final {name} val:', trainer.summarize_metrics(val_logs))
print(f'Best {name} val:', trainer.summarize_metrics(best_val_logs))<train_model>
%%time
group = train_data.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train_data.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True )
img_size *= 2
crop_size *= 2
name = f'{arch}-{crop_size}'
print(f'{name} image size: {img_size}, crop size: {crop_size}' )<create_dataframe>
%%time
group = train_data.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True )
train_transform, valid_transform, tta_transform = get_transform(img_size=img_size, crop_size=crop_size)
 train_dataset = PlantDataset(train_df, dataset_path, train_transform)
 train_sampler = torch.utils.data.sampler.WeightedRandomSampler(sampler_weights, len(train_df))
train_loader = torch.utils.data.DataLoader(
 train_dataset, batch_size=batch_size, num_workers=4,
sampler=train_sampler, shuffle=False 
)
 val_dataset = PlantDataset(val_df, dataset_path, valid_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False )<choose_model_class>
matrix['month'] = matrix['date_block_num'] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
criterion = bce
optimizer = torch.optim.SGD(model.parameters() , lr=lr, weight_decay=1e-5)
 lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True )<train_model>
%%time
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for row in matrix.itertuples() :
idx = getattr(row,'Index')
 item_id = getattr(row,'item_id')
 shop_id = getattr(row,'shop_id')
 date_block_num = getattr(row,'date_block_num')
 item_cnt_month = getattr(row,'item_cnt_month')
 key = str(item_id)+' '+str(shop_id)
 if key not in cache:
if item_cnt_month!=0:
cache[key] = date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = date_block_num - last_date_block_num
cache[key] = date_block_num
tracked_metrics = [
metrics.MultilabelAccuracy(apply_sigmoid=True),
metrics.MultilabelRocAuc(apply_sigmoid=True),
metrics.MultilabelConfusionMatrix()
]
best_model = trainer.fit(
 model, train_dataloader=train_loader, val_dataloader=val_loader,
test_dataloader=None, test_every=0, criterion=criterion,
optimizer=optimizer, scheduler=lr_scheduler, metrics=tracked_metrics, n_epochs=n_epochs_448,
metric_choice='col-auc', mode='max',
name=name, device=device, weight={'train': train_weight, 'val': val_weight}
)<train_on_grid>
%%time
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for row in matrix.itertuples() :
idx = getattr(row,'Index')
 key = getattr(row,'item_id')
 date_block_num = getattr(row,'date_block_num')
 item_cnt_month = getattr(row,'item_cnt_month')
 if key not in cache:
if item_cnt_month!=0:
cache[key] = date_block_num
else:
last_date_block_num = cache[key]
if date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = date_block_num - last_date_block_num
cache[key] = date_block_num
tracked_metrics = [
metrics.MultilabelAccuracy(metric='top1-acc', apply_sigmoid=False),
metrics.MultilabelRocAuc(apply_sigmoid=False),
metrics.MultilabelConfusionMatrix()
]
softmaxer = Softmaxer(model)
 softmaxer_best = Softmaxer(best_model)
 val_dataset = PlantDataset(val_df, dataset_path, tta_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False)
 val_logs = trainer.test(softmaxer, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True)
 best_val_logs = trainer.test(softmaxer_best, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True )<train_model>
%%time
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min' )
print(f'Final {name} val:', trainer.summarize_metrics(val_logs))
print(f'Best {name} val:', trainer.summarize_metrics(best_val_logs))<load_pretrained>
%%time
matrix = matrix[matrix.date_block_num > 11]
test_dataset = PlantDataset(test_df, dataset_path, tta_transform)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=10, shuffle=False )<find_best_params>
%%time
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix )
def get_test_preds(model, test_loader):
outputs = []
for batch_idx,(data, labels)in enumerate(tqdm(test_loader)) :
batch_size, n_crops, c, h, w = data.size()
data = data.view(-1, c, h, w)
 with torch.no_grad() :
output = model(data.to(device))
output = output.view(batch_size, n_crops, -1 ).mean(1)
 outputs.append(output.cpu())
 return torch.cat(outputs, dim=0 ).numpy()<prepare_output>
numerical_features = [col for col in matrix.columns if(col.find('cnt')!=-1 or col.find('cnt')!=-1 or col.find('price')!=-1 or col.find('revenue')!=-1 or col.find('sale')!=-1)]
matrix[numerical_features].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )
def make_submission_df(df, preds):
classes = ['healthy', 'multiple_diseases', 'rust', 'scab']
for idx, name in enumerate(classes):
df[name] = pd.Series(preds[:, idx])
 return df<save_to_csv>
matrix[numerical_features].apply(lambda x: kurt(x.dropna())).sort_values(ascending=False )
softmaxer.eval()
test_preds = get_test_preds(softmaxer, test_loader)
 test_submission = make_submission_df(test_df, test_preds)
 test_submission.to_csv(f'submission-{name}.csv', index=False )<save_to_csv>
matrix.to_pickle('data.pkl')
 del matrix, cache, group, train_data, shop_data, item_data, item_category_data
gc.collect() ;
softmaxer_best.eval()
test_preds = get_test_preds(softmaxer_best, test_loader)
 test_submission = make_submission_df(test_df, test_preds)
 test_submission.to_csv(f'submission-best-{name}.csv', index=False )<import_modules>
1/0
import numpy as np
import pandas as pd
import os
import random, re, math
import tensorflow as tf, tensorflow.keras.backend as K
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers
from kaggle_datasets import KaggleDatasets
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.applications import ResNet152V2, InceptionResNetV2, InceptionV3, Xception, VGG19
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt<install_modules>
data = pd.read_pickle('data.pkl' )
!pip install efficientnet 
<set_options>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<train_model>
del data
gc.collect()
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_0.jpg')
 print(img.shape)
 plt.imshow(img )<load_from_csv>
%%time
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=10086)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10 )
path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<define_variables>
Y_pred = model.predict(X_valid)
 Y_pred = np.expm1(Y_pred ).clip(0, 20)
 print('Expm1 after RMSE = ', np.sqrt(sklearn.metrics.mean_squared_error(np.expm1(Y_valid),Y_pred)) )
nb_classes = 4
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 40
SEED = 123<normalization>
Y_valid_expm1 = np.expm1(Y_valid)
 Y_pred_correct = Y_pred*(np.mean(Y_valid_expm1)/np.mean(Y_pred))
Y_pred_correct_add = Y_pred+(np.mean(Y_valid_expm1-Y_pred))
print('Mean of pred - valid:', np.mean(Y_pred - Y_valid_expm1))
print('Mean of pred_correct - valid:', np.mean(Y_pred_correct - Y_valid_expm1))
print('Mean of pred_correct_add - valid:', np.mean(Y_pred_correct_add - Y_valid_expm1))
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
fig = plt.figure(figsize=(22, 6))
compare_df = pd.DataFrame({'Y_pred':Y_pred[:500],
'Y_valid':Y_valid_expm1[:500],
'Y_pred_correct':Y_pred_correct[:500],
'Y_pred_correct_add':Y_pred_correct_add[:500]})
 sns.lineplot(data=compare_df)
 plt.show()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<categorify>
print('Expm1 after and Correct Factor RMSE = ', np.sqrt(sklearn.metrics.mean_squared_error(Y_valid_expm1,Y_pred_correct)))
 print('Expm1 after and Correct Add RMSE = ', np.sqrt(sklearn.metrics.mean_squared_error(Y_valid_expm1,Y_pred_correct_add)) )
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
Y_test = np.expm1(model.predict(X_test)).clip(0, 20)+np.mean(Y_valid_expm1-Y_pred)
 submission = pd.DataFrame({
"ID": test_data.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )