Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<choose_model_class>
%matplotlib inline
sns.set(palette=sns.color_palette('Set2',9))
class ResNetFPN(mx.gluon.HybridBlock):
def __init__(self, backbone= 'resnet50', backbone_lr_mult=0.1, **kwargs):
super(ResNetFPN, self ).__init__()
self.backbone_name = backbone
self.backbone_lr_mult = backbone_lr_mult
self._kwargs = kwargs
with self.name_scope() :
self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)
 self.head = _FPNHead(output_channels=256, **kwargs)
 def load_pretrained_weights(self):
pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)
 backbone_params = self.backbone.collect_params()
pretrained_weights = pretrained.collect_params()
for k, v in pretrained_weights.items() :
param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]
backbone_params[param_name].set_data(v.data())
 self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)
 def hybrid_forward(self,F, x):
c1, c2, c3, c4 = self.backbone(x)
 p1, p2, p3, p4 = self.head(c1, c2, c3, c4)
 return p1, p2, p3, p4
class ResNetUnet(mx.gluon.HybridBlock):
def __init__(self, backbone= 'resnet50', backbone_lr_mult=0.1, cls_branch=False, **kwargs):
super(ResNetUnet, self ).__init__()
self.backbone_name = backbone
self.backbone_lr_mult = backbone_lr_mult
self.cls_branch = cls_branch
self._kwargs = kwargs
with self.name_scope() :
self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)
 self.head = _UnetHead(**kwargs)
 def load_pretrained_weights(self):
pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)
 backbone_params = self.backbone.collect_params()
pretrained_weights = pretrained.collect_params()
for k, v in pretrained_weights.items() :
param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]
backbone_params[param_name].set_data(v.data())
 self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)
 def hybrid_forward(self,F, x):
c1, c2, c3, c4 = self.backbone(x)
 out = self.head(c1, c2, c3, c4)
 if self.cls_branch:
logits = F.max(F.mean(out, axis=1), axis=(1, 2))
return out, logits
return out
class _DecoderBlock(HybridBlock):
def __init__(self, output_channels, norm_layer=nn.BatchNorm):
super(_DecoderBlock, self ).__init__()
with self.name_scope() :
self.block = nn.HybridSequential()
self.block.add(ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer))
self.block.add(ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer))
def hybrid_forward(self, F, x, y=None):
if y is not None:
x = F.contrib.BilinearResize2D(x, scale_height=2, scale_width=2)
 x = F.concat(x, y, dim=1)
 out = self.block(x)
 return out
class _UnetHead(HybridBlock):
def __init__(self, num_classes, output_channels=[256, 128, 64, 32], scale=4, norm_layer=nn.BatchNorm):
super(_UnetHead, self ).__init__()
self.scale = scale
with self.name_scope() :
self.block4 = _DecoderBlock(output_channels[0], norm_layer=norm_layer)
 self.block3 = _DecoderBlock(output_channels[1], norm_layer=norm_layer)
 self.block2 = _DecoderBlock(output_channels[2], norm_layer=norm_layer)
 self.block1 = _DecoderBlock(output_channels[3], norm_layer=norm_layer)
 self.postprocess_block = nn.Conv2D(num_classes, kernel_size=1)
 def hybrid_forward(self, F, c1, c2, c3, c4):
p4 = self.block4(c4)
 p3 = self.block3(p4, c3)
 p2 = self.block2(p3, c2)
 p1 = self.block1(p2, c1)
 if self.scale > 1:
p1 = F.contrib.BilinearResize2D(p1, scale_height=self.scale, scale_width=self.scale)
 out = self.postprocess_block(p1)
 return out
class _FPNHead(HybridBlock):
def __init__(self, output_channels=256, norm_layer=nn.BatchNorm):
super(_FPNHead, self ).__init__()
self._hdsize = {}
with self.name_scope() :
self.block4 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.block3 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.block2 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.block1 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 def hybrid_forward(self, F, c1, c2, c3, c4):
p4 = self.block4(c4)
 p3 = self._resize_as(F, 'id_1', p4, c3)+ self.block3(c3)
 p2 = self._resize_as(F, 'id_2', p3, c2)+ self.block2(c2)
 p1 = self._resize_as(F, 'id_3', p2, c1)+ self.block1(c1)
 return p1, p2, p3, p4
def _resize_as(self, F, name, x, y):
h_key = name + '_h'
w_key = name + '_w'
if hasattr(y, 'shape'):
_, _, h, w = y.shape
_, _, h2, w2 = x.shape
if h == h2 and w == w2:
h = 0
w = 0
self._hdsize[h_key] = h
self._hdsize[w_key] = w
else:
h, w = self._hdsize[h_key], self._hdsize[w_key]
if h == 0 and w == 0:
return x
else:
return F.contrib.BilinearResize2D(x, height=h, width=w)
 class SemanticFPNHead(HybridBlock):
def __init__(self, num_classes, output_channels=128, norm_layer=nn.BatchNorm):
super(SemanticFPNHead, self ).__init__()
self._hdsize = {}
with self.name_scope() :
self.block4_1 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block4_2 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block4_3 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block3_1 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block3_2 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block2 = ConvBlock(output_channels, kernel_size=3, padding=1, norm_layer=norm_layer)
 self.block1 = ConvBlock(output_channels, kernel_size=1, norm_layer=norm_layer)
 self.postprocess_block = nn.Conv2D(num_classes, kernel_size=1)
 def hybrid_forward(self, F, c1, c2, c3, c4):
out4 = self._resize_as(F, 'id_1', self.block4_1(c4), c3)
 out4 = self._resize_as(F, 'id_2', self.block4_2(out4), c2)
 out4 = self._resize_as(F, 'id_3', self.block4_3(out4), c1)
 out3 = self._resize_as(F, 'id_4', self.block3_1(c3), c2)
 out3 = self._resize_as(F, 'id_5', self.block3_2(out3), c1)
 out2 = self._resize_as(F, 'id_6', self.block2(c2), c1)
 out1 = self.block1(c1)
 out = out1 + out2 + out3 + out4
out = self.postprocess_block(out)
 out = F.contrib.BilinearResize2D(out,scale_height=4,scale_width=4)
 return out
def _resize_as(self, F,name, x, y):
h_key = name + '_h'
w_key = name + '_w'
if hasattr(y, 'shape'):
_, _, h, w = y.shape
_, _, h2, w2 = x.shape
if h == h2 and w == w2:
h = 0
w = 0
self._hdsize[h_key]=h
self._hdsize[w_key]=w
else:
h, w = self._hdsize[h_key], self._hdsize[w_key]
if h == 0 and w == 0:
return x
else:
return F.contrib.BilinearResize2D(x,height=h,width=w)
 class ConvBlock(HybridBlock):
def __init__(self, output_channels, kernel_size, padding=0, activation='relu', norm_layer=nn.BatchNorm):
super().__init__()
self.body = nn.HybridSequential()
self.body.add(
 nn.Conv2D(output_channels, kernel_size=kernel_size, padding=padding, activation=activation),
norm_layer(in_channels=output_channels)
)
 def hybrid_forward(self, F, x):
return self.body(x)
<load_pretrained>
train_path='/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv'
test_path='/kaggle/input/competitive-data-science-predict-future-sales/test.csv'
train=pd.read_csv(train_path)
 test=pd.read_csv(test_path)
 items_path='/kaggle/input/competitive-data-science-predict-future-sales/items.csv'
item_categories_path='/kaggle/input/competitive-data-science-predict-future-sales/item_categories.csv'
shops_path='/kaggle/input/competitive-data-science-predict-future-sales/shops.csv'
items=pd.read_csv(items_path)
 item_cat=pd.read_csv(item_categories_path)
 shops=pd.read_csv(shops_path)
 sample_submission_path='/kaggle/input/competitive-data-science-predict-future-sales/sample_submission.csv'
sample_submission=pd.read_csv(sample_submission_path )
class ResNetSteel(mx.gluon.HybridBlock):
def __init__(self, backbone= 'resnet50', num_classes=4, backbone_lr_mult=0.1, **kwargs):
super(ResNetSteel, self ).__init__()
self.backbone_name = backbone
self.backbone_lr_mult = backbone_lr_mult
self._kwargs = kwargs
with self.name_scope() :
self.backbone = ResNetBackbone(backbone=self.backbone_name, pretrained_base=False, dilated=False, **kwargs)
 self.head = Classification_head(output_channels=256, num_classes=num_classes)
 def load_pretrained_weights(self):
pretrained = ResNetBackbone(backbone=self.backbone_name, pretrained_base=True, dilated=False, **self._kwargs)
 backbone_params = self.backbone.collect_params()
pretrained_weights = pretrained.collect_params()
for k, v in pretrained_weights.items() :
param_name = backbone_params.prefix + k[len(pretrained_weights.prefix):]
backbone_params[param_name].set_data(v.data())
 self.backbone.collect_params().setattr('lr_mult', self.backbone_lr_mult)
 def hybrid_forward(self,F, x):
c1, c2, c3, c4 = self.backbone(x)
 logits = self.head(c4)
 return logits
class Classification_head(HybridBlock):
def __init__(self, output_channels=256, num_classes=4):
super(Classification_head, self ).__init__()
with self.name_scope() :
self.cls_head = nn.HybridSequential()
self.cls_head.add(ConvBlock(output_channels, kernel_size=1))
self.cls_head.add(nn.GlobalAvgPool2D())
 self.cls_head.add(nn.Conv2D(num_classes, kernel_size=1))
def hybrid_forward(self, F, x):
logits = self.cls_head(x)
 return F.squeeze(logits)
<train_on_grid>
train=train[train.item_price<100000]
train=train[train.item_cnt_day<1100]
ctx = mx.gpu()
cls_net = ResNetSteel(num_classes=4)
 cls_net.collect_params().initialize()
cls_net.load_parameters('.. /input/mxnet-gluon-classification/unet_4_0.0.params')
 cls_net.collect_params().reset_ctx(ctx )<choose_model_class>
train[train.item_price<0]
unet = ResNetUnet(output_channels=[256, 128, 64, 32], num_classes=5)
 unet.load_parameters('.. /input/mxnet-gluon-baseline/unet_34_-1.params')
 unet.collect_params().reset_ctx(ctx )<categorify>
median=train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].median()
train.loc[train.item_price<0,'item_price']=median
def mask2rle(mask):
if np.sum(mask)== 0: return ''
ar = mask.flatten(order='F')
 EncodedPixel = ''
l = 0
for i in range(len(ar)) :
if ar[i] == 0:
if l > 0:
if EncodedPixel != '': EncodedPixel += ' '
EncodedPixel += str(st+1)+' '+str(l)
 l = 0
else:
if l == 0: st = i
l += 1
return EncodedPixel<categorify>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
def remove_small_one(predict, min_size):
H,W = predict.shape
num_component, component = cv2.connectedComponents(predict.astype(np.uint8))
predict = np.zeros(( H,W), np.bool)
 for c in range(1,num_component):
p =(component==c)
 if p.sum() >min_size:
predict[p] = True
return predict 
<save_to_csv>
train[train.item_price.isnull() ]
submission = pd.read_csv(".. /input/severstal-steel-defect-detection/sample_submission.csv")
 print(len(ImageId),len(submission['ImageId_ClassId']))
for i, encoded in zip(ImageId_ClassIds,EncodedPixels):
submission.loc[submission['ImageId_ClassId']==i,["EncodedPixels"]] = encoded
submission.to_csv('submission.csv',index=False )<install_modules>
train[(train.item_id==2973)&(train.shop_id==32)]
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/senetunetmodelcode'
sys.path.append(package_path )<import_modules>
train.loc[train.item_price.isnull() ,'item_price']=1249.0
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.pytorch import ToTensor
import torch.utils.data as data
from senet_unet_model_code import Unet<categorify>
shops.loc[shops.shop_name=='Сергиев Посад ТЦ "7Я"','shop_name']='СергиевПосад ТЦ "7Я"'
shops['city']=shops['shop_name'].str.split(' ' ).map(lambda x:x[0])
 shops['city_code']=LabelEncoder().fit_transform(shops['city'])
 shops=shops[['shop_id','city_code']]
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
item_cat['split']=item_cat['item_category_name'].str.split('-')
 item_cat['type']=item_cat['split'].map(lambda x:x[0].strip())
 item_cat['type_code']=LabelEncoder().fit_transform(item_cat['type'])
 def subtype(x):
if len(x)>2:
return x[1].strip() +x[2].strip()
elif len(x)>1:
return x[1].strip()
else:
return x[0].strip()
item_cat['subtype']=item_cat['split'].map(subtype)
 item_cat['subtype_code']=LabelEncoder().fit_transform(item_cat['subtype'])
 item_cat=item_cat[['item_category_id','type_code','subtype_code']]
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
items.drop(['item_name'],axis=1,inplace=True )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
matrix=[]
cols=['date_block_num','shop_id','item_id']
for i in range(34):
sales=train[train.date_block_num==i]
matrix.append(np.array(list(product([i],sales.shop_id.unique() ,sales.item_id.unique()))))
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
matrix=pd.DataFrame(np.vstack(matrix),columns=cols)
 matrix.sort_values(cols,inplace=True )
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<load_pretrained>
train['revenue']=train['item_price']*train['item_cnt_day']
ckpt_path = ".. /input/senetmodels/senext50_30_epochs.pth"
device = torch.device("cuda")
 model = Unet('se_resnext50_32x4d', encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )<save_to_csv>
group=train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':'sum'})
 group.columns=['item_cnt_month']
group.reset_index(inplace=True )
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )<load_from_csv>
matrix=pd.merge(matrix,group,on=cols,how='left')
 matrix['item_cnt_month']=matrix['item_cnt_month'].fillna(0 ).clip(0,20 )
train_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',index_col=0)
 test_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col=0)
 train_df.head()<count_missing_values>
test['date_block_num']=34
test.drop(['ID'],axis=1,inplace=True)
 matrix=pd.concat([matrix,test],ignore_index=True)
 matrix['item_cnt_month'].fillna(0,inplace=True )
missing_data = train_df.isnull().sum().sort_values(ascending=False ).head(20)
 percentage_missing_data = train_df.isnull().sum() /train_df.isnull().count().sort_values(ascending = False)* 100
missing_values = pd.concat([missing_data, percentage_missing_data], axis=1, keys=['missing_data', 'percentage'])
 missing_values.head(20 )<drop_column>
matrix=pd.merge(matrix,shops,on='shop_id',how='left')
 matrix=pd.merge(matrix,items,on='item_id',how='left' )
train_df.drop(['MiscFeature', 'PoolQC', 'Alley', 'Fence'], axis=1, inplace=True)
 train_df.shape<feature_engineering>
matrix=pd.merge(matrix,item_cat,on='item_category_id',how='left' )
train_df['LotFrontage'] = train_df['LotFrontage'].fillna(train_df['LotFrontage'].mean() )<feature_engineering>
matrix[['date_block_num','shop_id','city_code',
'item_category_id','type_code','subtype_code']]=matrix[['date_block_num','shop_id','city_code',
'item_category_id','type_code','subtype_code']].astype(np.int8)
 matrix[['item_id']]=matrix[['item_id']].astype(np.int16)
 matrix[['item_cnt_month']]=matrix[['item_cnt_month']].astype(np.float16 )
train_df['GarageCond'] = train_df['GarageCond'].fillna(train_df['GarageCond'].mode() [0])
 train_df['FireplaceQu'] = train_df['FireplaceQu'].fillna(train_df['FireplaceQu'].mode() [0])
 train_df['GarageType'] = train_df['GarageType'].fillna(train_df['GarageType'].mode() [0])
 train_df['GarageYrBlt'] = train_df['GarageYrBlt'].fillna(train_df['GarageYrBlt'].mode() [0])
 train_df['GarageFinish'] = train_df['GarageFinish'].fillna(train_df['GarageFinish'].mode() [0])
 train_df['GarageQual'] = train_df['GarageQual'].fillna(train_df['GarageQual'].mode() [0])
 train_df['BsmtExposure'] = train_df['BsmtExposure'].fillna(train_df['BsmtExposure'].mode() [0])
 train_df['BsmtFinType2'] = train_df['BsmtFinType2'].fillna(train_df['BsmtFinType2'].mode() [0])
 train_df['BsmtFinType1'] = train_df['BsmtFinType1'].fillna(train_df['BsmtFinType1'].mode() [0])
 train_df['BsmtCond'] = train_df['BsmtCond'].fillna(train_df['BsmtCond'].mode() [0])
 train_df['BsmtQual'] = train_df['BsmtQual'].fillna(train_df['BsmtQual'].mode() [0])
 train_df['MasVnrArea'] = train_df['MasVnrArea'].fillna(train_df['MasVnrArea'].mode() [0])
 train_df['MasVnrType'] = train_df['MasVnrType'].fillna(train_df['MasVnrType'].mode() [0])
 train_df['Electrical'] = train_df['Electrical'].fillna(train_df['Electrical'].mode() [0] )<sort_values>
def lag_feature(df,lags,col):
tmp=df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted=tmp.copy()
shifted.columns=['date_block_num','shop_id','item_id',col+'_lag'+str(i)]
shifted['date_block_num']+=i
df=pd.merge(df,shifted,on=['date_block_num','shop_id','item_id'],how='left')
 return df
missing_test_data = test_df.isnull().sum().sort_values(ascending=False)
 percentage_mtd = test_df.isnull().sum() /test_df.isnull().count().sort_values(ascending=False)*100
missing_test_values = pd.concat([missing_test_data, percentage_mtd], axis=1, keys=['missing_test_data', 'percentage'])
 missing_test_values.head(34 )<drop_column>
matrix=lag_feature(matrix,[1,2,3,6,12],'item_cnt_month' )
test_df.drop(['PoolQC', 'Fence', 'Alley', 'MiscFeature'], axis=1, inplace=True )<feature_engineering>
def add_mean_feature(df,addname,grouplist,time):
group_tmp=df.groupby(grouplist ).agg({'item_cnt_month':'mean'})
 group_tmp.columns=addname
group_tmp.reset_index(inplace=True)
 df=pd.merge(df,group_tmp,on=grouplist,how='left')
 df=lag_feature(df,time,addname[0])
 df.drop(addname,axis=1,inplace=True)
 return df
test_df['LotFrontage'] = test_df['LotFrontage'].fillna(test_df['LotFrontage'].mean() )<feature_engineering>
matrix=add_mean_feature(matrix,['date_avg_item_cnt'],['date_block_num'],[1])
 matrix=add_mean_feature(matrix,['date_item_avg_item_cnt'],['date_block_num','item_id'],[1,2,3,6,12])
 matrix=add_mean_feature(matrix,['date_shop_avg_item_cnt'],['date_block_num','shop_id'],[1,2,3,6,12])
 matrix=add_mean_feature(matrix,['date_cat_avg_item_cnt'],['date_block_num','item_category_id'],[1])
 matrix=add_mean_feature(matrix,['date_type_avg_item_cnt'],['date_block_num','type_code'],[1])
 matrix=add_mean_feature(matrix,['date_subtype_avg_item_cnt'],['date_block_num','subtype_code'],[1])
 matrix=add_mean_feature(matrix,['date_city_avg_item_cnt'],['date_block_num','city_code'],[1])
 matrix=add_mean_feature(matrix,['date_shop_cat_avg_item_cnt'],['date_block_num','shop_id','item_category_id'],[1])
 matrix=add_mean_feature(matrix,['date_shop_type_avg_item_cnt'],['date_block_num','shop_id','type_code'],[1])
 matrix=add_mean_feature(matrix,['date_shop_subtype_avg_item_cnt'],['date_block_num','shop_id','subtype_code'],[1])
 matrix=add_mean_feature(matrix,['date_item_city_avg_item_cnt'],['date_block_num','item_id','city_code'],[1] )
test_df['FireplaceQu'] = test_df['FireplaceQu'].fillna(test_df['FireplaceQu'].mode() [0])
 test_df['GarageCond'] = test_df['GarageCond'].fillna(test_df['GarageCond'].mode() [0])
 test_df['GarageFinish'] = test_df['GarageFinish'].fillna(test_df['GarageFinish'].mode() [0])
 test_df['GarageYrBlt'] = test_df['GarageYrBlt'].fillna(test_df['GarageYrBlt'].mode() [0])
 test_df['GarageQual'] = test_df['GarageQual'].fillna(test_df['GarageQual'].mode() [0])
 test_df['GarageType'] = test_df['GarageType'].fillna(test_df['GarageType'].mode() [0])
 test_df['BsmtCond'] = test_df['BsmtCond'].fillna(test_df['BsmtCond'].mode() [0])
 test_df['BsmtExposure'] = test_df['BsmtExposure'].fillna(test_df['BsmtExposure'].mode() [0])
 test_df['BsmtQual'] = test_df['BsmtQual'].fillna(test_df['BsmtQual'].mode() [0])
 test_df['BsmtFinType1'] = test_df['BsmtFinType1'].fillna(test_df['BsmtFinType1'].mode() [0])
 test_df['BsmtFinType2'] = test_df['BsmtFinType2'].fillna(test_df['BsmtFinType2'].mode() [0])
 test_df['MasVnrType'] = test_df['MasVnrType'].fillna(test_df['MasVnrType'].mode() [0])
 test_df['MasVnrArea'] = test_df['MasVnrArea'].fillna(test_df['MasVnrArea'].mode() [0])
 test_df['MSZoning'] = test_df['MSZoning'].fillna(test_df['MSZoning'].mode() [0])
 test_df['BsmtHalfBath'] = test_df['BsmtHalfBath'].fillna(test_df['BsmtHalfBath'].mode() [0])
 test_df['Utilities'] = test_df['Utilities'].fillna(test_df['Utilities'].mode() [0])
 test_df['Functional'] = test_df['Functional'].fillna(test_df['Functional'].mode() [0])
 test_df['BsmtFullBath'] = test_df['BsmtFullBath'].fillna(test_df['BsmtFullBath'].mode() [0])
 test_df['BsmtFinSF2'] = test_df['BsmtFinSF2'].fillna(test_df['BsmtFinSF2'].mode() [0])
 test_df['BsmtFinSF1'] = test_df['BsmtFinSF1'].fillna(test_df['BsmtFinSF1'].mode() [0])
 test_df['BsmtUnfSF'] = test_df['BsmtUnfSF'].fillna(test_df['BsmtUnfSF'].mode() [0])
 test_df['TotalBsmtSF'] = test_df['TotalBsmtSF'].fillna(test_df['TotalBsmtSF'].mode() [0])
 test_df['Exterior2nd'] = test_df['Exterior2nd'].fillna(test_df['Exterior2nd'].mode() [0])
 test_df['SaleType'] = test_df['SaleType'].fillna(test_df['SaleType'].mode() [0])
 test_df['Exterior1st'] = test_df['Exterior1st'].fillna(test_df['Exterior1st'].mode() [0])
 test_df['KitchenQual'] = test_df['KitchenQual'].fillna(test_df['KitchenQual'].mode() [0])
 test_df['GarageArea'] = test_df['GarageArea'].fillna(test_df['GarageArea'].mode() [0])
 test_df['GarageCars'] = test_df['GarageCars'].fillna(test_df['GarageCars'].mode() [0] )<count_missing_values>
train_f=train.drop(['date','item_cnt_day'],axis=1)
 train_f[['date_block_num','shop_id']]=train_f[['date_block_num','shop_id']].astype(np.int8)
 train_f[['item_id']]=train_f[['item_id']].astype(np.int16)
 train_f[['item_price','revenue']]=train_f[['item_price','revenue']].astype(np.float16)
 train_f.info()
test_df.isnull().sum().max()<concatenate>
def add_price_feature(mat,df,addname,grouplist):
group_tmp=df.groupby(grouplist ).agg({'item_price':'mean'})
 group_tmp.columns=addname
group_tmp[addname]=group_tmp[addname].astype(np.float16)
 group_tmp.reset_index(inplace=True)
 mat=pd.merge(mat,group_tmp,on=grouplist,how='left')
 return mat
frames = [train_df, test_df]
full_data = pd.concat(frames, axis=0)
 full_data['SalePrice']<categorify>
matrix=add_price_feature(matrix,train_f,['item_avg_item_price'],['item_id'])
 matrix=add_price_feature(matrix,train_f,['date_item_avg_item_price'],['date_block_num','item_id'])
 lags=[1,2,3,4,5,6]
matrix=lag_feature(matrix,lags,'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag'+str(i)]=(matrix['date_item_avg_item_price_lag'+str(i)]-matrix['item_avg_item_price'])/matrix['item_avg_item_price']
full_data = pd.get_dummies(full_data, drop_first=True )<feature_engineering>
def select_trend(row):
for i in lags:
if row['delta_price_lag'+str(i)]:
return row['delta_price_lag'+str(i)]
return 0
matrix['delta_price_lag']=matrix.apply(select_trend,axis=1)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 matrix['delta_price_lag']=matrix['delta_price_lag'].astype(np.float16)
 drop_list=['item_avg_item_price','date_item_avg_item_price']
for i in lags:
drop_list.append('date_item_avg_item_price_lag'+str(i))
drop_list.append('delta_price_lag'+str(i))
matrix.drop(drop_list,axis=1,inplace=True )
full_data['GrLivArea'] = np.log(full_data['GrLivArea'] )<feature_engineering>
def fun_first_item_shop(x):
d={}
d['first_sale_item_shop']=x.loc[x.item_cnt_month>0,'date_block_num'].min()
return pd.Series(d)
 def fun_first_item(x):
d={}
d['first_sale_item']=x.loc[x.item_cnt_month>0,'date_block_num'].min()
return pd.Series(d)
 def first(mat):
group1=mat.groupby(['item_id','shop_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_first_item_shop)
 group2=mat.groupby(['item_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_first_item)
 mat=pd.merge(mat,group1,on=['item_id','shop_id'],how='left')
 mat=pd.merge(mat,group2,on=['item_id'],how='left')
 mat['first_sale_item_shop_period']=mat['date_block_num']-mat['first_sale_item_shop']
mat['first_sale_item_period']=mat['date_block_num']-mat['first_sale_item']
mat.loc[mat.first_sale_item_shop_period<0,'first_sale_item_shop_period']=0
mat.loc[mat.first_sale_item_period<0,'first_sale_item_period']=0
mat.drop(['first_sale_item_shop','first_sale_item'],axis=1,inplace=True)
 return mat
full_data['HasBsmt'] = pd.Series(len(full_data['TotalBsmtSF']), index=full_data.index)
 full_data['HasBsmt'] = 0
full_data.loc[full_data['TotalBsmtSF'] > 0, 'HasBsmt'] = 1
full_data.loc[full_data['HasBsmt']==1, 'TotalBsmtSF'] = np.log(full_data['TotalBsmtSF'])
 full_data.drop(['HasBsmt'], axis=1, inplace=True )<split>
def fun_last_item_shop(x):
d={}
d['last_sale_item_shop']=x.loc[x.item_cnt_month>0,'date_block_num'].max()
return pd.Series(d)
 def fun_last_item(x):
d={}
d['last_sale_item']=x.loc[x.item_cnt_month>0,'date_block_num'].max()
return pd.Series(d)
 def last(mat):
group1=mat.groupby(['item_id','shop_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_last_item_shop)
 group2=mat.groupby(['item_id'])[['date_block_num','item_cnt_month','item_id','shop_id']].apply(fun_last_item)
 mat=pd.merge(mat,group1,on=['item_id','shop_id'],how='left')
 mat=pd.merge(mat,group2,on=['item_id'],how='left')
 mat['last_sale_item_shop_period']=mat['date_block_num']-mat['last_sale_item_shop']
mat['last_sale_item_period']=mat['date_block_num']-mat['last_sale_item']
mat.loc[mat.last_sale_item_shop_period<0,'last_sale_item_shop_period']=0
mat.loc[mat.last_sale_item_period<0,'last_sale_item_period']=0
mat.drop(['last_sale_item_shop','last_sale_item'],axis=1,inplace=True)
 return mat
df_train = full_data.iloc[:1460, :]
df_test = full_data.iloc[1460:,:]<count_missing_values>
matrix=first(matrix)
 matrix=last(matrix )
df_test['SalePrice'].isnull().sum() 
<drop_column>
frls_list=['first_sale_item_shop_period','first_sale_item_period','last_sale_item_shop_period','last_sale_item_period']
df_test.drop(['SalePrice'], axis=1, inplace=True )<prepare_x_and_y>
matrix[frls_list]=matrix[frls_list].fillna(0 )
y_train = df_train['SalePrice']
y_train.head()
y_train.shape<drop_column>
matrix[frls_list]=matrix[frls_list].astype(np.int8 )
x_train = df_train.drop(['SalePrice'], axis=1)
 x_train.head()<import_modules>
matrix['month'] = matrix['date_block_num'] % 12
import xgboost as xgb
from sklearn.metrics import mean_squared_error<define_search_model>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
xg_reg = xgb.XGBRegressor()
xg_reg<import_modules>
matrix.to_pickle('data.pkl')
from sklearn.model_selection import RandomizedSearchCV 
<define_search_space>
data = pd.read_pickle('data.pkl' )
max_depth = [1,3,5,7,9,11,13,15,17,19,21]
n_estimators = [100, 200, 300, 400, 500, 600, 700]
booster = ['gbtree', 'gblinear']
learning_rate = [0.01, 0.05, 0.1, 0.2,0.3,0.4,0.5,0.6,0.7,0.8]
min_child_weight = [1,3,5,7,9,11,13,15,17,19,21]
hyperparameter_grid = {
"max_depth" : max_depth,
"n_estimators" : n_estimators,
"booster" : booster,
"learning_rate" : learning_rate,
"min_child_weight" : min_child_weight
}<choose_model_class>
data = data[data.date_block_num > 11]
random_cv = RandomizedSearchCV(estimator = xg_reg, param_distributions=hyperparameter_grid,
cv=5, n_iter=50,
scoring = 'neg_mean_absolute_error',n_jobs = 4,
verbose = 5,
return_train_score = True,
random_state=42 )<train_on_grid>
data=data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag1',
'item_cnt_month_lag2',
'item_cnt_month_lag3',
'item_cnt_month_lag6',
'item_cnt_month_lag12',
'date_avg_item_cnt_lag1',
'date_item_avg_item_cnt_lag1',
'date_item_avg_item_cnt_lag2',
'date_item_avg_item_cnt_lag3',
'date_item_avg_item_cnt_lag6',
'date_item_avg_item_cnt_lag12',
'date_shop_avg_item_cnt_lag1',
'date_shop_avg_item_cnt_lag2',
'date_shop_avg_item_cnt_lag3',
'date_shop_avg_item_cnt_lag6',
'date_shop_avg_item_cnt_lag12',
'date_cat_avg_item_cnt_lag1',
'date_city_avg_item_cnt_lag1',
'date_shop_cat_avg_item_cnt_lag1',
'date_item_city_avg_item_cnt_lag1',
'delta_price_lag',
'first_sale_item_shop_period',
'first_sale_item_period',
'month'
]]
random_cv.fit(x_train, y_train )<find_best_params>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
random_cv.best_estimator_<choose_model_class>
model = XGBRegressor(
 tree_method='gpu_hist',
max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
random_state=42 )
xg_reg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.05, max_delta_step=0, max_depth=5,
min_child_weight=5, missing=None, monotone_constraints='() ',
n_estimators=700, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None )<train_model>
model.fit(
 X_train,
Y_train,
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
eval_metric="rmse",
verbose=True,
early_stopping_rounds = 10 )
xg_reg.fit(x_train, y_train )<predict_on_test>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )