Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<predict_on_test>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from itertools import product
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgbm
from lightgbm import plot_importance
from sklearn.metrics import mean_squared_error
import joblib
from math import sqrt
from time import time
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
train_sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 train_sales['date'] = pd.to_datetime(train_sales['date'], dayfirst = True)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 df_items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 df_shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 df_categories = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv' )
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
max_cnt = train_sales["item_cnt_day"].quantile(0.999)
 max_price = train_sales["item_price"].quantile(0.999)
 print(max_cnt)
 train_sales = train_sales[train_sales["item_cnt_day"]<=max_cnt]
print(max_price)
 train_sales = train_sales[(train_sales["item_price"]>0)&(train_sales["item_price"]<=max_price)]
print(train_sales.shape )
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))<define_variables>
group_sales = train_sales.groupby(['date_block_num', 'shop_id', 'item_id'],
as_index=False)[['item_cnt_day']].sum()
group_sales = group_sales.rename(columns={"item_cnt_day":"item_cnt_month"})
 train_monthly = train_monthly.merge(group_sales,
on=['date_block_num', 'shop_id', 'item_id'],
how='left')
 train_monthly['item_cnt_month'] = train_monthly['item_cnt_month'].fillna(0 ).clip(0, 20 )
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15<save_to_csv>
test['date_block_num'] = 34
train_test_monthly = pd.concat([train_monthly, test] ).reset_index(drop = True )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<set_options>
prices = train_sales.groupby(by = ['item_id'],
as_index=False ).agg({'item_price':['mean']})
 prices.columns = ['item_id', 'item_price_mean']
prices.fillna(0, inplace = True)
 train_test_monthly = train_test_monthly.merge(prices,
on=['item_id'],
how='left' )
sns.set_style('whitegrid')
 warnings.simplefilter(action='ignore')
<load_from_csv>
train_test_monthly.loc[train_test_monthly['shop_id']==0, 'shop_id'] = 57
train_test_monthly.loc[train_test_monthly['shop_id']==1, 'shop_id'] = 58
train_test_monthly.loc[train_test_monthly['shop_id']==39, 'shop_id'] = 40
a = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 b = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<set_options>
df_shops.loc[df_shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
df_shops['city'] = df_shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 df_shops.loc[df_shops['city'] == '!Якутск', 'city'] = 'Якутск'
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )<train_model>
train_test_monthly = train_test_monthly.merge(df_shops[['shop_id', 'city']],
how = 'left',
on = 'shop_id' )
print('train : %sの要素と%sの特徴量があります。'%(a.shape[0],a.shape[1]))
print('test : %sの要素と%sの特徴量があります'%(b.shape[0],b.shape[1]))
print('testはtrainよりも特徴量が1つ少なくなっており、その残り1つが予測対象であるSalePriceです。' )<prepare_x_and_y>
train_test_monthly['city'] = LabelEncoder().fit_transform(train_test_monthly['city'] )
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"データサイズ : {c1.shape}" )<compute_test_metric>
df_categories['type'] = df_categories['item_category_name'].str.\
split('-' ).map(lambda x: x[0].strip())
 df_categories['sub_type'] = df_categories['item_category_name'].str.\
split('-' ).map(lambda x: x[1].strip() if len(x)> 1
else x[0].strip())
 df_categories.head()
msv1(c1, 80, color=('black', 'deeppink'))<correct_missing_values>
df_items = df_items.merge(df_categories[['item_category_id', 'type', 'sub_type']],
on = 'item_category_id',
how = 'left' )
c=c1.dropna(thresh=len(c1)*0.2, axis=1)
 print(f"trainとtestの結合データ(combination)から{c1.shape[1] - c.shape[1]}つの特徴量を削除しました。" )<compute_test_metric>
train_test_monthly = train_test_monthly.merge(df_items[['item_id', 'item_category_id', 'type', 'sub_type']],
on = 'item_id',
how = 'left' )
msv2(c )<data_type_conversions>
train_test_monthly['type'] = LabelEncoder().fit_transform(train_test_monthly['type'])
 train_test_monthly['sub_type'] = LabelEncoder().fit_transform(train_test_monthly['sub_type'] )
NA=c[allna.index.to_list() ]<feature_engineering>
train_test_monthly['month'] =(train_test_monthly['date_block_num'] % 12)+ 1
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )<categorify>
train_test_monthly['shop_item_id'] = train_test_monthly['shop_id'].astype(str)+ '_' +\
train_test_monthly['item_id'].astype(str)
 train_test_monthly['shop_cat_id'] = train_test_monthly['shop_id'].astype(str)+ '_' +\
train_test_monthly['item_category_id'].astype(str)
 train_test_monthly['shop_type_id'] = train_test_monthly['shop_id'].astype(str)+ '_' +\
train_test_monthly['type'].astype(str)
 train_test_monthly['item_city_id'] = train_test_monthly['item_id'].astype(str)+ '_' +\
train_test_monthly['city'].astype(str)
 train_test_monthly['item_city_month'] = train_test_monthly['item_city_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str)
 train_test_monthly['shop_item_month'] = train_test_monthly['shop_item_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str)
 train_test_monthly['shop_cat_month'] = train_test_monthly['shop_cat_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str)
 train_test_monthly['shop_type_month'] = train_test_monthly['shop_type_id'].astype(str)+ '_' +\
train_test_monthly['month'].astype(str )
ffill_cols = ['KitchenQual', 'Electrical', 'SaleType', 'Exterior1st',
'Exterior2nd', 'Utilities', 'Functional', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data<compute_test_metric>
train_test_monthly.drop(columns=['month'], inplace=True )
msv2(c )<concatenate>
train_test_monthly['shop_item_id'] = LabelEncoder().fit_transform(train_test_monthly['shop_item_id'])
 train_test_monthly['shop_cat_id'] = LabelEncoder().fit_transform(train_test_monthly['shop_cat_id'])
 train_test_monthly['shop_type_id'] = LabelEncoder().fit_transform(train_test_monthly['shop_type_id'])
 train_test_monthly['item_city_id'] = LabelEncoder().fit_transform(train_test_monthly['item_city_id'])
 train_test_monthly['item_city_month'] = LabelEncoder().fit_transform(train_test_monthly['item_city_month'])
 train_test_monthly['shop_item_month'] = LabelEncoder().fit_transform(train_test_monthly['shop_item_month'])
 train_test_monthly['shop_cat_month'] = LabelEncoder().fit_transform(train_test_monthly['shop_cat_month'])
 train_test_monthly['shop_type_month'] = LabelEncoder().fit_transform(train_test_monthly['shop_type_month'] )
d=filling_NA(c, ffill_cols )<compute_test_metric>
train_test_monthly.isna().sum()
msv2(d )<data_type_conversions>
group_mean_type = train_test_monthly.groupby('type')['item_price_mean'].mean()
train_test_monthly.loc[train_test_monthly['item_price_mean'].isna() , 'item_price_mean'] =\
train_test_monthly.loc[train_test_monthly['item_price_mean'].isna() , 'type'].map(group_mean_type )
fill_cols = ['KitchenQual', 'Electrical', 'SaleType', 'Exterior1st',
'Exterior2nd', 'Utilities', 'Functional', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )<data_type_conversions>
print('%i cities'%(train_test_monthly[train_test_monthly['date_block_num']<34]['city'].nunique()))
print('%i types'%(train_test_monthly[train_test_monthly['date_block_num']<34]['type'].nunique()))
print('%i sub_types'%(train_test_monthly[train_test_monthly['date_block_num']<34]['sub_type'].nunique()))
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )<data_type_conversions>
print('%i cities'%(train_test_monthly[train_test_monthly['date_block_num']==34]['city'].nunique()))
print('%i types'%(train_test_monthly[train_test_monthly['date_block_num']==34]['type'].nunique()))
print('%i sub_types'%(train_test_monthly[train_test_monthly['date_block_num']==34]['sub_type'].nunique()))
for col in NAcols:
if c[col].dtype != "object":
c[col]= c[col].fillna(0 )<feature_engineering>
def col_months_lag(col, dataset, date_block_mean=True, use_similar=False):
lag_months = [1, 2, 3, 12]
df = dataset.copy()
if date_block_mean:
group_mean = df.groupby(['date_block_num',
col],
as_index=False)['item_cnt_month'].mean().fillna(0)
 group_mean.rename(columns={'item_cnt_month':col+'_date_block_mean'},
inplace = True)
 for lag in lag_months:
group_mean_lag = group_mean.copy()
group_mean_lag['date_block_num'] += lag
group_mean_lag.rename(columns={col+'_date_block_mean':col+'_date_block_mean_lag_'+str(lag)},
inplace = True)
 df = df.merge(group_mean_lag,
on=['date_block_num', col],
how='left')
 df[col+'_date_block_mean_lag_'+str(lag)].fillna(0, inplace = True)
 df[col+'_3_months_mean'] = df[[col+'_date_block_mean_lag_1',
col+'_date_block_mean_lag_2',
col+'_date_block_mean_lag_3']].mean(axis = 1)
 df[col+'_3_months_std'] = df[[col+'_date_block_mean_lag_1',
col+'_date_block_mean_lag_2',
col+'_date_block_mean_lag_3']].std(axis = 1)
 df[col+'_diff_1_2_months'] = df[col+'_date_block_mean_lag_1'] - df[col+'_date_block_mean_lag_2']
df.drop(columns=[col+'_date_block_mean_lag_3'],
inplace=True)
 else:
for lag in lag_months:
df[col+'_month_lag_'+str(lag)] =\
df.groupby([col])['item_cnt_month'].shift(lag)
 df[col+'_month_lag_'+str(lag)].fillna(0, inplace = True)
 df[col+'_3_months_mean'] = df[[col+'_month_lag_1',
col+'_month_lag_2',
col+'_month_lag_3']].mean(axis = 1)
 df[col+'_3_months_std'] = df[[col+'_month_lag_1',
col+'_month_lag_2',
col+'_month_lag_3']].std(axis = 1)
 df[col+'_diff_1_2_months'] = df[col+'_month_lag_1'] - df[col+'_month_lag_2']
df.drop(columns=[col+'_month_lag_3'],
inplace=True)
 return df
c['TotalArea'] = c['TotalBsmtSF'] + c['1stFlrSF'] + c['2ndFlrSF'] + c['GrLivArea'] +c['GarageArea']
c['Bathrooms'] = c['FullBath'] + c['HalfBath']*0.5
c['Year average']=(c['YearRemodAdd']+c['YearBuilt'])/2<data_type_conversions>
train_test_monthly = col_months_lag('shop_item_id', train_test_monthly, False)
 train_test_monthly = col_months_lag('shop_type_id', train_test_monthly)
 train_test_monthly = col_months_lag('item_id', train_test_monthly )
c['MoSold'] = c['MoSold'].astype(str)
 c['MSSubClass'] = c['MSSubClass'].apply(str)
 c['YrSold'] = c['YrSold'].astype(str )<categorify>
similar_group_mean = train_test_monthly.groupby(by=['date_block_num',
'shop_id',
'item_id'],
as_index=False)['item_cnt_month'].last().fillna(0)
 similar_group_mean.rename(columns={'item_cnt_month':'shop_similar_item_id_month_lag_1'},
inplace = True)
 similar_group_mean['item_id'] += 1
similar_group_mean['date_block_num'] += 1
train_test_monthly = train_test_monthly.merge(similar_group_mean,
on=['date_block_num', 'shop_id', 'item_id'],
how='left' )
cb=pd.get_dummies(c)
 print(f"the shape of the original dataset {c.shape}")
 print(f"the shape of the encoded dataset {cb.shape}")
 print(f"We have {cb.shape[1]- c.shape[1]} new encoded features" )<split>
def RMSE(y_true, y_pred):
return sqrt(mean_squared_error(y_true, y_pred))
Train = cb[:na]
Test = cb[na:]<sort_values>
def split_date_block_num(df, date_block_num):
split_1 = df[df['date_block_num'] < date_block_num].copy()
split_2 = df[df['date_block_num'] == date_block_num].copy().reset_index(drop=True)
 return split_1, split_2
a['GrLivArea'].sort_values(ascending=False ).head(2 )<sort_values>
train_sales, test = split_date_block_num(train_test_monthly, 34 )
a['TotalBsmtSF'].sort_values(ascending=False ).head(1 )<sort_values>
train_sales = train_sales[train_sales['date_block_num']>2]
a['MasVnrArea'].sort_values(ascending=False ).head(1 )<sort_values>
select_feats = ['shop_id',
'item_id',
'shop_item_id',
'shop_cat_id',
'shop_type_id',
'item_city_month',
'shop_item_month',
'shop_item_id_month_lag_1',
'shop_item_id_month_lag_2',
'shop_item_id_month_lag_12',
'shop_item_id_3_months_mean',
'shop_item_id_3_months_std',
'shop_item_id_diff_1_2_months',
'shop_type_id_date_block_mean_lag_1',
'shop_type_id_date_block_mean_lag_2',
'shop_type_id_date_block_mean_lag_12',
'shop_type_id_3_months_mean',
'shop_type_id_3_months_std',
'shop_type_id_diff_1_2_months',
'item_id_date_block_mean_lag_1',
'item_id_date_block_mean_lag_2',
'item_id_date_block_mean_lag_12',
'item_id_3_months_mean',
'item_id_3_months_std',
'item_id_diff_1_2_months',
'shop_similar_item_id_month_lag_1'
]
a['1stFlrSF'].sort_values(ascending=False ).head(1 )<sort_values>
params = {
'num_iterations':140,
'max_depth':15,
'objective': 'rmse',
'metric': 'rmse',
'num_leaves':(2 ** 11)- 1,
'learning_rate': 0.025,
'feature_fraction': 0.4,
'bagging_fraction': 0.2,
'bagging_freq': 5,
'seed': 1,
'verbose': 1
}
cat_cols = ['shop_id', 'item_id', 'shop_item_id', 'shop_cat_id',
'shop_type_id', 'item_city_month', 'shop_item_month']
a['TotRmsAbvGrd'].sort_values(ascending=False ).head(1 )<filter>
errors = []
for block_num in range(24, 34):
train, validation = split_date_block_num(train_sales, block_num)
 x_train = train[select_feats].values
y_train = train['item_cnt_month'].values
x_val = validation[select_feats].values
y_val = validation['item_cnt_month'].values
del train
del validation
print('TRAINING %i MONTHS:'%(block_num))
start = time()
lgbmr_train = lgbm.Dataset(x_train, y_train)
 lgbmr_val = lgbm.Dataset(x_val, y_val, reference=lgbmr_train)
 lgbmr_trained = lgbm.train(params,
lgbmr_train,
feature_name=select_feats,
categorical_feature=cat_cols)
 end = time()
del lgbmr_train
del lgbmr_val
y_train_pred = lgbmr_trained.predict(x_train ).clip(0, 20)
 train_rmse = RMSE(y_train, y_train_pred)
 print('Train RMSE = %.5f'%(train_rmse))
y_val_pred = lgbmr_trained.predict(x_val ).clip(0, 20)
 val_rmse = RMSE(y_val, y_val_pred)
 print('Validation RMSE = %.5f'%(val_rmse))
errors.append(val_rmse)
 print('Training time = %.2f s'%(end - start))
print()
del x_train
del y_train
del x_val
del y_val
del lgbmr_trained
del y_train_pred
del y_val_pred
train=Train[(Train['GrLivArea'] < 4676)&(Train['MasVnrArea'] < 1600)]
print(f'{Train.shape[0]- train.shape[0]}つの外れ値を削除しました。' )<prepare_x_and_y>
print('RMSE = %.5f +/- %.5f'%(np.array(errors ).mean() , np.array(errors ).std()))
target=a[['SalePrice']]
target.loc[1298]<filter>
x_train = train_sales[select_feats].values
y_train = train_sales['item_cnt_month'].values
x_test = test[select_feats].values
target.loc[523]<drop_column>
start = time()
lgbmr_train = lgbm.Dataset(x_train, y_train)
 lgbmr_trained = lgbm.train(params,
lgbmr_train,
feature_name=select_feats,
categorical_feature=cat_cols)
 end = time()
print('Training time = %.2f s'%(end - start))
pos = [1298,523, 297]
target.drop(target.index[pos], inplace=True )<train_model>
test['ID'] = test.index
test['item_cnt_month'] = lgbmr_trained.predict(x_test ).clip(0, 20 )
print('外れ値を除去した後、特徴量とターゲットの行番号が同じであることを確認します。')
 print('Train: ',train.shape[0],'列')
 print('Target:', target.shape[0],'列' )<import_modules>
test[['ID', 'item_cnt_month']].to_csv("lgbm_submission.csv", index = False )
<feature_engineering><EOS> 
joblib.dump(lgbmr_trained, 'lgbm_model.joblib' )