Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<groupby>
import numpy as np
import pandas as pd
import scipy
import sklearn
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from itertools import product
from sklearn.cluster import KMeans
import lightgbm as lgb
avg_payments = payments.groupby("SK_ID_CURR" ).mean()
avg_payments2 = payments.groupby("SK_ID_CURR" ).max()
avg_payments3 = payments.groupby("SK_ID_CURR" ).min()<drop_column>
dpath = '.. /input/competitive-data-science-predict-future-sales/'
adpath ='.. /input/predict-future-sales/'
del avg_payments["SK_ID_PREV"]<merge>
df_train = pd.read_csv(dpath + 'sales_train.csv')
 df_test = pd.read_csv(dpath + 'test.csv', index_col='ID')
 df_shops = pd.read_csv(dpath + 'shops.csv', index_col='shop_id')
 df_items = pd.read_csv(dpath + 'items.csv', index_col='item_id')
 df_itemcat = pd.read_csv(dpath + 'item_categories.csv', index_col='item_category_id')
 sample_submission = pd.read_csv(dpath + 'sample_submission.csv', index_col='ID' )
data = data.merge(right=avg_prev.reset_index() , how="left", on="SK_ID_CURR")
 test = test.merge(right=avg_prev.reset_index() , how="left", on="SK_ID_CURR")
<merge>
calendar = pd.read_csv(adpath + 'calendar.csv', dtype='int16' )
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR' )<merge>
def shop_name2city(sn):
sn = sn.split() [0]
if sn == 'Цифровой' or sn == 'Интернет-магазин': sn = 'Internet'
if sn[0] == '!': sn = sn[1:]
return sn
df_shops['city'] = df_shops['shop_name'].apply(shop_name2city)
 df_shops['city_enc'] = LabelEncoder().fit_transform(df_shops['city'] ).astype('int8')
 city_info = pd.read_pickle(adpath + 'city_info.pkl')
 df_shops['city_size'] = df_shops['city'].map(city_info['city_size'] )
data = data.merge(right=POS_CASH.groupby("SK_ID_CURR" ).mean().reset_index() , how="left", on="SK_ID_CURR")
 test = test.merge(right=POS_CASH.groupby("SK_ID_CURR" ).mean().reset_index() , how="left", on="SK_ID_CURR" )<merge>
class Items() :
def __init__(self, df_items, df_itemcat):
self.df_items = df_items
self.df_itemcat = df_itemcat
self.set_hl_cat()
self.make_items_ext()
self.item_features = ['item_category_id', 'hl_cat_id']
def set_hl_cat(self):
self.df_itemcat['hl_cat_id'] = self.df_itemcat['item_category_name'].str.split(n=1, expand=True)[0]
self.df_itemcat['hl_cat_id'] = LabelEncoder().fit_transform(self.df_itemcat['hl_cat_id'])
 def make_items_ext(self):
self.df_items = df_items.merge(self.df_itemcat, how = 'left',
left_on = 'item_category_id', right_index = True)
 def get_items_df(self):
return self.df_items[self.item_features].astype('int32' )
data = data.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
<merge>
items = Items(df_items, df_itemcat )
data = data.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
<merge>
class TT_Extended() :
def __init__(self, df_train, df_test, items, df_shops, calendar, cmode, verbose=True):
self.info = verbose
self.df_train = df_train.copy()
self.df_test = df_test.copy()
self.df_shops = df_shops.copy()
self.calendar = self.set_calender(calendar.copy())
 self.idx_columns = ['date_block_num', 'shop_id', 'item_id']
self.df_test['date_block_num'] = 34
self.df_test['item_cnt_month'] = 0.
 self.df_test['item_cnt_month'] = self.df_test['item_cnt_month'].astype('float32')
 self.df_train[self.idx_columns] = self.df_train[self.idx_columns].astype('int32')
 self.df_test[self.idx_columns] = self.df_test[self.idx_columns].astype('int32')
 self.df_train_cleaning(cmode)
 self.item_mean_features = []
self.shop_mean_features = []
self.lag_names_to_clip = []
self.df_items = items.get_items_df()
self.item_ext_features = list(self.df_items.columns)
 self.df_items_ext = self.items_ext()
self.df_bb = self.build_bb()
self.df_train_ext = self.df_train_agg(cmin = 0, cmax = 1000, drop = None)
 self.add_test_df()
self.df_train_ext = self.df_train_extention()
def df_train_cleaning(self, mode):
assert mode in ['keep', 'drop', 'block', 'total', 't+b']
if self.info: print('Cleaning train dataframe...( Mode -', mode, ')')
 shop_idx = self.df_train[(self.df_train['shop_id'] == 9)| 
(self.df_train['shop_id'] == 20)].index
self.df_train.drop(shop_idx, inplace=True)
 self.df_train = self.df_train[(self.df_train['item_price'] > 0)& 
(self.df_train['item_price'] < 51000)]
self.df_train = self.df_train[self.df_train['item_cnt_day'] <= 1000]
shop_repl_dict = {0 : 57, 1 : 58, 11 : 10, 40 : 39}
self.df_train['shop_id'] = self.df_train['shop_id'].apply(
 lambda s: shop_repl_dict.get(s)if s in shop_repl_dict else s)
 if mode == 'drop':
self.df_train = self.df_train[self.df_train['item_cnt_day'] > 0]
elif mode == 'block':
item_block_cnt = self.df_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_block_cnt[item_block_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['date_block_num', 'shop_id', 'item_id'] ).index.isin(items_to_drop)]
elif mode == 'total':
item_total_cnt = self.df_train.groupby(['shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_total_cnt[item_total_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['shop_id', 'item_id'] ).index.isin(items_to_drop)]
elif mode == 't+b':
item_total_cnt = self.df_train.groupby(['shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_total_cnt[item_total_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['shop_id', 'item_id'] ).index.isin(items_to_drop)]
item_block_cnt = self.df_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum()
items_to_drop = item_block_cnt[item_block_cnt <= 0].index
self.df_train = self.df_train[~self.df_train.set_index(
 ['date_block_num', 'shop_id', 'item_id'] ).index.isin(items_to_drop)]
return
def set_calender(self, calendar):
calendar['date_block_num'] =(calendar['year'] - 2013)*12 +(calendar['month'] - 1)
 calendar['hdays'] = calendar['mdays'] - calendar['wdays']
calendar.set_index('date_block_num', inplace=True)
 return calendar
def items_ext(self):
dfi = self.df_items.copy()
dfi['fsb'] = self.df_train.groupby('item_id')['date_block_num'].min()
dfi['fsb'].fillna(34, inplace=True)
 dfi['fsb'] = dfi['fsb'].astype('int8')
 self.item_ext_features += ['fsb']
return dfi
def build_bb(self):
if self.info: print('Building index dataframe...')
 df_work = []
for block_num in self.df_train['date_block_num'].unique() :
cur_shops = self.df_train.loc[self.df_train['date_block_num'] == block_num, 'shop_id'].unique()
cur_items = self.df_train.loc[self.df_train['date_block_num'] == block_num, 'item_id'].unique()
df_work.append(np.array(list(product(*[[block_num], cur_shops, cur_items])) , dtype='int32'))
df_work = pd.DataFrame(np.vstack(df_work), columns = self.idx_columns)
 return df_work
def df_train_agg(self, cmin = 0, cmax = 20, drop = None):
if self.info: print('Aggregation...')
 df_work = self.df_train.groupby(self.idx_columns ).agg({'item_price' : np.mean,
'item_cnt_day': np.sum})
 df_work.reset_index(inplace=True)
 df_work = df_work.rename(columns={'item_cnt_day': 'item_cnt_month'})
 df_work = pd.merge(self.df_bb, df_work, on=self.idx_columns, how='left')
 df_work['item_cnt_month'] = df_work['item_cnt_month'].astype('float32' ).fillna(0.).clip(cmin, cmax)
 df_work['item_price'] = df_work['item_price'].astype('float32' ).fillna(0.)
df_tmp = self.df_train[self.df_train['item_cnt_day'] > 0].groupby(self.idx_columns ).agg({'item_cnt_day': 'count'})
 df_tmp.reset_index(inplace=True)
 df_tmp = df_tmp.rename(columns={'item_cnt_day': 'item_rate_month'})
 df_work = pd.merge(df_work, df_tmp, on=self.idx_columns, how='left')
 df_work['item_rate_month'] = df_work['item_rate_month'].astype('float32' ).fillna(0.)
del df_tmp
if drop: df_work.drop(drop, axis=1, inplace=True)
 return df_work
def add_test_df(self):
self.df_train_ext = pd.concat([self.df_train_ext, self.df_test], ignore_index=True,
sort=False, keys=self.idx_columns)
 def add_item_means(self, df, feature = None):
if feature == None :
group_items = ['date_block_num','item_id']
feature = 'item_cnt'
else:
group_items = ['date_block_num','item_id'] + [feature]
feature_mean_name = feature + '_mean'
if self.info: print('Adding item means for', feature, '...')
 df_tmp = df.groupby(group_items)['item_cnt_month'].mean()
df_tmp = df_tmp.reset_index().rename(columns = {'item_cnt_month': feature_mean_name})
 df = pd.merge(df, df_tmp, on=group_items, how='left')
 self.item_mean_features.append(feature_mean_name)
 del df_tmp
return df
def add_shop_means(self, df, feature):
group_items = ['date_block_num', 'shop_id'] + [feature]
feature_mean_name = feature + '_mean'
if self.info: print('Adding shop means for', feature, '...')
 df_tmp = df.groupby(group_items)['item_cnt_month'].mean()
df_tmp = df_tmp.reset_index().rename(columns = {'item_cnt_month': feature_mean_name})
 df = pd.merge(df, df_tmp, on=group_items, how='left')
 self.shop_mean_features.append(feature)
 del df_tmp
return df
def df_train_extention(self, test_cat_only = False):
df_work = self.df_train_ext.merge(df_shops[['city_enc', 'city_size']], how = 'left', on = 'shop_id')
 df_work = df_work.merge(self.calendar[['mdays', 'wdays', 'hdays']],
how = 'left', left_on = 'date_block_num', right_index = True)
 df_work = df_work.merge(self.df_items_ext[self.item_ext_features],
how = 'left', left_on = 'item_id', right_index = True)
 ssbn = self.df_train.groupby('shop_id')['date_block_num'].min().astype('int8')
 ssbn.name = 'ssbn'
df_work = df_work.merge(ssbn, how = 'left', on = 'shop_id')
 if test_cat_only:
test_cat = df_work[df_work['date_block_num'] == 34]['item_category_id'].unique()
df_work = df_work[df_work['item_category_id'].isin(test_cat)]
return df_work
def make_base_df(self, keepnans = False):
df_work = self.df_train_ext.copy()
if keepnans:
fill_value = None
else:
fill_value = 0
df_work = pd.pivot_table(df_work, values='item_cnt_month', index=['shop_id', 'item_id'],
columns = 'date_block_num', aggfunc=np.sum, fill_value = fill_value)
 df_work.columns.name = ''
return df_work
def add_total_cnt(self, df):
if self.info: print('Adding total count...')
 df_base = self.make_base_df()
for i in range(1, 33):
df_base[i + 1] += df_base[i]
df_base = df_base.shift(1, axis=1 ).loc[:, 1:].astype('int32')
 df_base = df_base.melt(var_name='date_block_num', value_name='total_cnt', ignore_index=False)
 df = df.merge(df_base, how='left', on=self.idx_columns)
 del df_base
return df
def add_item_lags(self, df, feature_name, nlags=3, keepnans=False, dnc=False):
if self.info: print('Adding item lags for', feature_name, '...')
 df_tmp = df[['date_block_num', 'shop_id', 'item_id', feature_name]]
for i in range(nlags, 0, -1):
lag_feature_name = feature_name +'_lag-' + str(i)
 if not dnc: self.lag_names_to_clip.append(lag_feature_name)
 df_shifted = df_tmp.copy()
df_shifted.columns = ['date_block_num', 'shop_id', 'item_id', lag_feature_name]
df_shifted['date_block_num'] += i
df = pd.merge(df, df_shifted, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 if keepnans:
df[lag_feature_name] = df[lag_feature_name].astype('float32')
 else:
df[lag_feature_name] = df[lag_feature_name].fillna(0 ).astype('float32')
 del df_tmp
return df
def add_shop_lags(self, df, feature_name, nlags=3, dnc=False):
mean_feature_name = feature_name + '_mean'
if self.info: print('Adding lags for', mean_feature_name, '...')
 df_tmp = df[['date_block_num', 'shop_id', feature_name, mean_feature_name]]
for i in range(nlags, 0, -1):
lag_feature_name = mean_feature_name + '_lag-' + str(i)
 if not dnc: self.lag_names_to_clip.append(lag_feature_name)
 df_shifted = df_tmp.copy()
df_shifted.columns = ['date_block_num', 'shop_id', feature_name, lag_feature_name]
df_shifted['date_block_num'] += i
df = pd.merge(df, df_shifted.drop_duplicates() , on=['date_block_num', 'shop_id', feature_name], how='left')
 df[lag_feature_name] = df[lag_feature_name].fillna(0 ).astype('float32')
 del df_tmp
del df_shifted
return df
def shop_clustering(self, df_work):
print('No shop clusters provided')
 return df_work
def build_work_db(self, hd, item_mean_features = [], shop_mean_features = [], add_total_cnt = False):
if self.info: print('Building work dataframe...')
 df_work = self.df_train_ext.copy()
df_work = self.add_item_means(df_work)
 for mf in item_mean_features:
df_work = self.add_item_means(df_work, mf)
 for mf in shop_mean_features:
df_work = self.add_shop_means(df_work, mf)
 df_work = self.add_item_lags(df_work, 'item_cnt_month', hd, keepnans=False)
 df_work = self.add_item_lags(df_work, 'item_rate_month', hd, dnc=True)
 df_work = self.add_item_lags(df_work, 'item_price', hd, dnc=True)
 for mf in self.item_mean_features:
df_work = self.add_item_lags(df_work, mf, hd)
 for mf in self.shop_mean_features:
df_work = self.add_shop_lags(df_work, mf, hd)
 df_work.drop(df_work[df_work['date_block_num'] < hd].index, inplace=True)
 df_work.drop(self.item_mean_features, axis=1, inplace=True)
 df_work.drop(['item_category_id_mean', 'item_price', 'item_rate_month'], axis=1, inplace=True)
 self.item_mean_features = []
self.shop_mean_features = []
df_work['qmean'] = df_work[['item_cnt_month_lag-1',
'item_cnt_month_lag-2',
'item_cnt_month_lag-3']].mean(skipna=True, axis=1)
 df_work['new'] = df_work['fsb'] == df_work['date_block_num']
df_work['fsb'] = df_work['date_block_num'] - df_work['fsb']
df_work['ssbn'] = df_work['date_block_num'] - df_work['ssbn']
df_work['month'] =(df_work['date_block_num']%12 ).astype('int8')
 idx = df_work[(df_work['item_price_lag-1'] == 0)&(df_work['item_price_lag-2'] != 0)].index
df_work.loc[idx, 'item_price_lag-1'] = df_work.loc[idx, 'item_price_lag-2']
idx = df_work[(df_work['item_price_lag-2'] == 0)&(df_work['item_price_lag-3'] != 0)].index
df_work.loc[idx, 'item_price_lag-2'] = df_work.loc[idx, 'item_price_lag-3']
df_work['grad-1'] = df_work['item_cnt_month_lag-1']/df_work['item_cnt_month_lag-2']
df_work['grad-1'] = df_work['grad-1'].replace([np.inf, -np.inf], np.nan ).fillna(0.)
df_work['grad-2'] = df_work['item_cnt_month_lag-2']/df_work['item_cnt_month_lag-3']
df_work['grad-2'] = df_work['grad-2'].replace([np.inf, -np.inf], np.nan ).fillna(0.)
df_work = self.shop_clustering(df_work)
 if add_total_cnt: df_work = self.add_total_cnt(df_work)
 col2clip = ['item_cnt_month', 'qmean'] + self.lag_names_to_clip
df_work[col2clip] = df_work[col2clip].clip(0, 20)
 return df_work
def get_work_db(self, hd = 3, item_mean_features = [],
shop_mean_features = [],
drop_features = None,
add_total_cnt = False):
df_work = self.build_work_db(hd, item_mean_features, shop_mean_features, add_total_cnt)
 if drop_features == None:
return df_work
else:
return df_work.drop(drop_features, axis = 1 )
data = data.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR' )<merge>
%%time
pfs = TT_Extended(df_train, df_test, items, df_shops, calendar, cmode='total' )
data = data.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR' )<filter>
pfs.shop_clustering = simple_shop_clustering
test = test[test.columns[data.isnull().mean() < 0.85]]
data = data[data.columns[data.isnull().mean() < 0.85]]<define_variables>
%%time
df_work = pfs.get_work_db(hd = 3,
item_mean_features = ['city_enc'],
shop_mean_features = ['item_category_id'],
drop_features = ['wdays', 'hdays', 'ssbn'],
add_total_cnt = False ).copy()
excluded_feats = ["SK_ID_CURR"]
features = [f for f in data.columns if f not in excluded_feats]<init_hyperparams>
df_work['city_size'] = df_work['city_size'].round(1 )
<init_hyperparams>
X_train = df_work[df_work.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = df_work[df_work.date_block_num < 33]['item_cnt_month']
X_valid = df_work[df_work.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_valid = df_work[df_work.date_block_num == 33]['item_cnt_month']
X_test = df_work[df_work.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
lgbm_params = {
"boosting":"dart",
"application":"binary",
"learning_rate": 0.1,
'reg_alpha':0.01,
'reg_lambda': 0.01,
"n_estimators":10000,
"max_depth":7,
"num_leaves":100,
"max_bin":225,
"drop_rate":0.02
}<choose_model_class>
del df_work
model = lgb.LGBMClassifier(application="binary", boosting_type=lgbm_params["boosting"],
learning_rate=lgbm_params["learning_rate"],n_estimators=lgbm_params["n_estimators"],drop_rate=lgbm_params["drop_rate"],
num_leaves=lgbm_params["num_leaves"], max_depth=lgbm_params["max_depth"],
max_bin=lgbm_params["max_bin"])
<train_model>
%%time
feature_names = X_train.columns.tolist()
params = {
'objective': 'mse',
'metric': 'rmse',
'num_leaves':100,
'learning_rate': 0.0001,
'feature_fraction': 0.5,
'bagging_fraction': 0.5,
'bagging_freq': 5,
'n_estimators':1000,
'seed': 1,
'verbose': 1,
'force_row_wise' : True
}
categorical_feature_names = [
'item_category_id',
'hl_cat_id',
'city_enc',
'month',
'shop_group',
'shop_id'
]
lgb_train = lgb.Dataset(X_train[feature_names], y_train, categorical_feature=None)
 lgb_eval = lgb.Dataset(X_valid[feature_names], y_valid, categorical_feature=None, reference=lgb_train)
 evals_result = {}
gbm = lgb.train(
 params,
lgb_train,
num_boost_round = 10000,
valid_sets =(lgb_train, lgb_eval),
feature_name = feature_names,
categorical_feature = categorical_feature_names,
verbose_eval = 200,
evals_result = evals_result,
early_stopping_rounds = 1000 )
<filter><EOS> 
sample_submission['item_cnt_month'] = gbm.predict(X_test[feature_names] ).clip(0, 20)
 sample_submission.to_csv('submission_k4.csv' )