Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<prepare_x_and_y>
%matplotlib inline
x=train
y=np.array(target )<split>
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv' )
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0 )<normalization>
train = sales[(sales.item_price < 100000)&(sales.item_price > 0)]
train = train[sales.item_cnt_day < 1001]
scaler= RobustScaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test )<compute_train_metric>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 40, 'shop_id'] = 39
test.loc[test.shop_id == 40, 'shop_id'] = 39
lreg=LinearRegression()
MSEs=ms.cross_val_score(lreg, x, y, scoring='neg_mean_squared_error', cv=5)
 meanMSE=np.mean(MSEs)
 print(meanMSE)
 print('RMSE = '+str(math.sqrt(-meanMSE)) )<compute_test_metric>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 df = pd.concat([df, test], ignore_index=True, sort=False, keys=index_cols)
 df.fillna(0, inplace=True )
def score(y_pred):
return str(math.sqrt(sklm.mean_squared_error(y_test, y_pred)) )<train_on_grid>
shops['city'] = shops['shop_name'].apply(lambda x: x.split() [0].lower())
 shops.loc[shops.city == '!якутск', 'city'] = 'якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 coords = dict()
coords['якутск'] =(62.028098, 129.732555, 4)
 coords['адыгея'] =(44.609764, 40.100516, 3)
 coords['балашиха'] =(55.8094500, 37.9580600, 1)
 coords['волжский'] =(53.4305800, 50.1190000, 3)
 coords['вологда'] =(59.2239000, 39.8839800, 2)
 coords['воронеж'] =(51.6720400, 39.1843000, 3)
 coords['выездная'] =(0, 0, 0)
 coords['жуковский'] =(55.5952800, 38.1202800, 1)
 coords['интернет-магазин'] =(0, 0, 0)
 coords['казань'] =(55.7887400, 49.1221400, 4)
 coords['калуга'] =(54.5293000, 36.2754200, 4)
 coords['коломна'] =(55.0794400, 38.7783300, 4)
 coords['красноярск'] =(56.0183900, 92.8671700, 4)
 coords['курск'] =(51.7373300, 36.1873500, 3)
 coords['москва'] =(55.7522200, 37.6155600, 1)
 coords['мытищи'] =(55.9116300, 37.7307600, 1)
 coords['н.новгород'] =(56.3286700, 44.0020500, 4)
 coords['новосибирск'] =(55.0415000, 82.9346000, 4)
 coords['омск'] =(54.9924400, 73.3685900, 4)
 coords['ростовнадону'] =(47.2313500, 39.7232800, 3)
 coords['спб'] =(59.9386300, 30.3141300, 2)
 coords['самара'] =(53.2000700, 50.1500000, 4)
 coords['сергиев'] =(56.3000000, 38.1333300, 4)
 coords['сургут'] =(61.2500000, 73.4166700, 4)
 coords['томск'] =(56.4977100, 84.9743700, 4)
 coords['тюмень'] =(57.1522200, 65.5272200, 4)
 coords['уфа'] =(54.7430600, 55.9677900, 4)
 coords['химки'] =(55.8970400, 37.4296900, 1)
 coords['цифровой'] =(0, 0, 0)
 coords['чехов'] =(55.1477000, 37.4772800, 4)
 coords['ярославль'] =(57.6298700, 39.8736800, 2)
 shops['city_coord_1'] = shops['city'].apply(lambda x: coords[x][0])
 shops['city_coord_2'] = shops['city'].apply(lambda x: coords[x][1])
 shops['country_part'] = shops['city'].apply(lambda x: coords[x][2])
 shops = shops[['shop_id', 'city_code', 'city_coord_1', 'city_coord_2', 'country_part']]
ridge=Ridge()
parameters= {'alpha':[x for x in range(1,101)]}
ridge_reg=ms.GridSearchCV(ridge, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 ridge_reg.fit(x_train,y_train)
 print(f"ベストパラメータ : {ridge_reg.best_params_}")
 print(f"ベストスコア : {math.sqrt(-ridge_reg.best_score_)}")
 ridge_pred=math.sqrt(-ridge_reg.best_score_ )<compute_train_metric>
df = pd.merge(df, shops, on=['shop_id'], how='left' )
ridge_mod=Ridge(alpha=17)
 ridge_mod.fit(x_train,y_train)
 y_pred_train=ridge_mod.predict(x_train)
 y_pred_test=ridge_mod.predict(x_test)
 print(f'RMSE : train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_pred_train)))}')
 print(f'RMSE : test = {score(y_pred_test)}' )<train_on_grid>
map_dict = {
'Чистые носители(штучные)': 'Чистые носители',
'Чистые носители(шпиль)' : 'Чистые носители',
'PC ': 'Аксессуары',
'Служебные': 'Служебные '
}
items = pd.merge(items, item_cats, on='item_category_id')
 items['item_category'] = items['item_category_name'].apply(lambda x: x.split('-')[0])
 items['item_category'] = items['item_category'].apply(lambda x: map_dict[x] if x in map_dict.keys() else x)
 items['item_category_common'] = LabelEncoder().fit_transform(items['item_category'])
 items['item_category_code'] = LabelEncoder().fit_transform(items['item_category_name'])
 items = items[['item_id', 'item_category_common', 'item_category_code']]
parameters= {'alpha':[10, 1, 0.1, 0.01, 0.001, 0.0001, 0.0003, 0.0005, 0.0007, 0.00075]}
lasso=Lasso()
lasso_reg=ms.GridSearchCV(lasso, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 lasso_reg.fit(x_train,y_train)
 print(f'ベストパラメータ: {lasso_reg.best_params_}' )<compute_train_metric>
df = pd.merge(df, items, on=['item_id'], how='left' )
lasso_mod=Lasso(alpha=0.0007)
 lasso_mod.fit(x_train,y_train)
 y_lasso_train=lasso_mod.predict(x_train)
 y_lasso_test=lasso_mod.predict(x_test)
 print(f'RMSE : train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_lasso_train)))}')
 print(f'RMSE : test = {score(y_lasso_test)}' )<compute_train_metric>
def count_days(date_block_num):
year = 2013 + date_block_num // 12
month = 1 + date_block_num % 12
weeknd_count = len([1 for i in calendar.monthcalendar(year, month)if i[6] != 0])
 days_in_month = calendar.monthrange(year, month)[1]
return weeknd_count, days_in_month, month
map_dict = {i: count_days(i)for i in range(35)}
df['weeknd_count'] = df['date_block_num'].apply(lambda x: map_dict[x][0])
 df['days_in_month'] = df['date_block_num'].apply(lambda x: map_dict[x][1] )
alphas = [10, 1, 0.1, 0.01, 0.001, 0.0001, 0.0003, 0.0005, 0.0006, 0.00055]
l1ratio = [0.1, 0.3, 0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print('RMSE : test = '+ str(math.sqrt(sklm.mean_squared_error(y_test, ela_pred))))
print(elastic_cv.alpha_ )<find_best_params>
first_item_block = df.groupby(['item_id'])['date_block_num'].min().reset_index()
first_item_block['item_first_interaction'] = 1
first_shop_item_buy_block = df[df['date_block_num'] > 0].groupby(['shop_id', 'item_id'])['date_block_num'].min().reset_index()
first_shop_item_buy_block['first_date_block_num'] = first_shop_item_buy_block['date_block_num']
alphas = [0.00055]
l1ratio = [0.1, 0.3, 0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print(f'RMSE : test = {score(ela_pred)}')
 print(elastic_cv.alpha_)
 print(elastic_cv.l1_ratio_)
<train_model>
df = pd.merge(df, first_item_block[['item_id', 'date_block_num', 'item_first_interaction']], on=['item_id', 'date_block_num'], how='left')
 df = pd.merge(df, first_shop_item_buy_block[['item_id', 'shop_id', 'first_date_block_num']], on=['item_id', 'shop_id'], how='left')
 df['first_date_block_num'].fillna(100, inplace=True)
 df['shop_item_sold_before'] =(df['first_date_block_num'] < df['date_block_num'] ).astype('int8')
 df.drop(['first_date_block_num'], axis=1, inplace=True)
 df['item_first_interaction'].fillna(0, inplace=True)
 df['shop_item_sold_before'].fillna(0, inplace=True)
 df['item_first_interaction'] = df['item_first_interaction'].astype('int8')
 df['shop_item_sold_before'] = df['shop_item_sold_before'].astype('int8' )
def regularization(x,y,modelo=Ridge, scaler=RobustScaler):
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0)
 scaler= scaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test)
 if modelo != ElasticNetCV:
if modelo == Ridge:
parameters= {'alpha':[x for x in range(1,101)]}
elif modelo == Lasso:
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
model=modelo()
model=ms.GridSearchCV(model, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 model.fit(x_train,y_train)
 y_pred= model.predict(x_test)
 print("best_score : %s, RMSE = %s" %(model.best_params_,
score(y_pred)))
 elif modelo == ElasticNetCV:
alphas = [0.00055]
l1ratio = [0.1, 0.3, 0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print("best_score : alpha %s and l1_ratio %s, RMSE = %s"
%(elastic_cv.alpha_,elastic_cv.l1_ratio_, score(ela_pred)))
<compute_test_metric>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 df[col+'_lag_'+str(i)] = df[col+'_lag_'+str(i)].astype('float16')
 return df
regularization(x,y,Ridge )<compute_test_metric>
index_cols = ['shop_id', 'item_id', 'date_block_num']
group = train.groupby(index_cols)['item_price'].mean().reset_index().rename(columns={"item_price": "avg_shop_price"}, errors="raise")
 df = pd.merge(df, group, on=index_cols, how='left')
 df['avg_shop_price'] =(df['avg_shop_price'] 
.fillna(0)
.astype(np.float16))
index_cols = ['item_id', 'date_block_num']
group = train.groupby(['date_block_num','item_id'])['item_price'].mean().reset_index().rename(columns={"item_price": "avg_item_price"}, errors="raise")
 df = pd.merge(df, group, on=index_cols, how='left')
 df['avg_item_price'] =(df['avg_item_price'] 
.fillna(0)
.astype(np.float16))
df['item_shop_price_avg'] =(df['avg_shop_price'] - df['avg_item_price'])/ df['avg_item_price']
df['item_shop_price_avg'].fillna(0, inplace=True)
 df = lag_feature(df, [1, 2, 3], 'item_shop_price_avg')
 df.drop(['avg_shop_price', 'avg_item_price', 'item_shop_price_avg'], axis=1, inplace=True )
regularization(x,y, Lasso )<compute_test_metric>
item_id_target_mean = df.groupby(['date_block_num','item_id'])['item_cnt_month'].mean().reset_index().rename(columns={"item_cnt_month": "item_target_enc"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_id'], how='left')
 df['item_target_enc'] =(df['item_target_enc'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'item_target_enc')
 df.drop(['item_target_enc'], axis=1, inplace=True )
regularization(x,y, ElasticNetCV )<init_hyperparams>
item_id_target_mean = df.groupby(['date_block_num','item_id', 'city_code'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "item_loc_target_enc"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_id', 'city_code'], how='left')
 df['item_loc_target_enc'] =(df['item_loc_target_enc'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'item_loc_target_enc')
 df.drop(['item_loc_target_enc'], axis=1, inplace=True )
from xgboost.sklearn import XGBRegressor 
<choose_model_class>
item_id_target_mean = df.groupby(['date_block_num','item_id', 'shop_id'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "item_shop_target_enc"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_id', 'shop_id'], how='left')
 df['item_shop_target_enc'] =(df['item_shop_target_enc'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'item_shop_target_enc')
 df.drop(['item_shop_target_enc'], axis=1, inplace=True )
xgb= XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=3, min_child_weight=0, missing=None, n_estimators=4000,
n_jobs=1, nthread=None, objective='reg:squarederror', random_state=0,
reg_alpha=0.0001, reg_lambda=0.01, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 xgmod=xgb.fit(x_train,y_train)
 xg_pred=xgmod.predict(x_test)
 print(f'RMSE : test = {score(xg_pred)}' )<predict_on_test>
item_id_target_mean = df[df['item_first_interaction'] == 1].groupby(['date_block_num','item_category_code'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "new_item_cat_avg"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_category_code'], how='left')
 df['new_item_cat_avg'] =(df['new_item_cat_avg'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'new_item_cat_avg')
 df.drop(['new_item_cat_avg'], axis=1, inplace=True )
vote_mod = VotingRegressor([('Ridge', ridge_mod),('Lasso', lasso_mod),('Elastic', elastic_cv), 
('XGBRegressor', xgb)])
 vote= vote_mod.fit(x_train, y_train.ravel())
 vote_pred=vote.predict(x_test)
 print(f'RMSE : test = {score(vote_pred)}' )<train_model>
item_id_target_mean = df[df['item_first_interaction'] == 1].groupby(['date_block_num','item_category_code', 'shop_id'])['item_cnt_month'].mean().reset_index().rename(columns={
"item_cnt_month": "new_item_shop_cat_avg"}, errors="raise")
 df = pd.merge(df, item_id_target_mean, on=['date_block_num','item_category_code', 'shop_id'], how='left')
 df['new_item_shop_cat_avg'] =(df['new_item_shop_cat_avg'] 
.fillna(0)
.astype(np.float16))
df = lag_feature(df, [1, 2, 3], 'new_item_shop_cat_avg')
 df.drop(['new_item_shop_cat_avg'], axis=1, inplace=True )
stregr = StackingRegressor(regressors=[elastic_cv,ridge_mod, lasso_mod, vote_mod],
meta_regressor=xgb, use_features_in_secondary=True 
)
 stack_mod=stregr.fit(x_train, y_train.ravel())
 stacking_pred=stack_mod.predict(x_test)
 print(f'RMSE : test = {score(stacking_pred)}' )<compute_test_metric>
def lag_feature_adv(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)+'_adv']
shifted['date_block_num'] += i
shifted['item_id'] -= 1
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 df[col+'_lag_'+str(i)+'_adv'] = df[col+'_lag_'+str(i)+'_adv'].astype('float16')
 return df
df = lag_feature_adv(df, [1, 2, 3], 'item_cnt_month' )
final_test=(0.3*vote_pred + 0.5*stacking_pred + 0.2*y_lasso_test)
 print(f'RMSE : test= {score(final_test)}')
<predict_on_test>
df.fillna(0, inplace=True)
 df = df[(df['date_block_num'] > 2)]
df.head()
vote_test = vote_mod.predict(X_test)
 final1=np.expm1(vote_test)
 stack_test = stregr.predict(X_test)
 final2=np.expm1(stack_test)
 lasso_test = lasso_mod.predict(X_test)
 final3=np.expm1(lasso_test)
<save_to_csv>
df.drop(['ID'], axis=1, inplace=True, errors='ignore')
 df.to_pickle('df.pkl' )
final=(0.2*final1 + 0.6*final2 + 0.2*final3)
 submission = pd.DataFrame({
"Id": b["Id"],
"SalePrice": final
})
 submission.to_csv("./submission.csv", index=False)
 submission.head()<set_options>
df = pd.read_pickle('df.pkl')
 df.info()
%matplotlib inline
use_cuda = torch.cuda.is_available()
use_cuda<normalization>
X_train = df[df.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = df[df.date_block_num < 33]['item_cnt_month']
X_valid = df[df.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = df[df.date_block_num == 33]['item_cnt_month']
X_test = df[df.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del df
manualSeed = 999
def fixSeed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if use_cuda:
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 fixSeed(manualSeed )<define_variables>
feature_name = X_train.columns.tolist()
params = {
'objective': 'mse',
'metric': 'rmse',
'num_leaves': 2 ** 7 - 1,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'seed': 1,
'verbose': 1
}
feature_name_indexes = [
'country_part',
'item_category_common',
'item_category_code',
'city_code',
]
lgb_train = lgb.Dataset(X_train[feature_name], Y_train)
 lgb_eval = lgb.Dataset(X_valid[feature_name], Y_valid, reference=lgb_train)
 evals_result = {}
gbm = lgb.train(
 params,
lgb_train,
num_boost_round=3000,
valid_sets=(lgb_train, lgb_eval),
feature_name = feature_name,
categorical_feature = feature_name_indexes,
verbose_eval=5,
evals_result = evals_result,
early_stopping_rounds = 100)
<prepare_x_and_y><EOS> 
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 Y_test = gbm.predict(X_test[feature_name] ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('gbm_submission.csv', index=False )