Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<predict_on_test>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100 )
train.loc[:, 'pickup_cluster'] = kmeans.predict(train[['pickup_latitude', 'pickup_longitude']])
 train.loc[:, 'dropoff_cluster'] = kmeans.predict(train[['dropoff_latitude', 'dropoff_longitude']])
 test.loc[:, 'pickup_cluster'] = kmeans.predict(test[['pickup_latitude', 'pickup_longitude']])
 test.loc[:, 'dropoff_cluster'] = kmeans.predict(test[['dropoff_latitude', 'dropoff_longitude']])
 t1 = dt.datetime.now()
print('Time till clustering: %i seconds' %(t1 - t0 ).seconds )<merge>
sales = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 sales.name = 'sales'
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar.name = 'calendar'
prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 prices.name = 'prices'
for gby_col in ['pickup_hour', 'pickup_date', 'pickup_dt_bin',
'pickup_week_hour', 'pickup_cluster', 'dropoff_cluster']:
gby = train.groupby(gby_col ).mean() [['avg_speed_h', 'avg_speed_m', 'log_trip_duration']]
gby.columns = ['%s_gby_%s' %(col, gby_col)for col in gby.columns]
train = pd.merge(train, gby, how='left', left_on=gby_col, right_index=True)
 test = pd.merge(test, gby, how='left', left_on=gby_col, right_index=True)
 for gby_cols in [['center_lat_bin', 'center_long_bin'],
['pickup_hour', 'center_lat_bin', 'center_long_bin'],
['pickup_hour', 'pickup_cluster'], ['pickup_hour', 'dropoff_cluster'],
['pickup_cluster', 'dropoff_cluster']]:
coord_speed = train.groupby(gby_cols ).mean() [['avg_speed_h']].reset_index()
coord_count = train.groupby(gby_cols ).count() [['id']].reset_index()
coord_stats = pd.merge(coord_speed, coord_count, on=gby_cols)
 coord_stats = coord_stats[coord_stats['id'] > 100]
coord_stats.columns = gby_cols + ['avg_speed_h_%s' % '_'.join(gby_cols), 'cnt_%s' % '_'.join(gby_cols)]
train = pd.merge(train, coord_stats, how='left', on=gby_cols)
 test = pd.merge(test, coord_stats, how='left', on=gby_cols )<merge>
for d in range(1942,1970):
col = 'd_' + str(d)
 sales[col] = 0
sales[col] = sales[col].astype(np.int16 )
group_freq = '60min'
df_all = pd.concat(( train, test)) [['id', 'pickup_datetime', 'pickup_cluster', 'dropoff_cluster']]
train.loc[:, 'pickup_datetime_group'] = train['pickup_datetime'].dt.round(group_freq)
 test.loc[:, 'pickup_datetime_group'] = test['pickup_datetime'].dt.round(group_freq)
 df_counts = df_all.set_index('pickup_datetime')[['id']].sort_index()
df_counts['count_60min'] = df_counts.isnull().rolling(group_freq ).count() ['id']
train = train.merge(df_counts, on='id', how='left')
 test = test.merge(df_counts, on='id', how='left')
 dropoff_counts = df_all \ 
.set_index('pickup_datetime')\ 
.groupby([pd.TimeGrouper(group_freq), 'dropoff_cluster'])\ 
.agg({'id': 'count'})\ 
.reset_index().set_index('pickup_datetime')\ 
.groupby('dropoff_cluster' ).rolling('240min' ).mean() \ 
.drop('dropoff_cluster', axis=1)\ 
.reset_index().set_index('pickup_datetime' ).shift(freq='-120min' ).reset_index() \ 
.rename(columns={'pickup_datetime': 'pickup_datetime_group', 'id': 'dropoff_cluster_count'})
 train['dropoff_cluster_count'] = train[['pickup_datetime_group', 'dropoff_cluster']].merge(dropoff_counts, on=['pickup_datetime_group', 'dropoff_cluster'], how='left')['dropoff_cluster_count'].fillna(0)
 test['dropoff_cluster_count'] = test[['pickup_datetime_group', 'dropoff_cluster']].merge(dropoff_counts, on=['pickup_datetime_group', 'dropoff_cluster'], how='left')['dropoff_cluster_count'].fillna(0 )<load_from_csv>
def downcast(df):
cols = df.dtypes.index.tolist()
types = df.dtypes.values.tolist()
for i,t in enumerate(types):
if 'int' in str(t):
if df[cols[i]].min() > np.iinfo(np.int8 ).min and df[cols[i]].max() < np.iinfo(np.int8 ).max:
df[cols[i]] = df[cols[i]].astype(np.int8)
 elif df[cols[i]].min() > np.iinfo(np.int16 ).min and df[cols[i]].max() < np.iinfo(np.int16 ).max:
df[cols[i]] = df[cols[i]].astype(np.int16)
 elif df[cols[i]].min() > np.iinfo(np.int32 ).min and df[cols[i]].max() < np.iinfo(np.int32 ).max:
df[cols[i]] = df[cols[i]].astype(np.int32)
 else:
df[cols[i]] = df[cols[i]].astype(np.int64)
 elif 'float' in str(t):
if df[cols[i]].min() > np.finfo(np.float16 ).min and df[cols[i]].max() < np.finfo(np.float16 ).max:
df[cols[i]] = df[cols[i]].astype(np.float16)
 elif df[cols[i]].min() > np.finfo(np.float32 ).min and df[cols[i]].max() < np.finfo(np.float32 ).max:
df[cols[i]] = df[cols[i]].astype(np.float32)
 else:
df[cols[i]] = df[cols[i]].astype(np.float64)
 elif t == np.object:
if cols[i] == 'date':
df[cols[i]] = pd.to_datetime(df[cols[i]], format='%Y-%m-%d')
 else:
df[cols[i]] = df[cols[i]].astype('category')
 return df
sales = downcast(sales)
 prices = downcast(prices)
 calendar = downcast(calendar )
<feature_engineering>
df = pd.melt(sales, id_vars=['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name='d', value_name='sold' ).dropna()
df = pd.merge(df, calendar, on='d', how='left')
 df = pd.merge(df, prices, on=['store_id','item_id','wm_yr_wk'], how='left' )
feature_stats = pd.DataFrame({'feature': feature_names})
 feature_stats.loc[:, 'train_mean'] = np.nanmean(train[feature_names].values, axis=0 ).round(4)
 feature_stats.loc[:, 'test_mean'] = np.nanmean(test[feature_names].values, axis=0 ).round(4)
 feature_stats.loc[:, 'train_std'] = np.nanstd(train[feature_names].values, axis=0 ).round(4)
 feature_stats.loc[:, 'test_std'] = np.nanstd(test[feature_names].values, axis=0 ).round(4)
 feature_stats.loc[:, 'train_nan'] = np.mean(np.isnan(train[feature_names].values), axis=0 ).round(3)
 feature_stats.loc[:, 'test_nan'] = np.mean(np.isnan(test[feature_names].values), axis=0 ).round(3)
 feature_stats.loc[:, 'train_test_mean_diff'] = np.abs(feature_stats['train_mean'] - feature_stats['test_mean'])/ np.abs(feature_stats['train_std'] + feature_stats['test_std'])* 2
feature_stats.loc[:, 'train_test_nan_diff'] = np.abs(feature_stats['train_nan'] - feature_stats['test_nan'])
 feature_stats = feature_stats.sort_values(by='train_test_mean_diff')
 feature_stats[['feature', 'train_test_mean_diff']].tail()<sort_values>
d_id = dict(zip(df.id.cat.codes, df.id))
d_item_id = dict(zip(df.item_id.cat.codes, df.item_id))
d_dept_id = dict(zip(df.dept_id.cat.codes, df.dept_id))
d_cat_id = dict(zip(df.cat_id.cat.codes, df.cat_id))
d_store_id = dict(zip(df.store_id.cat.codes, df.store_id))
d_state_id = dict(zip(df.state_id.cat.codes, df.state_id))
feature_stats = feature_stats.sort_values(by='train_test_nan_diff')
 feature_stats[['feature', 'train_nan', 'test_nan', 'train_test_nan_diff']].tail()<split>
list1=['event_name_1','event_type_1','event_name_2','event_type_2']
for i in list1:
df[i] = df[i].cat.add_categories("nan" ).fillna("nan")
 df[i]=LabelEncoder().fit_transform(df[i] ).astype(np.int8)
 df[i]=df[i].astype('category')
 df.d = df['d'].apply(lambda x: x.split('_')[1] ).astype(np.int16)
 cols = df.dtypes.index.tolist()
types = df.dtypes.values.tolist()
for i,type in enumerate(types):
if type.name == 'category':
df[cols[i]] = df[cols[i]].cat.codes
df['date'] = df['date'].apply(lambda x: x.strftime('%d')).astype(np.int8 )
Xtr, Xv, ytr, yv = train_test_split(train[feature_names].values, y, test_size=0.2, random_state=1987)
 dtrain = xgb.DMatrix(Xtr, label=ytr)
 dvalid = xgb.DMatrix(Xv, label=yv)
 dtest = xgb.DMatrix(test[feature_names].values)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
xgb_pars = {'min_child_weight': 50, 'eta': 0.3, 'colsample_bytree': 0.3, 'max_depth': 10,
'subsample': 0.8, 'lambda': 1., 'nthread': 4, 'booster' : 'gbtree', 'silent': 1,
'eval_metric': 'rmse', 'objective': 'reg:linear'}<train_model>
lags = [28,35,42,49]
for lag in lags:
df['sold_lag_'+str(lag)] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['sold'].shift(lag ).astype(np.float16)
 lags2 = [1,2]
for lag in lags2:
df['event1_lag_'+str(lag)] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['event_name_1'].shift(lag ).astype(np.float16)
 df['event1_lag_'+str(lag)].fillna(100, inplace=True)
 df['event1_lag_'+str(lag)]=df['event1_lag_'+str(lag)].astype(np.int8)
 df['event1_lag_'+str(lag)]=df['event1_lag_'+str(lag)].astype('category' )
model = xgb.train(xgb_pars, dtrain, 60, watchlist, early_stopping_rounds=50,
maximize=False, verbose_eval=10 )<train_model>
df['item_sold_avg'] = df.groupby('item_id')['sold'].transform('mean' ).astype(np.float16)
 df['cat_sold_avg'] = df.groupby('cat_id')['sold'].transform('mean' ).astype(np.float16)
 df['dept_sold_avg'] = df.groupby('dept_id')['sold'].transform('mean' ).astype(np.float16)
 df['cat_dept_sold_avg'] = df.groupby(['cat_id','dept_id'])['sold'].transform('mean' ).astype(np.float16)
 df['store_item_sold_avg'] = df.groupby(['store_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['cat_item_sold_avg'] = df.groupby(['cat_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['dept_item_sold_avg'] = df.groupby(['dept_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['store_cat_dept_sold_avg'] = df.groupby(['store_id','cat_id','dept_id'])['sold'].transform('mean' ).astype(np.float16)
 df['wm_yr_wk_linear']=LabelEncoder().fit_transform(df['wm_yr_wk'].values ).astype(np.int16)
 df.drop(['wm_yr_wk'], axis=1, inplace=True)
 df['price_lag'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['sell_price'].shift(7 ).astype(np.float16)
 df['price-diff']=df['price_lag']-df['sell_price']
df.drop(['price_lag'], axis=1, inplace=True)
 df['sell_price'].fillna(-1,inplace=True)
 df['decimal']=df['sell_price'].apply(lambda x: 100*(x-int(x)) ).astype(np.int16)
 df['sell_price'].replace(-1,np.nan,inplace=True)
 df['expanding_price_mean'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sell_price'].transform(lambda x: x.expanding(2 ).mean() ).astype(np.float16)
 df['diff_moving_mean']=df['expanding_price_mean']-df['sell_price']
df.drop(['expanding_price_mean'], axis=1, inplace=True)
 df['price-diff']=df['price-diff'].astype(np.float16)
 df.drop(['wday'], axis=1, inplace=True)
 df['decimal']=df['decimal'].astype(np.int8)
 df['year']=LabelEncoder().fit_transform(df['year'] ).astype(np.int8)
 df['daily_avg_sold'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id','d'])['sell_price'].transform('mean' ).astype(np.float16)
 df['avg_sold'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sell_price'].transform('mean' ).astype(np.float16)
 df['selling_trend'] =(df['daily_avg_sold'] - df['avg_sold'] ).astype(np.float16)
 df.drop(['daily_avg_sold','avg_sold'],axis=1,inplace=True)
 df['price_max'] = df.groupby(['store_id','item_id'])['sell_price'].transform('max')
 df['price_norm'] = df['sell_price']/df['price_max']
df['price_momentum_m'] = df['sell_price']/df.groupby(['store_id','item_id','month'])['sell_price'].transform('mean')
 df['price_momentum_y'] = df['sell_price']/df.groupby(['store_id','item_id','year'])['sell_price'].transform('mean')
print('Modeling RMSLE %.5f' % model.best_score)
 t1 = dt.datetime.now()
print('Training time: %i seconds' %(t1 - t0 ).seconds )<save_to_csv>
list3=['cat_id','state_id']
for i in list3:
df.drop([i], axis=1, inplace=True )
ytest = model.predict(dtest)
 print('Test shape OK.')if test.shape[0] == ytest.shape[0] else print('Oops')
 test['trip_duration'] = np.exp(ytest)- 1
test[['id', 'trip_duration']].to_csv('mrig_xgb_submission.csv', index=False)
 print('Valid prediction mean: %.3f' % ypred.mean())
 print('Test prediction mean: %.3f' % ytest.mean())
 fig, ax = plt.subplots(nrows=2, sharex=True, sharey=True)
 sns.distplot(ypred, ax=ax[0], color='blue', label='validation prediction')
 sns.distplot(ytest, ax=ax[1], color='green', label='test prediction')
 ax[0].legend(loc=0)
 ax[1].legend(loc=0)
 plt.show()
t1 = dt.datetime.now()
print('Total time: %i seconds' %(t1 - t0 ).seconds )<define_search_space>
df = df[df['d']>=49]
df.to_pickle('data.pkl')
 del df, sales, prices, calendar
gc.collect()
FOREVER_COMPUTING_FLAG = False
xgb_pars = []
for MCW in [10, 20, 50, 75, 100]:
for ETA in [0.05, 0.1, 0.15]:
for CS in [0.3, 0.4, 0.5]:
for MD in [6, 8, 10, 12, 15]:
for SS in [0.5, 0.6, 0.7, 0.8, 0.9]:
for LAMBDA in [0.5, 1., 1.5, 2., 3.]:
xgb_pars.append({'min_child_weight': MCW, 'eta': ETA,
'colsample_bytree': CS, 'max_depth': MD,
'subsample': SS, 'lambda': LAMBDA,
'nthread': -1, 'booster' : 'gbtree', 'eval_metric': 'rmse',
'silent': 1, 'objective': 'reg:linear'})
 while FOREVER_COMPUTING_FLAG:
xgb_par = np.random.choice(xgb_pars, 1)[0]
print(xgb_par)
 model = xgb.train(xgb_par, dtrain, 2000, watchlist, early_stopping_rounds=50,
maximize=False, verbose_eval=100)
 print('Modeling RMSLE %.5f' % model.best_score )<set_options>
data = pd.read_pickle('data.pkl')
 valid_csv=data[(data['d']>=1914)&(data['d']<1942)][['id','d','sold']]
test = data[data['d']>=1942][['id','d','sold']]
eval_preds = test['sold']
valid_preds_csv=valid_csv['sold']
%matplotlib inline<load_from_csv>
cat_column=[]
for i in data.columns:
if(str(data.dtypes[i])=='category'):
cat_column.append(i )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<count_missing_values>
for store in d_store_id:
df = data[data['store_id']==store]
X_train, y_train = df[df['d']<1914].drop('sold',axis=1), df[df['d']<1914]['sold']
X_valid_csv, y_valid_csv = df[(df['d']>=1914)&(df['d']<1942)].drop('sold',axis=1), df[(df['d']>=1914)&(df['d']<1942)]['sold']
X_test = df[df['d']>=1942].drop('sold',axis=1)
 model = LGBMRegressor(
 learning_rate= 0.05,
subsample=0.6,
feature_fraction=0.6,
num_iterations = 1200,
max_bin=350,
num_leaves= 100,
lambda_l2=0.003,
max_depth=200,
min_data_in_leaf= 80,
force_row_wise= True, 
)
 print('*****Prediction for Store: {}*****'.format(d_store_id[store]))
model.fit(X_train, y_train, eval_set=[(X_train,y_train),(X_valid_csv,y_valid_csv)],
eval_metric='rmse', verbose=100, early_stopping_rounds=20,categorical_feature=cat_column)
 valid_preds_csv[X_valid_csv.index] = model.predict(X_valid_csv)
 eval_preds[X_test.index] = model.predict(X_test)
 filename = 'model'+str(d_store_id[store])+'.pkl'
joblib.dump(model, filename)
 del model, X_train, y_train, X_valid_csv, y_valid_csv
gc.collect()
train.isnull().sum()<count_missing_values>
feature_importance_df = pd.DataFrame()
features = [f for f in data.columns if f != 'sold']
for filename in os.listdir('/kaggle/working/'):
if 'model' in filename:
model = joblib.load(filename)
 store_importance_df = pd.DataFrame()
store_importance_df["feature"] = features
store_importance_df["importance"] = model.feature_importances_
store_importance_df["store"] = filename[5:9]
feature_importance_df = pd.concat([feature_importance_df, store_importance_df], axis=0)
 def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(averaged over store predictions)')
 plt.tight_layout()
display_importances(feature_importance_df )
<count_missing_values><EOS> 
valid_csv['sold'] = valid_preds_csv
validation = valid_csv[['id','d','sold']]
validation = pd.pivot(validation, index='id', columns='d', values='sold' ).reset_index()
validation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
validation.id = validation.id.map(d_id ).str.replace('evaluation','validation')
 test['sold'] = eval_preds
evaluation = test[['id','d','sold']]
evaluation = pd.pivot(evaluation, index='id', columns='d', values='sold' ).reset_index()
evaluation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation.id = evaluation.id.map(d_id)
 submit = pd.concat([validation,evaluation] ).reset_index(drop=True)
 for i in range(1,29):
submit['F'+str(i)] *= 1.04
submit.to_csv('submission.csv',index=False )