Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<predict_on_test>
MODEL_VERSION = 'v15_param_tuning'
svml_pred = svml_best.predict(x_test )<compute_train_metric>
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 TRAIN_SPLIT = '2016-03-27'
EVAL_SPLIT = '2016-04-24'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'
random_state = 2
DT = DecisionTreeRegressor(random_state=random_state)
 MSLE = cross_val_score(DT,x_train, y = y_train, scoring = sklearn.metrics.make_scorer(mean_squared_log_error), cv = kfold , n_jobs =-1)
 mean_squared_log_err = np.mean(np.sqrt(MSLE))
print('mean squared log error is : ',mean_squared_log_err)
<compute_test_metric>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32'
}
DT.fit(x_train,y_train)
 math.sqrt(mean_squared_log_error(y_train,DT.predict(x_train)) )<predict_on_test>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_validation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
DT_pred = DT.predict(x_test )<train_on_grid>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['id', 'sales']].groupby('id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'rmean_{lag_col}_{window}'
strain[window_col] = strain[['id', lag_col]].groupby('id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales for done for window {window}')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday"
}
for date_feat_name, date_feat_func in date_features.items() :
if not date_feat_name in dt.columns:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
RFC = RandomForestRegressor(random_state = random_state)
 rf_param_grid = {"max_depth": [2],
"min_samples_split": [5],
"min_samples_leaf": [2],
"bootstrap": [True],
"n_estimators" :[200],
}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring= sklearn.metrics.make_scorer(mean_squared_log_error), n_jobs= -1, verbose = 1)
 gsRFC.fit(x_train,y_train)
 print(math.sqrt(gsRFC.best_score_))
print(gsRFC.best_params_ )<find_best_params>
%%time
sales_train_validation = load_data()
RFC_best = gsRFC.best_estimator_<predict_on_test>
sales_train_validation["sale"] =(( sales_train_validation['sell_price'] * 100 % 10)< 6 ).astype('int8' )
rfc_pred = RFC_best.predict(x_test )<train_on_grid>
%%time
make_lag_features(sales_train_validation)
 make_date_features(sales_train_validation )
GBC = GradientBoostingRegressor(random_state = random_state)
 gb_param_grid = {}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring= sklearn.metrics.make_scorer(mean_squared_log_error), n_jobs= -1, verbose = 1)
 gsGBC.fit(x_train,y_train)
 print(math.sqrt(gsGBC.best_score_))<find_best_params>
total = len(sales_train_validation['event_type_2'])
 zero = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 0])
 one = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 1])
 two = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 2])
 print('0: '+ str(zero))
print('1: ' + str(one))
print('2: ' + str(two))
print('-'*7 + '
Total: ' + str(total))
del total; del zero; del one; del two
GBC_best = gsGBC.best_estimator_<predict_on_test>
sales_train_validation.isnull().sum()
gbc_pred = GBC_best.predict(x_test )<train_on_grid>
before = len(sales_train_validation)
 sales_train_validation.dropna(inplace = True)
 after = len(sales_train_validation)
 print(f"Reduced {(before-after)/before}%")
DT1 = DecisionTreeRegressor(random_state=random_state)
 adaDTC = AdaBoostRegressor(DT1, random_state=random_state)
 ada_param_grid = {"n_estimators":[100,500],
"learning_rate": [ 0.01]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring= sklearn.metrics.make_scorer(mean_squared_log_error), n_jobs= -1, verbose = 1)
 gsadaDTC.fit(x_train,y_train)
<find_best_params>
print("Splitting data into train, validation, evaluation set")
Ada_best = gsadaDTC.best_estimator_<compute_train_metric>
%%time
np.random.seed(777)
 validation_idx = np.random.choice(sales_train_validation.index.values, 2_000_000, replace = False)
 train_idx = np.setdiff1d(sales_train_validation.index.values, validation_idx)
 train = sales_train_validation.loc[train_idx]
validation = sales_train_validation.loc[validation_idx]
evaluation = sales_train_validation[(sales_train_validation['date'] > EVAL_SPLIT)]
print(Ada_best.get_params())
 math.sqrt(cross_val_score(Ada_best, x_train, y = y_train, scoring = sklearn.metrics.make_scorer(mean_squared_log_error), cv = kfold , n_jobs =-1 ).mean() )<predict_on_test>
print("Train:", train.shape)
 print("Validation:", validation.shape)
 print("Evaluation:", evaluation.shape )
ada_pred = Ada_best.predict(x_test )<choose_model_class>
categorical_features = [
'item_id', 'dept_id', 'store_id', 'cat_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]
xgb =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2500,
n_jobs=1, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
<compute_train_metric>
train_pool = lgb.Dataset(
 data=train[train_columns],
label=train["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 del train; gc.collect()
MSLE = cross_val_score(xgb,x_train, y = y_train, scoring = sklearn.metrics.make_scorer(mean_squared_log_error), cv = kfold , n_jobs =-1)
 mean_squared_log_err = np.mean(np.sqrt(MSLE))
print('mean squared log error is : ',mean_squared_log_err )<predict_on_test>
val_pool = lgb.Dataset(
 data=validation[train_columns],
label=validation["sales"],
categorical_feature=categorical_features,
free_raw_data=False 
)
 del validation; del evaluation; gc.collect()
xgb.fit(x_train,y_train)
 xgb_pred = xgb.predict(x_test )<choose_model_class>
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 2000
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
'early_stopping': 10
}
lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=11000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4, 
)
<compute_train_metric>
%%time
if TASK_TYPE=='GPU':
model = "todo"
else:
model = lgb.train(
 params,
train_pool,
valid_sets = val_pool,
verbose_eval=20 )
MSLE = cross_val_score(lgbm,x_train, y = y_train, scoring = sklearn.metrics.make_scorer(mean_squared_log_error), cv = kfold , n_jobs =-1)
 mean_squared_log_err = np.mean(np.sqrt(MSLE))
print('mean squared log error is : ',mean_squared_log_err )<train_model>
model.save_model("model_{}.lgb".format(MODEL_VERSION))
lgbm.fit(x_train,y_train,eval_metric= sklearn.metrics.make_scorer(mean_squared_log_error))
lgbm_pred = lgbm.predict(x_test )<choose_model_class>
del train_pool, val_pool; gc.collect()
models = [ linear_reg,
Ridge(alpha = 1),
Lasso(alpha = 0.1),
ElasticNet(alpha = 0.5, l1_ratio = 1e-5),
SVR(kernel = 'rbf', C= 500, gamma = 500),
SVR(kernel = 'linear',C = 10, gamma = 0.001),
DT,
RandomForestRegressor(random_state=2, n_jobs=-1,
min_samples_split= 5,
min_samples_leaf = 2,
bootstrap = True,
n_estimators=120,
max_features = 200,
max_depth= 2),
GradientBoostingRegressor(random_state = random_state),
AdaBoostRegressor(DT1, random_state=random_state, learning_rate = 0.01),
xgb,
lgbm
]<compute_train_metric>
df = load_data(train=False)
 make_date_features(df )
meta_model1 = LinearRegression()
meta_model1.fit(S_train, y_train)
 y_pred1 = meta_model1.predict(S_train)
 stack_pred1 = meta_model1.predict(S_test)
 print('Final train prediction score: [%.8f]' % math.sqrt(mean_squared_log_error(y_train, y_pred1)) )<define_variables>
df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8' )
weighted_pred =(0.45*xgb_pred + 0.55*lgbm_pred )<choose_model_class>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'id'])
 dt.loc[dt['date'] == str(day),f'rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print("Lag features done")
 print(type(dt))
models1 = [
GradientBoostingRegressor(random_state = random_state),
xgb,
lgbm
]<concatenate>
%%time
END_DATE = EVAL_SPLIT
ZERO_THRESHOLD = 0.01
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 prediction_data = prediction_data.loc[prediction_data['date'] == str(pred_date), train_columns]
prediction = model.predict(prediction_data)
 print("Prediction", prediction.size, prediction)
 df.loc[df['date'] == str(pred_date), 'sales'] = prediction
S1_train, S1_test = stacking(models1,
x_train, y_train, x_test,
regression= True,
mode='oof_pred_bag',
needs_proba=False,
save_dir=None,
metric= mean_squared_error,
n_folds=5,
stratified=False,
shuffle=True,
random_state=0,
verbose=2 )<compute_train_metric>
del prediction_data
gc.collect()
meta_model2 = LinearRegression()
meta_model2.fit(S1_train, y_train)
 y_pred2 = meta_model2.predict(S1_train)
 stack_pred2 = meta_model2.predict(S1_test)
 print('Final train prediction score: [%.8f]' % math.sqrt(mean_squared_log_error(y_train, y_pred2)) )<compute_train_metric>
submission_val = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_val.loc[submission_val['sales'] < 0, 'sales'] = 0
submission_val.sort_values('id', inplace=True)
 submission_val['day'] = submission_val['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_val.columns )
MSLE = cross_val_score(meta_model2,S1_train, y = y_train, scoring = sklearn.metrics.make_scorer(mean_squared_log_error), cv = kfold , n_jobs =-1)
 mean_squared_log_err = np.mean(np.sqrt(MSLE))
print('mean squared log error is : ',mean_squared_log_err )<save_to_csv>
submission_eval = submission_val.copy()
submission_eval['id'] = submission_eval['id'].str.replace('validation', 'evaluation')
 submission = pd.concat([submission_val, submission_eval], axis=0, sort=False)
 print(submission.columns)
 print(submission.head(1))
sample_submission['SalePrice'] =(0.55*stack_pred2 + 0.45*weighted_pred)
 sample_submission.to_csv('stacking.csv',index = False )<set_options>
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )