Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<feature_engineering>
import os
import gc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta, date
from tqdm.notebook import tqdm
import lightgbm as lgb
df_train_add = df_train.copy()
df_train_add['TotalSF']=df_train_add['TotalBsmtSF'] + df_train_add['1stFlrSF'] + df_train_add['2ndFlrSF']
df_train_add['Total_Bathrooms'] =(df_train_add['FullBath'] +(0.5 * df_train_add['HalfBath'])+
df_train_add['BsmtFullBath'] +(0.5 * df_train_add['BsmtHalfBath']))
df_train_add['Total_porch_sf'] =(df_train_add['OpenPorchSF'] + df_train_add['3SsnPorch'] +
df_train_add['EnclosedPorch'] + df_train_add['ScreenPorch'] +
df_train_add['WoodDeckSF'])
 df_test_add = df_test.copy()
df_test_add['TotalSF']=df_test_add['TotalBsmtSF'] + df_test_add['1stFlrSF'] + df_test_add['2ndFlrSF']
df_test_add['Total_Bathrooms'] =(df_test_add['FullBath'] +(0.5 * df_test_add['HalfBath'])+
df_test_add['BsmtFullBath'] +(0.5 * df_test_add['BsmtHalfBath']))
df_test_add['Total_porch_sf'] =(df_test_add['OpenPorchSF'] + df_test_add['3SsnPorch'] +
df_test_add['EnclosedPorch'] + df_test_add['ScreenPorch'] +
df_test_add['WoodDeckSF'] )<feature_engineering>
TEST_RUN = False
MODEL_VERSION = 'final'
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
SUBMISSION_PATH = "/kaggle/input/lgbmindividualbestsubmission/submission.csv"
MODEL_PATH = "/kaggle/input/lgbmindividualbestsubmission/model_v13_param_tuning.lgb"
BACKWARD_LAG = 60
END_DAY = 1913 + 28
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 EVAL_SPLIT = '2016-05-22'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'
df_train_add['haspool'] = df_train_add['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['has2ndfloor'] = df_train_add['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['hasgarage'] = df_train_add['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['hasbsmt'] = df_train_add['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_train_add['hasfireplace'] = df_train_add['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['haspool'] = df_test_add['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['has2ndfloor'] = df_test_add['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['hasgarage'] = df_test_add['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['hasbsmt'] = df_test_add['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_test_add['hasfireplace'] = df_test_add['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<filter>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32',
'sales': 'float32'
}
df_train[df_train["SalePrice"] > 600000 ]<define_variables>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
categorical_values_train = list(cat_train.index)
 categorical_values_test = list(cat_test.index)
 print(categorical_values_train )<categorify>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['item_id', 'sales']].groupby('item_id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'item_id_rmean_{lag_col}_{window}'
strain[window_col] = strain[['item_id', lag_col]].groupby('item_id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales done for window {window} per item_id')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday",
"month": "month",
"year": "year"
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
df_train_add = df_train.copy()
df_test_add = df_test.copy()
lb_make = LabelEncoder()
for i in categorical_values_train:
df_train_add[i] = lb_make.fit_transform(df_train[i])
 for i in categorical_values_test:
df_test_add[i] = lb_make.fit_transform(df_test[i] )<feature_engineering>
%%time
sales_train_evaluation = load_data()
for i in categorical_values_train:
feature_set = set(df_train[i])
 for j in feature_set:
feature_list = list(feature_set)
 df_train.loc[df_train[i] == j, i] = feature_list.index(j)
 df_train_add.loc[df_train[i] == j, i] = feature_list.index(j)
 for i in categorical_values_test:
feature_set2 = set(df_test[i])
 for j in feature_set2:
feature_list2 = list(feature_set2)
 df_test.loc[df_test[i] == j, i] = feature_list2.index(j)
 df_test_add.loc[df_test[i] == j, i] = feature_list2.index(j )<import_modules>
%%time
sales_train_evaluation["sale"] =(( sales_train_evaluation['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_lag_features(sales_train_evaluation)
 make_date_features(sales_train_evaluation )
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score, mean_squared_error
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score, KFold<prepare_x_and_y>
before = len(sales_train_evaluation)
 sales_train_evaluation.dropna(inplace = True)
 after = len(sales_train_evaluation)
 print(f"Reduced {(before-after)/before}%" )
X_train = df_train_add.drop(["SalePrice","LogPrice"], axis=1)
 y_train = df_train_add["LogPrice"]<split>
%%time
print("Splitting data into train, validation, evaluation set")
 np.random.seed(777)
 SIZE = 2_000_000
validation_idx = np.random.choice(sales_train_evaluation.index.values, SIZE, replace = False)
 train_idx = np.setdiff1d(sales_train_evaluation.index.values, validation_idx)
 train = sales_train_evaluation.loc[train_idx]
validation = sales_train_evaluation.loc[validation_idx]
del sales_train_evaluation; gc.collect()
print("Train:", train.shape)
 print("Validation:", validation.shape )
X_training, X_valid, y_training, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0 )<train_model>
categorical_features = [
'item_id', 'dept_id', 'store_id', 'cat_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]
lm = LinearRegression()
lm.fit(X_training,y_training)
 print(lm )<predict_on_test>
train_pool = lgb.Dataset(
 data=train[train_columns],
label=train["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 del train; gc.collect()
predictions = lm.predict(X_valid)
 predictions= predictions.reshape(-1,1 )<prepare_output>
val_pool = lgb.Dataset(
 data=validation[train_columns],
label=validation["sales"],
categorical_feature=categorical_features,
free_raw_data=False 
)
 del validation; gc.collect()
submission_predictions = np.exp(predictions )<compute_test_metric>
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 1200
params = {
"objective" : "poisson",
"metric" : "rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
'verbosity': 1,
'num_iterations': ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
"early_stopping": 10
}
print('MAE:', metrics.mean_absolute_error(y_valid, submission_predictions))
print('MSE:', metrics.mean_squared_error(y_valid, submission_predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, submission_predictions)) )<train_on_grid>
%%time
model = lgb.train(
 params,
train_pool,
valid_sets = val_pool,
verbose_eval=20 )
linreg = LinearRegression()
parameters_lin = {"fit_intercept" : [True, False], "normalize" : [True, False], "copy_X" : [True, False]}
grid_linreg = GridSearchCV(linreg, parameters_lin, verbose=1 , scoring = "r2")
 grid_linreg.fit(X_training, y_training)
 print("Best LinReg Model: " + str(grid_linreg.best_estimator_))
print("Best Score: " + str(grid_linreg.best_score_))<compute_test_metric>
model.save_model("model_{}.lgb".format(MODEL_VERSION))
del train_pool, val_pool; gc.collect()
linreg = grid_linreg.best_estimator_
linreg.fit(X_training, y_training)
 lin_pred = linreg.predict(X_valid)
 r2_lin = r2_score(y_valid, lin_pred)
 rmse_lin = np.sqrt(mean_squared_error(y_valid, lin_pred))
print("R^2 Score: " + str(r2_lin))
print("RMSE Score: " + str(rmse_lin))<compute_test_metric>
df = load_data(train=False)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_date_features(df )
scores_lin = cross_val_score(linreg, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_lin)) )<train_on_grid>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("item_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'item_id'])
 dt.loc[dt['date'] == str(day),f'item_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
lag = 28
lag_col = 'lag_28'
window = 28
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("store_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'store_id'])
 dt.loc[dt['date'] == str(day),f'store_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print(f"Features done for day {str(day)}")
ridge = Ridge()
parameters_ridge = {"fit_intercept" : [True, False], "normalize" : [True, False], "copy_X" : [True, False], "solver" : ["auto"]}
grid_ridge = GridSearchCV(ridge, parameters_ridge, verbose=1, scoring="r2")
 grid_ridge.fit(X_training, y_training)
 print("Best Ridge Model: " + str(grid_ridge.best_estimator_))
print("Best Score: " + str(grid_ridge.best_score_))<compute_test_metric>
%%time
END_DATE = EVAL_SPLIT
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 prediction_data = prediction_data.loc[prediction_data['date'] == str(pred_date), train_columns]
prediction = model.predict(prediction_data)
 print("Prediction", prediction.size, prediction)
 df.loc[df['date'] == str(pred_date), 'sales'] = prediction
ridge = grid_ridge.best_estimator_
ridge.fit(X_training, y_training)
 ridge_pred = ridge.predict(X_valid)
 r2_ridge = r2_score(y_valid, ridge_pred)
 rmse_ridge = np.sqrt(mean_squared_error(y_valid, ridge_pred))
print("R^2 Score: " + str(r2_ridge))
print("RMSE Score: " + str(rmse_ridge))<compute_test_metric>
del prediction_data
gc.collect()
scores_ridge = cross_val_score(ridge, X_training, y_training, cv=10, scoring="r2")
 print("Cross Validation Score: " + str(np.mean(scores_ridge)) )<import_modules>
sales_ = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'))
submission = pd.read_csv(os.path.join(SUBMISSION_PATH))
from sklearn import ensemble<train_model>
submission_eval = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_eval.loc[submission_eval['sales'] < 0, 'sales'] = 0
submission_eval.sort_values('id', inplace=True)
 submission_eval['day'] = submission_eval['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_eval.columns)
 print(submission_eval.head() , submission_eval.tail() , sep="
" )
params = {'n_estimators': 20000, 'max_depth': 5, 'min_samples_split': 2,
'learning_rate': 0.05, 'loss': 'ls' , 'max_features' : 20}
clf = ensemble.GradientBoostingRegressor(**params)
 clf.fit(X_training, y_training )<compute_test_metric>
submission.iloc[30490:, 1:] = submission_eval.iloc[:,1:].to_numpy()
print(submission.head() , submission.tail() , sep="
" )
clf_pred=clf.predict(X_valid)
 clf_pred= clf_pred.reshape(-1,1)
 r2_clf = r2_score(y_valid, clf_pred)
 rmse_clf = np.sqrt(mean_squared_error(y_valid, clf_pred))
print("R^2 Score: " + str(r2_clf))
print("RMSE Score: " + str(rmse_clf))<compute_test_metric>
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )