Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<count_missing_values>
import os
import gc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta, date
from tqdm.notebook import tqdm
import lightgbm as lgb
df_train.isna().sum()<prepare_x_and_y>
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
SUBMISSION_PATH = "/kaggle/input/lgbmensemblebestvalidationsubmission/submission.csv"
PATH_TO_ALL_CATS = "/kaggle/input/single-model-evaluation-submission/model_final.lgb"
PATH_TO_CAT0 = "/kaggle/input/category-models-evaluation-submission/model_cat0_final.lgb"
PATH_TO_CAT1 = "/kaggle/input/category-models-evaluation-submission/model_cat1_final.lgb"
PATH_TO_CAT2 = "/kaggle/input/category-models-evaluation-submission/model_cat2_final.lgb"
PATH_MODELS = [PATH_TO_ALL_CATS, PATH_TO_CAT0, PATH_TO_CAT1, PATH_TO_CAT2]
KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 28+28
END_DAY = 1913+28
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 EVAL_SPLIT = '2016-05-22'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
with_n = len(df_train[df_train['store_and_fwd_flag'] == 'N'].index)
 with_y = len(df_train[df_train['store_and_fwd_flag'] == 'Y'].index)
 len(df_train[df_train['store_and_fwd_flag'] == 'N'].index), len(df_train[df_train['store_and_fwd_flag'] == 'Y'].index )<load_from_csv>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32',
'sales': 'float32'
}
df_test = pd.read_csv(os.path.join('.. ', 'input', 'test.csv'), index_col=0)
 df_test.head()<filter>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
max_trip_duration_flag_y = df_train[df_train['store_and_fwd_flag'] == 'Y']['trip_duration'].max()
row_count_before_deletion = len(df_train.index)
 print('{} lines before deletion'.format(row_count_before_deletion))
df_train = df_train[~(( df_train['store_and_fwd_flag'] == 'N')&(df_train['trip_duration'] > max_trip_duration_flag_y)) ]
row_count_after_deletion = len(df_train.index)
 print('{} lines after deletion'.format(row_count_after_deletion))
print('{} lines deleted'.format(row_count_before_deletion - row_count_after_deletion))<count_values>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['item_id', 'sales']].groupby('item_id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'item_id_rmean_{lag_col}_{window}'
strain[window_col] = strain[['item_id', lag_col]].groupby('item_id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales done for window {window} per item_id')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday",
"month": "month",
"year": "year"
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
df_train[(( df_train['store_and_fwd_flag'] == 'Y')&(df_train['trip_duration'] > 6500)) ]['trip_duration'].count()<filter>
model_all = lgb.Booster(model_file = PATH_MODELS[0])
 model_cat0 = lgb.Booster(model_file = PATH_MODELS[1])
 model_cat1 = lgb.Booster(model_file = PATH_MODELS[2])
 model_cat2 = lgb.Booster(model_file = PATH_MODELS[3])
 model_per = [lgb.Booster(model_file = PATH_MODELS[category])for category in range(1, 3+1)]
row_count_before_deletion = len(df_train.index)
 print('{} lines before deletion'.format(row_count_before_deletion))
df_train = df_train[df_train['trip_duration'] >= 10]
row_count_after_deletion = len(df_train.index)
 print('{} lines after deletion'.format(row_count_after_deletion))
print('{} lines deleted'.format(row_count_before_deletion - row_count_after_deletion))<feature_engineering>
df = load_data(train=False)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_lag_features(df)
 make_date_features(df )
<feature_engineering>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("item_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'item_id'])
 dt.loc[dt['date'] == str(day),f'item_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
lag = 28
lag_col = 'lag_28'
window = 28
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("store_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'store_id'])
 dt.loc[dt['date'] == str(day),f'store_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print(f"Features done for day {str(day)}")
def create_datetime_based_columns(df):
df['datetime'] = pd.to_datetime(df['pickup_datetime'])
 df['year'] = df['datetime'].dt.year
df['month'] = df['datetime'].dt.month
df['day'] = df['datetime'].dt.day
df['dayofweek'] = df['datetime'].dt.weekday
df['hour'] = df['datetime'].dt.hour
return df<prepare_x_and_y>
%%time
END_DATE = EVAL_SPLIT
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 print(pred_begin_date)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 prediction_data = prediction_data.loc[prediction_data['date'] == str(pred_date), train_columns]
prediction_all_cats = model_all.predict(prediction_data)
 prediction_per_cat = pd.DataFrame(data=None, index=prediction_data.index, columns=["sales"])
 for category in range(3):
prediction_data_cat = prediction_data.loc[prediction_data["cat_id"] == category].drop("cat_id", axis = 1)
 prediction = model_per[category].predict(prediction_data_cat)
 prediction_per_cat.loc[prediction_per_cat.index.isin(prediction_data_cat.index), "sales"] = prediction
prediction_per_cat = prediction_per_cat['sales'].to_numpy(dtype='float64')
 prediction = np.average([prediction_all_cats, prediction_per_cat], axis = 0, weights=[1./2, 1./2])
 print("Prediction", prediction.size, prediction)
 df.loc[df['date'] == str(pred_date), 'sales'] = prediction
def filter_feature_columns(df):
selected_columns = []
selected_columns = ['pickup_longitude', 'pickup_latitude']
selected_columns += ['dropoff_longitude', 'dropoff_latitude']
selected_columns += ['month', 'dayofweek', 'hour']
return df[selected_columns]
def filter_target_column(df):
return df['trip_duration']
def filter_split_dataset(df):
X = filter_feature_columns(df)
 y = filter_target_column(df)
 return X, y 
<split>
del prediction_data
gc.collect()
X_train, y_train = filter_split_dataset(df_train_copy)
 X_train.shape, y_train.shape<import_modules>
sales_ = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'))
submission = pd.read_csv(os.path.join(SUBMISSION_PATH))
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV<choose_model_class>
submission_eval = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_eval.loc[submission_eval['sales'] < 0, 'sales'] = 0
submission_eval.sort_values('id', inplace=True)
 submission_eval['day'] = submission_eval['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_eval.columns)
 print(submission_eval.head() , submission_eval.tail() , sep="
" )
rfr = RandomForestRegressor(n_estimators=30, min_samples_leaf=10, min_samples_split=15,
max_features='auto', max_depth=90, bootstrap=True)
 s_split = ShuffleSplit(n_splits=4, train_size=.12, test_size=.6)
<train_model>
submission.iloc[30490:, 1:] = submission_eval.iloc[:,1:].to_numpy()
print(submission.head() , submission.tail() , sep="
" )
<prepare_output><EOS> 
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )