Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<categorify>
import os
import gc
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta, date
from tqdm.notebook import tqdm
import lightgbm as lgb
for c in obj_col:
le = LabelEncoder()
le.fit(train_x[c].fillna('NA'))
train_x[c] = le.transform(train_x[c].fillna('NA'))
test_x[c] = le.transform(test_x[c].fillna('NA'))<define_search_space>
TEST_RUN = False
MODEL_VERSION = 'final'
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
SUBMISSION_PATH = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/submission.csv"
PATH_TO_CAT0 = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/model_cat0_v10.lgb"
PATH_TO_CAT1 = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/model_cat1_v10.lgb"
PATH_TO_CAT2 = "/kaggle/input/rausnaus-lightgbm-per-category-best-notebook/model_cat2_v10.lgb"
PATH_MODELS = [PATH_TO_CAT0, PATH_TO_CAT1, PATH_TO_CAT2]
KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913+28
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 EVAL_SPLIT = '2016-05-22'
TRAIN_SPLIT = str(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d')- timedelta(days=28)) [:10]
print(f"Train split: {datetime.strptime(TRAIN_SPLIT, '%Y-%m-%d')}")
 print(f"Eval split: {datetime.strptime(EVAL_SPLIT, '%Y-%m-%d')}" )
<import_modules>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32',
}
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score<import_modules>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
from sklearn.metrics import mean_absolute_error<init_hyperparams>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['item_id', 'sales']].groupby('item_id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'item_id_rmean_{lag_col}_{window}'
strain[window_col] = strain[['item_id', lag_col]].groupby('item_id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales done for window {window} per item_id')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday",
"month": "month",
"year": "year"
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
<define_search_space>
%%time
sales_train_evaluation = load_data()
best_param = [16,8,2]<define_variables>
%%time
sales_train_evaluation["sale"] =(( sales_train_evaluation['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_lag_features(sales_train_evaluation)
 make_date_features(sales_train_evaluation )
params = {"metric": "mae",
"max_depth":best_param[0],
"num_leaves":best_param[1],
"min_data_in_leaf":best_param[2]
}
train_scores = []
valid_scores = []
pred = np.zeros(test_x.shape[0])
 pred_valid = np.zeros(train_x.shape[0])
 kf = KFold(n_splits = 5, shuffle = True, random_state=0)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
lgb_train = lgb.Dataset(tr_x, tr_y)
 lgb_eval = lgb.Dataset(va_x, va_y)
 gbm = lgb.train(params,lgb_train,valid_sets=lgb_eval,num_boost_round=10000,early_stopping_rounds=100,verbose_eval=500)
 tr_pred = gbm.predict(tr_x)
 va_pred = gbm.predict(va_x)
 pred += gbm.predict(test_x)/ 5
pred_valid[va_idx] = va_pred
train_MAE = mean_absolute_error(tr_y,tr_pred)
 valid_MAE = mean_absolute_error(va_y,va_pred)
 train_scores.append(train_MAE)
 valid_scores.append(valid_MAE)
 print(f'fold{i}: train_MAE={train_MAE} valid_MAE={valid_MAE}')
 mean_train_scores = np.mean(train_scores)
 mean_valid_scores = np.mean(valid_scores)
 print(f'mean train MAE={mean_train_scores} mean valid MAE={mean_valid_scores}' )<compute_test_metric>
before = len(sales_train_evaluation)
 sales_train_evaluation.dropna(inplace = True)
 after = len(sales_train_evaluation)
 print(f"Reduced {(before-after)/before}%" )
True_Pred_map(tr_y,tr_pred )<compute_test_metric>
print("Splitting data into train, validation, evaluation set")
 train = sales_train_evaluation[(sales_train_evaluation['date'] >= BEGIN_DATE)&(sales_train_evaluation['date'] <= TRAIN_SPLIT)]
validation = sales_train_evaluation[(sales_train_evaluation['date'] > TRAIN_SPLIT)&(sales_train_evaluation['date'] <= EVAL_SPLIT)]
del sales_train_evaluation; gc.collect()
True_Pred_map(va_y,va_pred )<import_modules>
print(train["cat_id"].value_counts())
 print(validation["cat_id"].value_counts() )
<prepare_output>
categorical_features = [
'item_id', 'dept_id', 'store_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]
train_tmp = train_x.copy()
train_tmp["Weekly_Sales"] = train_y
train_tmp["pred"] = pred_valid
train_tmp["pred_diff"] = train_tmp['Weekly_Sales']-train_tmp["pred"]
train_tmp<filter>
train_sets = [train[train["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del train; gc.collect()
val_sets = [validation[validation["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del validation; gc.collect()
train_tmp[(train_tmp["pred_diff"]<-200000)|(train_tmp["pred_diff"]>200000)]<filter>
train_pools = [lgb.Dataset(
 data=train_sets[category][train_columns],
label=train_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del train_sets; gc.collect()
data_join2[(data_join2["Store"] == 1)&(data_join2["Dept"] == 1)&(data_join2["Holiday_name"].notnull())]<data_type_conversions>
val_pools = [lgb.Dataset(
 data=val_sets[category][train_columns],
label=val_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del val_sets; gc.collect()
test_pred = test.copy(deep=True)
 test_pred["Weekly_Sales"] = pred
test_pred[['Store', 'Dept', 'Date']] = test_pred[['Store', 'Dept', 'Date']].astype(str)
 test_pred["id"] = test_pred["Store"] + "_" + test_pred["Dept"] + "_" + test_pred["Date"] 
<feature_engineering>
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 1200
params = {
"objective" : "poisson",
"metric" : "rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
'verbosity': 1,
'num_iterations': ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
}
test_pred["Date"] = pd.to_datetime(test_pred["Date"])
 test_pred["YEAR"] = test_pred["Date"].dt.year
test_pred["MONTH"] = test_pred["Date"].dt.month
test_pred["WEEK"] = test_pred["Date"].dt.week
test_pred<data_type_conversions>
%%time
models = [lgb.train(params,
train_pools[category],
valid_sets = val_pools[category],
verbose_eval=20)
 for category in range(3)]
for cat,model in enumerate(models):
model.save_model(f"model_cat{cat}_{MODEL_VERSION}.lgb" )
test_pred[['Store', 'Dept']] = test_pred[['Store', 'Dept']].astype(int)
 test_pred_bef = pd.merge(test_pred, data_sales, on=["Dept","Store","YEAR","WEEK"], how="left")
 test_pred_bef[(test_pred_bef["Dept"] == 72)&(test_pred_bef["WEEK"] == 47)]<feature_engineering>
del train_pools, val_pools; gc.collect()
test_pred_bef["Weekly_Sales"] = np.where(( test_pred_bef["Dept"] == 72)&(test_pred_bef["WEEK"] == 47), test_pred_bef["Weekly_Sale_before"], test_pred_bef["Weekly_Sales"])
 test_pred_bef[(test_pred_bef["Dept"] == 72)&(test_pred_bef["WEEK"] == 47)]<feature_engineering>
df = load_data(train=False)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_date_features(df )
test_pred2 = test_pred_bef.copy()
test_pred2["Weekly_Sales"] = np.where(( test_pred2["YEAR"]==2012)&(test_pred2["WEEK"]==52)&(test_pred2["Weekly_Sales"].shift(1)> 2*test_pred2["Weekly_Sales"])\
,test_pred2["Weekly_Sales"]+test_pred2["Weekly_Sales"].shift(1)*2.5/7, test_pred2["Weekly_Sales"] )<prepare_output>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("item_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'item_id'])
 dt.loc[dt['date'] == str(day),f'item_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
lag = 28
lag_col = 'lag_28'
window = 28
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("store_id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'store_id'])
 dt.loc[dt['date'] == str(day),f'store_id_rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print(f"Features done for day {str(day)}")
test_pred3 = test_pred2[["id","Weekly_Sales"]]
test_pred3.head()<save_to_csv>
%%time
END_DATE = EVAL_SPLIT
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 print(pred_begin_date)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 for i in range(3):
prediction_data_cat = prediction_data.loc[(prediction_data['date'] == str(pred_date)) &(prediction_data["cat_id"] == i), train_columns]
prediction = models[i].predict(prediction_data_cat)
 df.loc[(df['date'] == str(pred_date)) &(df.index.isin(prediction_data_cat.index)) , 'sales'] = prediction
print("Prediction", prediction.size, prediction )
submission = test_pred3[["id","Weekly_Sales"]]
submission.to_csv('submission15.csv', index=False )<load_from_csv>
del prediction_data
gc.collect()
train = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/train.csv.zip",parse_dates=["Date"])
 test = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/test.csv.zip",parse_dates=["Date"] )<load_from_csv>
sales_ = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_evaluation.csv'))
submission = pd.read_csv(os.path.join(SUBMISSION_PATH))
add = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/features.csv.zip")
 add.head()<load_from_csv>
submission_eval = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_eval.loc[submission_eval['sales'] < 0, 'sales'] = 0
submission_eval.sort_values('id', inplace=True)
 submission_eval['day'] = submission_eval['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_eval.columns)
 print(submission_eval.head() , submission_eval.tail() , sep="
" )
store = pd.read_csv("/kaggle/input/walmart-recruiting-store-sales-forecasting/stores.csv")
 store<merge>
submission.iloc[30490:, 1:] = submission_eval.iloc[:,1:].to_numpy()
print(submission.head() , submission.tail() , sep="
" )
<feature_engineering><EOS> 
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )