Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<data_type_conversions>
MODEL_VERSION = 'v10'
df_train["GarageYrBlt"].fillna(df_train["GarageYrBlt"].mean() , inplace=True)
 df_test["GarageYrBlt"].fillna(df_test["GarageYrBlt"].mean() , inplace=True)
 data["GarageYrBlt"].fillna(data["GarageYrBlt"].mean() , inplace=True)
 df_train["MasVnrArea"].fillna(df_train["MasVnrArea"].mean() , inplace=True)
 df_test["MasVnrArea"].fillna(df_test["MasVnrArea"].mean() , inplace=True)
 data["MasVnrArea"].fillna(data["MasVnrArea"].mean() , inplace=True)
 df_train["MasVnrType"].fillna("None", inplace=True)
 df_test["MasVnrType"].fillna("None", inplace=True)
 data["MasVnrType"].fillna("None", inplace=True )<define_variables>
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
PATH_TO_CAT0="/kaggle/input/models-per-cat-with-sale/model_cat0_v10.lgb"
PATH_TO_CAT1="/kaggle/input/models-per-cat-with-sale/model_cat1_v10.lgb"
PATH_TO_CAT2="/kaggle/input/models-per-cat-with-sale/model_cat2_v10.lgb"
PATH_TO_ALL_CATS = "/kaggle/input/lgbmindividualbestsubmission/model_v13_param_tuning.lgb"
BACKWARD_LAG = 60
END_DAY = 1913
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 TRAIN_SPLIT = '2016-03-27'
EVAL_SPLIT = '2016-04-24'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'
PATH_MODELS = [PATH_TO_ALL_CATS, PATH_TO_CAT0, PATH_TO_CAT1, PATH_TO_CAT2]
types_train = df_train.dtypes
num_train = types_train[(types_train == int)|(types_train == float)]
cat_train = types_train[types_train == object]
types_test = df_test.dtypes
num_test = types_test[(types_test == int)|(types_test == float)]
cat_test = types_test[types_test == object]<define_variables>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32'
}
numerical_values_train = list(num_train.index)
 numerical_values_test = list(num_test.index)
 fill_num = numerical_values_train+numerical_values_test
print(fill_num )<data_type_conversions>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_validation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
for i in fill_num:
df_train[i].fillna(df_train[i].mean() , inplace=True )<drop_column>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['id', 'sales']].groupby('id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'rmean_{lag_col}_{window}'
strain[window_col] = strain[['id', lag_col]].groupby('id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales for done for window {window}')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday"
}
for date_feat_name, date_feat_func in date_features.items() :
if not date_feat_name in dt.columns:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
fill_num.remove('SalePrice' )<data_type_conversions>
model_all = lgb.Booster(model_file = PATH_MODELS[0])
 model_cat0 = lgb.Booster(model_file = PATH_MODELS[1])
 model_cat1 = lgb.Booster(model_file = PATH_MODELS[2])
 model_cat2 = lgb.Booster(model_file = PATH_MODELS[3])
 model_per = [model_cat0, model_cat1, model_cat2]
for i in fill_num:
df_test[i].fillna(df_test[i].mean() , inplace=True)
 data[i].fillna(data[i].mean() , inplace=True )<define_variables>
df = load_data(train=False)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8')
 make_lag_features(df)
 make_date_features(df )
categorical_values_train = list(cat_train.index)
 categorical_values_test = list(cat_test.index )<define_variables>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'id'])
 dt.loc[dt['date'] == str(day),f'rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print("Lag features done")
 print(type(dt))
fill_cat = []
for i in categorical_values_train:
if i in list(null_few.index):
fill_cat.append(i)
 print(fill_cat )<count_unique_values>
%%time
END_DATE = EVAL_SPLIT
ZERO_THRESHOLD = 0.01
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 print(pred_begin_date)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 prediction_data = prediction_data.loc[prediction_data['date'] == str(pred_date), train_columns]
prediction_all_cats = model_all.predict(prediction_data)
 print("Single model done")
 prediction_per_cat = pd.DataFrame(data=None, index=prediction_data.index, columns=["sales"])
 for category in range(3):
prediction_data_cat = prediction_data.loc[prediction_data["cat_id"] == category, train_columns].drop("cat_id", axis = 1)
 prediction = model_per[category].predict(prediction_data_cat)
 prediction_per_cat.loc[prediction_per_cat.index.isin(prediction_data_cat.index), "sales"] = prediction
prediction_per_cat = prediction_per_cat['sales'].to_numpy(dtype='float64')
 prediction = np.average([prediction_all_cats, prediction_per_cat], axis = 0, weights=[1./3, 2./3])
 print("Prediction", prediction.size, prediction)
 df.loc[df['date'] == str(pred_date), 'sales'] = prediction
def most_common_term(lst):
lst = list(lst)
 return max(set(lst), key=lst.count)
 most_common = []
for i in fill_cat:
most_common.append(most_common_term(data[i]))
most_common<define_variables>
del prediction_data
gc.collect()
most_common_dictionary = {fill_cat[0]: [most_common[0]], fill_cat[1]: [most_common[1]], fill_cat[2]: [most_common[2]], fill_cat[3]: [most_common[3]],
fill_cat[4]: [most_common[4]], fill_cat[5]: [most_common[5]], fill_cat[6]: [most_common[6]], fill_cat[7]: [most_common[7]],
fill_cat[8]: [most_common[8]]}
most_common_dictionary<data_type_conversions>
submission_val = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_val.loc[submission_val['sales'] < 0, 'sales'] = 0
submission_val.sort_values('id', inplace=True)
 submission_val['day'] = submission_val['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_val.columns )
k = 0
for i in fill_cat:
df_train[i].fillna(most_common[k], inplace=True)
 df_test[i].fillna(most_common[k], inplace=True)
 data[i].fillna(most_common[k], inplace=True)
 k += 1<concatenate>
submission_eval = submission_val.copy()
submission_eval['id'] = submission_eval['id'].str.replace('validation', 'evaluation')
 submission = pd.concat([submission_val, submission_eval], axis=0, sort=False)
 print(submission.columns)
 print(submission.head(1))
<feature_engineering><EOS> 
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )