Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<prepare_x_and_y>
MODEL_VERSION = 'v10'
y = df_train.label.to_numpy()
X = df_train.drop(columns=["label"] ).to_numpy(np.float64)
 X /= 255.0<train_model>
TEST_RUN = False
pd.set_option('display.max_columns', 50)
 KAGGLE_DATA_FOLDER = '/kaggle/input/m5-forecasting-accuracy'
BACKWARD_LAG = 60
END_DAY = 1913
BEGIN_DATE = '2013-08-01'
BEGIN_DAY = str(( datetime.strptime(BEGIN_DATE, '%Y-%m-%d')- datetime.strptime('2011-01-29', '%Y-%m-%d')).days)
 TRAIN_SPLIT = '2016-03-27'
EVAL_SPLIT = '2016-04-24'
print(datetime.strptime(EVAL_SPLIT, '%Y-%m-%d'))
TASK_TYPE='CPU'
X_totrain = X.reshape(X.shape[0], 28, 28, 1 )<split>
CALENDAR_DTYPES = {
'date': 'str',
'wm_yr_wk': 'int16',
'weekday': 'category',
'wday': 'int16',
'month': 'int16',
'year': 'int16',
'd': 'object',
'event_name_1': 'category',
'event_type_1': 'category',
'event_name_2': 'category',
'event_type_2': 'category',
'snap_CA': 'int16',
'snap_TX': 'int16',
'snap_WI': 'int16'
}
PARSE_DATES = ['date']
SALES_PRICES_DTYPES = {
'store_id': 'category',
'item_id': 'category',
'wm_yr_wk': 'int16',
'sell_price': 'float32'
}
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y)
 X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28, 1 )<categorify>
def load_data(train=True):

 print("Loading train and validation data")
 numcols = [f"d_{day}" for day in range(int(BEGIN_DAY),END_DAY+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 sales_train_validation = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sales_train_validation.csv'),
usecols=catcols+numcols, dtype=dtype)
 for col in catcols:
if col != "id":
sales_train_validation[col] = sales_train_validation[col].cat.codes.astype("int16")
 sales_train_validation[col] -= sales_train_validation[col].min()
if not train:
for day in range(END_DAY+1, END_DAY+28+1):
sales_train_validation[f"d_{day}"] = 0
value_vars = [column for column in sales_train_validation.columns
if(column.startswith('d_')and int(column.replace('d_', '')) >= END_DAY - BACKWARD_LAG)]
else:
value_vars = [col for col in sales_train_validation.columns
if(col.startswith('d_')and(int(col.replace('d_', '')) >= int(BEGIN_DAY)))]
print("Shape:", sales_train_validation.shape)
 print("Memory usage(Mb)before melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 sales_train_validation = pd.melt(
 sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
value_vars=value_vars,
var_name = 'day',
value_name = 'sales')
 print("Completed melting, new shape:", sales_train_validation.shape)
 print("Colums after melting:", sales_train_validation.columns)
 print("Memory usage(Mb)after melting:", sales_train_validation.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
print("Loading calendar")
 calendar = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'calendar.csv'), dtype=CALENDAR_DTYPES, parse_dates=['date'])
 print("Calendar columns: ", calendar.columns)
 print("Memory usage(Mb)calendar: ", calendar.memory_usage().sum() / 1024**2)
 calendar.rename(columns={'d':'day'}, inplace=True)
 for col, col_dtype in CALENDAR_DTYPES.items() :
if col_dtype == "category":
calendar[col] = calendar[col].cat.codes.astype("int16")
 calendar[col] -= calendar[col].min()
sales_train_validation = sales_train_validation.merge(calendar, on="day", copy=False)
 del calendar; gc.collect()
print("Merged calendar(in place)")
 print("Memory usage(Mb)after merging calendar:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 sell_prices = pd.read_csv(os.path.join(KAGGLE_DATA_FOLDER, 'sell_prices.csv'), dtype=SALES_PRICES_DTYPES)
 for col, col_dtype in SALES_PRICES_DTYPES.items() :
if col_dtype == "category":
sell_prices[col] = sell_prices[col].cat.codes.astype("int16")
 sell_prices[col] -= sell_prices[col].min()
print("Memory usage(Mb)sell prices:", sell_prices.memory_usage().sum() / 1024**2)
 columns = ['item_id', 'store_id', 'sell_price']
for feature in columns:
if feature == 'sell_price':
sell_prices[feature].fillna(0, inplace=True)
 sales_train_validation = sales_train_validation.merge(sell_prices, on=["store_id","item_id","wm_yr_wk"], copy=False)
 del sell_prices; gc.collect()
print("Merged sales prices(in place)")
 print("Memory usage(Mb)after merging sales:", sales_train_validation.memory_usage().sum() / 1024**2)
 print("Colums after merge:", sales_train_validation.columns)
 return sales_train_validation
y_cat = to_categorical(y, 10)
 y_train_cat = to_categorical(y_train, 10)
 y_test_cat = to_categorical(y_test, 10 )<data_type_conversions>
def make_lag_features(strain):

 print('in dataframe:', strain.shape)
 print("headers:", strain.columns)
 lags = [7, 28]
lag_cols = ['lag_{}'.format(lag)for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
strain[lag_col] = strain[['id', 'sales']].groupby('id')['sales'].shift(lag)
 print('lag sales done')
 windows= [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
window_col = f'rmean_{lag_col}_{window}'
strain[window_col] = strain[['id', lag_col]].groupby('id')[lag_col].transform(
 lambda x: x.rolling(window ).mean() 
)
 print(f'Rolling mean sales for done for window {window}')
 def make_date_features(dt):
date_features = {
'week': 'weekofyear',
'quarter': 'quarter',
'mday': 'day',
"wday": "weekday"
}
for date_feat_name, date_feat_func in date_features.items() :
if not date_feat_name in dt.columns:
dt[date_feat_name] = getattr(dt['date'].dt, date_feat_func ).astype('int16')
 print('date features done')
 dt['day'] = dt['day'].apply(lambda x: int(x.replace('d_', '')))
 print('out dataframe:', dt.shape )
test = df_test.to_numpy(np.float64)
 test = test.reshape(test.shape[0], 28, 28, 1)
 test /= 255.0<choose_model_class>
%%time
sales_train_validation = load_data()
def convNeuralNetwork(filters=256, kernel_size=(3, 3), pool_size=(2, 2), units=128, dropout=0.2):
cnn = Sequential()
cnn.add(Conv2D(filters=filters, kernel_size=kernel_size, strides=(1, 1),
input_shape=(28, 28, 1), activation="relu", padding="same"))
cnn.add(MaxPool2D(pool_size=pool_size, padding="same"))
cnn.add(Conv2D(filters=filters, kernel_size=kernel_size, strides=(1, 1),
activation="relu", padding="same"))
cnn.add(MaxPool2D(pool_size=pool_size, padding="same"))
cnn.add(Flatten())
 cnn.add(Dense(units=units, activation="relu"))
cnn.add(Dropout(dropout))
cnn.add(Dense(units=units, activation="relu"))
cnn.add(Dropout(dropout))
cnn.add(Dense(units=10, activation="softmax"))
cnn.compile(optimizer="adamax", loss="categorical_crossentropy", metrics=["accuracy"])
 return cnn<train_model>
sales_train_validation["sale"] =(( sales_train_validation['sell_price'] * 100 % 10)< 6 ).astype('int8' )
early_stopping = EarlyStopping(monitor="val_loss", patience=10, verbose=1,
restore_best_weights=True)
 cnn = convNeuralNetwork(filters=2048, units=1024)
 cnn_hist = cnn.fit(X_train, y_train_cat, validation_data=(X_test, y_test_cat),
epochs=50, batch_size=256, callbacks=[early_stopping] )<train_on_grid>
%%time
make_lag_features(sales_train_validation)
 make_date_features(sales_train_validation )
early_stopping = EarlyStopping(monitor="loss", patience=10, restore_best_weights=True)
 model_checkpoint = ModelCheckpoint(filepath="./", monitor="loss", verbose=1,
save_best_only=True, save_weights_only=True)
 model = convNeuralNetwork(filters=2048, units=1024)
 model_hist = model.fit(X_totrain, y_cat, batch_size=256, epochs=50,
callbacks=[early_stopping, model_checkpoint] )<predict_on_test>
total = len(sales_train_validation['event_type_2'])
 zero = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 0])
 one = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 1])
 two = len(sales_train_validation.loc[sales_train_validation['event_type_2'] == 2])
 print('0: '+ str(zero))
print('1: ' + str(one))
print('2: ' + str(two))
print('-'*7 + '
Total: ' + str(total))
del total; del zero; del one; del two
y_pred = model.predict(test ).argmax(1)
 doSubmission(y_pred )<import_modules>
sales_train_validation.isnull().sum()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from sklearn.model_selection import train_test_split<load_from_csv>
before = len(sales_train_validation)
 sales_train_validation.dropna(inplace = True)
 after = len(sales_train_validation)
 print(f"Reduced {(before-after)/before}%")
training = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 testing = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<prepare_x_and_y>
print("Splitting data into train, validation, evaluation set")
 train = sales_train_validation[(sales_train_validation['date'] >= BEGIN_DATE)&(sales_train_validation['date'] <= TRAIN_SPLIT)]
validation = sales_train_validation[(sales_train_validation['date'] > TRAIN_SPLIT)&(sales_train_validation['date'] <= EVAL_SPLIT)]
evaluation = sales_train_validation[(sales_train_validation['date'] > EVAL_SPLIT)]
del sales_train_validation; gc.collect()
X = training.drop(columns = 'label' ).values
y = training['label'].values<train_model>
categorical_features = [
'item_id', 'dept_id', 'store_id', 'state_id',
'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2'
]
X = X/255
X = X.reshape(X.shape[0], *(28, 28, 1))
print(X.shape )<normalization>
drop = ["id", "date", "sales", "day", "wm_yr_wk", "weekday", "cat_id"]
train_columns = train.columns[~train.columns.isin(drop)]
print(train_columns )
testing = testing.values
testing = testing/255
testing = testing.reshape(testing.shape[0], *(28, 28, 1))<split>
train_sets = [train[train["cat_id"] == category].drop("cat_id", axis = 1)for category in range(3)]
del train; gc.collect()
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, random_state = 0 )<find_best_params>
train_pools = [lgb.Dataset(
 data=train_sets[category][train_columns],
label=train_sets[category]["sales"],
categorical_feature=categorical_features,
free_raw_data=False)
 for category in range(3)]
del train_sets; gc.collect()
cnn = tf.keras.models.Sequential()<choose_model_class>
val_pool = lgb.Dataset(
 data=validation[train_columns],
label=validation["sales"],
categorical_feature=categorical_features,
free_raw_data=False 
)
 del validation; del evaluation; gc.collect()
cnn.add(layer = tf.keras.layers.Conv2D(input_shape =(28, 28, 1),
filters=64,
kernel_size =(3,3),
activation = 'relu'))<choose_model_class>
if TEST_RUN:
ITERATIONS=1
else:
ITERATIONS = 1200
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : ITERATIONS,
'num_leaves': 128,
"min_data_in_leaf": 100,
}
cnn.add(layer = tf.keras.layers.MaxPool2D(pool_size=(2, 2), strides = 2))<choose_model_class>
%%time
models = [lgb.train(params,
train_pools[category],
valid_sets = val_pool,
verbose_eval=20)
 for category in range(3)]
for cat,model in enumerate(models):
model.save_model(f"model_cat{cat}_{MODEL_VERSION}.lgb" )
cnn.add(layer = tf.keras.layers.Conv2D(filters=64,
kernel_size =(3,3),
activation = 'relu'))<choose_model_class>
del train_pools, val_pool; gc.collect()
cnn.add(layer = tf.keras.layers.MaxPool2D(pool_size=(2, 2), strides = 2))<choose_model_class>
df = load_data(train=False)
 make_date_features(df)
 df["sale"] =(( df['sell_price'] * 100 % 10)< 6 ).astype('int8' )
cnn.add(layer = tf.keras.layers.Flatten() )<choose_model_class>
def lag_features_for_day(dt, day):
print(type(dt))
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt['date'] == str(day), lag_col] = \
dt.loc[dt['date'] == str(day-timedelta(days=lag)) , 'sales'].values
windows = [7, 28]
for window in windows:
for lag, lag_col in zip(lags, lag_cols):
df_window = dt[(dt['date'] <= str(day-timedelta(days=lag)))&(dt['date'] > str(day-timedelta(days=lag+window)))]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt['date']==str(day),'id'])
 dt.loc[dt['date'] == str(day),f'rmean_{lag_col}_{window}'] = df_window_grouped.sales.values
print("Lag features done")
 print(type(dt))
cnn.add(layer = tf.keras.layers.Dropout(0.5))<choose_model_class>
%%time

 END_DATE = EVAL_SPLIT
ZERO_THRESHOLD = 0.01
PREDICT_DAYS = 28
for f_day in tqdm(range(1,PREDICT_DAYS+1)) :
pred_date =(datetime.strptime(END_DATE, '%Y-%m-%d')+ timedelta(days=f_day)).date()
print(f"Forecasting day {END_DAY+f_day}, date: {str(pred_date)}")
 pred_begin_date = pred_date - timedelta(days=BACKWARD_LAG+1)
 prediction_data = df[(df['date'] >= str(pred_begin_date)) &(df['date'] <= str(pred_date)) ].copy()
lag_features_for_day(prediction_data, pred_date)
 for i in range(3):
prediction_data_cat = prediction_data.loc[(prediction_data['date'] == str(pred_date)) &(prediction_data["cat_id"] == i), train_columns]
prediction = models[i].predict(prediction_data_cat)
 df.loc[(df['date'] == str(pred_date)) &(df.index.isin(prediction_data_cat.index)) , 'sales'] = prediction
cnn.add(layer = tf.keras.layers.Dense(units = 108, activation='relu'))<choose_model_class>
del prediction_data
gc.collect()
cnn.add(layer = tf.keras.layers.Dropout(0.5))<choose_model_class>
submission_val = df.loc[df['date'] > END_DATE, ['id', 'day', 'sales']].copy()
submission_val.loc[submission_val['sales'] < 0, 'sales'] = 0
submission_val.sort_values('id', inplace=True)
 submission_val['day'] = submission_val['day'].apply(lambda x: 'F{}'.format(x - END_DAY))
print(submission_val.columns )
cnn.add(layer = tf.keras.layers.Dense(units = 108, activation='relu'))<choose_model_class>
submission_eval = submission_val.copy()
submission_eval['id'] = submission_eval['id'].str.replace('validation', 'evaluation')
 submission = pd.concat([submission_val, submission_eval], axis=0, sort=False)
 print(submission.columns)
 print(submission.head(1))
<choose_model_class><EOS> 
submission.to_csv('submission.csv', index=False)
 print('Submission shape', submission.shape )