Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<feature_engineering>
!pip install catboost
!pip install ipywidgets
!jupyter nbextension enable --py widgetsnbextension
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 50)
 pd.set_option('display.width', 1000)
df_test['Weekly_Sales'] = y_pred
df_test['DatasetType'] = 'Predicted'
df_test['Date'] = input_dfs['test']['Date'].values
df_train['DatasetType'] = 'Actual'
df_train['Date'] = input_dfs['train']['Date'].values
df_result = df_train.append(df_test)
 graph_relation_to_weekly_sale('DatasetType', df_result, x='Date' )<save_to_csv>
X_train = pd.read_pickle('.. /input/featureengineering/df_data_features.pkl')
 day_ini = X_train['d'].min()
day_train_last = 1913
print('start day',day_ini)
 day_train_first = day_train_last - 1*365
X_train = X_train.loc[(X_train['d'] >= day_train_first), :]
X_train = X_train.loc[(( X_train['Lag28_rmean28_id'].notna())&(X_train['type'] =='train')) |(X_train['type'] =='val'), :]
print('First training day: ', day_train_first)
 print('Last training day: ', day_train_last)
 print(X_train.groupby(['year', 'month'])['d'].count())
 X_train.columns = [c.replace('[','')for c in X_train.columns]
X_train.columns = [c.replace(']','')for c in X_train.columns]
X_train.columns = [c.replace(' ','_')for c in X_train.columns]
X_train.columns = [c.replace(',','')for c in X_train.columns]
X_train.columns = [c.replace("'",'')for c in X_train.columns]
print(X_train.info())
 X_train.head()
input_dfs['sampleSubmission']['Weekly_Sales'] = y_pred
input_dfs['sampleSubmission'].to_csv('submission.csv',index=False)
 input_dfs['sampleSubmission']<set_options>
def repeat_data(df):
rep_month = 0
rep_event = 0
rep_event_test = 0
months = [5]
events_test = ['Pesach End', 'OrthodoxEaster', 'Cinco De Mayo', "Mother's day"]
events2 = df[(df['event_name_2'].notna())]
ts = events2.loc[:,'event_name_2']
events2.loc[:,'event_name_1'] = ts
ts = events2.loc[:,'event_type_1']
events2.loc[:,'event_type_2'] = ts
df = df.append(events2, ignore_index=True)
 df = df.drop(['event_name_2', 'event_type_2'], axis='columns')
 df_month = df.loc[df['month'].isin(months), :]
df_events_test = df.loc[df['event_name_1'].isin(events_test), :]
df_events = df.loc[df['event_name_1'].notna() , :]
for i in range(rep_month):
df = df.append(df_month,ignore_index=True)
 for i in range(rep_event):
df = df.append(df_events,ignore_index=True)
 for i in range(rep_event_test):
df = df.append(df_events_test,ignore_index=True)
 df[['event_name_1', 'event_type_1']] = df[['event_name_1', 'event_type_1']].astype('category')
 return df
%matplotlib inline
%precision 4<load_from_csv>
def addTrendFeatures(df):
df.loc[df['Lag1_rmean7_id'] > 0, 'Lag1_rmean1_id_div_Lag1_rmean7_id'] = df['Lag1_rmean1_id']/df['Lag1_rmean7_id']
df.loc[(df['Lag1_rmean7_id'] == 0)&(df['Lag1_rmean1_id'] > 0), 'Lag1_rmean1_id_div_Lag1_rmean7_id'] = df['Lag1_rmean1_id']
df.loc[(df['Lag1_rmean7_id'] == 0)&(df['Lag1_rmean1_id'] == 0), 'Lag1_rmean1_id_div_Lag1_rmean7_id'] = -1
df.loc[df['Lag1_rmean28_id'] > 0, 'Lag1_rmean7_id_div_Lag1_rmean28_id'] = df['Lag1_rmean7_id']/df['Lag1_rmean28_id']
df.loc[(df['Lag1_rmean28_id'] == 0)&(df['Lag1_rmean7_id'] > 0), 'Lag1_rmean7_id_div_Lag1_rmean28_id'] = df['Lag1_rmean7_id']
df.loc[(df['Lag1_rmean28_id'] == 0)&(df['Lag1_rmean7_id'] == 0), 'Lag1_rmean7_id_div_Lag1_rmean28_id'] = -1
df['Lag1_rmean1_id_div_Lag1_rmean7_id'] = df['Lag1_rmean1_id_div_Lag1_rmean7_id'].astype('float32')
 df['Lag1_rmean7_id_div_Lag1_rmean28_id'] = df['Lag1_rmean7_id_div_Lag1_rmean28_id'].astype('float32')
 print(df.head())
 return df
def addDateTypeAvg(df, cols):
print(f'Calculating item average over {cols} grouping')
 df[f'{cols}_Avg'] = df.groupby(cols)['saleCount'].transform(lambda x : np.round(x.mean() ,2))
return df
features = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/features.csv.zip')
 train = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/train.csv.zip')
 stores = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/stores.csv')
 test = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/test.csv.zip')
 holidays = pd.read_csv('.. /input/holidays/holidays.csv' )<data_type_conversions>
best_cols = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id',
'wday', 'month', 'year', 'event_name_1', 'event_type_1',
'snap_CA', 'snap_TX', 'snap_WI', 'sell_price',
'available',
'Lag1_rmean1_id', 'Lag7_rmean1_id', 'Lag28_rmean1_id',
'Lag1_rmean7_id', 'Lag7_rmean7_id', 'Lag28_rmean7_id',
'Lag1_rmean28_id', 'Lag28_rmean28_id',
"id_wday_Avg", "id_month_Avg",
"item_id_wday_Avg", "item_id_month_Avg"]
cols_drop = ['event_name_2', 'event_type_2','available', 'cat_id','state_id', 'dept_id', 'year']
X_train = X_train.drop(cols_drop, axis='columns')
 X_train.info()
holidays["Date"]=pd.to_datetime(holidays["Date"])
 features["IsHoliday"] = np.where(features["IsHoliday"]==True,1,0)
 features["Date"] = pd.to_datetime(features["Date"])
 train["Date"] = pd.to_datetime(train["Date"])
 test["Date"] = pd.to_datetime(test["Date"] )<merge>
cols_ignore = ['id', 'd', 'saleCount', 'type',
'event_name_2', 'event_type_2', 'available']
col_feature = [i
for i in X_train.columns
if i not in cols_ignore]
mask_val =(X_train['d'].between(1913 - 365, 1913-365 +28)) |(X_train['d'].between(1913-28, 1913))
mask_train = X_train['type']=='train'
print('Feature columns:
',col_feature)
 cat_cols = X_train[col_feature].select_dtypes(include=['category'] ).columns
for c in X_train.select_dtypes(include=['category'] ).columns:
if c not in ['id','type']:
X_train.loc[:,c] = X_train.loc[:,c].cat.codes
X_train.info()
train["category"] = "train"
test["category"] = "test"
data_join = pd.concat([train,test])
 data_join = pd.merge(data_join, features, on=["Store","Date", "IsHoliday"], how= "left")
 data_join = pd.merge(data_join, stores, on="Store", how= "inner")
 data_join = pd.merge(data_join, holidays, on="Date", how= "left")
 data_join["IsHoliday"] = data_join["IsHoliday"].astype(int)
 data_join.shape<feature_engineering>
X_train[(X_train['id'] == 'FOODS_3_318_CA_1_validation')& mask_train]
data_join["YEAR"] = data_join["Date"].dt.year
data_join["MONTH"] = data_join["Date"].dt.month
data_join["WEEK"] = data_join["Date"].dt.week
data_join<merge>
def repNAN(df, cols, val):
print('Setting NAN values to {}'.format(val))
for c in cols:
print(f'Col.{c} contains {df[c].isna().sum() } NAN')
 df.loc[df[c].isna() , c] = val
return df
cols_rep = ['Lag1_rmean1_id', 'Lag7_rmean1_id', 'Lag28_rmean1_id',
'Lag1_rmean7_id', 'Lag7_rmean7_id', 'Lag28_rmean7_id',
'Lag1_rmean28_id', 'Lag28_rmean28_id']
X_train = repNAN(X_train, ['sell_price'], -1)
 X_train = repNAN(X_train, ['saleCount'], 0)
data_sales = data_join[["Dept","Store","YEAR","WEEK","Weekly_Sales"]]
data_sales = data_sales.rename(columns={"Weekly_Sales":"Weekly_Sale_before"})
 data_sales["YEAR"] = data_sales["YEAR"] + 1
data_join2 = pd.merge(data_join, data_sales, on=["Dept","Store","YEAR","WEEK"], how="left")
 data_join2<feature_engineering>
def run_nn() :
n_feat = X_train[col_feature].shape[1]
n_items = X_train['id'].nunique()
print(n_feat, n_items)
 model = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape=(n_feat,)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(n_feat*3, activation='sigmoid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(n_feat*2, activation='sigmoid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(n_feat*1, activation='sigmoid'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(1, activation=None)])
 model.summary()
model.compile(optimizer="Adam", loss="mse")
 callback = tf.keras.callbacks.EarlyStopping(monitor='mse', mode='auto',min_delta=0.01, patience=10)
 t0 = time.time()
history = model.fit(X_train.loc[mask_train, col_feature],
X_train.loc[mask_train, 'saleCount'],
batch_size=n_items,
epochs=30)
 plt.figure()
plt.plot(history.history['loss'],label='Training')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend()
plt.show()
return model
data_join2["FLAG_D72&W47"] = np.where(( data_join2["Dept"]==72)&(data_join2["WEEK"]==47),1,0)
 data_join2[data_join2["FLAG_D72&W47"] == 1]<count_missing_values>
def run_xgboost(max_depth=6,n_estimators=1000,min_child_weight=1, learning_rate=0.01, gamma=0, subsample=1,colsample_bytree = 1):
model = xgb.XGBRegressor(
 learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth = max_depth,
min_child_weight = min_child_weight,
gamma = gamma,
subsample=subsample,
colsample_bytree = colsample_bytree,
random_state =1,
verbosity=2, 
)
 model.fit(X_train.loc[mask_train, col_feature],
X_train.loc[mask_train, 'saleCount'],
verbose=True)
 return model
data_join2.isnull().sum()<feature_engineering>
def run_rf(n_range=1):
score_train = []
score_val = []
train_time = []
for c in X_train.select_dtypes(include=['category'] ).columns:
if c not in ['id','type']:
X_train.loc[:,c] = X_train.loc[:,c].cat.codes
print(f'training with trees {n_range}')
 for n in n_range:
model = RandomForestRegressor(n_estimators = n,
max_depth = None,
random_state=1,
n_jobs = -1,
bootstrap = True,
max_features='sqrt',
min_samples_leaf = 10,
verbose=10)
 t0 = time.time()
print('
',f'Training RF with n={n} initiated at {time.ctime(t0)}')
 model.fit(X_train.loc[~mask_val & mask_train, col_feature], X_train.loc[~mask_val & mask_train, 'saleCount'])
 dt =(time.time() - t0)/60
train_time.append(dt)
 print('Tree depths: ', [estimator.get_depth() for estimator in model.estimators_])
 score_train.append(model.score(X_train.loc[~mask_val & mask_train, col_feature], X_train.loc[~mask_val & mask_train, 'saleCount']))
score_val.append(model.score(X_train.loc[mask_val & mask_train, col_feature], X_train.loc[mask_val & mask_train, 'saleCount']))
for i,(t, v)in enumerate(zip(score_train, score_val)) :
print(f'{n_range[i]} trees(training, validation):({np.round(t,2)}, {np.round(v,2)}), {np.round(train_time[i],3)}')
 if n != n_range[-1]:
del model
gc.collect()
return model
data_join2["Weekly_Sale_before"] = data_join2["Weekly_Sale_before"].fillna(0)
 data_join2["Weekly_Sale_before"] = np.where(data_join2["Weekly_Sale_before"]==0,data_join2["Weekly_Sales"],data_join2["Weekly_Sale_before"])
 data_join2<drop_column>
def run_cb_val() :
model = CatBoostRegressor(iterations=2000,
learning_rate=0.05,
depth=16,
random_strength = 0,
l2_leaf_reg = 3,
min_data_in_leaf = 1,
boosting_type='Plain',
border_count=256,
task_type='GPU',
random_state=1,
one_hot_max_size = 1)
 cat_cols = X_train[col_feature].select_dtypes(include=['category'] ).columns
print('Converting categorical datatype to object')
 print(X_train[col_feature].columns)
 cat_idx = []
model.fit(X_train.loc[~mask_val & mask_train, col_feature],
X_train.loc[~mask_val & mask_train, 'saleCount'],
eval_set =(X_train.loc[mask_val & mask_train, col_feature],
X_train.loc[mask_val & mask_train, 'saleCount']),
early_stopping_rounds = 10,
cat_features = cat_idx, plot=True)
 return model
def run_cb_train() :
model = CatBoostRegressor(iterations=1000,
learning_rate=0.05,
depth=16,
random_strength = 0,
l2_leaf_reg = 5,
min_data_in_leaf = 1,
boosting_type='Plain',
border_count=512,
task_type='GPU',
logging_level='Verbose',
random_state=1,
one_hot_max_size = 1 
)
 print('Fitting model')
 model.fit(X_train.loc[mask_train, col_feature],
X_train.loc[mask_train, 'saleCount'])
 return model
data_join2.drop(columns=["Temperature","Fuel_Price","MarkDown1","MarkDown2","MarkDown3","MarkDown4","MarkDown5","CPI","Unemployment"],axis=1,inplace=True)
 data_join2<feature_engineering>
def addLaggedRollingAvg(df, grp_cols, val_col, windows, lag):
for w in windows:
for l in lag:
col_name = f'Lag{l}_rmean{w}_{grp_cols}'
df[col_name] = df.groupby(grp_cols)[val_col] \ 
.transform(lambda x : np.round(x.shift(l ).rolling(w, min_periods=1 ).mean() ,2))
return df
X_test = X_train.loc[X_train['d'] >= 1913-(28+1), :]
d_min_val = X_train.loc[X_train['type'] == 'val', 'd'].min()
d_max_val = X_train.loc[X_train['type'] == 'val', 'd'].max()
X_test.loc[X_test['id'] == 'FOODS_3_318_CA_3_validation', :].head(100)
data_join2.loc[(data_join2.YEAR==2010)&(data_join2.WEEK==13), 'IsHoliday'] = True
data_join2.loc[(data_join2.YEAR==2011)&(data_join2.WEEK==16), 'IsHoliday'] = True
data_join2.loc[(data_join2.YEAR==2012)&(data_join2.WEEK==14), 'IsHoliday'] = True
data_join2.loc[(data_join2.YEAR==2013)&(data_join2.WEEK==13), 'IsHoliday'] = True<feature_engineering>
for d in range(d_min_val,d_max_val+1):
print(f'Day {d}')
 X_test = addLaggedRollingAvg(X_test, grp_cols='id', val_col = 'saleCount', windows=[1], lag = [1,7])
 X_test = addLaggedRollingAvg(X_test, grp_cols='id', val_col = 'saleCount', windows=[7], lag = [1, 7])
 X_test = addLaggedRollingAvg(X_test, grp_cols='id', val_col = 'saleCount', windows=[28], lag = [1])
 y_pred = model.predict(X_test.loc[X_test['d'] == d, col_feature])
 print('Negative sale predictions: ', sum(y_pred < 0))
y_pred[y_pred < 0] = 0
X_test.loc[X_test['d'] == d, 'saleCount'] = y_pred
X_test.loc[(X_test['id'] == 'FOODS_3_318_CA_3_validation')&(X_test['type'] == 'val'), :].head(30 )
<import_modules>
X_test[(X_test['d'] == d)&(X_test['saleCount'] < 0)]['d'].count()
import lightgbm as lgb 
<prepare_x_and_y>
print('Generating forecast')
 X_forecast = X_test.loc[X_test['type']=='val',['id', 'd', 'saleCount']].copy()
X_forecast.loc[(X_forecast['saleCount'] < 0), 'saleCount'] = 0
X_forecast.loc[:,'saleCount'] = np.round(X_forecast.loc[:,'saleCount'],3)
 days = X_forecast['d'].unique()
n_days = len(days)
 day_min = min(days)
 print(days)
 assert n_days == 28, 'Wrong number of forecastet days'
day_1 = days.min()
X_forecast['d'] = 'F'+ X_forecast['d'].add(- day_min + 1 ).astype('str')
 print('Done' )
<define_variables><EOS> 
df_sub = pd.read_csv('.. /input/m5-forecasting-accuracy/sample_submission.csv')
 fcols = [f for f in df_sub.columns if 'F' in f]
for f in fcols:
X_forecast_f = X_forecast[X_forecast['d'] == f]
df_sub[f] = df_sub.merge(X_forecast_f,
how = 'right',
left_on = 'id',
right_on = 'id')['saleCount']
df_sub.fillna(0)
 df_sub.to_csv('submission.csv', index=False, na_rep=0)
 print('Done' )