Kaggle competition: M5 Forecasting - Accuracy
standardScalarX = StandardScaler().fit(X)
 X = standardScalarX.transform(X )<normalization>
warnings.filterwarnings(action='once' )
standardScalarY = StandardScaler().fit(y)
 y = standardScalarY.transform(y )<choose_model_class>
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv', parse_dates=['date'], index_col='date')
 calendar.head(5 )
model = Sequential()
model.add(Dense(128, activation = 'relu',input_shape=(X.shape[1],)))
 model.add(Dense(128, activation = 'relu'))
model.add(Dense(64, activation = 'relu'))
model.add(Dense(64, activation = 'relu'))
model.add(Dense(1))<choose_model_class>
calendar.fillna('Regular', inplace=True)
 label_encoder = LabelEncoder()
label_cols = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for col in label_cols:
calendar[col] = label_encoder.fit_transform(calendar[col] )
model.compile(optimizer=Adam(lr=0.00001),
metrics=['mean_squared_error'],
loss='mean_squared_error' )<train_model>
calendar['is_weekend'] = calendar['wday'].apply(lambda x: 1 if x == 1 or x == 2 else 0)
 seasons = {1: 1, 2: 1, 12: 1, 3: 2, 4: 2, 5: 2, 6: 3, 7: 3, 8: 3, 9: 4, 10: 4, 11: 4 }
calendar['season'] = calendar['month'].apply(lambda x: seasons[x])
 calendar.head()
history = model.fit(X, y,
batch_size=32,
validation_split=0.2,
epochs=50,
callbacks=callbacks_list )<save_model>
train_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 train_data.head()
model.save('model.hdf5' )<load_from_csv>
prices = pd.read_csv('.. /input/m5-forecasting-accuracy/sell_prices.csv')
 prices['wm_yr_wk'] = prices['wm_yr_wk'].astype(np.int16)
 prices['sell_price'] = prices['sell_price'].astype(np.float32)
 prices['wm_yr_wk'] = prices['wm_yr_wk'].astype(np.int16)
 prices.set_index(['store_id', 'item_id', 'wm_yr_wk'], inplace=True)
 prices = prices.sort_index()
prices
test_df = pd.read_csv(base_path+"test.csv" )<feature_engineering>
def get_dataset_simple(df, label_column):
test_df = df.loc['d_1914':,:].copy()
valid_models_df = df.loc['d_1914': 'd_1941',:].copy()
_df = df.loc[df.index[0]: 'd_1941',:].copy()
samples_idx = random.sample(range(_df.shape[0]), int(0.2 * _df.shape[0]))
valid_df = _df.iloc[samples_idx]
idx_train = list(set(list(range(0, item_df.shape[0])))- set(samples_idx))
train_df = df.iloc[idx_train].copy()
return train_df, valid_df, test_df, valid_models_df
test_df['pickup_datetime'] = test_df['pickup_datetime'].apply(strtodatetime )<feature_engineering>
evaluation_data = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 evaluation_data.loc[:,'id'] = evaluation_data['id'].apply(lambda x: x[:-10] + 'validation')
 evaluation_data.set_index('id', inplace=True)
 evaluation_data = evaluation_data.loc[:,'d_1914': 'd_1941'].copy()
def get_rmse(submission_validation_boost):
sub = submission_validation_boost.set_index('id')
 error = mean_squared_error(sub.values, evaluation_data.loc[sub.index,:].values)
 return error
test_df['day_of_week'] = test_df['pickup_datetime'].apply(dayofweek )<feature_engineering>
def apply_target_encoder(item_df, cat_features_to_encoding):
target_enc = ce.TargetEncoder(cols=cat_features_to_encoding)
 target_enc.fit(item_df.loc[:'d_1913'][cat_features_to_encoding], item_df.loc[:'d_1913']['sales'])
 return item_df.join(target_enc.transform(item_df[cat_features_to_encoding] ).add_suffix('_target'))
test_df['sin_day'] ,test_df['cos_day'] = zip(*test_df['day_of_week'].map(dayofweektosincosday))<feature_engineering>
def create_sales_features(item_df, horizon=7):
lag = horizon // 7
item_df[f'sales_shift_{horizon}'] = item_df['sales'].shift(horizon)
 item_df[f'sales_shift_{horizon}_shift_{horizon}'] = item_df['sales'].shift(horizon * 2)
 item_df['sales_mean_rolling_4_wday'] = item_df.groupby(['wday'])['sales'].transform(lambda x: x.rolling(4 ).mean())
 item_df[f'sales_mean_rolling_4_wday_shift_{lag}'] = item_df.groupby(['wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(lag))
item_df[f'sales_mean_rolling_shift_{lag}_4_wday_shift_{lag}'] = item_df.groupby(['wday'])['sales_mean_rolling_4_wday'].transform(lambda x: x.shift(lag * 2))
item_df[f'sales_diff_rolling_shift_{lag}_4_wday_shift_{lag}'] = item_df[f'sales_mean_rolling_4_wday_shift_{lag}'] - item_df[f'sales_mean_rolling_shift_{lag}_4_wday_shift_{lag}']
test_df['dist'] = test_df[['pickup_latitude',
'pickup_longitude',
'dropoff_latitude',
'dropoff_longitude']].apply(lambda x:dist(x), axis=1 )<categorify>
def create_price_features(item_df, horizon=7):
lag = horizon // 7
item_df['sell_price_diff_shift_1'] = item_df['sell_price'] - item_df['sell_price'].shift(1)
 item_df[f'sell_price_diff_shift_{horizon}'] = item_df['sell_price'] - item_df['sell_price'].shift(horizon)
 item_df['sell_price_diff_rolling_7'] = item_df['sell_price'] - item_df['sell_price'].rolling(7 ).mean()
item_df[f'sell_price_diff_shift_{horizon}_shift_1'] = item_df['sell_price'].shift(horizon)- item_df['sell_price'].shift(horizon + 1)
 item_df[f'sell_price_diff_shift_{horizon}_shift_{horizon}'] = item_df['sell_price'].shift(horizon)- item_df['sell_price'].shift(horizon * 2)
 item_df[f'sell_price_diff_shift_{horizon}_rolling_7'] = item_df['sell_price'].shift(28)- item_df['sell_price'].shift(horizon ).rolling(7 ).mean()
item_df[f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}'] = item_df['sell_price_diff_rolling_7'] - item_df['sell_price_diff_rolling_7'].shift(horizon )
test_df['dist'].replace(to_replace=0, value=1, inplace=True )<feature_engineering>
def create_categorical_features(item_df, horizon=7):
item_df[f'month_target_diff_{horizon}'] = item_df['month_target'] - item_df['month_target'].shift(horizon)
 item_df[f'season_target_diff_{horizon}'] = item_df['season_target'] - item_df['season_target'].shift(horizon)
 item_df[f'event_name_1_target_diff_{horizon}'] = item_df['event_name_1_target'] - item_df['event_name_1_target'].shift(horizon)
 item_df[f'event_type_1_target_diff_{horizon}'] = item_df['event_type_1_target'] - item_df['event_type_1_target'].shift(horizon)
 item_df[f'event_name_2_target_diff_{horizon}'] = item_df['event_name_2_target'] - item_df['event_name_2_target'].shift(horizon)
 item_df[f'event_type_2_target_diff_{horizon}'] = item_df['event_type_2_target'] - item_df['event_type_2_target'].shift(horizon)
 item_df[f'snap_CA_target_diff_{horizon}'] = item_df['snap_CA_target'] - item_df['snap_CA_target'].shift(horizon)
 item_df[f'snap_TX_target_diff_{horizon}'] = item_df['snap_TX_target'] - item_df['snap_TX_target'].shift(horizon)
 item_df[f'snap_WI_target_diff_{horizon}'] = item_df['snap_WI_target'] - item_df['snap_WI_target'].shift(horizon)
 item_df[f'season_target_diff_{horizon}_shift{horizon}'] = item_df['season_target'].shift(horizon)- item_df['season_target'].shift(horizon)
 item_df[f'event_name_1_target_diff_{horizon}_shift{horizon}'] = item_df['event_name_1_target'].shift(horizon)- item_df['event_name_1_target'].shift(horizon * 2)
 item_df[f'event_type_1_target_diff_{horizon}_shift{horizon}'] = item_df['event_type_1_target'].shift(horizon)- item_df['event_type_1_target'].shift(horizon * 2)
 item_df[f'event_name_2_target_diff_{horizon}_shift{horizon}'] = item_df['event_name_2_target'].shift(horizon)- item_df['event_name_2_target'].shift(horizon * 2)
 item_df[f'event_type_2_target_diff_{horizon}_shift{horizon}'] = item_df['event_type_2_target'].shift(horizon)- item_df['event_type_2_target'].shift(horizon * 2)
 item_df[f'snap_CA_target_diff_{horizon}_shift{horizon}'] = item_df['snap_CA_target'].shift(horizon)- item_df['snap_CA_target'].shift(horizon * 2)
 item_df[f'snap_TX_target_diff_{horizon}_shift{horizon}'] = item_df['snap_TX_target'].shift(horizon)- item_df['snap_TX_target'].shift(horizon * 2)
 item_df[f'snap_WI_target_diff_{horizon}_shift{horizon}'] = item_df['snap_WI_target'].shift(horizon)- item_df['snap_WI_target'].shift(horizon * 2 )
test_df['dist'] = test_df['dist'].apply(math.log )<categorify>
def get_created_features(horizon=7):
lag = horizon // 7
linear_feature_columns = [
f'sales_mean_rolling_4_wday_shift_{lag}',
f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}',
f'month_target_diff_{horizon}',
f'season_target_diff_{horizon}',
f'event_name_1_target_diff_{horizon}',
f'event_type_1_target_diff_{horizon}',
f'event_name_2_target_diff_{horizon}',
f'event_type_2_target_diff_{horizon}',
f'snap_CA_target_diff_{horizon}',
f'snap_TX_target_diff_{horizon}',
f'snap_WI_target_diff_{horizon}',
]
feature_columns = [
f'sales_mean_rolling_4_wday_shift_{lag}',
f'sales_mean_rolling_shift_{lag}_4_wday_shift_{lag}',
f'sales_diff_rolling_shift_{lag}_4_wday_shift_{lag}',
'sell_price_diff_shift_1',
'sell_price_diff_rolling_7',
f'sell_price_diff_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_shift_1',
f'sell_price_diff_shift_{horizon}_shift_{horizon}',
f'sell_price_diff_shift_{horizon}_rolling_7',
f'sell_price_diff_rolling_7_diff_rolling_7_shift{horizon}',
f'month_target_diff_{horizon}',
f'season_target_diff_{horizon}',
f'event_name_1_target_diff_{horizon}',
f'event_type_1_target_diff_{horizon}',
f'event_name_2_target_diff_{horizon}',
f'event_type_2_target_diff_{horizon}',
f'snap_CA_target_diff_{horizon}',
f'snap_TX_target_diff_{horizon}',
f'snap_WI_target_diff_{horizon}',
f'season_target_diff_{horizon}_shift{horizon}',
f'event_name_1_target_diff_{horizon}_shift{horizon}',
f'event_type_1_target_diff_{horizon}_shift{horizon}',
f'event_name_2_target_diff_{horizon}_shift{horizon}',
f'event_type_2_target_diff_{horizon}_shift{horizon}',
f'snap_CA_target_diff_{horizon}_shift{horizon}',
f'snap_TX_target_diff_{horizon}_shift{horizon}',
f'snap_WI_target_diff_{horizon}_shift{horizon}',
]
return feature_columns, linear_feature_columns
test_df['pickup_sin_sec'] ,test_df['pickup_cos_sec'] = zip(*test_df['pickup_datetime'].map(timetosectosincosday))<drop_column>
%%time
idx_feature = ['id']
categorical_feature = [
'wday',
'month',
'year',
'event_name_1',
'event_type_1',
'snap_CA',
'snap_TX',
'snap_WI',
'event_name_2',
'event_type_2',
'is_weekend',
'season',
]
cat_features_to_encoding = [
'wday',
'month',
'year',
'event_name_1',
'event_type_1',
'snap_CA',
'snap_TX',
'snap_WI',
'event_name_2',
'event_type_2',
'is_weekend',
'season',
]
encoded_cat_features = [i + '_target' for i in cat_features_to_encoding]
label_column = 'sales'
sub_columns = ['id'] + ['F%s' % i for i in range(1, 29)]
submission_validation = pd.DataFrame(columns=sub_columns)
 submission_evaluation = pd.DataFrame(columns=sub_columns)
 submission_validation_lgb = pd.DataFrame(columns=sub_columns)
 submission_evaluation_lgb = pd.DataFrame(columns=sub_columns)
 submission_validation_sgd = pd.DataFrame(columns=sub_columns)
 submission_evaluation_sgd = pd.DataFrame(columns=sub_columns)
 submission_validation_linear = pd.DataFrame(columns=sub_columns)
 submission_evaluation_linear = pd.DataFrame(columns=sub_columns)
 submission_validation_mean = pd.DataFrame(columns=sub_columns)
 submission_evaluation_mean = pd.DataFrame(columns=sub_columns)
 random.seed(2)
 samples_idx = random.sample(range(train_data.shape[0]), 300)
 models = {'linear': 0, 'lgb': 0, 'mean': 0, 'linearSGD': 0}
feature_importance_init =False
feature_importance = None
feature_importance_n = 0
for iteration, i in enumerate(range(train_data.shape[0])) :
if iteration % 1000 == 1:
print(iteration, train_data.shape[0])
 print('mean', get_rmse(submission_validation_mean))
print('linear', get_rmse(submission_validation_linear))
print('sgd', get_rmse(submission_validation_sgd))
print('lgb', get_rmse(submission_validation_lgb))
print('pred', get_rmse(submission_validation))
row = train_data.loc[i]
all_id = row.id
item_id = row.item_id
dept_id = row.dept_id
cat_id = row.cat_id
store_id = row.store_id
state_id = row.state_id
sales = row['d_1':]
item_df = calendar.join(sales.to_frame('sales'), on='d')
 item_df = item_df.join(prices.loc[store_id, item_id], on=['wm_yr_wk'])
 item_df.sales.fillna(0, inplace=True)
 item_df.loc[:,'sales'] = item_df.sales.astype('int64')
 item_df = item_df.set_index('d')
 first_sale = item_df[item_df.sales!=0].index[0]
first_sale_int = int(first_sale[2:])
 if first_sale_int > 1914 - 90:
first_sale = 'd_{}'.format(1914 - 90)
 item_df = item_df.loc[first_sale:,:]
item_df = apply_target_encoder(item_df, cat_features_to_encoding)
 feature_columns = []
linear_feature_columns = []
for horizon in [28,]:
lag = horizon // 7
create_price_features(item_df, horizon=horizon)
 create_categorical_features(item_df, horizon=horizon)
 create_sales_features(item_df, horizon=horizon)
 features, linear_features = get_created_features(horizon=horizon)
 feature_columns += features
linear_feature_columns = linear_features
feature_columns = feature_columns + encoded_cat_features + categorical_feature
item_df.dropna(inplace=True)
 predictions_list = []
try:
train_df, valid_df, test_df, valid_models_df = get_dataset_simple(item_df[feature_columns + [label_column]], label_column=label_column)
 sc_X = StandardScaler()
sc_y = StandardScaler()
X_train = sc_X.fit_transform(train_df[linear_feature_columns])
 y_train = sc_y.fit_transform(train_df[label_column].values.reshape(-1,1))
X_test = sc_X.fit_transform(test_df[linear_feature_columns])
 X_valid = sc_X.fit_transform(valid_df[linear_feature_columns])
 y_valid = sc_y.fit_transform(valid_df[label_column].values.reshape(-1,1))
model_sgd = SGDRegressor(max_iter=3000)
 model_sgd.fit(X_train, y_train.reshape(-1))
prediction_sgd = sc_y.inverse_transform(model_sgd.predict(X_test ).reshape(-1,1)).reshape(-1 ).tolist()
predictions_list.append(['linearSGD', prediction_sgd, mean_squared_error(valid_models_df.sales.values, prediction_sgd[0:28])])
 model = LinearRegression()
model.fit(X_train, y_train.reshape(-1))
prediction_linear = sc_y.inverse_transform(model.predict(X_test ).reshape(-1,1)).reshape(-1 ).tolist()
predictions_list.append(['linear', prediction_linear, mean_squared_error(valid_models_df.sales.values, prediction_linear[0:28])])
 dtrain = lgb.Dataset(train_df[feature_columns], label=train_df[label_column], categorical_feature=categorical_feature)
 dvalid = lgb.Dataset(valid_df[feature_columns], label=valid_df[label_column], categorical_feature=categorical_feature)
 param = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 64,
'min_data_in_leaf': min(1024, test_df.shape[0] // 4),
'feature_fraction': 0.1,
'boost_from_average': False,
'verbose': -1,
'lambda_l1': 0.8,
}
bst = lgb.train(param, dtrain, valid_sets=[dvalid], num_boost_round = 2400, early_stopping_rounds=500, verbose_eval=False, categorical_feature=categorical_feature)
 if not feature_importance_init:
feature_importance_init = True
feature_importance = bst.feature_importance()
else:
feature_importance += bst.feature_importance()
feature_importance_n += 1
prediction_lgb = bst.predict(test_df[feature_columns] ).tolist()
predictions_list.append(['lgb', prediction_lgb, mean_squared_error(valid_models_df.sales.values, prediction_lgb[0:28])])
 except Exception as e:
print(i, all_id, item_df.shape)
 print(e)
 prediction_lgb = test_df[f'sales_mean_rolling_4_wday_shift_{lag}'].values.tolist()
prediction_mean =(test_df[f'sales_mean_rolling_4_wday_shift_{lag}'] ).values.tolist()
predictions_list.append(['mean', prediction_mean, mean_squared_error(valid_models_df.sales.values, prediction_mean[0:28])])
 best_predictions = sorted(predictions_list, key=lambda x: x[2])[0]
models[best_predictions[0]] += 1
key = all_id[:-10] + 'validation'
submission_validation.loc[len(submission_validation)] = [key] + best_predictions[1][0:28]
submission_evaluation.loc[len(submission_evaluation)] = [all_id] + best_predictions[1][28:56]
submission_validation_lgb.loc[len(submission_validation_lgb)] = [key] + prediction_lgb[0:28]
submission_evaluation_lgb.loc[len(submission_evaluation_lgb)] = [all_id] + prediction_lgb[28:56]
submission_validation_sgd.loc[len(submission_validation_sgd)] = [key] + prediction_sgd[0:28]
submission_evaluation_sgd.loc[len(submission_evaluation_sgd)] = [all_id] + prediction_sgd[28:56]
submission_validation_linear.loc[len(submission_validation_linear)] = [key] + prediction_linear[0:28]
submission_evaluation_linear.loc[len(submission_evaluation_linear)] = [all_id] + prediction_linear[28:56]
submission_validation_mean.loc[len(submission_validation_mean)] = [key] + prediction_mean[0:28]
submission_evaluation_mean.loc[len(submission_evaluation_mean)] = [all_id] + prediction_mean[28:56]
print('Final')
 print('mean', get_rmse(submission_validation_mean))
print('linear', get_rmse(submission_validation_linear))
print('sgd', get_rmse(submission_validation_sgd))
print('lgb', get_rmse(submission_validation_lgb))
print('pred', get_rmse(submission_validation))
print(models )
test_df_processed = test_df[['vendor_id',
'passenger_count',
'pickup_sin_sec',
'pickup_cos_sec',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'sin_day',
'cos_day',
'dist']]<normalization>
submission = submission_validation.append(submission_evaluation)
 submission.to_csv('/kaggle/working/my_submission.csv', index=False )
test_data = standardScalarX.transform(test_df_processed )<predict_on_test>
submission_mean = submission_validation_mean.append(submission_evaluation_mean)
 submission_mean.to_csv('/kaggle/working/my_submission_mean.csv', index=False )
y_pred = model.predict(test_data )<normalization>
submission_lgb = submission_validation_lgb.append(submission_evaluation_lgb)
 submission_lgb.to_csv('/kaggle/working/my_submission_lgb.csv', index=False )
y_pred =standardScalarY.inverse_transform(y_pred)
 y_pred = np.exp(y_pred )<define_variables>
submission_linear = submission_validation_linear.append(submission_evaluation_linear)
 submission_linear.to_csv('/kaggle/working/my_submission_linear.csv', index=False )
<create_dataframe><EOS> 
submission_sgd = submission_validation_sgd.append(submission_evaluation_sgd)
 submission_sgd.to_csv('/kaggle/working/my_submission_sgd.csv', index=False )