Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<prepare_x_and_y>
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 warnings.filterwarnings('ignore' )
train_df = X.sample(frac=0.6,random_state=200)
 evaluate_df = X.drop(train_df.index)
 X_train = train_df.drop(['Id','SalePrice'], axis=1)
 Y_train = train_df["SalePrice"]
X_train_full = X.drop(['Id','SalePrice'], axis=1)
 Y_train_full = X["SalePrice"]
X_eval = evaluate_df.drop(['Id','SalePrice'], axis=1)
 Y_eval = evaluate_df["SalePrice"]
X_test = test_df.drop('Id', axis=1)
 len(X_train_full.columns )<choose_model_class>
INPUT_DIR_PATH = '.. /input/m5-forecasting-accuracy/'
grad_boost = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=1.5, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 grad_boost_full = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=1.5, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 num_feat_to_drop = 1
dropped_features = []
i = 0
best_acc = 1
while len(X_train.columns)> 5:
grad_boost.fit(X_train, Y_train)
 grad_boost_full.fit(X_train_full, Y_train_full)
 acc_train = np.sqrt(mean_squared_log_error(Y_train, grad_boost.predict(X_train)))
 acc_eval = np.sqrt(mean_squared_log_error(Y_eval, grad_boost.predict(X_eval)))
 acc_full = np.sqrt(mean_squared_log_error(Y_train_full, grad_boost_full.predict(X_train_full)))
 avg_acc =(acc_train + acc_eval + acc_full)/ 3
if(avg_acc >= best_acc):
print(f'{i}: Train = {acc_train} Evaluate = {acc_eval} Full = {acc_full} Average = {avg_acc}')
 else:
best_acc = avg_acc
print(f'{i}: Train = {acc_train} Evaluate = {acc_eval} Full = {acc_full} Average = {avg_acc} - NEW BEST!!!')
 print(f' Dropped: {dropped_features}')
 importances = pd.DataFrame({'feature':X_train.columns,'importance':np.round(grad_boost.feature_importances_,3)})
 importances = importances.sort_values('importance',ascending=True ).set_index('feature')
 features_to_drop = importances[0:num_feat_to_drop].index.tolist()
if len(features_to_drop)<= 0:
break
X_train.drop(features_to_drop, axis=1, inplace=True)
 X_eval.drop(features_to_drop, axis=1, inplace=True)
 X_train_full.drop(features_to_drop, axis=1, inplace=True)
 dropped_features.append(features_to_drop)
 i += num_feat_to_drop<define_variables>
sell_prices_df, calendar_df, sales_train_validation_df, submission_df = read_data()
best_dropped = [['LotConfig_4'], ['Foundation_3'], ['Neighborhood_3'], ['BldgType_2'], ['Neighborhood_21'], ['Neighborhood_12'], ['Neighborhood_8'], ['HouseStyle_8'], ['BldgType_3'], ['HouseStyle_2'], ['Neighborhood_10']]
best_dropped = [x for inner in best_dropped for x in inner]
best_dropped<prepare_x_and_y>
NUM_ITEMS = sales_train_validation_df.shape[0]
DAYS_PRED = 28
nrows = 365 * 2 * NUM_ITEMS
X = pd.read_csv("/kaggle/working/new_train_1h.csv")
 test_df = pd.read_csv("/kaggle/working/new_test_1h.csv")
 train_df = X.sample(frac=0.6,random_state=200)
 evaluate_df = X.drop(train_df.index)
 X_train = train_df.drop(['Id','SalePrice'], axis=1)
 Y_train = train_df["SalePrice"]
X_train_full = X.drop(['Id','SalePrice'], axis=1)
 Y_train_full = X["SalePrice"]
X_eval = evaluate_df.drop(['Id','SalePrice'], axis=1)
 Y_eval = evaluate_df["SalePrice"]
X_test = test_df.drop('Id', axis=1 )<count_missing_values>
def encode_categorical(df, cols):
for col in cols:
le = preprocessing.LabelEncoder()
not_null = df[col][df[col].notnull() ]
df[col] = pd.Series(le.fit_transform(not_null), index=not_null.index)
 return df
calendar_df = encode_categorical(calendar_df, ["event_name_1", "event_type_1", "event_name_2", "event_type_2"] ).pipe(reduce_mem_usage)
 sales_train_validation_df = encode_categorical(sales_train_validation_df, ["item_id", "dept_id", "cat_id", "store_id", "state_id"] ).pipe(reduce_mem_usage)
 sell_prices_df = encode_categorical(sell_prices_df, ["item_id", "store_id"] ).pipe(reduce_mem_usage )
for(columnName, columnData)in X_test.iteritems() :
print(f'{columnName}: {X_test[columnName].isna().sum() }' )<drop_column>
def melt_and_merge(calendar, sell_prices, sales_train_validation, submission, nrows = 55000000, merge = False):
sales_train_validation = pd.melt(sales_train_validation, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')
 print('Melted sales train validation has {} rows and {} columns'.format(sales_train_validation.shape[0], sales_train_validation.shape[1]))
sales_train_validation = reduce_mem_usage(sales_train_validation)
 sales_train_validation = sales_train_validation.iloc[-nrows:,:]
test1_rows = [row for row in submission['id'] if 'validation' in row]
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test1 = submission[submission['id'].isin(test1_rows)]
test2 = submission[submission['id'].isin(test2_rows)]
test1.columns = ['id', 'd_1914', 'd_1915', 'd_1916', 'd_1917', 'd_1918', 'd_1919', 'd_1920', 'd_1921', 'd_1922', 'd_1923', 'd_1924', 'd_1925', 'd_1926', 'd_1927', 'd_1928', 'd_1929', 'd_1930', 'd_1931',
'd_1932', 'd_1933', 'd_1934', 'd_1935', 'd_1936', 'd_1937', 'd_1938', 'd_1939', 'd_1940', 'd_1941']
test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959',
'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']
product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
test2['id'] = test2['id'].str.replace('_evaluation','_validation')
 test1 = test1.merge(product, how = 'left', on = 'id')
 test2 = test2.merge(product, how = 'left', on = 'id')
 test2['id'] = test2['id'].str.replace('_validation','_evaluation')
 test1 = pd.melt(test1, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')
 test2 = pd.melt(test2, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')
 sales_train_validation['part'] = 'train'
test1['part'] = 'test1'
test2['part'] = 'test2'
data = pd.concat([sales_train_validation, test1, test2], axis = 0)
 del sales_train_validation, test1, test2
print(data.shape)
 calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)
 data = data[data['part'] != 'test2']
if merge:
data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1)
 data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 print('Our final dataset to train has {} rows and {} columns'.format(data.shape[0], data.shape[1]))
else:
pass
gc.collect()
return data
drop_list = ['LotConfig_4', 'Foundation_3', 'Neighborhood_3', 'BldgType_2', 'Neighborhood_21', 'Neighborhood_12', 'Neighborhood_8', 'HouseStyle_8', 'BldgType_3', 'HouseStyle_2', 'Neighborhood_10']
X_train.drop(drop_list, axis=1,inplace=True)
 X_eval.drop(drop_list, axis=1,inplace=True)
 X_train_full.drop(drop_list, axis=1,inplace=True)
 X_test.drop(drop_list, axis=1,inplace=True )<train_model>
nrows = 27500000
data = melt_and_merge(calendar_df, sell_prices_df, sales_train_validation_df, submission_df, nrows = nrows, merge = True)
lr = linear_model.LinearRegression()
lr.fit(X_train, Y_train)
 acc_train_lr = np.sqrt(mean_squared_log_error(Y_train, lr.predict(X_train)))
 acc_eval_lr = np.sqrt(mean_squared_log_error(Y_eval, lr.predict(X_eval)))
 print(f'LR: Train = {acc_train_lr} Evaluate = {acc_eval_lr}' )<compute_train_metric>
def transform(data):
nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in nan_features:
data[feature].fillna('unknown', inplace = True)
 cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in cat:
encoder = preprocessing.LabelEncoder()
data[feature] = encoder.fit_transform(data[feature])
 return data
def simple_fe(data):
for val in [28, 29, 30]:
data[f"shift_t{val}"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(val))
for val in [7, 30, 60, 90, 180]:
data[f"rolling_std_t{val}"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(val ).std())
 for val in [7, 30, 60, 90, 180]:
data[f"rolling_mean_t{val}"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(val ).mean())
 data["rolling_skew_t30"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(30 ).skew())
 data["rolling_kurt_t30"] = data.groupby(["id"])["demand"].transform(lambda x: x.shift(28 ).rolling(30 ).kurt())
 data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))
data['price_change_t1'] =(data['lag_price_t1'] - data['sell_price'])/(data['lag_price_t1'])
 data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1 ).rolling(365 ).max())
 data['price_change_t365'] =(data['rolling_price_max_t365'] - data['sell_price'])/(data['rolling_price_max_t365'])
 data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7 ).std())
 data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30 ).std())
 data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1)
 data['date'] = pd.to_datetime(data['date'])
 attrs = ["year", "quarter", "month", "week", "day", "dayofweek", "is_year_end", "is_year_start", "is_quarter_end", \
"is_quarter_start", "is_month_end","is_month_start",
]
for attr in attrs:
dtype = np.int16 if attr == "year" else np.int8
data[attr] = getattr(data['date'].dt, attr ).astype(dtype)
 data["is_weekend"] = data["dayofweek"].isin([5, 6] ).astype(np.int8)
 return data
sgd = linear_model.SGDClassifier(random_state=1, max_iter=5, tol=None)
 sgd.fit(X_train, Y_train)
 acc_train_sgd = np.sqrt(mean_squared_log_error(Y_train, sgd.predict(X_train)))
 acc_eval_sgd = np.sqrt(mean_squared_log_error(Y_eval, sgd.predict(X_eval)))
 print(f'SGD: Train = {acc_train_sgd} Evaluate = {acc_eval_sgd}' )<compute_train_metric>
features = [
"item_id", "dept_id", "cat_id", "store_id", "state_id", "event_name_1", "event_type_1", "snap_CA", "snap_TX", \
"snap_WI", "sell_price", \
"shift_t28", "rolling_std_t7", "rolling_std_t30", "rolling_std_t90", "rolling_std_t180", \
"rolling_mean_t7", "rolling_mean_t30", "rolling_mean_t60", \
"price_change_t1", "price_change_t365", "rolling_price_std_t7",
"year", "month", "dayofweek",
] 
("wday", "month", "year",
"event_name_1", "event_type_1",
"snap_CA", "snap_TX", "snap_WI",
"sell_price", "sell_price_rel_diff", "sell_price_cumrel", "sell_price_roll_sd7",
"lag_t28", "rolling_mean_t7", "rolling_mean_t30", "rolling_mean_t60",
"rolling_mean_t90", "rolling_mean_t180", "rolling_sd_t7", "rolling_sd_t30",
"item_id", "dept_id", "cat_id", "store_id", "state_id" )
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 acc_train_rf = np.sqrt(mean_squared_log_error(Y_train, random_forest.predict(X_train)))
 acc_eval_rf = np.sqrt(mean_squared_log_error(Y_eval, random_forest.predict(X_eval)))
 print(f'RF: Train = {acc_train_rf} Evaluate = {acc_eval_rf}' )<compute_train_metric>
def run_lgb(data):
x_train = data[data['date'] <= '2016-03-27']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
y_val = x_val['demand']
test = data[(data['date'] > '2016-04-24')]
del data
gc.collect()
params = {
'metric': 'rmse',
'objective': 'poisson',
'n_jobs': -1,
'seed': 20,
'learning_rate': 0.1,
'alpha': 0.1,
'lambda': 0.1,
'bagging_fraction': 0.66,
'bagging_freq': 2,
'colsample_bytree': 0.77}
train_set = lgb.Dataset(x_train[features], y_train)
 val_set = lgb.Dataset(x_val[features], y_val)
 del x_train, y_train
model = lgb.train(params, train_set, num_boost_round = 2000, early_stopping_rounds = 200, valid_sets = [train_set, val_set], verbose_eval = 100)
 joblib.dump(model, 'lgbm_0.sav')
 val_pred = model.predict(x_val[features], num_iteration=model.best_iteration)
 val_score = np.sqrt(metrics.mean_squared_error(val_pred, y_val))
print(f'Our val rmse score is {val_score}')
 y_pred = model.predict(test[features], num_iteration=model.best_iteration)
 test['demand'] = y_pred
return test
def predict(test, submission):
predictions = test[['id', 'date', 'demand']]
predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand' ).reset_index()
predictions.columns = ['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation_rows = [row for row in submission['id'] if 'evaluation' in row]
evaluation = submission[submission['id'].isin(evaluation_rows)]
validation = submission[['id']].merge(predictions, on = 'id')
 final = pd.concat([validation, evaluation])
 final.to_csv('submission.csv', index = False)
 def transform_train_and_eval(data):
data = simple_fe(data)
 data = reduce_mem_usage(data)
 test = run_lgb(data)
 predict(test, submission_df)
poly_features = PolynomialFeatures(degree=1)
 X_train_poly = poly_features.fit_transform(X_train)
 X_eval_poly = poly_features.fit_transform(X_eval)
 poly_model = LinearRegression()
poly_model.fit(X_train_poly, Y_train)
 acc_train_pr = np.sqrt(mean_squared_log_error(Y_train, poly_model.predict(X_train_poly)))
 acc_eval_pr = np.sqrt(mean_squared_log_error(Y_eval, abs(poly_model.predict(X_eval_poly))))
print(f'PR: Train = {acc_train_pr} Evaluate = {acc_eval_pr}' )<compute_train_metric>
transform_train_and_eval(data )