Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<compute_train_metric>
warnings.filterwarnings("ignore")
 pd.set_option("display.max_columns", 500)
 pd.set_option("display.max_rows", 500)
 register_matplotlib_converters()
sns.set()
linear_svc = LinearSVC(random_state=1)
 linear_svc.fit(X_train, Y_train)
 acc_train_svm = np.sqrt(mean_squared_log_error(Y_train, linear_svc.predict(X_train)))
 acc_eval_svm = np.sqrt(mean_squared_log_error(Y_eval, linear_svc.predict(X_eval)))
 print(f'SVM: Train = {acc_train_svm} Evaluate = {acc_eval_svm}' )<compute_train_metric>
def on_kaggle() :
return "KAGGLE_KERNEL_RUN_TYPE" in os.environ
decision_tree = DecisionTreeClassifier(random_state=1)
 decision_tree.fit(X_train, Y_train)
 acc_train_dt = np.sqrt(mean_squared_log_error(Y_train, decision_tree.predict(X_train)))
 acc_eval_dt = np.sqrt(mean_squared_log_error(Y_eval, decision_tree.predict(X_eval)))
 print(f'DT: Train = {acc_train_dt} Evaluate = {acc_eval_dt}' )<compute_train_metric>
if on_kaggle() :
os.system("pip install --quiet mlflow_extend" )
random_forest_reg = RandomForestRegressor(random_state=1, n_estimators=100)
 random_forest_reg.fit(X_train, Y_train)
 acc_train_rfr = np.sqrt(mean_squared_log_error(Y_train, random_forest_reg.predict(X_train)))
 acc_eval_rfr = np.sqrt(mean_squared_log_error(Y_eval, random_forest_reg.predict(X_eval)))
 print(f'RFR: Train = {acc_train_rfr} Evaluate = {acc_eval_rfr}' )<train_model>
def reduce_mem_usage(df, verbose=False):
start_mem = df.memory_usage().sum() / 1024 ** 2
int_columns = df.select_dtypes(include=["int"] ).columns
float_columns = df.select_dtypes(include=["float"] ).columns
for col in int_columns:
df[col] = pd.to_numeric(df[col], downcast="integer")
 for col in float_columns:
df[col] = pd.to_numeric(df[col], downcast="float")
 end_mem = df.memory_usage().sum() / 1024 ** 2
if verbose:
print(
 "Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)".format(
 end_mem, 100 *(start_mem - end_mem)/ start_mem 
)
)
 return df
grad_boost = GradientBoostingRegressor(random_state=1, n_estimators=500, max_features= 0.3)
 grad_boost.fit(X_train, Y_train)
 acc_train_gb = np.sqrt(mean_squared_log_error(Y_train, grad_boost.predict(X_train)))
 acc_eval_gb = np.sqrt(mean_squared_log_error(Y_eval, grad_boost.predict(X_eval)))
 print(f'GB: Train = {acc_train_gb} Evaluate = {acc_eval_gb}' )<compute_train_metric>
def read_data() :
INPUT_DIR = "/kaggle/input" if on_kaggle() else "input"
INPUT_DIR = f"{INPUT_DIR}/m5-forecasting-accuracy"
print("Reading files...")
 calendar = pd.read_csv(f"{INPUT_DIR}/calendar.csv" ).pipe(reduce_mem_usage)
 prices = pd.read_csv(f"{INPUT_DIR}/sell_prices.csv" ).pipe(reduce_mem_usage)
 sales = pd.read_csv(f"{INPUT_DIR}/sales_train_validation.csv", ).pipe(
 reduce_mem_usage 
)
 submission = pd.read_csv(f"{INPUT_DIR}/sample_submission.csv" ).pipe(
 reduce_mem_usage 
)
 print("sales shape:", sales.shape)
 print("prices shape:", prices.shape)
 print("calendar shape:", calendar.shape)
 print("submission shape:", submission.shape)
 return sales, prices, calendar, submission
ada_boost = AdaBoostRegressor(random_state=1, n_estimators=100)
 ada_boost.fit(X_train, Y_train)
 acc_train_ab = np.sqrt(mean_squared_log_error(Y_train, ada_boost.predict(X_train)))
 acc_eval_ab = np.sqrt(mean_squared_log_error(Y_eval, ada_boost.predict(X_eval)))
 print(f'AdaBoost: Train = {acc_train_ab} Evaluate = {acc_eval_ab}' )<compute_train_metric>
sales, prices, calendar, submission = read_data()
NUM_ITEMS = sales.shape[0]
DAYS_PRED = submission.shape[1] - 1
ex_trees = ExtraTreesRegressor(random_state=1, n_estimators=100)
 ex_trees.fit(X_train, Y_train)
 acc_train_et = np.sqrt(mean_squared_log_error(Y_train, ex_trees.predict(X_train)))
 acc_eval_et = np.sqrt(mean_squared_log_error(Y_eval, ex_trees.predict(X_eval)))
 print(f'ET: Train = {acc_train_et} Evaluate = {acc_eval_et}' )<choose_model_class>
def encode_categorical(df, cols):
for col in cols:
le = LabelEncoder()
not_null = df[col][df[col].notnull() ]
df[col] = pd.Series(le.fit_transform(not_null), index=not_null.index)
 return df
calendar = encode_categorical(
 calendar, ["event_name_1", "event_type_1", "event_name_2", "event_type_2"]
).pipe(reduce_mem_usage)
 sales = encode_categorical(
 sales, ["item_id", "dept_id", "cat_id", "store_id", "state_id"],
).pipe(reduce_mem_usage)
 prices = encode_categorical(prices, ["item_id", "store_id"] ).pipe(reduce_mem_usage )
reg1 = GradientBoostingRegressor(random_state=1, n_estimators=100)
 reg2 = RandomForestRegressor(random_state=1, n_estimators=100)
 reg3 = ExtraTreesRegressor(random_state=1, n_estimators=100)
 reg4 = AdaBoostRegressor(random_state=1, n_estimators=100)
 reg5 = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 ereg = VotingRegressor(estimators=[('gb', reg1),('rf', reg2),('et', reg3),('ad', reg4),('xg', reg5)])
 ereg = ereg.fit(X_train, Y_train)
 acc_train_vr = np.sqrt(mean_squared_log_error(Y_train, ereg.predict(X_train)))
 acc_eval_vr = np.sqrt(mean_squared_log_error(Y_eval, ereg.predict(X_eval)))
 print(f'VR: Train = {acc_train_vr} Evaluate = {acc_eval_vr}' )<choose_model_class>
def extract_num(ser):
return ser.str.extract(r"(\d+)" ).astype(np.int16)
 def reshape_sales(sales, submission, d_thresh=0, verbose=True):
id_columns = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]
product = sales[id_columns]
sales = sales.melt(id_vars=id_columns, var_name="d", value_name="demand",)
 sales = reduce_mem_usage(sales)
 vals = submission[submission["id"].str.endswith("validation")]
evals = submission[submission["id"].str.endswith("evaluation")]
vals.columns = ["id"] + [f"d_{d}" for d in range(1914, 1914 + DAYS_PRED)]
evals.columns = ["id"] + [f"d_{d}" for d in range(1942, 1942 + DAYS_PRED)]
evals["id"] = evals["id"].str.replace("_evaluation", "_validation")
 vals = vals.merge(product, how="left", on="id")
 evals = evals.merge(product, how="left", on="id")
 evals["id"] = evals["id"].str.replace("_validation", "_evaluation")
 if verbose:
print("validation")
 display(vals)
 print("evaluation")
 display(evals)
 vals = vals.melt(id_vars=id_columns, var_name="d", value_name="demand")
 evals = evals.melt(id_vars=id_columns, var_name="d", value_name="demand")
 sales["part"] = "train"
vals["part"] = "validation"
evals["part"] = "evaluation"
data = pd.concat([sales, vals, evals], axis=0)
 del sales, vals, evals
data["d"] = extract_num(data["d"])
 data = data[data["d"] >= d_thresh]
data = data[data["part"] != "evaluation"]
gc.collect()
if verbose:
print("data")
 display(data)
 return data
def merge_calendar(data, calendar):
calendar = calendar.drop(["weekday", "wday", "month", "year"], axis=1)
 return data.merge(calendar, how="left", on="d")
 def merge_prices(data, prices):
return data.merge(prices, how="left", on=["store_id", "item_id", "wm_yr_wk"] )
est_cnt = 100
seed = 42
final_estimator = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 final_layer = StackingRegressor(
 estimators=[ 
('abar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('adar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('gbrt', GradientBoostingRegressor(random_state=seed, n_estimators=est_cnt))
],
final_estimator=final_estimator 
)
 second_layer = StackingRegressor(
 estimators=[ 
('abar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('extr', ExtraTreesRegressor(random_state=seed, n_estimators=est_cnt))
,('adar', AdaBoostRegressor(random_state=seed, n_estimators=est_cnt))
,('gbrt', GradientBoostingRegressor(random_state=seed, n_estimators=est_cnt))
,('rf', RandomForestRegressor(random_state=seed, n_estimators=est_cnt))
],
final_estimator=final_layer 
)
 multi_layer_regressor = StackingRegressor(
 estimators=[ 
('lasso', LassoCV(random_state=seed))
,('svr', SVR(C=1, gamma=1e-6, kernel='rbf'))
,('etr', ExtraTreesRegressor(random_state=seed, n_estimators=est_cnt))
],
final_estimator=second_layer 
)
 multi_layer_regressor.fit(X_train, Y_train)
 acc_train_sg = np.sqrt(mean_squared_log_error(Y_train, multi_layer_regressor.predict(X_train)))
 acc_eval_sg = np.sqrt(mean_squared_log_error(Y_eval, multi_layer_regressor.predict(X_eval)))
 print(f'SG: Train = {acc_train_sg} Evaluate = {acc_eval_sg}' )<train_model>
def add_demand_features(df):
for diff in [0, 1, 2]:
shift = DAYS_PRED + diff
df[f"shift_t{shift}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(shift)
)
 for window in [7, 30, 60, 90, 180]:
df[f"rolling_std_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).std() 
)
 for window in [7, 30, 60, 90, 180]:
df[f"rolling_mean_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).mean() 
)
 for window in [7, 30, 60]:
df[f"rolling_min_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).min() 
)
 for window in [7, 30, 60]:
df[f"rolling_max_t{window}"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(window ).max() 
)
 df["rolling_skew_t30"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(30 ).skew() 
)
 df["rolling_kurt_t30"] = df.groupby(["id"])["demand"].transform(
 lambda x: x.shift(DAYS_PRED ).rolling(30 ).kurt() 
)
 return df
def add_price_features(df):
df["shift_price_t1"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.shift(1)
)
 df["price_change_t1"] =(df["shift_price_t1"] - df["sell_price"])/(
 df["shift_price_t1"] 
)
 df["rolling_price_max_t365"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.shift(1 ).rolling(365 ).max() 
)
 df["price_change_t365"] =(df["rolling_price_max_t365"] - df["sell_price"])/(
 df["rolling_price_max_t365"] 
)
 df["rolling_price_std_t7"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.rolling(7 ).std() 
)
 df["rolling_price_std_t30"] = df.groupby(["id"])["sell_price"].transform(
 lambda x: x.rolling(30 ).std() 
)
 return df.drop(["rolling_price_max_t365", "shift_price_t1"], axis=1)
 def add_time_features(df, dt_col):
df[dt_col] = pd.to_datetime(df[dt_col])
 attrs = [
"year",
"quarter",
"month",
"week",
"day",
"dayofweek",
]
for attr in attrs:
dtype = np.int16 if attr == "year" else np.int8
df[attr] = getattr(df[dt_col].dt, attr ).astype(dtype)
 df["is_weekend"] = df["dayofweek"].isin([5, 6] ).astype(np.int8)
 return df
gbm = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.7, gamma=2,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=0, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.3, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1 ).fit(X_train, Y_train)
 acc_train_xg = np.sqrt(mean_squared_log_error(Y_train, gbm.predict(X_train)))
 acc_eval_xg = np.sqrt(mean_squared_log_error(Y_eval, gbm.predict(X_eval)))
 print(f'XG: Train = {acc_train_xg} Evaluate = {acc_eval_xg}' )<train_model>
class CustomTimeSeriesSplitter:
def __init__(self, n_splits=5, train_days=80, test_days=20, day_col="d"):
self.n_splits = n_splits
self.train_days = train_days
self.test_days = test_days
self.day_col = day_col
def split(self, X, y=None, groups=None):
SEC_IN_DAY = 3600 * 24
sec =(X[self.day_col] - X[self.day_col].iloc[0])* SEC_IN_DAY
duration = sec.max()
train_sec = self.train_days * SEC_IN_DAY
test_sec = self.test_days * SEC_IN_DAY
total_sec = test_sec + train_sec
if self.n_splits == 1:
train_start = duration - total_sec
train_end = train_start + train_sec
train_mask =(sec >= train_start)&(sec < train_end)
 test_mask = sec >= train_end
yield sec[train_mask].index.values, sec[test_mask].index.values
else:
step = DAYS_PRED * SEC_IN_DAY
for idx in range(self.n_splits):
shift =(self.n_splits -(idx + 1)) * step
train_start = duration - total_sec - shift
train_end = train_start + train_sec
test_end = train_end + test_sec
train_mask =(sec > train_start)&(sec <= train_end)
 if idx == self.n_splits - 1:
test_mask = sec > train_end
else:
test_mask =(sec > train_end)&(sec <= test_end)
 yield sec[train_mask].index.values, sec[test_mask].index.values
def get_n_splits(self):
return self.n_splits
best_reg = xgb.XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=5, min_child_weight=1.5, n_estimators=1900,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 best_reg.fit(X_train_full, Y_train_full)
 acc_train_full = np.sqrt(mean_squared_log_error(Y_train_full, best_reg.predict(X_train_full)))
 print(f'Best(previous): Train = {acc_train_xg} Evaluate = {acc_eval_xg}')
 print(f'Best FULL Train = {acc_train_full}')
 predictions = best_reg.predict(X_test)
 output = pd.DataFrame({'Id': test_df.Id, 'SalePrice': predictions})
 output.to_csv('my_submission.csv', index=False )<set_options>
def show_cv_days(cv, X, dt_col, day_col):
for ii,(tr, tt)in enumerate(cv.split(X)) :
print(f"----- Fold:({ii + 1} / {cv.n_splits})-----")
 tr_start = X.iloc[tr][dt_col].min()
tr_end = X.iloc[tr][dt_col].max()
tr_days = X.iloc[tr][day_col].max() - X.iloc[tr][day_col].min() + 1
tt_start = X.iloc[tt][dt_col].min()
tt_end = X.iloc[tt][dt_col].max()
tt_days = X.iloc[tt][day_col].max() - X.iloc[tt][day_col].min() + 1
df = pd.DataFrame(
 {
"start": [tr_start, tt_start],
"end": [tr_end, tt_end],
"days": [tr_days, tt_days],
},
index=["train", "test"], 
)
 display(df)
 def plot_cv_indices(cv, X, dt_col, lw=10):
n_splits = cv.get_n_splits()
_, ax = plt.subplots(figsize=(20, n_splits))
for ii,(tr, tt)in enumerate(cv.split(X)) :
indices = np.array([np.nan] * len(X))
indices[tt] = 1
indices[tr] = 0
ax.scatter(
 X[dt_col],
[ii + 0.5] * len(indices),
c=indices,
marker="_",
lw=lw,
cmap=plt.cm.coolwarm,
vmin=-0.2,
vmax=1.2, 
)
 MIDDLE = 15
LARGE = 20
ax.set_xlabel("Datetime", fontsize=LARGE)
 ax.set_xlim([X[dt_col].min() , X[dt_col].max() ])
 ax.set_ylabel("CV iteration", fontsize=LARGE)
 ax.set_yticks(np.arange(n_splits)+ 0.5)
 ax.set_yticklabels(list(range(n_splits)))
 ax.invert_yaxis()
ax.tick_params(axis="both", which="major", labelsize=MIDDLE)
 ax.set_title("{}".format(type(cv ).__name__), fontsize=LARGE)
 return ax
%matplotlib inline
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn<load_from_csv>
features = [
"item_id",
"dept_id",
"cat_id",
"store_id",
"state_id",
"event_name_1",
"event_type_1",
"event_name_2",
"event_type_2",
"snap_CA",
"snap_TX",
"snap_WI",
"sell_price",
"shift_t28",
"shift_t29",
"shift_t30",
"rolling_std_t7",
"rolling_std_t30",
"rolling_std_t60",
"rolling_std_t90",
"rolling_std_t180",
"rolling_mean_t7",
"rolling_mean_t30",
"rolling_mean_t60",
"rolling_mean_t90",
"rolling_mean_t180",
"rolling_min_t7",
"rolling_min_t30",
"rolling_min_t60",
"rolling_max_t7",
"rolling_max_t30",
"rolling_max_t60",
"rolling_skew_t30",
"rolling_kurt_t30",
"price_change_t1",
"price_change_t365",
"rolling_price_std_t7",
"rolling_price_std_t30",
"year",
"quarter",
"month",
"week",
"day",
"dayofweek",
"is_weekend",
]
is_train = data["d"] < 1914
X_train = data[is_train][[day_col] + features].reset_index(drop=True)
 y_train = data[is_train]["demand"].reset_index(drop=True)
 X_test = data[~is_train][features].reset_index(drop=True)
 id_date = data[~is_train][["id", "date"]].reset_index(drop=True)
 del data
gc.collect()
print("X_train shape:", X_train.shape)
 print("X_test shape:", X_test.shape )
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<feature_engineering>
def train_lgb(bst_params, fit_params, X, y, cv, drop_when_train=None):
models = []
if drop_when_train is None:
drop_when_train = []
for idx_fold,(idx_trn, idx_val)in enumerate(cv.split(X, y)) :
print(f"
----- Fold:({idx_fold + 1} / {cv.get_n_splits() })-----
")
 X_trn, X_val = X.iloc[idx_trn], X.iloc[idx_val]
y_trn, y_val = y.iloc[idx_trn], y.iloc[idx_val]
train_set = lgb.Dataset(
 X_trn.drop(drop_when_train, axis=1),
label=y_trn,
categorical_feature=["item_id"], 
)
 val_set = lgb.Dataset(
 X_val.drop(drop_when_train, axis=1),
label=y_val,
categorical_feature=["item_id"], 
)
 model = lgb.train(
 bst_params,
train_set,
valid_sets=[train_set, val_set],
valid_names=["train", "valid"],
**fit_params, 
)
 models.append(model)
 del idx_trn, idx_val, X_trn, X_val, y_trn, y_val
gc.collect()
return models
train["SalePrice"] = np.log1p(train["SalePrice"])
<concatenate>
bst_params = {
"boosting_type": "gbdt",
"metric": "rmse",
"objective": "regression",
"n_jobs": -1,
"seed": 42,
"learning_rate": 0.1,
"bagging_fraction": 0.75,
"bagging_freq": 10,
"colsample_bytree": 0.75,
}
fit_params = {
"num_boost_round": 100_000,
"early_stopping_rounds": 50,
"verbose_eval": 100,
}
models = train_lgb(
 bst_params, fit_params, X_train, y_train, cv, drop_when_train=[day_col] 
)
 del X_train, y_train
gc.collect()
data = pd.concat([train, test] )<rename_columns>
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
data.set_index('Id', inplace = True )<count_missing_values>
imp_type = "gain"
importances = np.zeros(X_test.shape[1])
 preds = np.zeros(X_test.shape[0])
 for model in models:
preds += model.predict(X_test)
 importances += model.feature_importance(imp_type)
 preds = preds / cv.get_n_splits()
importances = importances / cv.get_n_splits()
data.isnull().sum() [:50]<data_type_conversions>
def make_submission(test, submission):
preds = test[["id", "date", "demand"]]
preds = preds.pivot(index="id", columns="date", values="demand" ).reset_index()
preds.columns = ["id"] + ["F" + str(d + 1)for d in range(DAYS_PRED)]
vals = submission[["id"]].merge(preds, how="inner", on="id")
 evals = submission[submission["id"].str.endswith("evaluation")]
final = pd.concat([vals, evals])
 assert final.drop("id", axis=1 ).isnull().sum().sum() == 0
assert final["id"].equals(submission["id"])
 final.to_csv("submission.csv", index=False )
data['PoolQC'].fillna('None', inplace=True)
 data['MiscFeature'].fillna('None', inplace=True)
 data['Alley'].fillna('None', inplace=True)
 data['Fence'].fillna('None', inplace=True)
 data['FireplaceQu'].fillna('None', inplace=True )<data_type_conversions>
make_submission(id_date.assign(demand=preds), submission )