Kaggle competition: M5 Forecasting - Uncertainty
<SOS> metric: WeightedRowwisePinballLoss Kaggle data source: m5-forecasting-uncertainty<feature_engineering>
import pandas as pd, numpy as np
from matplotlib import pyplot as plt
import scipy.stats as stats
a = list()
for i in df_test['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_test['CompetitionDistance'] = a<feature_engineering>
pd.options.display.max_columns = 50
a = list()
for i in df_train['Store']:
a.append(float(df_store['CompetitionDistance'][df_store['Store'] == i]))
df_train['CompetitionDistance'] = a
df_train['CompetitionDistance'] = df_train['CompetitionDistance'].fillna(df_train['CompetitionDistance'].mean() )<feature_engineering>
best = pd.read_csv(".. /input/accuracy-best-public-lbs/kkiller_first_public_notebook_under050_v5.csv")
 best.head()
df_train['CompetitionDistance'] = np.log(df_train['CompetitionDistance'])
 df_test['CompetitionDistance'] = np.log(df_test['CompetitionDistance'] )<groupby>
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_validation.csv")
 sales.head()
train_stores = dict(list(df_train.groupby('Store')))
 test_stores = dict(list(df_test.groupby('Store')) )<prepare_x_and_y>
sub = best.merge(sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]], on = "id")
 sub["_all_"] = "Total"
sub.shape
best_list_max_depth = []
best_list_n_estimators = []
for i in test_stores:
store = train_stores[i]
X_train = store.drop(["Sales", "Store", "Customers"],axis=1)
 Y_train = store["Sales"]
X_test = test_stores[i].copy()
store_ids = X_test["Id"]
X_test.drop(["Id","Store"], axis=1,inplace=True)
 X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean())
 estimator = RandomForestRegressor(random_state=123, criterion = 'mse')
 params = {'n_estimators': range(5, 20), 'max_depth': range(5, 25)}
grid = GridSearchCV(estimator, params ).fit(X_train, Y_train)
 best_list_max_depth.append(grid.best_params_['max_depth'])
 best_list_n_estimators.append(grid.best_params_['n_estimators'])
 print("score", grid.best_score_)
 print("params", grid.best_params_ )<find_best_params>
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 qs.shape
def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 level_coef_dict = {"id": get_ratios(coef=0.3), "item_id": get_ratios(coef=0.15),
"dept_id": get_ratios(coef=0.08), "cat_id": get_ratios(coef=0.07),
"store_id": get_ratios(coef=0.08), "state_id": get_ratios(coef=0.07), "_all_": get_ratios(coef=0.05), 
("state_id", "item_id"): get_ratios(coef=0.19),("state_id", "dept_id"): get_ratios(coef=0.1), 
("store_id","dept_id"): get_ratios(coef=0.11),("state_id", "cat_id"): get_ratios(coef=0.08), 
("store_id","cat_id"): get_ratios(coef=0.1)
 }
<find_best_params><EOS> 
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_validation', '')}_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_validation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df = pd.concat([df,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index >= len(df.index)//2, "id"] = df.loc[df.index >= len(df.index)//2, "id"].str.replace(
 "_validation$", "_evaluation")
 df.shape
df.head()
df.to_csv("submission01.csv", index = False )