Kaggle competition: M5 Forecasting - Uncertainty
<SOS> metric: WeightedRowwisePinballLoss Kaggle data source: m5-forecasting-uncertainty<not_enough_vertices>
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error, mean_absolute_error
data_average_sales_customers = data_average_sales_customers.reset_index()<merge>
class MultipleRegression() :
def __init__(self, reg=0.01):
self.reg = reg
self.BETA = None
def fit(self, Y, Z):
F = Z.shape[1]
U = np.linalg.inv(Z.T.dot(Z)+self.reg*np.eye(F))
self.BETA = Y.dot(Z ).dot(U)
 def predict(self, Z):
return self.BETA.dot(Z.T)
data_stores_average = data_average_sales_customers.join(data_store.set_index('Store'), on='Store' )<merge>
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
data_stores_new = data_total_customers_for_store.join(data_store.set_index('Store'), on='Store' )<groupby>
maps, mods = {}, {}
calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 calendar = preprocess_calendar(calendar)
 calendar['nb'] = calendar.index + 1
calendar['n_week'] =(calendar.index - 2)// 7
average_store_type = data_stores_new.groupby('StoreType')['Sales', 'Customers', 'CompetitionDistance'].mean()<create_dataframe>
%%time
FE = ['snap_CA','snap_TX','snap_WI']
for col in tqdm(['wday','month','year','event_name','nday']):
_temp = pd.get_dummies(calendar[col], prefix=col)
 new_cols = list(_temp.columns)
 FE += new_cols
calendar = calendar.join(_temp)
Data_cmp = pd.DataFrame()
Data_cmp['Customers'] = average_store_type['Sales']
Data_cmp['Sales'] = average_store_type['Customers']
Data_cmp['Comp'] = average_store_type['CompetitionDistance']<groupby>
sales = pd.read_csv(".. /input/walmartadd/sales_aug.csv" )
average_assort = data_stores_new.groupby('Assortment')['Sales', 'Customers'].mean()<load_from_csv>
START = 1000
H = 28
B = 28
COLS = [f"d_{i}" for i in range(START, 1942)]
calendar = calendar[calendar.nb>=START]
ids = sales.id.values
data_train = pd.read_csv(".. /input/train.csv")
 data_test = pd.read_csv(".. /input/test.csv")
 data_store = pd.read_csv(".. /input/store.csv" )<feature_engineering>
sc = sales['scale1'].values
closed_store_data = data_test["Id"][data_test["Open"] == 0].values
data_train.StateHoliday = data_train.StateHoliday.replace(0,'0')
 data_test.StateHoliday = data_test.StateHoliday.replace(0,'0' )<data_type_conversions>
data_train['Year'] = data_train['Date'].apply(lambda x: int(x[:4]))
data_train['Month'] = data_train['Date'].apply(lambda x: int(x[5:7]))
data_train["HolidayBin"] = data_train.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1} )<drop_column>
X = sales[COLS].values
sw = sales['sales1'].values
sc = sales['scale1'].values
Z = calendar[FE].values
Xs = X / sc[:, np.newaxis]
print(X.shape, Z.shape )
del data_train['Date']
del data_train['StateHoliday']<data_type_conversions>
%%time
step1 = MultipleRegression(reg=0.001)
 step1.fit(Xs, Z[:-28] )
data_test['Year'] = data_test['Date'].apply(lambda x: int(x[:4]))
data_test['Month'] = data_test['Date'].apply(lambda x: int(x[5:7]))
data_test["HolidayBin"] = data_test.StateHoliday.map({"0": 0, "a": 1, "b": 1, "c": 1} )<drop_column>
Xp = step1.predict(Z )
del data_test['Date']
del data_test['StateHoliday']<filter>
print("mae:", mean_absolute_error(Xs[:, -28:], Xp[:,-56:-28]))
print("mse:", mean_squared_error(Xs[:, -28:], Xp[:,-56:-28], squared=False))
data_test = data_test[data_test["Open"] != 0]<feature_engineering>
resid = Xs - Xp[:, :-28]
arr_tmp = []
for i in data_test['Store']:
arr_tmp.append(float(data_store['CompetitionDistance'][data_store['Store'] == i]))
data_test['CompetitionDistance'] = arr_tmp<feature_engineering>
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
cp = np.quantile(resid, qs, axis=1 ).T
arr_tmp = []
for i in data_train['Store']:
arr_tmp.append(float(data_store['CompetitionDistance'][data_store['Store'] == i]))
data_train['CompetitionDistance'] = arr_tmp
data_train['CompetitionDistance'] = data_train['CompetitionDistance'].fillna(data_train['CompetitionDistance'].mean() )<groupby>
cp = cp[:,np.newaxis, :]
train_stores = dict(list(data_train.groupby('Store')))
 test_stores = dict(list(data_test.groupby('Store')) )<import_modules>
Zq = Xp[:,:,np.newaxis] + cp
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV<prepare_x_and_y>
Zq = Zq.clip(0 )
<train_model>
pv = Zq[:,-56:-28, :]
pe = Zq[:,-28:, :]
result = pd.Series()
for i in test_stores:
store = train_stores[i]
X_train = store.drop(["Sales", "Store", "Customers"],axis=1)
 Y_train = store["Sales"]
X_test = test_stores[i].copy()
store_ind = X_test["Id"]
X_test.drop(["Id","Store"], axis=1,inplace=True)
 X_train = X_train.fillna(X_train.mean())
 X_test = X_test.fillna(X_train.mean())
 estimator = RandomForestRegressor(n_estimators=10, max_depth=13, criterion = 'mse')
 estimator.fit(X_train, Y_train)
 Y_pred = estimator.predict(X_test)
 result = result.append(pd.Series(Y_pred, index=store_ind))
result = result.append(pd.Series(0, index=closed_store_data))
result = pd.DataFrame({ "Id": result.index, "Sales": result.values})
 result.to_csv('result_new.csv', index=False )<set_options>
sc= sc[:, np.newaxis]
%matplotlib inline<load_from_csv>
names = [f"F{i+1}" for i in range(28)]
piv = pd.DataFrame(ids, columns=["id"] )
df_train = pd.read_csv(".. /input/train.csv")
 df_store = pd.read_csv(".. /input/store.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df_train.head()<filter>
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sc, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*sc, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
closed_store_ids = df_test["Id"][df_test["Open"] == 0].values
closed_store_ids<feature_engineering>
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2
<categorify><EOS> 
sub.to_csv("submission.csv", index=False )