Kaggle competition: M5 Forecasting - Accuracy
data_df['dist'] = data_df['dist'].apply(math.log )<feature_engineering>
submission0 = pd.read_csv('/kaggle/input/m5-final-models/submission_LSTM.csv')
 submission1 = pd.read_csv('/kaggle/input/m5-final-models/submission_XGBoost.csv')
 submission2 = pd.read_csv('/kaggle/input/m5-final-models/submission_LGBM.csv')
 submission3 = pd.read_csv('/kaggle/input/m5-final-models/submission_prophet.csv')
 submission4 = pd.read_csv('/kaggle/input/m5-final-models/submission_SARIMAX.csv' )
MAX_SECONDS_IN_DAY = 24*60*60
def timetosectosincosday(x):
initial_date = datetime(x.year, x.month, x.day, 0 , 0, 0)
 sec =(x-initial_date ).total_seconds()
sin = math.sin(2*math.pi*(sec/MAX_SECONDS_IN_DAY))
cos = math.cos(2*math.pi*(sec/MAX_SECONDS_IN_DAY))
return sin, cos<categorify>
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 sample_sub = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv' )
data_df['pickup_sin_sec'] ,data_df['pickup_cos_sec'] = zip(*data_df['pickup_datetime'].map(timetosectosincosday))<feature_engineering>
def perfect_sub() :
submission = OperateBaseModels(submission0,submission1, a=0, b=1)
 diference = validation.merge(submission, how='right')
 shift= 56
perfect_submission = diference[diference.columns[-shift:-shift+28]]
col = { 'd_'+str(1914+i):'F'+str(i+1)for i in range(28)}
perfect_submission = perfect_submission.rename(columns=col)
 perfect_submission.insert(loc=0, column='id', value= submission0['id'])
 perfect_submission = perfect_submission.fillna(0)
 return perfect_submission
def OperateBaseModels(ModelA,ModelB, a,b):
submissionMerge = pd.merge(ModelA, ModelB, on='id', how='left', suffixes=('_x', '_y'))
submissionMerge
submission = pd.DataFrame()
submission.insert(loc=0, column='id', value= ModelB['id'])
 for j in range(28):
i =j+1
submission.insert(loc=i, column='F'+str(i), value=(( submissionMerge[submissionMerge.columns[i]]*a)+(submissionMerge[submissionMerge.columns[i+28]])*b)/(a+b))
return submission
MAX_DAY_IN_WEEK = 6
def dayofweektosincosday(day):
sin = math.sin(2*math.pi*(day/MAX_DAY_IN_WEEK))
cos = math.cos(2*math.pi*(day/MAX_DAY_IN_WEEK))
return sin, cos<feature_engineering>
from typing import Union
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm_notebook as tqdm
data_df['sin_day'] ,data_df['cos_day'] = zip(*data_df['day_of_week'].map(dayofweektosincosday))<drop_column>
class WRMSSEEvaluator(object):
def __init__(self, train_df: pd.DataFrame, valid_df: pd.DataFrame, calendar: pd.DataFrame, prices: pd.DataFrame):
train_y = train_df.loc[:, train_df.columns.str.startswith('d_')]
train_target_columns = train_y.columns.tolist()
weight_columns = train_y.iloc[:, -28:].columns.tolist()
train_df['all_id'] = 0
id_columns = train_df.loc[:, ~train_df.columns.str.startswith('d_')].columns.tolist()
valid_target_columns = valid_df.loc[:, valid_df.columns.str.startswith('d_')].columns.tolist()
if not all([c in valid_df.columns for c in id_columns]):
valid_df = pd.concat([train_df[id_columns], valid_df], axis=1, sort=False)
 self.train_df = train_df
self.valid_df = valid_df
self.calendar = calendar
self.prices = prices
self.weight_columns = weight_columns
self.id_columns = id_columns
self.valid_target_columns = valid_target_columns
weight_df = self.get_weight_df()
self.group_ids =(
 'all_id',
'state_id',
'store_id',
'cat_id',
'dept_id',
['state_id', 'cat_id'],
['state_id', 'dept_id'],
['store_id', 'cat_id'],
['store_id', 'dept_id'],
'item_id',
['item_id', 'state_id'],
['item_id', 'store_id'] 
)
 for i, group_id in enumerate(tqdm(self.group_ids)) :
train_y = train_df.groupby(group_id)[train_target_columns].sum()
scale = []
for _, row in train_y.iterrows() :
series = row.values[np.argmax(row.values != 0):]
scale.append(((series[1:] - series[:-1])** 2 ).mean())
 setattr(self, f'lv{i + 1}_scale', np.array(scale))
setattr(self, f'lv{i + 1}_train_df', train_y)
 setattr(self, f'lv{i + 1}_valid_df', valid_df.groupby(group_id)[valid_target_columns].sum())
 lv_weight = weight_df.groupby(group_id)[weight_columns].sum().sum(axis=1)
 setattr(self, f'lv{i + 1}_weight', lv_weight / lv_weight.sum())
 def get_weight_df(self)-> pd.DataFrame:
day_to_week = self.calendar.set_index('d')['wm_yr_wk'].to_dict()
weight_df = self.train_df[['item_id', 'store_id'] + self.weight_columns].set_index(['item_id', 'store_id'])
 weight_df = weight_df.stack().reset_index().rename(columns={'level_2': 'd', 0: 'value'})
 weight_df['wm_yr_wk'] = weight_df['d'].map(day_to_week)
 weight_df = weight_df.merge(self.prices, how='left', on=['item_id', 'store_id', 'wm_yr_wk'])
 weight_df['value'] = weight_df['value'] * weight_df['sell_price']
weight_df = weight_df.set_index(['item_id', 'store_id', 'd'] ).unstack(level=2)['value']
weight_df = weight_df.loc[zip(self.train_df.item_id, self.train_df.store_id), :].reset_index(drop=True)
 weight_df = pd.concat([self.train_df[self.id_columns], weight_df], axis=1, sort=False)
 return weight_df
def rmsse(self, valid_preds: pd.DataFrame, lv: int)-> pd.Series:
valid_y = getattr(self, f'lv{lv}_valid_df')
 score =(( valid_y - valid_preds)** 2 ).mean(axis=1)
 scale = getattr(self, f'lv{lv}_scale')
 return(score / scale ).map(np.sqrt)
 def score(self, valid_preds: Union[pd.DataFrame, np.ndarray])-> float:
assert self.valid_df[self.valid_target_columns].shape == valid_preds.shape
if isinstance(valid_preds, np.ndarray):
valid_preds = pd.DataFrame(valid_preds, columns=self.valid_target_columns)
 valid_preds = pd.concat([self.valid_df[self.id_columns], valid_preds], axis=1, sort=False)
 all_scores = []
for i, group_id in enumerate(self.group_ids):
lv_scores = self.rmsse(valid_preds.groupby(group_id)[self.valid_target_columns].sum() , i + 1)
 weight = getattr(self, f'lv{i + 1}_weight')
 lv_scores = pd.concat([weight, lv_scores], axis=1, sort=False ).prod(axis=1)
 all_scores.append(lv_scores.sum())
 return np.mean(all_scores)
 df_train_full = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv")
 df_calendar = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv")
 df_prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv")
 df_train = df_train_full.iloc[:, :-28]
df_valid = df_train_full.iloc[:, -28:]
evaluator = WRMSSEEvaluator(df_train, df_valid, df_calendar, df_prices)
 def WRMSSEE(sub):
val_preds = sample_sub[['id']].merge(sub, on = 'id')
 val_preds.columns = ['id'] + list(df_valid.columns)
 valid_preds = val_preds.iloc[:30490, -28:]
return evaluator.score(valid_preds )
df_processed = data_df[['vendor_id',
'passenger_count',
'pickup_sin_sec',
'pickup_cos_sec',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'sin_day',
'cos_day',
'dist',
'trip_duration']]<prepare_x_and_y>
def function(solution):
solution = np.abs(solution)
 submission = OperateBaseModels(submission0,submission1, a=solution[0], b=solution[1])
 submission = OperateBaseModels(submission,submission2, a=1, b=solution[2])
 submission = OperateBaseModels(submission,submission3, a=1, b=solution[3])
 submission = OperateBaseModels(submission,submission4, a=1, b=solution[4])
 error = WRMSSEE(submission)
 return error
X = data_df[['vendor_id',
'passenger_count',
'pickup_sin_sec',
'pickup_cos_sec',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'sin_day',
'cos_day',
'dist']]<prepare_x_and_y>
diference = validation.merge(submission,how='left')
 error = np.mean(np.mean(np.abs(perfect_sub._get_numeric_data() -submission._get_numeric_data())) )
y = data_df[['trip_duration']]<normalization>
submission.to_csv("submission.csv", index=False)
 submission