Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<feature_engineering>
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb
combined['NeighborhoodLotArea'] = combined['Neighborhood'] * combined['LotArea']
for d in all_data:
d['NeighborhoodLotArea'] = d['Neighborhood'] * d['LotArea']
d['NeighborhoodLotArea'] =(d['NeighborhoodLotArea'] - combined['NeighborhoodLotArea'].min())\
/(combined['NeighborhoodLotArea'].max() -combined['NeighborhoodLotArea'].min())
 combined['NeighborhoodLotArea'] =(combined['NeighborhoodLotArea'] - combined['NeighborhoodLotArea'].min())\
/(combined['NeighborhoodLotArea'].max() -combined['NeighborhoodLotArea'].min() )<feature_engineering>
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }
subclass_map = train_df.groupby('BldgType')['SalePrice'].mean().sort_values()
for d in all_data:
for idx, s in enumerate(subclass_map.index):
d.loc[d['BldgType'] == s,'BldgType'] = idx+1
for idx, s in enumerate(subclass_map.index):
combined.loc[combined['BldgType'] == s,'BldgType'] = idx+1
train_df.groupby('BldgType')['SalePrice'].describe().sort_values('mean' )<feature_engineering>
pd.options.display.max_columns = 50
combined['NeighborhoodBldgType'] = combined['Neighborhood'] * combined['BldgType']
for d in all_data:
d['NeighborhoodBldgType'] = d['Neighborhood'] * d['BldgType']
d['NeighborhoodBldgType'] =(d['NeighborhoodBldgType'] - combined['NeighborhoodBldgType'].min())\
/(combined['NeighborhoodBldgType'].max() -combined['NeighborhoodBldgType'].min())
 combined['NeighborhoodBldgType'] =(combined['NeighborhoodBldgType'] - combined['NeighborhoodBldgType'].min())\
/(combined['NeighborhoodBldgType'].max() -combined['NeighborhoodBldgType'].min() )<data_type_conversions>
h = 28
max_lags = 70
tr_last = 1913
fday = datetime(2016,4, 25)
 fday
combined['TotalBsmtSF'] = combined['TotalBsmtSF'].fillna(0)
 combined['GarageArea'] = combined['GarageArea'].fillna(0)
 combined['TotalSquareFeet'] = combined['TotalBsmtSF'] + combined['1stFlrSF'] \
+ combined['2ndFlrSF'] + combined['GarageArea'] \
+ combined['WoodDeckSF']
for d in all_data:
d['TotalBsmtSF'] = d['TotalBsmtSF'].fillna(0)
 d['GarageArea'] = d['GarageArea'].fillna(0)
 d['TotalSquareFeet'] = d['TotalBsmtSF'] + d['1stFlrSF'] + d['2ndFlrSF'] + d['GarageArea'] + d['WoodDeckSF']
d['TotalSquareFeet'] =(d['TotalSquareFeet']-combined['TotalSquareFeet'].mean())/combined['TotalSquareFeet'].std()
combined['TotalSquareFeet'] =(combined['TotalSquareFeet']-combined['TotalSquareFeet'].mean())/combined['TotalSquareFeet'].std()<feature_engineering>
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt
subclass_map = train_df.groupby('HouseStyle')['SalePrice'].mean().sort_values()
for d in all_data:
for idx, s in enumerate(subclass_map.index):
d.loc[d['HouseStyle'] == s,'HouseStyle'] = idx+1
for idx, s in enumerate(subclass_map.index):
combined.loc[combined['HouseStyle'] == s,'HouseStyle'] = idx+1
train_df.groupby('HouseStyle')['SalePrice'].describe().sort_values('mean' )<feature_engineering>
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )
combined['NeighborhoodHouseStyle'] = combined['Neighborhood'] * combined['HouseStyle']
for d in all_data:
d['NeighborhoodHouseStyle'] = d['Neighborhood'] * d['HouseStyle']
d['NeighborhoodHouseStyle'] =(d['NeighborhoodHouseStyle'] - combined['NeighborhoodHouseStyle'].min())\
/(combined['NeighborhoodHouseStyle'].max() -combined['NeighborhoodHouseStyle'].min())
 combined['NeighborhoodHouseStyle'] =(combined['NeighborhoodHouseStyle'] - combined['NeighborhoodHouseStyle'].min())\
/(combined['NeighborhoodHouseStyle'].max() -combined['NeighborhoodHouseStyle'].min() )<feature_engineering>
FIRST_DAY = 800
combined['OverallQualNCond'] = combined['OverallQual'] * combined['OverallCond']
for d in all_data:
d['OverallQualNCond'] = d['OverallQual'] * d['OverallCond']
d['OverallQualNCond'] =(d['OverallQualNCond'] - combined['OverallQualNCond'].min())\
/(combined['OverallQualNCond'].max() -combined['OverallQualNCond'].min())
 combined['OverallQualNCond'] =(combined['OverallQualNCond'] - combined['OverallQualNCond'].min())\
/(combined['OverallQualNCond'].max() -combined['OverallQualNCond'].min() )<feature_engineering>
df.dropna(inplace = True)
 df.shape
combined['NeighborhoodOverallQualNCond'] = combined['Neighborhood'] * combined['OverallQualNCond']
for d in all_data:
d['NeighborhoodOverallQualNCond'] = d['Neighborhood'] * d['OverallQualNCond']
d['NeighborhoodOverallQualNCond'] =(d['NeighborhoodOverallQualNCond'] - combined['NeighborhoodOverallQualNCond'].min())\
/(combined['NeighborhoodOverallQualNCond'].max() -combined['NeighborhoodOverallQualNCond'].min())
 combined['NeighborhoodOverallQualNCond'] =(combined['NeighborhoodOverallQualNCond'] - combined['NeighborhoodOverallQualNCond'].min())\
/(combined['NeighborhoodOverallQualNCond'].max() -combined['NeighborhoodOverallQualNCond'].min() )<feature_engineering>
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]
max_year = combined['YrSold'].max()
max_month = combined.loc[combined['YrSold'] == max_year, 'MoSold'].max()
for d in all_data:
months_ago =(max_year-d['YearRemodAdd'])*12 + max_month
d['AdjRemodAdd'] =(months_ago - months_ago.min())/(months_ago.max() -months_ago.min())\
* -1 + 1
d.drop(['YearRemodAdd'], axis=1, inplace=True)
 months_ago =(max_year-combined['YearRemodAdd'])*12 + max_month
combined['AdjRemodAdd'] =(months_ago - months_ago.min())/(months_ago.max() -months_ago.min())\
* -1 + 1
combined.drop(['YearRemodAdd'], axis=1, inplace=True )<feature_engineering>
train_data = lgb.Dataset(X_train, label = y_train, categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_inds = np.random.choice(len(X_train), 1000000)
 fake_valid_data = lgb.Dataset(X_train.iloc[fake_valid_inds], label = y_train.iloc[fake_valid_inds],categorical_feature=cat_feats,
free_raw_data=False )
df = train_df.groupby(['YrSold','MoSold'])[['SalePrice']].mean()
df = df.rename({'SalePrice': 'MonthMean'}, axis=1)
 qt =(df[['MonthMean']] + df[['MonthMean']].shift(1)+ df[['MonthMean']].shift(-1)) / 3
df['QuarterMean'] = qt['MonthMean']
df['QuarterMean'] = df['QuarterMean'].fillna(df['MonthMean'])
 print(df )<merge>
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 2500,
}
for idx, d in enumerate(all_data):
all_data[idx] = pd.merge(d, df, how='left', left_on=['YrSold','MoSold'], right_on=['YrSold','MoSold'])
 all_data[idx].drop(['MoSold', 'YrSold'], axis=1, inplace=True)
 combined = pd.merge(combined, df, how='left', left_on=['YrSold','MoSold'], right_on=['YrSold','MoSold'])
 combined.drop(['MoSold', 'YrSold'], axis=1, inplace=True)
 train_df = all_data[0]
test_df = all_data[1]<feature_engineering>
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=100 )
for d in all_data:
d['QuarterMeanNorm'] =(d['QuarterMean']-combined['QuarterMean'].mean())/combined['QuarterMean'].std()
d['MonthMeanNorm'] =(d['MonthMean']-combined['MonthMean'].mean())/combined['MonthMean'].std()
combined['QuarterMeanNorm'] =(combined['QuarterMean']-combined['QuarterMean'].mean())/combined['QuarterMean'].std()
combined['MonthMeanNorm'] =(combined['MonthMean']-combined['MonthMean'].mean())/combined['MonthMean'].std() 
<categorify>
m_lgb.save_model("model.lgb" )
ex1_map = {'Other': 0, 'BrkFace': 1, 'CemntBd': 2, 'Plywood': 3, 'Wd Sdng': 4, 'MetalSd': 5, 'HdBoard': 6, 'VinylSd': 7}
ex2_map = {'Other': 0, 'AsbShng': 1, 'BrkFace': 2, 'Stucco': 3, 'Wd Shng': 4, 'CmentBd': 5, 'Plywood': 7, 'Wd Sdng': 8, 'HdBoard': 9, 'MetalSd': 10, 'VinylSd': 11}
max_map = {'None': 0, 'BrkCmn': 1, 'Stone': 2, 'BrkFace': 3}
exg_map = {'Po': 0.0, 'Fa': 0.25, 'TA': 0.5, 'Gd': 0.75, 'Ex': 1.0}
combined['MasVnrArea'] = combined['MasVnrArea'].fillna(0)
 for d in all_data:
d['Exterior1st'] = d['Exterior1st'].fillna(combined['Exterior1st'].mode() [0])
 d['Exterior1st'] = d['Exterior1st'].replace(['AsphShn','CBlock','ImStucc','BrkComm','Stone','AsbShng','Stucco','WdShing'], 'Other')
 d['Exterior1st'] = d['Exterior1st'].map(ex1_map)
 d['Exterior2nd'] = d['Exterior2nd'].fillna(combined['Exterior2nd'].mode() [0])
 d['Exterior2nd'] = d['Exterior2nd'].replace(['CBlock','AsphShn','Stone','Brk Cmn','ImStucc'], 'Other')
 d['Exterior2nd'] = d['Exterior2nd'].map(ex2_map)
 d['MasVnrType'] = d['MasVnrType'].fillna('None')
 d['MasVnrType'] = d['MasVnrType'].map(max_map)
 d['MasVnrArea'] = d['MasVnrArea'].fillna(0)
 d['MasVnrArea'] =(d['MasVnrArea'] - combined['MasVnrArea'].min())\
/(combined['MasVnrArea'].max() -combined['MasVnrArea'].min())
 d['ExterQual'] = d['ExterQual'].map(exg_map)
 d['ExterCond'] = d['ExterCond'].map(exg_map)
 combined['Exterior1st'] = combined['Exterior1st'].fillna(combined['Exterior1st'].mode() [0])
 combined['Exterior1st'] = combined['Exterior1st'].replace(['AsphShn','CBlock','ImStucc','BrkComm','Stone','AsbShng','Stucco','WdShing'], 'Other')
 combined['Exterior1st'] = combined['Exterior1st'].map(ex1_map)
 combined['Exterior2nd'] = combined['Exterior2nd'].fillna(combined['Exterior2nd'].mode() [0])
 combined['Exterior2nd'] = combined['Exterior2nd'].replace(['CBlock','AsphShn','Stone','Brk Cmn','ImStucc'], 'Other')
 combined['Exterior2nd'] = combined['Exterior2nd'].map(ex2_map)
 combined['MasVnrType'] = combined['MasVnrType'].fillna('None')
 combined['MasVnrType'] = combined['MasVnrType'].map(max_map)
 combined['MasVnrArea'] =(combined['MasVnrArea'] - combined['MasVnrArea'].min())\
/(combined['MasVnrArea'].max() -combined['MasVnrArea'].min())
 combined['ExterQual'] = combined['ExterQual'].map(exg_map)
 combined['ExterCond'] = combined['ExterCond'].map(exg_map )<categorify>
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()