Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<create_dataframe>
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb
dls = ImageDataLoaders.from_df(train_df,
splitter=RandomSplitter(0.2, seed=42),
label_col=0,
fn_col=1,
bs=bs,
item_tfms=item_tfms,
batch_tfms=batch_tfms )<choose_model_class>
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }
learn = cnn_learner(dls, resnet50, metrics=[error_rate, accuracy] ).to_native_fp16()<find_best_params>
pd.options.display.max_columns = 50
learn.freeze()
learn.fine_tune(1, cbs=[MixUp(0.5)] )<save_model>
h = 28
max_lags = 57
tr_last = 1941
fday = datetime(2016,4, 25)
 fday
learn.save('estagio-1' )<load_pretrained>
def create_dt(is_train = True, nrows = None, first_day = 1):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt
learn = learn.load('estagio-1' )<categorify>
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )
learn = learn.to_native_fp32()<save_model>
FIRST_DAY = 400
learn.save('estagio-2' )<load_from_csv>
df.dropna(inplace = True)
 df.shape
sample_df = pd.read_csv(dataset_path/'sample_submission.csv')
 sample_df.head()<categorify>
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]
_sample_df = sample_df.copy()
_sample_df['path'] = _sample_df['image_id'].map(lambda x:dataset_path/'test_images'/x)
 _sample_df = _sample_df.drop(columns=['image_id'])
 test_dl = dls.test_dl(_sample_df )<define_variables>
test_dl.show_batch()<normalization>
%%time
np.random.seed(777)
 train_inds = X_train.index.values
train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds], categorical_feature=cat_feats, free_raw_data=False)
preds, _ = learn.tta(dl=test_dl, n=8, beta=0 )<feature_engineering>
del df, X_train, y_train ; gc.collect()
sample_df['label'] = preds.argmax(dim=-1 ).numpy()<save_to_csv>
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.08,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1200,
'num_leaves': 128,
"min_data_in_leaf": 100,
}
sample_df.to_csv('submission.csv',index=False )<import_modules>
warnings.simplefilter(action='ignore', category=FutureWarning)
 pd.set_option('display.max_columns', 200)
 pd.set_option('display.max_rows', 20)
 pd.set_option('display.min_rows', 20)
 sns.set_style(style='whitegrid' )<compute_test_metric>
m_lgb = lgb.Booster(model_file='/kaggle/input/m5-accuracy-submission-file/model.lgb' )
def WMAE(dataset, real, predicted):

 weights = dataset.IsHoliday.apply(lambda x: 5 if x else 1)
 return np.round(np.sum(weights*abs(real-predicted)) /(np.sum(weights)) , 2 )<sort_values>
m_lgb.save_model("model.lgb" )
def create_groupped_analysis(col, df):

 unique_values = df[[col, 'Weekly_Sales']].groupby(by=col ).mean().size
print(f'The number of unique values on the category '{col}' is {unique_values}
')
 print('-' * 40)
 print(f'The top 5 '{col}' with higher average 'Weekly_Sales':')
 print('-' * 40)
 return(
 df[[col, 'Weekly_Sales']] 
.groupby(by=col)
.mean() 
.sort_values('Weekly_Sales', ascending=False)
.head()
)<choose_model_class>
fday = datetime(2016,5,23)
 fday
def model_factory(model_type, **kwargs):

 if model_type in ['RandomForestRegressor', 'ExtraTreesRegressor'] :
n_estimators = kwargs.get('n_estimators')
 verbose = kwargs.get('verbose')
 n_jobs = kwargs.get('n_jobs')
 if n_estimators is None:
n_estimators = 50
verbose = 0 if verbose is None else verbose
if model_type == 'LinearRegression':
return LinearRegression()
elif model_type == 'KNN':
return KNeighborsRegressor()
elif model_type == 'Ridge':
return RidgeCV()
elif model_type == 'Lasso':
return LassoCV()
elif model_type == 'XGBRegressor':
return XGBRegressor()
elif model_type == 'RandomForestRegressor':
return RandomForestRegressor(
 n_estimators = n_estimators,
verbose = verbose,
n_jobs = n_jobs 
)
 elif model_type == 'ExtraTreesRegressor':
return ExtraTreesRegressor(
 n_estimators = n_estimators,
verbose = verbose,
n_jobs = n_jobs 
)
 else:
raise ValueError('Model not defined on factory.' )<load_from_csv>
sub = pd.concat([sub2, sub], axis=0, sort=False)
 sub.to_csv("submission.csv",index=False )
base_path = '/kaggle/input/walmart-recruiting-store-sales-forecasting/'
input_paths = glob.glob(f'{base_path}*.csv*')
 input_dfs = {
re.search(r'forecasting\/ (.*)\.csv', path, re.IGNORECASE)[1] : pd.read_csv(path)
 for path in input_paths
}
input_dfs.keys()<merge>
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()
<count_missing_values><EOS> 
sub.id.nunique() , sub["id"].str.contains("evaluation$" ).sum()