Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<prepare_output>
warnings.filterwarnings('ignore')
sub["Weekly_Sales"] = result
sub.head()<save_to_csv>
sales = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 sales.name = 'sales'
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar.name = 'calendar'
prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 prices.name = 'prices'
sub.to_csv("walmart_predict_sub.csv",index=False )<set_options>
for d in range(1942,1970):
col = 'd_' + str(d)
 sales[col] = 0
sales[col] = sales[col].astype(np.int16 )
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
 tempo = time.time()
dados = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
<define_variables>
sales_bd = np.round(sales.memory_usage().sum() /(1024*1024),1)
 calendar_bd = np.round(calendar.memory_usage().sum() /(1024*1024),1)
 prices_bd = np.round(prices.memory_usage().sum() /(1024*1024),1 )
def verifica_nulos(dataset):
rows = []
for column in dataset.columns:
row = {'coluna': column, 'nans': dataset[column].isnull().sum() , 'frac_nans': dataset[column].isnull().sum() / dataset.shape[0]}
rows.append(row)
 res = pd.DataFrame(rows)
 res[res.nans>0].sort_values('nans', ascending=False)
 return res
features = [c for c in dados.columns if 'feature' in c]

<choose_model_class>
def downcast(df):
cols = df.dtypes.index.tolist()
types = df.dtypes.values.tolist()
for i,t in enumerate(types):
if 'int' in str(t):
if df[cols[i]].min() > np.iinfo(np.int8 ).min and df[cols[i]].max() < np.iinfo(np.int8 ).max:
df[cols[i]] = df[cols[i]].astype(np.int8)
 elif df[cols[i]].min() > np.iinfo(np.int16 ).min and df[cols[i]].max() < np.iinfo(np.int16 ).max:
df[cols[i]] = df[cols[i]].astype(np.int16)
 elif df[cols[i]].min() > np.iinfo(np.int32 ).min and df[cols[i]].max() < np.iinfo(np.int32 ).max:
df[cols[i]] = df[cols[i]].astype(np.int32)
 else:
df[cols[i]] = df[cols[i]].astype(np.int64)
 elif 'float' in str(t):
if df[cols[i]].min() > np.finfo(np.float16 ).min and df[cols[i]].max() < np.finfo(np.float16 ).max:
df[cols[i]] = df[cols[i]].astype(np.float16)
 elif df[cols[i]].min() > np.finfo(np.float32 ).min and df[cols[i]].max() < np.finfo(np.float32 ).max:
df[cols[i]] = df[cols[i]].astype(np.float32)
 else:
df[cols[i]] = df[cols[i]].astype(np.float64)
 elif t == np.object:
if cols[i] == 'date':
df[cols[i]] = pd.to_datetime(df[cols[i]], format='%Y-%m-%d')
 else:
df[cols[i]] = df[cols[i]].astype('category')
 return df
sales = downcast(sales)
 prices = downcast(prices)
 calendar = downcast(calendar )
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<prepare_x_and_y>
sales_ad = np.round(sales.memory_usage().sum() /(1024*1024),1)
 calendar_ad = np.round(calendar.memory_usage().sum() /(1024*1024),1)
 prices_ad = np.round(prices.memory_usage().sum() /(1024*1024),1 )
batch_size = 4096
hidden_units = [384, 896, 896, 394]
dropout_rates = [0.10143786981358652, 0.19720339053599725, 0.2703017847244654, 0.23148340929571917, 0.2357768967777311]
label_smoothing = 1e-2
learning_rate = 1e-3 
<load_pretrained>
df = pd.melt(sales, id_vars=['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name='d', value_name='sold' ).dropna()
num_models = 1
models = []
for i in range(num_models):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.load_weights(f'.. /input/testeupload/JSModel_{i}.hdf5')
 models.append(clf)
 f_mean = np.load('.. /input/testeupload/f_mean.npy')
<split>
df = pd.merge(df, calendar, on='d', how='left')
 df = pd.merge(df, prices, on=['store_id','item_id','wm_yr_wk'], how='left' )
env = janestreet.make_env()
env_iter = env.iter_test()<feature_engineering>
df['revenue'] = df['sold']*df['sell_price'].astype(np.float32 )
opt_th = 0.5
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = 0.
 for clf in models:
pred += clf(x_tt, training = False ).numpy().item() / num_models
pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df)
<set_options>
cal_data = group.copy()
cal_data = cal_data[cal_data.date <= '22-05-2016']
cal_data['week'] = cal_data.date.dt.weekofyear
cal_data['day_name'] = cal_data.date.dt.day_name()
pio.templates.default = "ggplot2"
env = janestreet.make_env()
iter_test = env.iter_test() 
<load_from_csv>
d_id = dict(zip(df.id.cat.codes, df.id))
d_item_id = dict(zip(df.item_id.cat.codes, df.item_id))
d_dept_id = dict(zip(df.dept_id.cat.codes, df.dept_id))
d_cat_id = dict(zip(df.cat_id.cat.codes, df.cat_id))
d_store_id = dict(zip(df.store_id.cat.codes, df.store_id))
d_state_id = dict(zip(df.state_id.cat.codes, df.state_id))
print("Reading dataset using CUDA dataframes...", end='')
 train_cudf = cudf.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train_data = train_cudf.to_pandas()
del train_cudf
meta_cudf = cudf.read_csv('/kaggle/input/jane-street-market-prediction/features.csv')
 meta_data = meta_cudf.to_pandas()
del meta_cudf
sample_prediction_df = pd.read_csv('.. /input/jane-street-market-prediction/example_sample_submission.csv')
 print('Finished.
 ')
 print(f'Train shape: {format(train_data.shape)}')
 print(f'Features meta shape: {format(meta_data.shape)}' )<data_type_conversions>
del group, group_price_cat, group_price_store, group_state, group_state_store, cal_data
gc.collect() ;
df.d = df['d'].apply(lambda x: x.split('_')[1] ).astype(np.int16)
 cols = df.dtypes.index.tolist()
types = df.dtypes.values.tolist()
for i,type in enumerate(types):
if type.name == 'category':
df[cols[i]] = df[cols[i]].cat.codes
df.drop('date',axis=1,inplace=True )
print('Preprocessing data...', end='')
 features = [c for c in train_data.columns if 'feature' in c]
train_data = train_data[train_data['weight'] > 0].reset_index(drop = True)
 train_data[features] = train_data[features].fillna(method = 'ffill' ).fillna(0)
 train_data['action'] =(train_data['resp'].values > 0 ).astype(int)
 print('Finished.' )<choose_model_class>
lags = [1,2,3,6,12,24,36]
for lag in lags:
df['sold_lag_'+str(lag)] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['sold'].shift(lag ).astype(np.float16 )
print('Creating XGBclassifier...
 ', end='')
 parameters = {'max_depth': 8,
'learning_rate': 0.015,
'random_state': 42,
'tree_method': 'gpu_hist',
'min_child_weight': 0.30,
'subsample': 0.46,
'colsample_bytree': 0.99,
'eval_metric': 'auc',
'gamma': 9.8,
'objective': 'binary:logistic'}
X_train = train_data.loc[train_data['date'] > 80, features].values
y_train = train_data.loc[train_data['date'] > 80, 'action'].values
d_train = xgb.DMatrix(X_train, y_train)
 %time clf = xgb.train(parameters, d_train, 1175)
 print('Finished training the classifier.' )<categorify>
df['iteam_sold_avg'] = df.groupby('item_id')['sold'].transform('mean' ).astype(np.float16)
 df['state_sold_avg'] = df.groupby('state_id')['sold'].transform('mean' ).astype(np.float16)
 df['store_sold_avg'] = df.groupby('store_id')['sold'].transform('mean' ).astype(np.float16)
 df['cat_sold_avg'] = df.groupby('cat_id')['sold'].transform('mean' ).astype(np.float16)
 df['dept_sold_avg'] = df.groupby('dept_id')['sold'].transform('mean' ).astype(np.float16)
 df['cat_dept_sold_avg'] = df.groupby(['cat_id','dept_id'])['sold'].transform('mean' ).astype(np.float16)
 df['store_item_sold_avg'] = df.groupby(['store_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['cat_item_sold_avg'] = df.groupby(['cat_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['dept_item_sold_avg'] = df.groupby(['dept_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['state_store_sold_avg'] = df.groupby(['state_id','store_id'])['sold'].transform('mean' ).astype(np.float16)
 df['state_store_cat_sold_avg'] = df.groupby(['state_id','store_id','cat_id'])['sold'].transform('mean' ).astype(np.float16)
 df['store_cat_dept_sold_avg'] = df.groupby(['store_id','cat_id','dept_id'])['sold'].transform('mean' ).astype(np.float16 )
@njit
def fast_fillna(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
train_data.loc[0, features[1:]] = fast_fillna(train_data.loc[0, features[1:]].values, 0 )<predict_on_test>
df['rolling_sold_mean'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sold'].transform(lambda x: x.rolling(window=7 ).mean() ).astype(np.float16 )
tmp = np.zeros(len(features))
for(test_df, sample_prediction_df)in tqdm(iter_test):
if test_df['weight'].item() > 0:
X_test = test_df.loc[:, features].values
X_test[0, :] = fast_fillna(X_test[0, :], tmp)
 tmp = X_test[0, :]
d_test = xgb.DMatrix(X_test)
 y_preds = clf.predict(d_test)
 sample_prediction_df.action = np.where(y_preds >= 0.5, 1, 0 ).astype(int)
 else:
sample_prediction_df.action = 0
env.predict(sample_prediction_df )<set_options>
df['expanding_sold_mean'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sold'].transform(lambda x: x.expanding(2 ).mean() ).astype(np.float16 )
warnings.filterwarnings('ignore')
 tf.random.set_seed(41)
<choose_model_class>
df['daily_avg_sold'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id','d'])['sold'].transform('mean' ).astype(np.float16)
 df['avg_sold'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sold'].transform('mean' ).astype(np.float16)
 df['selling_trend'] =(df['daily_avg_sold'] - df['avg_sold'] ).astype(np.float16)
 df.drop(['daily_avg_sold','avg_sold'],axis=1,inplace=True )
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<categorify>
df = df[df['d']>=36]
def create_mlp_wrapper(train, params, filename, features):
batch_size = params['batch_size']
hidden_units = [params['hidden_unit_1'], params['hidden_unit_2'], params['hidden_unit_3'], params['hidden_unit_4']]
dropout_rates = [params['dropout_rate_1'], params['dropout_rate_2'], params['dropout_rate_3'], params['dropout_rate_4'], params['dropout_rate_5']]
label_smoothing = params['label_smoothing']
learning_rate = params['learning_rate']
time='all'
oof = []
gkf = GroupKFold(n_splits = 5)
 scores = []
utility_noisy = pd.DataFrame(columns=features,index=range(5))
scores_noisy = pd.DataFrame(columns=features,index=range(5))
for fold,(tr, te)in enumerate(gkf.split(train['resp'].values, train['resp'].values, train['date'].values)) :
X_tr, X_val = train.iloc[tr][features].values, train.iloc[te][features].values
y_tr, y_val = train.iloc[tr]['action'].values, train.iloc[te]['action'].values
ckp_path = f'JSModel_{filename}_{fold}.hdf5'
model = create_mlp(X_tr.shape[1], 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 rlr = ReduceLROnPlateau(monitor = 'val_AUC', factor = 0.1, patience = 3, verbose = 0,
min_delta = 1e-4, mode = 'max')
 ckp = ModelCheckpoint(ckp_path, monitor = 'val_AUC', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'max')
 es = EarlyStopping(monitor = 'val_AUC', min_delta = 1e-4, patience = 7, mode = 'max',
baseline = None, restore_best_weights = True, verbose = 0)
 model.fit(X_tr, y_tr, validation_data =(X_val, y_val), epochs = 1000,
batch_size = batch_size, callbacks = [rlr, ckp, es], verbose = 0)
 rng = np.random.default_rng()
for i, feature in enumerate(features):
gc.collect()
X_val_noisy = X_val.copy()
X_val_noisy[:,i] = rng.permutation(X_val_noisy[:,i])
 pred = model.predict(X_val_noisy, batch_size = batch_size * 4 ).ravel()
scores_noisy.loc[fold, feature] = roc_auc_score(y_val,(pred>.5 ).astype(int))
utility_noisy.loc[fold, feature] = utility_score(train.iloc[te]['date'].values, train.iloc[te]['weight'].values, train.iloc[te]['resp'],(pred>.5 ).astype(int))
oof.append(model.predict(X_val, batch_size = batch_size * 4 ).ravel())
 score = roc_auc_score(y_val, oof[fold])
 print(f'Fold {fold} ROC AUC:\t', round(score, 4))
K.clear_session()
del model
rubbish = gc.collect()
return oof, scores_noisy, utility_noisy 
<compute_test_metric>
df.to_pickle('data.pkl')
 del df
gc.collect() ;
def utility_score(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u<load_from_csv>
data = pd.read_pickle('data.pkl')
 valid = data[(data['d']>=1914)&(data['d']<1942)][['id','d','sold']]
test = data[data['d']>=1942][['id','d','sold']]
eval_preds = test['sold']
valid_preds = valid['sold']
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 features = [c for c in train.columns if 'feature' in c]
f_mean = train[features[1:]].mean()
train = train.query('weight > 0' ).reset_index(drop = True)
 train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 params = {'batch_size': 4096, 'dropout_rate_1': 0.08949964830076425, 'dropout_rate_2': 0.09214922878769727, 'dropout_rate_3': 0.10007316489415027, 'dropout_rate_4': 0.2416444299467454, 'dropout_rate_5': 0.1338760637313039, 'hidden_unit_1': 245, 'hidden_unit_2': 678, 'hidden_unit_3': 759, 'hidden_unit_4': 471, 'label_smoothing': 0.0007423928500131863, 'learning_rate': 0.0003772663214527269}<compute_train_metric>
stores = sales.store_id.cat.codes.unique().tolist()
for store in stores:
df = data[data['store_id']==store]
X_train, y_train = df[df['d']<1914].drop('sold',axis=1), df[df['d']<1914]['sold']
X_valid, y_valid = df[(df['d']>=1914)&(df['d']<1942)].drop('sold',axis=1), df[(df['d']>=1914)&(df['d']<1942)]['sold']
X_test = df[df['d']>=1942].drop('sold',axis=1)
 model = LGBMRegressor(
 n_estimators=1000,
learning_rate=0.3,
subsample=0.8,
colsample_bytree=0.8,
max_depth=8,
num_leaves=50,
min_child_weight=300 
)
 print('*****Prediction for Store: {}*****'.format(d_store_id[store]))
model.fit(X_train, y_train, eval_set=[(X_train,y_train),(X_valid,y_valid)],
eval_metric='rmse', verbose=20, early_stopping_rounds=20)
 valid_preds[X_valid.index] = model.predict(X_valid)
 eval_preds[X_test.index] = model.predict(X_test)
 filename = 'model'+str(d_store_id[store])+'.pkl'
joblib.dump(model, filename)
 del model, X_train, y_train, X_valid, y_valid
gc.collect()
gc.collect()
oof_pred, scores_noisy, utility_noisy = create_mlp_wrapper(train, params, 'filename', features)
 gc.collect()<feature_engineering>
feature_importance_df = pd.DataFrame()
features = [f for f in data.columns if f != 'sold']
for filename in os.listdir('/kaggle/working/'):
if 'model' in filename:
model = joblib.load(filename)
 store_importance_df = pd.DataFrame()
store_importance_df["feature"] = features
store_importance_df["importance"] = model.feature_importances_
store_importance_df["store"] = filename[5:9]
feature_importance_df = pd.concat([feature_importance_df, store_importance_df], axis=0)
 def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:20].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(averaged over store predictions)')
 plt.tight_layout()
display_importances(feature_importance_df )
<split><EOS> 
actual = False
if actual == False:
validation = sales[['id']+['d_' + str(i)for i in range(1914,1942)]]
validation['id']=pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv' ).id
validation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
else:
valid['sold'] = valid_preds
validation = valid[['id','d','sold']]
validation = pd.pivot(validation, index='id', columns='d', values='sold' ).reset_index()
validation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
validation.id = validation.id.map(d_id ).str.replace('evaluation','validation')
 test['sold'] = eval_preds
evaluation = test[['id','d','sold']]
evaluation = pd.pivot(evaluation, index='id', columns='d', values='sold' ).reset_index()
evaluation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation.id = evaluation.id.map(d_id)
 submit = pd.concat([validation,evaluation] ).reset_index(drop=True)
 submit.to_csv('submission.csv',index=False )