Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<define_variables>
warnings.filterwarnings('ignore' )
ntrain = len(X_train)
 nval = len(X_val)
 batch_size = 32<choose_model_class>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df
model = Sequential()
model.add(Conv2D(32, kernel_size =(3, 3), activation='relu', input_shape=(150, 150, 3)))
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(96, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(96, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dense(12, activation = 'softmax'))<choose_model_class>
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle(AUX_MODELS+'test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]
model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc'] )<choose_model_class>
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'tweedie',
'tweedie_variance_power': 1.1,
'metric': 'rmse',
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 2**11-1,
'min_data_in_leaf': 2**12-1,
'feature_fraction': 0.5,
'max_bin': 100,
'n_estimators': 1400,
'boost_from_average': False,
'verbose': 1,
}
train_datagen = ImageDataGenerator(rescale=1./255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,)
 val_datagen = ImageDataGenerator(rescale=1./255 )<define_variables>
SEED = 42
seed_everything(SEED)
 lgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1913
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/m5-simple-fe/grid_part_1.pkl'
PRICE = '.. /input/m5-simple-fe/grid_part_2.pkl'
CALENDAR = '.. /input/m5-simple-fe/grid_part_3.pkl'
LAGS = '.. /input/m5-lags-features/lags_df_28.pkl'
MEAN_ENC = '.. /input/m5-custom-features/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/m5groupkfold/'
STORES_IDS = ['CA_1']
CV_FOLDS = [0,1,2]
SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )
train_generator = train_datagen.flow(X_train, y_train, batch_size=batch_size)
 val_generator = val_datagen.flow(X_val, y_val, batch_size=batch_size)
 print(ntrain)
 print(nval )<train_model>
for store_id in STORES_IDS:
print('Train', store_id)
 grid_df, features_columns = get_data_by_store(store_id)
 train_mask = grid_df['d']<=END_TRAIN
preds_mask = grid_df['d']>(END_TRAIN-100)
 folds = GroupKFold(n_splits=3)
 grid_df['groups'] = grid_df['tm_w'].astype(str)+ '_' + grid_df['tm_y'].astype(str)
 split_groups = grid_df[train_mask]['groups']
X,y = grid_df[train_mask][features_columns], grid_df[train_mask][TARGET]
grid_df = grid_df[preds_mask].reset_index(drop=True)
 keep_cols = [col for col in list(grid_df)if '_tmp_' not in col]
grid_df = grid_df[keep_cols]
grid_df.to_pickle('test_'+store_id+'.pkl')
 del grid_df
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X, y, groups=split_groups)) :
print('Fold:',fold_)
 print(len(trn_idx),len(val_idx))
tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]
v_X, v_y = X.iloc[val_idx,:], y[val_idx]
train_data = lgb.Dataset(tr_x, label=tr_y)
 valid_data = lgb.Dataset(v_X, label=v_y)
 seed_everything(SEED)
 estimator = lgb.train(
 lgb_params,
train_data,
valid_sets = [train_data, valid_data],
verbose_eval = 100 
)
 model_name = 'lgb_model_'+STORES_IDS[0]+'_'+str(fold_)+'.bin'
pickle.dump(estimator, open(model_name, 'wb'))
del train_data, valid_data, estimator
gc.collect()
MODEL_FEATURES = features_columns
history = model.fit_generator(train_generator,
steps_per_epoch=ntrain // batch_size,
epochs=1024,
validation_data=val_generator,
validation_steps=nval // batch_size )<load_pretrained>
for fold_ in CV_FOLDS:
print("FOLD:", fold_)
 all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 for store_id in STORES_IDS:
model_path = 'lgb_model_'+store_id+'_'+str(fold_)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb'))
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
all_preds = all_preds.reset_index(drop=True)
 print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
all_preds.to_csv('all_preds_'+str(fold_)+'.csv',index=False)
 del temp_df, all_preds
model.save_weights('model_wieghts.h5')
 model.save('model_keras.h5' )<predict_on_test>
all_preds_0=pd.read_csv(AUX_MODELS+'all_preds_0'+'.csv')
 all_preds_1=pd.read_csv(AUX_MODELS+'all_preds_1'+'.csv')
 all_preds_2=pd.read_csv(AUX_MODELS+'all_preds_2'+'.csv' )
X_test, y_test = read_and_process_image(test_imgs[10:20])
 x = np.array(X_test)
 test_datagen = ImageDataGenerator(rescale=1./255 )<predict_on_test>
final_all_preds = pd.DataFrame()
final_all_preds['id'] = all_preds_1['id']
for item in all_preds_1:
if item!='id':
final_all_preds[item]=(all_preds_0[item]*(1/3)) +(all_preds_1[item]*(1/3)) +(all_preds_2[item]*(1/3))
final_all_preds
<prepare_x_and_y><EOS> 
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(final_all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('submission'+'.csv', index=False )