Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<load_from_csv>
warnings.filterwarnings('ignore' )
train_df = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_labels = np.zeros(( train_df.shape[0], 1103))
for row_index, row in enumerate(train_df['attribute_ids']):
for label in row.split() :
train_labels[row_index, int(label)] = 1
labels_sum = np.sum(train_labels,axis=0 )<categorify>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
 train_index = None
valid_index = None
class data_generator(Sequence):
def mix_up(x, y):
x = np.array(x, np.float32)
 lam = np.random.beta(0.4, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def create_train(dataset_info, batch_size, shape, augument=True, mix=False):
assert shape[2] == 3
while True:
dataset_info = shuffle(dataset_info)
 for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), nb_classes))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(( image)/255.0)
 batch_labels[i][X_train_batch[i]['labels']] = 1
if(mix):
batch_images, batch_labels = data_generator.mix_up(batch_images, batch_labels)
 yield np.array(batch_images, np.float32), batch_labels
def create_valid(valid_dataset_info, batch_size, shape, augument=False):
global valid_index
assert shape[2] == 3
while True:
for start in range(0, len(valid_dataset_info), batch_size):
end = min(start + batch_size, len(valid_dataset_info))
batch_images = []
X_train_batch = valid_dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), nb_classes))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], shape)
 if augument:
image = data_generator.augment(image)
 batch_images.append(( image)/255.0)
 batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def load_image(path, shape):
img_cv = cv2.imread(path+'.png')
 img_cv = cv2.cvtColor(img_cv,cv2.COLOR_BGR2RGB)
 if img_cv.shape[1]==300 and img_cv.shape[0]>=450:
over_high = img_cv.shape[0]-300
start = np.random.randint(over_high//3,over_high,1)[0]
img_cv = img_cv[start:start+300,:,:]
elif img_cv.shape[0]==300 and img_cv.shape[1]>450:
over_width = img_cv.shape[1]-300
start = np.random.randint(over_width//3,over_width,1)[0]
img_cv = img_cv[:,start:start+300,:]
img_cv = cv2.resize(img_cv,(img_size,img_size))
return img_cv
def augment(image):
augment_img = iaa.Sequential([
iaa.SomeOf(( 0,4),[
iaa.ContrastNormalization(( 0.5, 1.5)) ,
iaa.Multiply(( 0.9, 1.1), per_channel=0.2),
iaa.Fliplr(0.5),
iaa.GaussianBlur(sigma=(0, 0.6)) ,
])], random_order=True)
 image_aug = augment_img.augment_image(image)
 return image_aug<import_modules>
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle('.. /input/darkerevapretrained/test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,Concatenate)
<import_modules>
lgb_params = {
'boosting_type': 'gbdt',
'metric': 'custom',
'objective': 'rmse',
'n_jobs': -1,
'n_estimators': 3000,
'verbose': -1,
}
class CyclicLR(Callback):

 def __init__(self, base_lr=1e-5, max_lr=1e-2, step_size=1000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )<choose_model_class>
VER = 1
SEED = 42
seed_everything(SEED)
 lgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1941
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/evalm5/grid_part_1.pkl'
PRICE = '.. /input/evalm5/grid_part_2.pkl'
CALENDAR = '.. /input/evalm5/grid_part_3.pkl'
LAGS = '.. /input/evalm5/lags_df_28.pkl'
MEAN_ENC = '.. /input/evalm5/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/darkerevapretrained/'
STORES_IDS = pd.read_csv('.. /input/m5-forecasting-accuracy/sales_train_validation.csv')['store_id']
STORES_IDS = list(STORES_IDS.unique())
 SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 model = applications.DenseNet121(weights=None,include_top=False,input_tensor=input_tensor)
 model.load_weights(model_weights_path)
 x0 = model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu', kernel_regularizer=l2(5e-4))(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='sigmoid', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model<choose_model_class>
MODEL_FEATURES = ['item_id',
'dept_id',
'cat_id',
'release',
'sell_price',
'price_max',
'price_min',
'price_std',
'price_mean',
'price_norm',
'price_nunique',
'item_nunique',
'price_momentum',
'price_momentum_m',
'price_momentum_y',
'event_name_1',
'event_type_1',
'event_name_2',
'event_type_2',
'snap_CA',
'snap_TX',
'snap_WI',
'tm_d',
'tm_w',
'tm_m',
'tm_y',
'tm_wm',
'tm_dw',
'tm_w_end',
'enc_cat_id_mean',
'enc_cat_id_std',
'enc_dept_id_mean',
'enc_dept_id_std',
'enc_item_id_mean',
'enc_item_id_std',
'sales_lag_1',
'sales_lag_2',
'sales_lag_3',
'sales_lag_4',
'sales_lag_5',
'sales_lag_6',
'sales_lag_7',
'sales_lag_8',
'sales_lag_9',
'sales_lag_10',
'sales_lag_11',
'sales_lag_12',
'sales_lag_13',
'sales_lag_14',
'sales_lag_15',
'sales_lag_16',
'sales_lag_17',
'sales_lag_18',
'sales_lag_19',
'sales_lag_20',
'sales_lag_21',
'sales_lag_22',
'sales_lag_23',
'sales_lag_24',
'sales_lag_25',
'sales_lag_26',
'sales_lag_27',
'rolling_mean_7',
'rolling_std_7',
'rolling_mean_14',
'rolling_std_14',
'rolling_mean_30',
'rolling_std_30',
'rolling_mean_60',
'rolling_std_60',
'rolling_mean_180',
'rolling_std_180',
'rolling_mean_tmp_1_7',
'rolling_mean_tmp_1_14',
'rolling_mean_tmp_1_30',
'rolling_mean_tmp_1_60',
'rolling_mean_tmp_7_7',
'rolling_mean_tmp_7_14',
'rolling_mean_tmp_7_30',
'rolling_mean_tmp_7_60',
'rolling_mean_tmp_14_7',
'rolling_mean_tmp_14_14',
'rolling_mean_tmp_14_30',
'rolling_mean_tmp_14_60',
'cluster']
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 17; warmup_batch_size = 128;mixup_batch_size = 64
batch_size = 64
checkpoint = ModelCheckpoint('denseNet121_focal.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=7)
 csv_logger = CSVLogger(filename='training_log.csv',
separator=',',
append=True)
 indexes = np.arange(train_dataset_info.shape[0])
 train_indexes, valid_indexes = train_test_split(indexes, test_size=0.15, random_state=8)
 train_generator = data_generator.create_train(
 train_dataset_info[train_indexes], batch_size,(img_size,img_size,3), augument=True, mix=False)
 train_mixup = data_generator.create_train(
 train_dataset_info[train_indexes], mixup_batch_size,(img_size,img_size,3), augument=False, mix=True)
 train_generator_warmup = data_generator.create_train(
 train_dataset_info[train_indexes], warmup_batch_size,(img_size,img_size,3), augument=False)
 validation_generator = data_generator.create_valid(
 train_dataset_info[valid_indexes], batch_size,(img_size,img_size,3), augument=False)
 lrsched = OneCycleScheduler(lr_max=1e-4, n_data_points=len(train_indexes),
epochs=1, batch_size=warmup_batch_size, verbose=0)
<train_model>
all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 print('Predict | Day:', PREDICT_DAY)
 for store_id in tqdm(STORES_IDS):
model_path = 'lgb_model_'+store_id+'_v'+str(VER)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb')) [0]
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES], num_iteration = estimator.best_iteration)
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
del temp_df
all_preds = all_preds.reset_index(drop=True )
<choose_model_class><EOS> 
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('darker_eval_v'+str(VER)+'.csv', index=False )