Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<categorify>
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb
%%time
input_columns = [
'comment_text'
]

 train = clean_data(train, input_columns)
 val = clean_data(val, input_columns)
<drop_column>
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }
%%time
input_columns = [
'content'
]
test_data = clean_data(test_data, input_columns)
 del tokenizer<compute_train_metric>
pd.options.display.max_columns = 50
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<categorify>
h = 28
max_lags = 57
tr_last = 1913
fday = datetime(2016,4, 25)
 fday
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<load_pretrained>
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt
MODEL = 'jplu/tf-xlm-roberta-large'
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 save_path = '/kaggle/working/xlmr_large/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
<prepare_x_and_y>
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )
%%time
x_train = regular_encode(train.comment_text.astype(str),
tokenizer, maxlen=maxlen)
 x_valid = regular_encode(val.comment_text.astype(str ).values,
tokenizer, maxlen=maxlen)
 x_test = regular_encode(test_data.content.astype(str ).values,
tokenizer, maxlen=maxlen)
 y_valid = val.toxic.values
y_train = train.toxic.values<categorify>
FIRST_DAY = 350
%%time
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<compute_train_metric>
df.dropna(inplace = True)
 df.shape
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed<choose_model_class>
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]
def build_model(transformer, loss='binary_crossentropy', max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.3 )(cls_token)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC() ])
 return model<load_pretrained>
%%time
np.random.seed(777)
 fake_valid_inds = np.random.choice(X_train.index.values, 2_000_000, replace = False)
 train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
 train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds],
categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_data = lgb.Dataset(X_train.loc[fake_valid_inds], label = y_train.loc[fake_valid_inds],
categorical_feature=cat_feats,
free_raw_data=False )
%%time
with strategy.scope() :
transformer_layer = transformers.TFXLMRobertaModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,loss='binary_crossentropy', max_len=maxlen)
 model.summary()<compute_train_metric>
del df, X_train, y_train, fake_valid_inds,train_inds ; gc.collect()
def callback() :
cb = []
reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',
factor=0.3, patience=2,
verbose=1, mode='auto',
epsilon=0.0001, cooldown=1, min_lr=0.000001)
 cb.append(reduceLROnPlat)
 log = CSVLogger('log.csv')
 cb.append(log)
 RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)
 cb.append(RocAuc)
 return cb<choose_model_class>
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1200,
'num_leaves': 2**11-1,
"min_data_in_leaf": 2**12-1,
}
def build_lrfn(lr_start=0.000001, lr_max=0.000002,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn<define_variables>
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=20 )
model_path = 'jigsawMultilingual.hdf5'
model_path1 = '/kaggle/working/jigsawMultilingual.hdf5'<choose_model_class>
m_lgb.save_model("model.lgb" )
checkpoint = ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)
 es = EarlyStopping(monitor='val_accuracy', mode='max', patience=2,
restore_best_weights=True, verbose=1)
 lr_callback = LearningRateScheduler(lrfn, verbose=1)
 callback_list = [checkpoint, lr_callback]<train_model>
def create_lag_features_for_test(dt, day):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt.date == day, lag_col] = \
dt.loc[dt.date ==day-timedelta(days=lag), 'sales'].values
windows = [7, 28]
for window in windows:
for lag in lags:
df_window = dt[(dt.date <= day-timedelta(days=lag)) &(dt.date > day-timedelta(days=lag+window)) ]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt.date==day,'id'])
 dt.loc[dt.date == day,f"rmean_{lag}_{window}"] = \
df_window_grouped.sales.values
%%time
N_STEPS = x_train.shape[0] // BATCH_SIZE
EPOCHS = 2
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=valid_dataset,
callbacks=callback_list,
epochs=EPOCHS
)<load_pretrained>
def create_date_features_for_test(dt):
date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(
 dt["date"].dt, date_feat_func ).astype("int16" )
if os.path.exists(model_path1):
model.load_weights(model_path1 )<train_model>
%%time
alphas = [1.028, 1.023, 1.018]
weights = [1/len(alphas)]*len(alphas)
 te0 = create_dt(False)
 create_date_features_for_test(te0)
 for icount,(alpha, weight)in enumerate(zip(alphas, weights)) :
te = te0.copy()
cols = [f"F{i}" for i in range(1, 29)]
for tdelta in range(0, 28):
day = fday + timedelta(days=tdelta)
 print(tdelta, day.date())
 tst = te[(te.date >= day - timedelta(days=max_lags))
&(te.date <= day)].copy()
create_lag_features_for_test(tst, day)
 tst = tst.loc[tst.date == day, train_cols]
te.loc[te.date == day, "sales"] = \
alpha * m_lgb.predict(tst)
 te_sub = te.loc[te.date >= fday, ["id", "sales"]].copy()
te_sub["F"] = [f"F{rank}" for rank in te_sub.groupby("id")[
"id"].cumcount() +1]
te_sub = te_sub.set_index(["id", "F"] ).unstack() [
"sales"][cols].reset_index()
te_sub.fillna(0., inplace=True)
 te_sub.sort_values("id", inplace=True)
 te_sub.reset_index(drop=True, inplace=True)
 te_sub.to_csv(f"submission_{icount}.csv", index=False)
 if icount == 0:
sub = te_sub
sub[cols] *= weight
else:
sub[cols] += te_sub[cols]*weight
print(icount, alpha, weight )
%%time
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
callbacks=callback_list,
epochs= EPOCHS
)<load_pretrained>
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()
<drop_column><EOS> 
sub2 = sub.copy()
sub2["id"] = sub2["id"].str.replace("validation$", "evaluation")
 sub = pd.concat([sub, sub2], axis=0, sort=False)
 sub.to_csv("submission.csv",index=False )