Kaggle competition: M5 Forecasting - Accuracy
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<train_model>
import pandas as pd
import numpy as np
import os
import lightgbm as lgb
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_0.jpg')
 print(img.shape)
 plt.imshow(img )<load_from_csv>
path = ".. /input/m5-forecasting-accuracy"
calendar = pd.read_csv(os.path.join(path, "calendar.csv"))
selling_prices = pd.read_csv(os.path.join(path, "sell_prices.csv"))
sample_submission = pd.read_csv(os.path.join(path, "sample_submission.csv"))
sales = pd.read_csv(os.path.join(path, "sales_train_validation.csv"))
path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<define_variables>
def prep_calendar(df):
df = df.drop(["date", "weekday", "event_type_1", "event_type_2"], axis=1)
 df = df.assign(d = df.d.str[2:].astype(int))
to_ordinal = ["event_name_1", "event_name_2"]
df[to_ordinal] = df[to_ordinal].fillna("1")
 df[to_ordinal] = OrdinalEncoder(dtype="int" ).fit_transform(df[to_ordinal])+ 1
to_int8 = ["wday", "month", "snap_CA", "snap_TX", "snap_WI"] + to_ordinal
df[to_int8] = df[to_int8].astype("int8")
 return df
nb_classes = 4
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 40
SEED = 12345
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [img_size, img_size]<categorify>
calendar = prep_calendar(calendar)
 calendar.head()
def mixup(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = nb_classes
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*float(lab1)+ a*float(lab2))
image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2<normalization>
LAGS = [7, 28]
WINDOWS = [7, 28]
FIRST = 1914
LENGTH = 28
def demand_features(df):

 for lag in LAGS:
df[f'lag_t{lag}'] = df.groupby('id')['demand'].transform(lambda x: x.shift(lag)).astype("float32")
 for w in WINDOWS:
df[f'rolling_mean_lag{lag}_w{w}'] = df.groupby('id')[f'lag_t{lag}'].transform(lambda x: x.rolling(w ).mean() ).astype("float32")
 return df
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 image = tf.image.random_brightness(image, max_delta=0.05, seed=seed)
 image = tf.image.random_contrast(image, lower=0.7, upper=1.3, seed=seed)
 if label is None:
return image
else:
return image, label<categorify>
def demand_features_eval(df):

 out = df.groupby('id', sort=False ).last()
for lag in LAGS:
out[f'lag_t{lag}'] = df.groupby('id', sort=False)['demand'].nth(-lag-1 ).astype("float32")
 for w in WINDOWS:
out[f'rolling_mean_lag{lag}_w{w}'] = df.groupby('id', sort=False)['demand'].nth(list(range(-lag-w, -lag)) ).groupby('id', sort=False ).mean().astype("float32")
 return out.reset_index()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.map(mixup)
.prefetch(AUTO)
 )<categorify>
def prep_data(df, drop_d=1000):

 df = df.drop(["d_" + str(i+1)for i in range(drop_d)], axis=1)
 df = df.assign(id=df.id.str.replace("_validation", ""))
df = df.reindex(columns=df.columns.tolist() + ["d_" + str(FIRST + i)for i in range(2 * LENGTH)])
 df = df.melt(id_vars=["id", "item_id", "store_id", "state_id", "dept_id", "cat_id"], var_name='d', value_name='demand')
 df = df.assign(d=df.d.str[2:].astype("int64"),
demand=df.demand.astype("float32"))
df = demand_features(df)
 df = df[df.d >(drop_d + max(LAGS)+ max(WINDOWS)) ]
df = df.merge(calendar, how="left", on="d")
 df = df.merge(selling_prices, how="left", on=["store_id", "item_id", "wm_yr_wk"])
 df = df.drop(["wm_yr_wk"], axis=1)
 for v in ["item_id", "store_id", "state_id", "dept_id", "cat_id"]:
df[v] = OrdinalEncoder(dtype="int" ).fit_transform(df[[v]] ).astype("int16")+ 1
x = list(set(df.columns)- {'id', 'd', 'demand'})
 test = df[df.d >= FIRST - max(LAGS)- max(WINDOWS)]
df = df[df.d < FIRST]
xtrain, xvalid, ytrain, yvalid = train_test_split(df[x], df["demand"], test_size=0.1, shuffle=True, random_state=54)
 train = lgb.Dataset(xtrain, label = ytrain)
 valid = lgb.Dataset(xvalid, label = yvalid)
 return train, valid, test, x
def fit_model(train, valid):

 params = {
'metric': 'rmse',
'objective': 'poisson',
'seed': 200,
'force_row_wise' : True,
'learning_rate' : 0.09,
'lambda': 0.1,
'num_leaves': 63,
'sub_row' : 0.7,
'bagging_freq' : 1,
'colsample_bytree': 0.77
}
fit = lgb.train(params,
train,
num_boost_round = 2000,
valid_sets = [valid],
early_stopping_rounds = 200,
verbose_eval = 100)
 lgb.plot_importance(fit, importance_type="gain", precision=0, height=0.5, figsize=(6, 10)) ;
return fit
def pred_to_csv(fit, test, x, cols=sample_submission.columns, file="submission.csv"):

 for i, day in enumerate(np.arange(FIRST, FIRST + LENGTH)) :
test_day = demand_features_eval(test[(test.d <= day)&(test.d >= day - max(LAGS)- max(WINDOWS)) ])
 test.loc[test.d == day, "demand"] = fit.predict(test_day[x])*(1.025 + 0.01 *(day > 1928))
test = test.assign(id=test.id + "_" + np.where(test.d < FIRST + LENGTH, "validation", "evaluation"),
F="F" +(test.d - FIRST + 1 - LENGTH *(test.d >= FIRST + LENGTH)).astype("str"))
submission = test.pivot(index="id", columns="F", values="demand" ).reset_index() [cols].fillna(1)
 submission.to_csv(file, index=False)
 return True
<init_hyperparams><EOS> 
train, valid, test, x = prep_data(sales, 1000)
 fit = fit_model(train, valid)
 pred_to_csv(fit, test, x )