Kaggle competition: M5 Forecasting - Uncertainty
<SOS> metric: WeightedRowwisePinballLoss Kaggle data source: m5-forecasting-uncertainty<concatenate>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc
x_train = pd.concat([_x_train,
svd_data_train,
], axis=1, copy=False)
 x_test = pd.concat([_x_test,
svd_data_test,
], axis=1, copy=False )<drop_column>
def preprocess_sales(sales, start=1400, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale","start"],
var_name='d', value_name='demand')
 print("generating order")
 if start is not None:
skip = start
else:
skip = 1
sales["nb"] =sales.index // 42840 + skip
return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1400, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 del calendar
print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 gc.collect()
return sales
<set_options>
%%time
CATEGORIZE = True;
START = 1400; UPPER = 1970;
maps = {}
mods = {}
sales = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )
gc.collect()<init_hyperparams>
sales["x"] = sales["demand"] / sales["scale"]
all_params = {'min_child_weight': [3],
'subsample': [1],
'subsample_freq': [1],
'seed': [114514],
'colsample_bytree': [0.8],
'learning_rate': [0.01],
'max_depth': [-1],
'min_split_gain': [0.01],
'reg_alpha': [0],
'max_bin': [63],
'num_leaves': [255],
'objective': ['regression_l1'],
'scale_pos_weight': [1],
'verbose': [-1],
'boosting_type': ['gbdt'],
'metric': ['rmse'],
}<split>
nb = sales['nb'].values
MAX_LAG = max(LAGS)
 tr_mask = np.logical_and(nb>START + MAX_LAG, nb<=1941)
 val_mask = np.logical_and(nb>1913, nb<=1941)
 te_mask = np.logical_and(nb>1941, nb<=1969 )
min_score = 10000
min_params = None
cv = KFold(n_splits=5, shuffle=True, random_state=114514)
 for params in tqdm(list(ParameterGrid(all_params))):
for test, train in cv.split(x_train, y_train):
trn_x = x_train.iloc[train]
val_x = x_train.iloc[test]
trn_y = y_train[train]
val_y = y_train[test]
train_data = lgb.Dataset(trn_x,
label=trn_y 
)
 test_data = lgb.Dataset(val_x,
label=val_y 
)
 clf = lgb.train(params,
train_data,
100000,
early_stopping_rounds=100,
valid_sets=[test_data],
verbose_eval=30 
)
 params['n_tree'] = clf.best_iteration
pred = clf.predict(val_x)
 score = np.sqrt(mean_squared_error(val_y, pred))
if min_score > score:
min_score = score
min_params = params
print(score, min_score)
 break<compute_test_metric>
scale = sales['scale'].values
ids = sales['id'].values
ys = sales['x'].values
Z = sales[FEATS].values
print(min_score, min_params )<train_model>
scale
train_data = lgb.Dataset(x_train,
label=y_train, 
)
 n_tree = int(clf.best_iteration * 1.1)
 clf = lgb.train(min_params,
train_data,
n_tree,
valid_sets=[train_data],
verbose_eval=10
)<save_to_csv>
sv = scale[val_mask]
se = scale[te_mask]
ids = ids[te_mask]
ids = ids.reshape(( -1, 28))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = deactivation(clf.predict(x_test))
sub.to_csv('submit.csv', index=False )<load_from_zip>
ca = sales[['snap_CA']].values
tx = sales[['snap_TX']].values
wi = sales[['snap_WI']].values
wday = sales[['wday']].values
month = sales[['month']].values
year = sales[['year']].values
event = sales[['event_name']].values
nday = sales[['nday']].values
lag1=sales[['x_28']].values
!mkdir fastai_lib;cd fastai_lib;git clone https://github.com/rpauli/fastai.git;ln -s fastai_lib/fastai/old/fastai/.. /<set_options>
item = sales[['item_id']].values
dept = sales[['dept_id']].values
cat = sales[['cat_id']].values
store = sales[['store_id']].values
state = sales[['state_id']].values
%load_ext autoreload
%autoreload<set_options>
def make_data(mask):
x = {"snap_CA":ca[mask], "snap_TX":tx[mask], "snap_WI":wi[mask], "wday":wday[mask],
"month":month[mask], "year":year[mask], "event":event[mask], "nday":nday[mask],
"item":item[mask], "dept":dept[mask], "cat":cat[mask], "store":store[mask],
"state":state[mask],"num":Z[mask]}
t = ys[mask]
return x, t
warnings.filterwarnings("ignore" )<concatenate>
xt, yt = make_data(tr_mask)
 xv, yv = make_data(val_mask)
 xe, ye = make_data(te_mask )
def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(path_base, path_train))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf
def calc_apk(pred,actual,k=3):

 pred=to_np(pred)
 actual=to_np(actual)
 top_idxs=top_k_preds(pred,k)
 idx_0=(np.where(actual==1)[1][:,None]-top_idxs)
 idx_correct=idx_0==0
weighted_scores=1/np.arange(1,k+1)
 score_per_data=weighted_scores[np.argmax(idx_correct,axis=1)]*np.max(idx_correct,axis=1)
 mean_score=np.mean(score_per_data)
 return mean_score
def top_k_preds(pred,k):
top_idxs=np.argsort(-to_np(pred),axis=1)[:,:k]
return top_idxs
def acc_metric(preds,actual):
return accuracy_score(np.where(actual==1)[1],np.argmax(to_np(preds),axis=1))<compute_train_metric>
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=1, reduce=True):
super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
self.reduce = reduce
def forward(self, inputs, targets):
BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
 pt = torch.exp(-BCE_loss)
 F_loss = self.alpha *(1-pt)**self.gamma * BCE_loss
if self.reduce:
return torch.mean(F_loss)
 else:
return F_loss<normalization>
q = tf.constant(np.array([qs]), dtype=tf.float32)
 q
def draw_cv2(raw_strokes, size=256, lw=6, time_color=True):
img = np.zeros(( BASE_SIZE, BASE_SIZE,3), np.float32)
 t_max=max(1,len(raw_strokes))
velos=[np.sqrt(np.sum(np.square(np.diff(stroke)) ,axis=0)).astype(int)for stroke in raw_strokes]
velo_min=np.min([np.min(velo)for velo in velos])
 velo_max=max(1,np.max([np.max(velo)for velo in velos]))
for t, stroke in enumerate(raw_strokes):
velo=velos[t]
velo=(velo)*0.9/(velo_max)+0.1
velo*=255
for i in range(len(stroke[0])- 1):
try:
color =(255 - int(t*255./t_max), int(t*255./t_max), int(velo[i])) if time_color else 255
except:
print(t,t_max,t*255./t_max,int(t*255./t_max),int(velo[i]))
_ = cv2.line(img,(stroke[0][i], stroke[1][i]), 
(stroke[0][i + 1], stroke[1][i + 1]), color, lw)
 img/=255
if size != BASE_SIZE:
return cv2.resize(img,(size, size))
else:
return img<prepare_x_and_y>
ca = L.Input(( 1,), name="snap_CA")
 ca
class DoodleDataset(Dataset):

 def __init__(self, x, y, transform=None,sz=64):
self.x = x
self.y = y
self.transform = transform
self.n = self.get_n()
self.c = self.get_c()
self.sz = sz
@abstractmethod
def get_x(self, i):
return draw_cv2(json.loads(self.x[i]),self.sz)
 @abstractmethod
def get_y(self, i):
y= np.zeros(self.c,dtype=np.float)
 y[int(self.y[i])]=1
return y
@property
def is_multi(self): return True
@property
def is_reg(self):return False
def get1item(self, idx):
x,y = self.get_x(idx),self.get_y(idx)
 return self.get(self.transform, x, y)
 def __getitem__(self, idx):
if isinstance(idx,slice):
xs,ys = zip(*[self.get1item(i)for i in range(*idx.indices(self.n)) ])
 return np.stack(xs),ys
return self.get1item(idx)
 def __len__(self): return self.n
def get(self, tfm, x, y):
return(x,y)if tfm is None else tfm(x,y)
 @abstractmethod
def get_n(self):
return len(self.y)
 @abstractmethod
def get_c(self):
return len(np.unique(self.y))
@abstractmethod
def get_sz(self):
return self.sz
@property
def is_multi(self):

 return True
class DoodleImageData(ImageData):
def get_dl(self, ds, shuffle):
if ds is None: return None
return DataLoader(ds, batch_size=self.bs, shuffle=shuffle,
num_workers=self.num_workers, pin_memory=False)
<choose_model_class>
def qloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def make_model(n_in):
num = L.Input(( n_in,), name="num")
 ca = L.Input(( 1,), name="snap_CA")
 tx = L.Input(( 1,), name="snap_TX")
 wi = L.Input(( 1,), name="snap_WI")
 wday = L.Input(( 1,), name="wday")
 month = L.Input(( 1,), name="month")
 year = L.Input(( 1,), name="year")
 event = L.Input(( 1,), name="event")
 nday = L.Input(( 1,), name="nday")
 item = L.Input(( 1,), name="item")
 dept = L.Input(( 1,), name="dept")
 cat = L.Input(( 1,), name="cat")
 store = L.Input(( 1,), name="store")
 state = L.Input(( 1,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
emb = L.Concatenate(name="embds" )(p)
 context = L.Flatten(name="context" )(emb)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.Dense(2048, activation="relu", name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.Dense(2048, activation="relu", name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.Dense(2048, activation="relu", name="d3" )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=qloss, optimizer="adam")
 return model
class DoodleConvnetBuilder(ConvnetBuilder):
def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, pretrained=True):
self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut
if xtra_fc is None: xtra_fc = [512]
if ps is None: ps = [0.25]*len(xtra_fc)+ [0.5]
self.ps,self.xtra_fc = ps,xtra_fc
if f in model_meta: cut,self.lr_cut = model_meta[f]
else: cut,self.lr_cut = 0,0
cut-=xtra_cut
layers = cut_model(f(pretrained), cut)
 layers[0] = nn.Conv2d(1,64,kernel_size=(7,7),stride=(2,2),padding=(3, 3), bias=False)
 self.nf = model_features[f] if f in model_features else(num_features(layers)*2)
 if not custom_head: layers += [AdaptiveConcatPool2d() , Flatten() ]
self.top_model = nn.Sequential(*layers)
 n_fc = len(self.xtra_fc)+1
if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
if custom_head: fc_layers = [custom_head]
else: fc_layers = self.get_fc_layers()
self.n_fc = len(fc_layers)
 self.fc_model = to_gpu(nn.Sequential(*fc_layers))
if not custom_head: apply_init(self.fc_model, kaiming_normal)
 self.model = to_gpu(nn.Sequential(*(layers+fc_layers)))
 class DoodleConvLearner(ConvLearner):
@classmethod
def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
pretrained=True, **kwargs):
models = DoodleConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=pretrained)
 return cls(data, models, precompute, **kwargs)
 @classmethod
def lsuv_learner(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False, **kwargs):
models = DoodleConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=False)
 convlearn=cls(data, models, precompute, **kwargs)
 convlearn.lsuv_init()
return convlearn<create_dataframe>
net = make_model(len(FEATS))
ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.001)
 es = EarlyStopping(monitor='val_loss', patience=4)
 print(net.summary() )
def get_stats(X_trn,X_val,y_trn,y_val,N=100):
bs=340
num_workers=4
sz=64
aug_tfms = None
tfms = tfms_from_stats(( np.array([0.,0.,0.]), np.array([1.,1.,1.])) , sz=sz, aug_tfms=aug_tfms,tfm_y=TfmType.NO)
 datasets=DoodleImageData.get_ds(DoodleDataset,(X_trn,y_trn),(X_val, y_val),tfms=tfms,sz=sz)
 doodleimageloader=DoodleImageData('.',datasets,bs,num_workers,enc.classes_)
 mn,var=[],[]
for i,(x,y)in tqdm(enumerate(iter(doodleimageloader.trn_dl))):
batch=to_np(x ).swapaxes(1,3 ).reshape([-1,3])
 mn.append(np.mean(batch,axis=0))
var.append(np.var(batch,axis=0))
if i>N:
break
channel_mean,channel_var=np.mean(mn,axis=0),np.mean(var,axis=0)
 return channel_mean,channel_var<load_from_csv>
net.fit(xt, yt, batch_size=50_000, epochs=75, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )
def get_set(N_cat,N_rows,N_skip):
if N_cats < len(list_all_categories()):
df = pd.concat([pd.read_csv(f'{path_base}/{path_train}/{cat}.csv',usecols=['drawing','word','key_id','recognized'],nrows=N_rows,skiprows=np.arange(1,N_skip+1)) for cat in tqdm(np.random.choice(list_all_categories() ,N_cats,replace=False)) ])
 elif N_cats == len(list_all_categories()):
df = pd.concat([pd.read_csv(f'{path_base}/{path_train}/{cat}.csv',usecols=['drawing','word','key_id','recognized'],nrows=N_rows,skiprows=np.arange(1,N_skip+1)) for cat in tqdm(list_all_categories())])
 else:
print('Somethings wrong')
 df_cleaned=df[df.recognized==True].drop(['recognized'],axis=1)
 df_cleaned.word=df_cleaned.word.str.replace(' ','_')
 return df<define_variables>
nett = make_model(len(FEATS))
nett.load_weights("w.h5" )
path_base='.. /input'
path_train='train_simplified' 
<define_variables>
pv = nett.predict(xv, batch_size=50_000, verbose=1)
 pe = nett.predict(xe, batch_size=50_000, verbose=1 )
np.random.seed(0)
 N_cats=len(list_all_categories())
 N_rows_train=10000
N_rows_val=100
N_cats<load_from_csv>
nett.evaluate(xv, yv, batch_size=50_000 )
test_file='test_simplified.csv'
df_test=pd.read_csv(f'{path_base}/{test_file}')
 df_test.info()<categorify>
names = [f"F{i+1}" for i in range(28)]
enc=LabelEncoder()
enc.fit(df_trn.word)
 df_trn=shuffle(df_trn)
 df_val=shuffle(df_val)
 y_trn=enc.transform(df_trn.word)
 y_val=enc.transform(df_val.word)
 X_trn=df_trn.drawing.values
X_val=df_val.drawing.values<init_hyperparams>
piv = pd.DataFrame(ids[:, 0], columns=["id"] )
bs=256
num_workers=4
BASE_SIZE=256
sz=128
arch=resnet18<compute_test_metric>
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
channel_mean,channel_var=get_stats(X_trn,X_val,y_trn,y_val,50 )<compute_train_metric>
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2
<train_model><EOS> 
sub.to_csv("submission.csv", index=False )