Kaggle competition: M5 Forecasting - Uncertainty
<SOS> metric: WeightedRowwisePinballLoss Kaggle data source: m5-forecasting-uncertainty<data_type_conversions>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc
charToInt(train_x_df,notNumTypeCol )<count_values>
def preprocess_sales(sales, start=1000, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale1",
"sales1","start","sales2","scale2"],
var_name='d', value_name='demand')
 return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar["nb"] = calendar.index + 1
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1000, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales_aug.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 calendar = calendar.loc[calendar.nb>=start]
gc.collect()
return sales, calendar
notNumValueCounter = {}
for key in col:
notNumValueCounter[key] = Counter(train_x_df[key] )<count_values>
def qloss_np(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = np.array([qs])
 e = y_true - y_pred
v = np.maximum(q*e,(q-1)*e)
 return np.mean(v )
notNumValueCounter = {}
for key in notNumTypeCol:
notNumValueCounter[key] = Counter(train_x_df[key] )<feature_engineering>
%%time
CATEGORIZE = True;
START = 1000; UPPER = 1970;
maps = {}
mods = {}
sales, calendar = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )
charToInt(train_x_df,notNumValueCounter )<feature_engineering>
sales["x"] = sales["demand"] / sales["scale1"]
df = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 df['Date']=pd.to_datetime(pd.Series(df['Original_Quote_Date']))
df['Year']=df['Date'].apply(lambda x: int(str(x)[:4]))
df['Month']=df['Date'].apply(lambda x: int(str(x)[5:7]))
df['Date']=df['Date'].apply(lambda x: int(str(x)[8:10]))
df['Field10'].apply(lambda x : int(x.replace(',','')))
 test['Date']=pd.to_datetime(pd.Series(test['Original_Quote_Date']))
test['Year']=test['Date'].apply(lambda x: int(str(x)[:4]))
test['Month']=test['Date'].apply(lambda x: int(str(x)[5:7]))
test['Date']=test['Date'].apply(lambda x: int(str(x)[8:10]))
test['Field10'].apply(lambda x : int(x.replace(',','')))
 label=df['QuoteConversion_Flag']
df.drop('QuoteConversion_Flag',axis=1,inplace=True)
 number=test['QuoteNumber']
drop_columns=['Original_Quote_Date','QuoteNumber']
for names in drop_columns:
df.drop(names,axis=1,inplace=True)
 test.drop(names,axis=1,inplace=True)
 clf=xgb.XGBClassifier(max_depth=7,learning_rate=0.03,n_estimators=650,subsample=0.86,seed=50 )<categorify>
NITEMS = 42840
scale = sales['scale1'].values.reshape(( NITEMS, -1))
ids = sales['id'].values.reshape(( NITEMS, -1))
ys = sales[['x','sales1']].values.reshape(( NITEMS, -1, 2))
Z = sales[FEATS].values.reshape(( NITEMS, -1, len(FEATS)) )
for f in df.columns:
if df[f].dtypes=='object':
encoder=preprocessing.LabelEncoder()
encoder.fit(list(df[f])+list(test[f]))
df[f]=encoder.transform(list(df[f].values))
test[f]=encoder.transform(list(test[f].values))
df.fillna(-1,inplace=True)
 test.fillna(-1,inplace=True )<predict_on_test>
LEN = 1969 - START + 1
MAX_LAG = max(LAGS)
 print(LEN, MAX_LAG )
clf.fit(df,label)
 output=clf.predict_proba(test)[:,1]<save_to_csv>
CATCOLS = ['snap_CA', 'snap_TX', 'snap_WI', 'wday', 'month', 'year', 'event_name', 'nday',
'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']
sample=pd.read_csv('.. /input/sample_submission.csv')
 sample.QuoteConversion_Flag=output
sample.to_csv('final.csv',index=False )<set_options>
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf
import itertools as it
%reload_ext autoreload
%autoreload 2
%matplotlib inline
warnings.filterwarnings("ignore" )<load_from_csv>
def make_data(c, z, y):
x = {"snap_CA":c[:,:,0], "snap_TX":c[:,:,1], "snap_WI":c[:,:,2], "wday":c[:,:,3],
"month":c[:,:,4], "year":c[:,:,5], "event":c[:,:,6], "nday":c[:,:,7],
"item":c[:,:,8], "dept":c[:,:,9], "cat":c[:,:,10], "store":c[:,:,11],
"state":c[:,:,12], "num":z}
t = y
return x, t
HEIGHT = 137
WIDTH = 236
SIZE = 128
bs = 128
stats =(0.0692, 0.2051)
 arch = models.densenet121
MODEL = '.. /input/grapheme-fast-ai-starter/model_0.pth'
nworkers = 2
TEST = ['/kaggle/input/bengaliai-cv19/test_image_data_0.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_1.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_2.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_3.parquet']
LABELS = '.. /input/bengaliai-cv19/train.csv'
df = pd.read_csv(LABELS)
 nunique = list(df.nunique())[1:-1]<choose_model_class>
class DataGenerator(tf.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, x, brks, batch_size=32, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.c = x[0]
self.z = x[1]
self.y = x[2]
self.brks = brks.copy()
self.list_IDs = np.array(range(42840))
self.shuffle = shuffle
self.nb_batch = int(np.ceil(len(self.list_IDs)/ self.batch_size))
self.n_windows = brks.shape[0]
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.list_IDs)/ self.batch_size)) *(self.n_windows)
 def __getitem__(self, index):
'Generate one batch of data'
idx, kx = self.ids[index]
batch_ids = self.list_IDs[idx*self.batch_size:(idx+1)*self.batch_size]
X, y = self.__data_generation(batch_ids, kx)
 return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.ids = list(it.product(np.arange(0, self.nb_batch), self.brks))
if self.shuffle == True:
np.random.shuffle(self.ids)
 def __data_generation(self, batch_ids, kx):
'Generates data containing batch_size samples'
x_batch, y_batch = make_data(self.c[batch_ids, kx-28:kx], self.z[batch_ids,kx-28:kx],
self.y[batch_ids, kx-28:kx])
 return x_batch, y_batch
class Head(nn.Module):
def __init__(self, nc, n, ps=0.5):
super().__init__()
layers = [AdaptiveConcatPool2d() , Mish() , Flatten() ] + \
bn_drop_lin(nc*2, 512, True, ps, Mish())+ \
bn_drop_lin(512, n, True, ps)
 self.fc = nn.Sequential(*layers)
 self._init_weight()
def _init_weight(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
torch.nn.init.kaiming_normal_(m.weight)
 elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
def forward(self, x):
return self.fc(x)
 class Dnet_1ch(nn.Module):
def __init__(self, arch=arch, n=nunique, pre=True, ps=0.5):
super().__init__()
m = arch(True)if pre else arch()
conv = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)
 w =(m.features.conv0.weight.sum(1)).unsqueeze(1)
 conv.weight = nn.Parameter(w)
 self.layer0 = nn.Sequential(conv, m.features.norm0, nn.ReLU(inplace=True))
self.layer1 = nn.Sequential(
 nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False),
m.features.denseblock1)
 self.layer2 = nn.Sequential(m.features.transition1,m.features.denseblock2)
 self.layer3 = nn.Sequential(m.features.transition2,m.features.denseblock3)
 self.layer4 = nn.Sequential(m.features.transition3,m.features.denseblock4,
m.features.norm5)
 nc = self.layer4[-1].weight.shape[0]
self.head1 = Head(nc,n[0])
 self.head2 = Head(nc,n[1])
 self.head3 = Head(nc,n[2])
 def forward(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x1 = self.head1(x)
 x2 = self.head2(x)
 x3 = self.head3(x)
 return x1,x2,x3<find_best_params>
xv, yv = make_data(C[:,LEN-56:LEN-28,:], Z[:,LEN-56:LEN-28,:], ys[:,LEN-56:LEN-28])
 xe, ye = make_data(C[:,LEN-28:LEN,:], Z[:,LEN-28:LEN,:], ys[:,LEN-28:LEN] )
model = Dnet_1ch(pre=False ).cuda()
model.load_state_dict(torch.load(MODEL, map_location=torch.device('cpu')));
model.eval() ;<data_type_conversions>
def wqloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true[:,:,:1] - y_pred
v = tf.maximum(q*e,(q-1)*e)
 psl = tf.reduce_mean(v, axis=[1,2])
 weights = y_true[:,0,1] / K.sum(y_true[:,0,1])
 return tf.reduce_sum(psl * weights)
 def qloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([[qs]]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def make_model(n_in):
seq_len = 28
num = L.Input(( seq_len, n_in,), name="num")
 ca = L.Input(( seq_len,), name="snap_CA")
 tx = L.Input(( seq_len,), name="snap_TX")
 wi = L.Input(( seq_len,), name="snap_WI")
 wday = L.Input(( seq_len,), name="wday")
 month = L.Input(( seq_len,), name="month")
 year = L.Input(( seq_len,), name="year")
 event = L.Input(( seq_len,), name="event")
 nday = L.Input(( seq_len,), name="nday")
 item = L.Input(( seq_len,), name="item")
 dept = L.Input(( seq_len,), name="dept")
 cat = L.Input(( seq_len,), name="cat")
 store = L.Input(( seq_len,), name="store")
 state = L.Input(( seq_len,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
context = L.Concatenate(name="context" )(p)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.GRU(300, return_sequences=True, name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.GRU(300, return_sequences=True, name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.GRU(300, return_sequences=True, name="d3" )(x)
 x = L.Dropout(0.3 )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=wqloss, optimizer="adam")
 return model
class GraphemeDataset(Dataset):
def __init__(self, fname):
self.df = pd.read_parquet(fname)
 self.data = 255 - self.df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
name = self.df.iloc[idx,0]
img =(self.data[idx]*(255.0/self.data[idx].max())).astype(np.uint8)
 img = crop_resize(img)
 img =(img.astype(np.float32)/255.0 - stats[0])/stats[1]
return img, name<install_modules>
net = make_model(Z.shape[2])
 ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1,
patience=5, min_lr=0.0005)
 es = EarlyStopping(monitor='val_loss', patience=20)
 print(net.summary() )
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
!pip install /kaggle/input/efficientnet-pytorch/efficientnet_pytorch-0.6.1-py3-none-any.whl > /dev/null<define_variables>
%%time
net.fit_generator(DataGenerator(( C, Z, ys), brks[:-1], batch_size=10_000), epochs=200,
validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )
BATCH_SIZE = 96
N_WORKERS = 4
HEIGHT = 137
WIDTH = 236
INPUT_PATH="/kaggle/input/bengaliai-cv19"
MODEL_BASE_PATH_EFNET_B3 = '/kaggle/input/bengaliaieffnetb3'
MODEL_BASE_PATH_DN161FULL = '/kaggle/input/densenet161full' 
<define_variables>
nett = make_model(Z.shape[2])
 nett.load_weights("w.h5" )
ENSEMBLES = [
{
'model': 'efficientnet_b3',
'model_state_file': MODEL_BASE_PATH_EFNET_B3 + '/efficientNet_fold2.pth',
},
{
'model': 'efficientnet_b3',
'model_state_file': MODEL_BASE_PATH_EFNET_B3 + '/efficientNet_fold4.pth',
},
{
'model': 'densenet161',
'model_state_file': MODEL_BASE_PATH_DN161FULL + '/densenet161_b30',
},
]<define_variables>
%%time
pv = nett.predict(xv, batch_size=500, verbose=1)
 pe = nett.predict(xe, batch_size=500, verbose=1 )
NUM_ENSEMBLE = len(ENSEMBLES )<feature_engineering>
nett.evaluate(xv, yv, batch_size=5000 )
for ensemble in ENSEMBLES:
ensemble['w_grapheme'] = 1 / NUM_ENSEMBLE
ensemble['w_vowel'] = 1 / NUM_ENSEMBLE
ensemble['w_conso'] = 1 / NUM_ENSEMBLE<load_from_csv>
sv = scale[:,LEN-56:LEN-28]
se = scale[:,LEN-28:LEN]
test_df = pd.read_csv(INPUT_PATH +('/test.csv'))
submission_df = pd.read_csv(INPUT_PATH + '/sample_submission.csv' )<import_modules>
names = [f"F{i+1}" for i in range(28)]
class Identity(nn.Module):
def __init__(self):
super(Identity, self ).__init__()
def forward(self, x):
return x
class EfficientNetWrapper(nn.Module):
def __init__(self):
super(EfficientNetWrapper, self ).__init__()
self.effNet = EfficientNet.from_name('efficientnet-b3')
 self.fc_root = nn.Linear(in_features=1000, out_features=168)
 self.fc_vowel = nn.Linear(in_features=1000, out_features=11)
 self.fc_consonant = nn.Linear(in_features=1000, out_features=7)
 def forward(self, X):
output = self.effNet(X)
 output_root = self.fc_root(output)
 output_vowel = self.fc_vowel(output)
 output_consonant = self.fc_consonant(output)
 return output_vowel, output_root, output_consonant
class DenseNet161(nn.Module):
def __init__(self, pretrained):
super(DenseNet161, self ).__init__()
if pretrained == True:
self.model = pretrainedmodels.__dict__["densenet161"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["densenet161"](pretrained=None)
 self.model.last_linear = Identity()
self.l0 = nn.Linear(2208, 11)
 self.l1 = nn.Linear(2208, 168)
 self.l2 = nn.Linear(2208, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2 
<create_dataframe>
piv = pd.DataFrame(ids[:, 0], columns=["id"] )
class ClsTestDataset(Dataset):
def __init__(self, num_samples=1, model='densenet161'):
self.num_samples = num_samples
self.images = np.zeros([num_samples, WIDTH * HEIGHT], dtype=np.uint8)
 self.aug = albumentations.Compose([
albumentations.Resize(int(HEIGHT), int(WIDTH), always_apply=True),
albumentations.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], always_apply=True)
 ])
 img_id = 0
for i in tqdm(range(4)) :
datafile = INPUT_PATH + '/test_image_data_{}.parquet'.format(i)
 parq = pq.read_pandas(datafile, columns=[str(x)for x in range(32332)] ).to_pandas()
parq = parq.iloc[:, :].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 for idx, image in enumerate(parq):
self.images[img_id,...] = image.reshape(-1 ).astype(np.uint8)
 img_id = img_id + 1
del parq
def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
img = self.images[idx]
img = img.reshape(HEIGHT, WIDTH ).astype(float)
 img = Image.fromarray(img ).convert("RGB")
 img = self.aug(image=np.array(img)) ["image"]
img = np.transpose(img,(2, 0, 1)).astype(np.float32)
 img = torch.tensor(img, dtype=torch.float)
 return {
"image": img,
"image_id": idx
}<create_dataframe>
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
bengali_dataset = ClsTestDataset(num_samples = test_df.shape[0] // 3 )<load_pretrained>
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2
data_loader_test = torch.utils.data.DataLoader(
 bengali_dataset,
batch_size=BATCH_SIZE,
num_workers=N_WORKERS,
sampler=SequentialSampler(bengali_dataset),
shuffle=False
)<define_variables>
sub.to_csv("submission.csv", index=False )