Kaggle competition: Don't Overfit! II
class DataGenerator(tf.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, x, brks, batch_size=32, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.c = x[0]
self.z = x[1]
self.y = x[2]
self.brks = brks.copy()
self.list_IDs = np.array(range(42840))
self.shuffle = shuffle
self.nb_batch = int(np.ceil(len(self.list_IDs)/ self.batch_size))
self.n_windows = brks.shape[0]
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.list_IDs)/ self.batch_size)) *(self.n_windows)
 def __getitem__(self, index):
'Generate one batch of data'
idx, kx = self.ids[index]
batch_ids = self.list_IDs[idx*self.batch_size:(idx+1)*self.batch_size]
X, y = self.__data_generation(batch_ids, kx)
 return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.ids = list(it.product(np.arange(0, self.nb_batch), self.brks))
if self.shuffle == True:
np.random.shuffle(self.ids)
 def __data_generation(self, batch_ids, kx):
'Generates data containing batch_size samples'
x_batch, y_batch = make_data(self.c[batch_ids, kx-28:kx], self.z[batch_ids,kx-28:kx],
self.y[batch_ids, kx-28:kx])
 return x_batch, y_batch 
<prepare_x_and_y>
import pandas as pd
import numpy as np
import random
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression,RidgeClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
xv, yv = make_data(C[:,LEN-56:LEN-28,:], Z[:,LEN-56:LEN-28,:], ys[:,LEN-56:LEN-28])
 xe, ye = make_data(C[:,LEN-28:LEN,:], Z[:,LEN-28:LEN,:], ys[:,LEN-28:LEN] )<compute_test_metric>
prob_auc_df = pd.read_csv(".. /input/300-probed-aucs-from-dont-overfit-ii/probed_aucs.csv")
 prob_auc_df.loc[:,"variable"]=prob_auc_df.loc[:,"variable"].transform(lambda x: x[1:] ).astype(int)
 prob_auc_df
def wqloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true[:,:,:1] - y_pred
v = tf.maximum(q*e,(q-1)*e)
 psl = tf.reduce_mean(v, axis=[1,2])
 weights = y_true[:,0,1] / K.sum(y_true[:,0,1])
 return tf.reduce_sum(psl * weights)
 def qloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([[qs]]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def make_model(n_in):
seq_len = 28
num = L.Input(( seq_len, n_in,), name="num")
 ca = L.Input(( seq_len,), name="snap_CA")
 tx = L.Input(( seq_len,), name="snap_TX")
 wi = L.Input(( seq_len,), name="snap_WI")
 wday = L.Input(( seq_len,), name="wday")
 month = L.Input(( seq_len,), name="month")
 year = L.Input(( seq_len,), name="year")
 event = L.Input(( seq_len,), name="event")
 nday = L.Input(( seq_len,), name="nday")
 item = L.Input(( seq_len,), name="item")
 dept = L.Input(( seq_len,), name="dept")
 cat = L.Input(( seq_len,), name="cat")
 store = L.Input(( seq_len,), name="store")
 state = L.Input(( seq_len,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
context = L.Concatenate(name="context" )(p)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.GRU(100, return_sequences=True, name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.GRU(100, return_sequences=True, name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.GRU(100, return_sequences=True, name="d3" )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=wqloss, optimizer="adam")
 return model<choose_model_class>
prob_auc_df.sort_values("variable",inplace=True)
 prob_auc_df.reset_index(drop=True,inplace=True )
net = make_model(Z.shape[2])
 ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.0005)
 es = EarlyStopping(monitor='val_loss', patience=20)
 print(net.summary() )<train_model>
prob_auc_df.loc[:,"public_auc"] = prob_auc_df.loc[:,"public_auc"]-0.5
prob_auc_df.loc[:,"public_auc"]=prob_auc_df.loc[:,"public_auc"].transform(lambda x: x if np.abs(x)>0.04 else 0)
 prob_auc_df
net.fit_generator(DataGenerator(( C, Z, ys), brks[:-1], batch_size=10_000), epochs=100,
validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )<load_pretrained>
score = np.array(prob_auc_df.loc[:,"public_auc"] )
nett = make_model(Z.shape[2])
 nett.load_weights("w.h5" )<predict_on_test>
train_df = pd.read_csv(".. /input/dont-overfit-ii/train.csv" ).drop('id', axis=1)
 train_df.head()
pv = nett.predict(xv, batch_size=500, verbose=1)
 pe = nett.predict(xe, batch_size=500, verbose=1 )<compute_test_metric>
test_df = pd.read_csv('.. /input/dont-overfit-ii/test.csv' ).drop('id', axis = 1)
 test_df.head()
nett.evaluate(xv, yv, batch_size=5000 )<find_best_params>
prob_df = test_df.copy()
prob_df.loc[:,"target"]= expit(np.dot(np.array(prob_df),score)) >0.5
sv = scale[:,LEN-56:LEN-28]
se = scale[:,LEN-28:LEN]<define_variables>
plt.figure(figsize=(12, 12))
sns.violinplot(data=test_df.dropna() ["0"],orient="h" )
names = [f"F{i+1}" for i in range(28)]<create_dataframe>
for col in test_df.columns:
z,p= scipy.stats.normaltest(test_df[col])
 print(p<0.005,z,p )
piv = pd.DataFrame(ids[:, 0], columns=["id"] )<feature_engineering>
x_corr = test_df.corr().sort_values("0" ).sort_values("0",axis=1)
 x_corr.head()
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
<create_dataframe>
kbest = TSNE(3)
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2<save_to_csv>
robust = RobustScaler().fit(np.concatenate(( train_df.drop('target', axis=1), test_df), axis=0))
y = train_df['target']
X = SimpleImputer(strategy='mean' ).fit_transform(train_df.drop('target', axis=1))
sub.to_csv("submission.csv", index=False )<import_modules>
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42,train_size=0.8)
import pandas as pd, numpy as np
from matplotlib import pyplot as plt
import scipy.stats as stats<set_options>
logreg = GaussianNB()
logreg.fit(X_train, y_train)
 test_score = logreg.score(X_test, y_test)
 test_score
pd.options.display.max_columns = 50<load_from_csv>
best_score = 0
best_std = 0
for penalty in ['l1']:
for C in [2.**(i/2)for i in range(-9*2,6*2)]:
score_ = []
for train_index, val_index in StratifiedShuffleSplit(n_splits=12, test_size=0.1, random_state=42 ).split(X, y):
X_train = X[train_index]
y_train = y[train_index]
X_test = X[val_index]
y_test = y[val_index]
seed = random.randint(0,2<<31)
 np.random.seed(seed)
 logreg = LogisticRegression(dual=False,max_iter=10**5,penalty=penalty, C=C)
 logreg.fit(X_train, y_train)
 score_ += [roc_auc_score(y_train,logreg.predict(X_train)) ]
score = np.mean(score_)
 if score > best_score:
best_std = np.std(score_)
 print(best_score,best_std)
 best_score = score
best_parameters = {'C': C, 'penalty': penalty}
s = seed
best = pd.read_csv(".. /input/m5-accuracy-submission-file/submission.csv")
 best.head()<load_from_csv>
reg_list = []
for train_index, val_index in StratifiedShuffleSplit(n_splits=30, test_size=0.1, random_state=42 ).split(X, y):
X_train = X[train_index]
y_train = y[train_index]
X_test = X[val_index]
y_test = y[val_index]
seed = random.randint(0,2<<31)
 np.random.seed(seed)
 logreg = LogisticRegression(dual=False,max_iter=10**5,**best_parameters)
 logreg.fit(X_train, y_train)
 test_score = logreg.score(X_test, y_test)
 reg_list.append(logreg)
 def auc_reg() :
return sum(map(lambda i:i.score(X_test, y_test), reg_list)) /len(reg_list)
 def score_reg() :
return sum(map(lambda i:roc_auc_score(y_test,i.predict(X_test)) , reg_list)) /len(reg_list)
 def sum_reg(x):
return sum(map(lambda i:i.predict_proba(x), reg_list)) /len(reg_list)
 test_score = score_reg()
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_evaluation.csv")
 sales.head()<merge>
X_prob = prob_df.drop("target",axis=1)
 Y_prob = prob_df["target"]
sub = best.merge(sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]], on = "id")
 sub["_all_"] = "Total"
sub.shape<define_search_space>
sum(map(lambda i:roc_auc_score(Y_prob,i.predict(X_prob)) , reg_list)) /len(reg_list )
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 qs.shape
def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 level_coef_dict = {"id": get_ratios(coef=0.3), "item_id": get_ratios(coef=0.15),
"dept_id": get_ratios(coef=0.08), "cat_id": get_ratios(coef=0.07),
"store_id": get_ratios(coef=0.08), "state_id": get_ratios(coef=0.07), "_all_": get_ratios(coef=0.05), 
("state_id", "item_id"): get_ratios(coef=0.19),("state_id", "dept_id"): get_ratios(coef=0.1), 
("store_id","dept_id"): get_ratios(coef=0.11),("state_id", "cat_id"): get_ratios(coef=0.08), 
("store_id","cat_id"): get_ratios(coef=0.1)
 } 
<statistical_test>
x = np.average([i.coef_ for i in reg_list],axis = 0)
 print(x,np.std(x),np.median(np.std([i.coef_ for i in reg_list],axis = 0)))
 logreg.coef_ = x
logreg.intercept_ = np.average([i.intercept_ for i in reg_list],axis = 0)
 print(logreg.score(X_test,y_test))
print(roc_auc_score(y_test,logreg.predict(X_test)))
 print(roc_auc_score(y_train,logreg.predict(X_train)) )
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_validation', '')}_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_validation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df2 = df.copy()<statistical_test>
print(roc_auc_score(Y_prob,logreg.predict(X_prob)) )
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_evaluation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_evaluation', '')}_{q:.3f}_evaluation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_evaluation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True )<save_to_csv>
print("Best score: {:.3f} {:.3f}".format(best_score,best_std))
print("Best parameters: {}".format(best_parameters))
print("Best score on test data: {:.3f}".format(test_score))
df = pd.concat([df2,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index < len(df.index)//2, "id"] = df.loc[df.index < len(df.index)//2, "id"].str.replace("_evaluation$", "_validation")
 df.shape
df.head()
df.to_csv("submission.csv", index = False )<import_modules>
sub = pd.read_csv('.. /input/dont-overfit-ii/sample_submission.csv')
 sub['target'] = sum_reg(robust.transform(test_df)) [:,1]
sub.to_csv('submission.csv', index=False )