Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<prepare_x_and_y>
%matplotlib inline
sns.set()
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 warnings.filterwarnings("ignore", category=RuntimeWarning )
xt, yt = make_data(tr_mask)
 xv, yv = make_data(val_mask)
 xe, ye = make_data(te_mask )<import_modules>
train = pd.read_csv(".. /input/train.csv", index_col=0)
 train.shape
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf<compute_test_metric>
test = pd.read_csv(".. /input/test.csv", index_col=0)
 test.shape
def qloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def make_model(n_in):
num = L.Input(( n_in,), name="num")
 ca = L.Input(( 1,), name="snap_CA")
 tx = L.Input(( 1,), name="snap_TX")
 wi = L.Input(( 1,), name="snap_WI")
 wday = L.Input(( 1,), name="wday")
 month = L.Input(( 1,), name="month")
 year = L.Input(( 1,), name="year")
 event = L.Input(( 1,), name="event")
 nday = L.Input(( 1,), name="nday")
 item = L.Input(( 1,), name="item")
 dept = L.Input(( 1,), name="dept")
 cat = L.Input(( 1,), name="cat")
 store = L.Input(( 1,), name="store")
 state = L.Input(( 1,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
emb = L.Concatenate(name="embds" )(p)
 context = L.Flatten(name="context" )(emb)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.Dense(500, activation="relu", name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.Dense(500, activation="relu", name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.Dense(500, activation="relu", name="d3" )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=qloss, optimizer="adam")
 return model<choose_model_class>
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission.head()
net = make_model(len(FEATS))
ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.001)
 es = EarlyStopping(monitor='val_loss', patience=3)
 print(net.summary() )<train_model>
train[train.target==0].shape[0] / train.shape[0]
net.fit(xt, yt, batch_size=50_000, epochs=20, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )<load_pretrained>
X = train.drop("target", axis=1 ).values
y = train.target.values
nett = make_model(len(FEATS))
nett.load_weights("w.h5" )<predict_on_test>
n_splits = 25
solver = "liblinear"
penalty = "l1"
C=0.1
class_weight="balanced"
pv = nett.predict(xv, batch_size=50_000, verbose=1)
 pe = nett.predict(xe, batch_size=50_000, verbose=1 )<compute_test_metric>
skf= StratifiedKFold(n_splits=n_splits, random_state=0)
 lr = LogisticRegression(solver=solver, penalty=penalty, C=C, class_weight=class_weight)
 scores = []
for train_idx, test_idx in skf.split(X, y):
x_train, x_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
lr.fit(x_train, y_train)
 proba = lr.predict_proba(x_test)
 scores.append(roc_auc_score(y_test, proba[:,1]))
nett.evaluate(xv, yv, batch_size=50_000 )<define_variables>
to_try = train.drop("target", axis=1)
 mean_drop_scores = []
std_drop_scores = []
for feat in importance[0:-1]:
to_try = to_try.drop(str(feat), axis=1)
 X = to_try.values
lr = LogisticRegression(solver=solver, C=C, penalty=penalty, class_weight=class_weight)
 stk = StratifiedKFold(n_splits=n_splits, random_state=0)
 scores = []
for train_idx, test_idx in stk.split(X, y):
x_train, x_test = X[train_idx], X[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
lr.fit(x_train, y_train)
 proba = lr.predict_proba(x_test)
 scores.append(roc_auc_score(y_test, proba[:,1]))
mean_score = np.mean(scores)
 std_score = np.std(scores)
 mean_drop_scores.append(mean_score)
 std_drop_scores.append(std_score )
names = [f"F{i+1}" for i in range(28)]<create_dataframe>
important_features = importance[250::]
important_features = important_features.astype(np.str)
 important_features
piv = pd.DataFrame(ids[:, 0], columns=["id"] )<feature_engineering>
chris_important_features = ["239", "217", "209", "199", "189",
"164", "117", "108", "91", "73",
"65", "63", "45", "33", "16"]
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
<create_dataframe>
class MySemiGMM:
def __init__(self, mu, sigma):
self.sigma = sigma
self.mu = mu
self.old_mus = []
self.old_sigmas = []
self.old_pis = []
self.train_roc_convergence = []
self.pi = [0.36, 0.64]
def set_data(self, x_test, x_train, y_train):
self.x_test = x_test
self.y_train = y_train
self.x_train = x_train
self.hot_data = x_train[np.where(y_train==1)]
self.cold_data = x_train[np.where(y_train==0)]
self.n_hot = len(self.hot_data)
 self.n_cold = len(self.cold_data)
 def update_Meff(self):
self.Meff = np.sum(self.gamma, axis=0)
 def update_pars(self):
denominator_cold = self.n_cold + self.Meff[0]
counter_cold = np.sum(self.cold_data, axis=0)
 counter_cold += np.einsum('id,i -> d', self.x_test, self.gamma[:,0])
 mu_cold = counter_cold/denominator_cold
denominator_hot = self.n_hot + self.Meff[1]
counter_hot = np.sum(self.hot_data, axis=0)
 counter_hot += np.einsum('id,i -> d', self.x_test, self.gamma[:,1])
 mu_hot = counter_hot/denominator_hot
self.mu[0] = mu_cold
self.mu[1] = mu_hot
cold_var = np.sum(np.power(self.x_train - self.mu[0], 2), axis=0)
 cold_var += np.einsum('id,i -> d', np.power(self.x_test-self.mu[0], 2), self.gamma[:,0])
 sigma_cold = np.sqrt(1/(self.n_cold + self.Meff[0])* cold_var)
 hot_var = np.sum(np.power(self.x_train - self.mu[1], 2), axis=0)
 hot_var += np.einsum('id,i -> d', np.power(self.x_test-self.mu[1], 2), self.gamma[:,1])
 sigma_hot = np.sqrt(1/(self.n_hot + self.Meff[1])* hot_var)
 self.sigma[0] = sigma_cold
self.sigma[1] = sigma_hot
self.pi[0] =(self.n_cold + self.Meff[0])/(self.x_train.shape[0] + self.x_test.shape[0])
 self.pi[1] =(self.n_hot + self.Meff[1])/(self.x_train.shape[0] + self.x_test.shape[0])
 def learn(self, n_iter=10):
self.old_mus.append(self.mu.copy())
 self.old_sigmas.append(self.sigma.copy())
 self.old_pis.append(self.pi.copy())
 for n in range(n_iter):
self.gamma = self.get_gamma(self.x_test)
 self.update_Meff()
self.update_pars()
self.old_mus.append(self.mu.copy())
 self.old_sigmas.append(self.sigma.copy())
 self.old_pis.append(self.pi.copy())
 score = roc_auc_score(self.y_train, self.predict(self.x_train)[:,1])
 if n % 1 == 0:
print(n, score)
 self.train_roc_convergence.append(score)
 return self.mu
def get_single_normal_proba(self, x, my_mu, my_sigma, my_pi):
sigma2 = np.power(my_sigma,2)
 return my_pi * np.prod(1/np.sqrt(2*np.pi*sigma2)* np.exp(-0.5 *(1/sigma2)* np.power(( x-my_mu),2)))
 def get_gamma(self, X):
probabilities = np.zeros(shape=(X.shape[0], 2))
for n in range(X.shape[0]):
norm_value = self.get_single_normal_proba(X[n,:], self.mu[0], self.sigma[0], self.pi[0])
 norm_value += self.get_single_normal_proba(X[n,:], self.mu[1], self.sigma[1], self.pi[1])
 for m in range(len(self.mu)) :
probabilities[n,m] = self.get_single_normal_proba(X[n,:], self.mu[m], self.sigma[m], self.pi[m])
 probabilities[n,m] /= norm_value
return probabilities
def predict(self, X):
return self.get_gamma(X )
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2<save_to_csv>
important_features = train.drop("target", axis=1 ).columns.values
sub.to_csv("submission.csv", index=False )<train_model>
m1 = train.loc[train.target==0, important_features].mean().values
m2 = train.loc[train.target==1, important_features].mean().values
mu = np.array([m1, m2])
 old_mu = mu.copy()
s1 = train.loc[train.target==0, important_features].std().values
s2 = train.loc[train.target==1, important_features].std().values
sigma = np.array([s1,s2])
 old_sigma = sigma.copy()
x_train = train.loc[:, important_features].values
y_train = train.target.values
x_test = test.loc[:, important_features].values
model = MySemiGMM(mu, sigma)
 model.set_data(x_test, x_train, y_train)
 new_mu = model.learn()
proba_test = model.predict(x_test)
 proba_train = model.predict(x_train);
<import_modules>
m1 = train.loc[train.target==0, chris_important_features].mean().values
m2 = train.loc[train.target==1, chris_important_features].mean().values
mu = np.array([m1, m2])
 old_mu = mu.copy()
s1 = train.loc[train.target==0, chris_important_features].std().values
s2 = train.loc[train.target==1, chris_important_features].std().values
sigma = np.array([s1,s2])
 old_sigma = sigma.copy()
x_train = train.loc[:, chris_important_features].values
y_train = train.target.values
x_test = test.loc[:, chris_important_features].values
model = MySemiGMM(mu, sigma)
 model.set_data(x_test, x_train, y_train)
 new_mu = model.learn()
proba_test = model.predict(x_test)
 proba_train = model.predict(x_train);