Kaggle competition: Don't Overfit! II
def make_model(n_in):
num = L.Input(( n_in,), name="num")
 ca = L.Input(( 1,), name="snap_CA")
 tx = L.Input(( 1,), name="snap_TX")
 wi = L.Input(( 1,), name="snap_WI")
 wday = L.Input(( 1,), name="wday")
 month = L.Input(( 1,), name="month")
 year = L.Input(( 1,), name="year")
 event = L.Input(( 1,), name="event")
 nday = L.Input(( 1,), name="nday")
 item = L.Input(( 1,), name="item")
 dept = L.Input(( 1,), name="dept")
 cat = L.Input(( 1,), name="cat")
 store = L.Input(( 1,), name="store")
 state = L.Input(( 1,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
emb = L.Concatenate(name="embds" )(p)
 context = L.Flatten(name="context" )(emb)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.Dense(500, activation="relu", name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.Dense(500, activation="relu", name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.Dense(500, activation="relu", name="d3" )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=qloss, optimizer="adam")
 return model<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv' )
net = make_model(len(FEATS))
ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.001)
 es = EarlyStopping(monitor='val_loss', patience=3)
 print(net.summary() )<train_model>
train_X = train.drop(['id','target'], axis=1 ).as_matrix()
train_y = train['target'].values
test_X = test.drop(['id'], axis=1 ).as_matrix()
net.fit(xt, yt, batch_size=50_000, epochs=20, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )<load_pretrained>
clf = KNeighborsClassifier(n_neighbors=5)
 clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_KNeighborsClassifier.csv', index=False)
 test = test.drop('target', axis=1 )
nett = make_model(len(FEATS))
nett.load_weights("w.h5" )<predict_on_test>
clf = DecisionTreeClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_DecisionTreeClassifier.csv', index=False)
 test = test.drop('target', axis=1 )
pv = nett.predict(xv, batch_size=50_000, verbose=1)
 pe = nett.predict(xe, batch_size=50_000, verbose=1 )<compute_test_metric>
clf = RandomForestClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_RandomForestClassifier.csv', index=False)
 test = test.drop('target', axis=1 )
nett.evaluate(xv, yv, batch_size=50_000 )<import_modules>
clf = AdaBoostClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_AdaBoostClassifier.csv', index=False)
 test = test.drop('target', axis=1 )
import matplotlib.pyplot as plt<feature_engineering>
clf = GaussianNB()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_GaussianNB.csv', index=False)
 test = test.drop('target', axis=1 )
names = [f"F{i+1}" for i in range(28)]
piv = pd.DataFrame(ids[:, 0], columns=["id"])
 QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2 )<save_to_csv>
clf = xgb.XGBClassifier()
clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_XGBClassifier.csv', index=False)
 test = test.drop('target', axis=1 )
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2
sub.to_csv("submission.csv", index=False )<import_modules>
clf = LogisticRegression(class_weight='balanced', penalty='l1', C=1.0, solver='liblinear')
 clf.fit(train_X, train_y)
 test['target'] = clf.predict(test_X)
 test[['id','target']].to_csv('submission_LogisticRegression.csv', index=False)
 test = test.drop('target', axis=1 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc<categorify>
for model_type in [KNeighborsClassifier, DecisionTreeClassifier, RandomForestClassifier,
AdaBoostClassifier, xgb.XGBClassifier, LogisticRegression]:
clf = model_type()
kfold = KFold(n_splits=5, shuffle=True)
 cv = ShuffleSplit(n_splits=100, test_size=0.3, random_state=0)
 scores = cross_val_score(clf, train_X, train_y, cv=cv, scoring='roc_auc')
 print("Print {} Accuracy: {}(+/- {})".format(model_type.__name__, scores.mean() , scores.std() * 2))
def preprocess_sales(sales, start=1400, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale1",
"sales1","start","sales2","scale2"],
var_name='d', value_name='demand')
 return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar["nb"] = calendar.index + 1
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1400, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales_aug.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 calendar = calendar.loc[calendar.nb>=start]
gc.collect()
return sales, calendar 
<compute_test_metric>
clf = LogisticRegression(class_weight='balanced', solver='liblinear')
 penalty = ['l1', 'l2']
C = uniform(loc=0, scale=4)
 hyperparameters = dict(C=C, penalty=penalty)
 rand_cv = RandomizedSearchCV(clf, hyperparameters, random_state=1, n_iter=100, cv=5, verbose=0, n_jobs=-1, scoring='roc_auc')
 best_model = rand_cv.fit(train_X, train_y)
 print('Best Penalty:', best_model.best_estimator_.get_params() ['penalty'])
 print('Best C:', best_model.best_estimator_.get_params() ['C'] )
def qloss_np(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = np.array([qs])
 e = y_true - y_pred
v = np.maximum(q*e,(q-1)*e)
 return np.mean(v )<create_dataframe>
cv_results = pd.DataFrame()
cv_results['params'] = best_model.cv_results_['params']
cv_results['mean_test_score'] = best_model.cv_results_['mean_test_score']
cv_results['std_test_score'] = best_model.cv_results_['std_test_score']
cv_results['rank_test_score'] = best_model.cv_results_['rank_test_score']
cv_results = pd.concat([cv_results.drop('params', axis=1), pd.DataFrame(cv_results['params'].tolist())], axis=1)
 cv_results.sort_values('rank_test_score' ).head()
%%time
CATEGORIZE = True;
START = 1400; UPPER = 1970;
maps = {}
mods = {}
sales, calendar = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )<feature_engineering>
C = [0.001, 0.01, 0.02, 0.05, 0.1, 0.12, 0.13, 0.15, 0.16, 0.17, 0.178, 0.179, 0.175, 0.2, 0.3]
hyperparameters = dict(C=C)
 clf = LogisticRegression(solver='liblinear', class_weight='balanced', penalty='l1')
 rand_cv = RandomizedSearchCV(clf, hyperparameters, random_state=1, n_iter=100, cv=5, verbose=0, n_jobs=-1, scoring='roc_auc')
 best_model = rand_cv.fit(train_X, train_y)
 print('Best Penalty:', best_model.best_estimator_.get_params() ['penalty'])
 print('Best C:', best_model.best_estimator_.get_params() ['C'])
 print('Best Score: {}'.format(best_model.best_score_))
sales["x"] = sales["demand"] / sales["scale1"]<prepare_x_and_y>
test['target'] = best_model.predict(test_X)
 test[['id','target']].to_csv('submission_LogisticRegression_randomCV.csv', index=False)
 test = test.drop('target', axis=1 )
NITEMS = 42840
scale = sales['scale1'].values.reshape(( NITEMS, -1))
ids = sales['id'].values.reshape(( NITEMS, -1))
ys = sales[['x','sales1']].values.reshape(( NITEMS, -1, 2))
Z = sales[FEATS].values.reshape(( NITEMS, -1, len(FEATS)) )<define_variables>
test['target'] = best_model.predict_proba(test_X)[:,1]
test[['id','target']].to_csv('submission_LogisticRegression_randomCV_proba.csv', index=False)
 test = test.drop('target', axis=1 )
LEN = 1969 - START + 1
MAX_LAG = max(LAGS)
 print(LEN, MAX_LAG )<define_variables>
clf = xgb.XGBClassifier(learning_rate=0.02, n_estimators=600, objective='binary:logistic')
 params = {
'min_child_weight': [1, 5, 10],
'gamma': [0.5, 1, 1.5, 2, 5],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'max_depth': [3, 4, 5]
}
rand_cv = RandomizedSearchCV(clf, params, random_state=1, n_iter=20, cv=5, verbose=0, n_jobs=-1, scoring='roc_auc')
 best_model = rand_cv.fit(train_X, train_y )
CATCOLS = ['snap_CA', 'snap_TX', 'snap_WI', 'wday', 'month', 'year', 'event_name', 'nday',
'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']<import_modules>
best_model.best_score_
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf
import itertools as it<categorify>
test['target'] = best_model.predict(test_X)
 test[['id','target']].to_csv('submission_XGBClassifier_randomCV.csv', index=False)
 test = test.drop('target', axis=1)
 test['target'] = best_model.predict_proba(test_X)[:,1]
test[['id','target']].to_csv('submission_XGBClassifier_randomCV_proba.csv', index=False)
 test = test.drop('target', axis=1 )
def make_data(c, z, y):
x = {"snap_CA":c[:,:,0], "snap_TX":c[:,:,1], "snap_WI":c[:,:,2], "wday":c[:,:,3],
"month":c[:,:,4], "year":c[:,:,5], "event":c[:,:,6], "nday":c[:,:,7],
"item":c[:,:,8], "dept":c[:,:,9], "cat":c[:,:,10], "store":c[:,:,11],
"state":c[:,:,12], "num":z}
t = y
return x, t<prepare_x_and_y>
clf = LogisticRegression(class_weight='balanced', penalty='l1', C=0.01, solver='liblinear' ).fit(train_X, train_y )