Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<categorify>
!pip install scikit-misc
warnings.filterwarnings("ignore")
 invlogit = lambda x: 1/(1 + tt.exp(-x))
def trace_predict(trace, X):
y_hat = np.apply_along_axis(np.mean, 1, expit(trace['alpha'] + np.dot(X, np.transpose(trace['beta']))))
return(y_hat)
 def posterior_predict(trace, model, n=1000, progressbar=True):
with model:
ppc = pm.sample_posterior_predictive(trace,n, progressbar=progressbar)
 return(np.mean(np.array(ppc['y_obs']), axis=0))
def gather(df, key, value, cols):
id_vars = [ col for col in df.columns if col not in cols ]
id_values = cols
var_name = key
value_name = value
return pd.melt(df, id_vars, id_values, var_name, value_name)
 def spread(df, index, columns, values):
return df.pivot(index, columns, values ).reset_index(level=index ).rename_axis(None,axis=1)
 def fit_loess(df, transform_logit=False):
l = loess(df["value"],df["target"])
 l.fit()
pred_obj = l.predict(df["value"],stderror=True)
 conf = pred_obj.confidence()
yhat = pred_obj.values
ll = conf.lower
ul = conf.upper
df["loess"] = np.clip(yhat,.001,.999)
 df["ll"] = np.clip(ll,.001,.999)
 df["ul"] = np.clip(ul,.001,.999)
 if transform_logit:
df["logit_loess"] = logit(df["loess"])
 df["logit_ll"] = logit(df["ll"])
 df["logit_ul"] = logit(df["ul"])
 return(df)
 def plot_loess(df, features):
z = gather(df[["id","target"]+features], "feature", "value", features)
 z = z.groupby("feature" ).apply(fit_loess, transform_logit=True)
 z["feature"] = pd.to_numeric(z["feature"])
 plot =(
 ggplot(z, aes("value","logit_loess",ymin="logit_ll",ymax="logit_ul")) +
geom_point(alpha=.5)+
geom_line(alpha=.5)+
geom_ribbon(alpha=.33)+
facet_wrap("~feature")
)
 return(plot)
 df = pd.read_csv(".. /input/train.csv")
 y = np.asarray(df.target)
 X = np.array(df.ix[:, 2:302])
 df2 = pd.read_csv('.. /input/test.csv')
 df2.head()
X2 = np.array(df2.ix[:, 1:301])
 print("training shape: ", X.shape)
 print("test shape: ", X2.shape )
random_results['hyperparameters'] = random_results['hyperparameters'].map(ast.literal_eval)
 bayes_results['hyperparameters'] = bayes_results['hyperparameters'].map(ast.literal_eval )<compute_train_metric>
def make_model(X, y, cauchy_scale):
model = pm.Model()
with model:
alpha = pm.Normal('alpha', mu=0, sd=3)
 beta = pm.Cauchy('beta', alpha=0, beta=cauchy_scale, shape=X.shape[1])
 mu = pm.math.dot(X, beta)
 y_obs = pm.Bernoulli('y_obs', p=invlogit(alpha + mu), observed=y)
 model.name = "linear_c_"+str(cauchy_scale)
 return(model)
train_set = lgb.Dataset(train, label = train_labels)
 hyperparameters = dict(**random_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS)
 print('The cross validation score on the full dataset for Random Search= {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )<save_to_csv>
comp = pm.stats.compare(model_dict, ic="LOO", method='BB-pseudo-BMA')
 for i in range(0,len(traces)) :
y_hat = trace_predict(traces[i], X)
 print("scale = ",cauchy_scale_pars[i],", training AUCROC:",roc_auc_score(y,y_hat))
comp
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_random_search.csv', index = False )<compute_train_metric>
model1 = models[5]
trace1 = traces[5]
model1.name
hyperparameters = dict(**bayes_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS)
 print('The cross validation score on the full dataset for Bayesian optimization = {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )<save_to_csv>
def generate_submission(trace, file_suffix=""):
test_predictions = trace_predict(trace, X2)
 submission = pd.DataFrame({'id':df2.id, 'target':test_predictions})
 submission.to_csv("submission_"+file_suffix+".csv", index = False)
 return(None)
 for model in model_dict.keys() :
generate_submission(model_dict[model], model.name )
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_bayesian_optimization.csv', index = False )<set_options>
comp_MA = pm.stats.compare(dict(zip(models[0:-3], traces[0:-3])) , ic="LOO", method='BB-pseudo-BMA')
 ppc_w = pm.sample_posterior_predictive_w(traces[0:-3], 4000, [make_model(X2,np.zeros(19750),c)for c in cauchy_scale_pars[0:-3]],
weights=comp_MA.weight.sort_index(ascending=True),
progressbar=True)
 y_hatMA = np.mean(np.array(ppc_w['y_obs']), axis=0)
 submission = pd.DataFrame({'id':df2.id, 'target':y_hatMA})
 submission.to_csv('submission_MA.csv', index = False )
<categorify><EOS> 
def make_polymodel(X, y):
with pm.Model() as model:
alpha = pm.Normal('alpha', mu=0, sd=3)
 beta1 = pm.Cauchy('beta', alpha=0, beta=.07, shape=X.shape[1])
 beta2 = pm.Cauchy('beta^2', alpha=0, beta=.07, shape=X.shape[1])
 beta3 = pm.Cauchy('beta^3', alpha=0, beta=.07, shape=X.shape[1])
 mu1 = pm.math.dot(X, beta1)
 mu2 = pm.math.dot(np.power(X,2), beta2)
 mu3 = pm.math.dot(np.power(X,3), beta3)
 p = invlogit(alpha + mu1 + mu2 + mu3)
 y_obs = pm.Bernoulli('y_obs', p=p, observed=y)
 return(model)