Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<train_model>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import pystan
from sklearn.preprocessing import RobustScaler
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.kernel_ridge import KernelRidge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, auc, classification_report, confusion_matrix, f1_score, fbeta_score, precision_recall_curve, precision_score, recall_score, roc_auc_score, roc_curve
train_y = df["y"].values
train_X = df.drop(['y'], axis=1)
 def xgb_r2_score(preds, final):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 xgb_params = {
'n_trees': 520,
'eta': 0.0045,
'max_depth': 4,
'subsample': 0.98,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
final = xgb.DMatrix(train_X, train_y, feature_names=train_X.columns.values)
 model = xgb.train(dict(xgb_params), final, num_boost_round=200, feval=xgb_r2_score, maximize=True)
 fig, ax = plt.subplots(figsize=(10,10))
xgb.plot_importance(model, max_num_features=40, height=0.8, ax=ax, color = 'coral')
 print("Feature Importance by XGBoost")
 plt.show()
model = RandomForestRegressor(n_estimators=200, max_depth=10, min_samples_leaf=4, max_features=0.2, n_jobs=-1, random_state=0)
 model.fit(train_X, train_y)
 feat_names = train_X.columns.values
importances = model.feature_importances_
std = np.std([tree.feature_importances_ for tree in model.estimators_], axis=0)
 indices = np.argsort(importances)[::-1][:40]
plt.subplots(figsize=(10,10))
plt.title("Feature importances by RandomForestRegressor")
 plt.ylabel("Features")
 plt.barh(range(len(indices)) , importances[indices], color="green", align="center")
 plt.yticks(range(len(indices)) , feat_names[indices], rotation='horizontal')
 plt.ylim([-1, len(indices)])
 plt.show()<feature_engineering>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
df['X314_plus_X315'] = df.apply(lambda row: row.X314 + row.X315, axis=1)
 test['X314_plus_X315'] = test.apply(lambda row: row.X314 + row.X315, axis=1 )<compute_test_metric>
df_train['datatype'], df_test['datatype'] = 'training', 'testing'
df_test.insert(1, 'target', np.nan)
 df_data = pd.concat([df_train, df_test], ignore_index=True )
print("Correalation between X314_plus_X315 and y is : ",df.y.corr(df['X314_plus_X315']))
print("Which makes it pretty much high !! Awesome !!" )<feature_engineering>
col_convert = ['target']
df_data[col_convert] = df_data[col_convert].astype('object' )
<feature_engineering>
df_data['target'] = df_data['target'].fillna(-1 )
df['X118_plus_X314_plus_X315'] = df.apply(lambda row: row.X118 + row.X314 + row.X315, axis=1)
 test['X118_plus_X314_plus_X315'] = test.apply(lambda row: row.X118 + row.X314 + row.X315, axis=1 )<feature_engineering>
df_data = pd.get_dummies(df_data, columns=['datatype'], drop_first=True )
df["X10_plus_X54"] = df.apply(lambda row: row.X10 + row.X54, axis=1)
 test["X10_plus_X54"] = test.apply(lambda row: row.X10 + row.X54, axis=1)
 print("Correalation between X10_plus_X54 and y is : ",df.y.corr(df['X10_plus_X54']))<feature_engineering>
col_convert = ['target']
df_data[col_convert] = df_data[col_convert].astype(int )
df["X10_plus_X29"] = df.apply(lambda row: row.X10 + row.X29, axis=1)
 test["X10_plus_X29"] = test.apply(lambda row: row.X10 + row.X29, axis=1)
 print("Correalation between X10_plus_X29 and y is : ",df.y.corr(df['X10_plus_X29']))<feature_engineering>
x = df_data[df_data['datatype_training'] == 1].drop(['id', 'target', 'datatype_training'], axis=1)
 y = df_data.loc[df_data['datatype_training'] == 1, 'target']
train_X['X314_plus_X315']=df['X314_plus_X315']
train_X['X118_plus_X314_plus_X315']=df['X118_plus_X314_plus_X315']
train_X["X10_plus_X54"] = df["X10_plus_X54"]
train_X["X10_plus_X29"] = df["X10_plus_X29"]<drop_column>
lassoreg = Lasso(alpha=1e-5 ).fit(x, y)
 feat = pd.DataFrame(data=lassoreg.coef_, index=x.columns, columns=['feature_importances'] ).sort_values(['feature_importances'], ascending=False )
booler = np.ones(400)
 for i in same_features:
if booler[int(i[1][1:])]==1:
booler[int(i[1][1:])]=0
df_num.drop(i[1],axis=1,inplace=True)
 df.drop(i[1],axis=1,inplace=True)
 test.drop(i[1],axis=1,inplace=True)
 train_X.drop(i[1],axis=1,inplace=True)
 elif booler[int(i[0][1:])]==1:
booler[int(i[0][1:])]=0
df_num.drop(i[0],axis=1,inplace=True)
 df.drop(i[0],axis=1,inplace=True)
 test.drop(i[0],axis=1,inplace=True)
 train_X.drop(i[0],axis=1,inplace=True )<categorify>
x = df_data.loc[df_data['datatype_training'] == 1, model_feat]
y = df_data.loc[df_data['datatype_training'] == 1, 'target']
<categorify>
scaler = RobustScaler()
x = scaler.fit_transform(x )
<split>
x_train, x_validate, y_train, y_validate = train_test_split(x, y, test_size=0.25, random_state=58 )
x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=420)
 d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(test)
 xgb_params = {
'n_trees': 500,
'eta': 0.0050,
'max_depth': 3,
'subsample': 0.95,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
def xgb_r2_score(preds, dtrain):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(xgb_params, d_train, 1050 , watchlist, early_stopping_rounds=70, feval=xgb_r2_score, maximize=True, verbose_eval=10 )<train_model>
model_linreg = LinearRegression()
model_linreg.fit(x_train, y_train)
 model_linreg_ypredict = model_linreg.predict(x_validate)
 model_linreg_rocaucscore = roc_auc_score(y_validate, model_linreg_ypredict)
 model_linreg_cvscores = cross_val_score(model_linreg, x, y, cv=20, scoring='roc_auc')
 print('linear regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_linreg_rocaucscore, model_linreg_cvscores.mean() , 2 * model_linreg_cvscores.std()))
d_train = xgb.DMatrix(train_X, label=train_y)
 d_test = xgb.DMatrix(test)
 xgb_params = {
'n_trees': 500,
'eta': 0.0050,
'max_depth': 3,
'subsample': 0.95,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
def xgb_r2_score(preds, dtrain):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 watchlist = [(d_train, 'train')]
clf = xgb.train(xgb_params, d_train, 1050 , watchlist, early_stopping_rounds=70, feval=xgb_r2_score, maximize=True, verbose_eval=10 )<save_to_csv>
model_lassoreg = Lasso(alpha=0.01)
 model_lassoreg.fit(x_train, y_train)
 model_lassoreg_ypredict = model_lassoreg.predict(x_validate)
 model_lassoreg_rocaucscore = roc_auc_score(y_validate, model_lassoreg_ypredict)
 model_lassoreg_cvscores = cross_val_score(model_lassoreg, x, y, cv=20, scoring='roc_auc')
 print('lasso regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_lassoreg_rocaucscore, model_lassoreg_cvscores.mean() , 2 * model_lassoreg_cvscores.std()))
Answer = clf.predict(d_test)
 sub = pd.DataFrame()
sub['ID'] = test.ID
sub['y'] = Answer
sub.to_csv('mercedes_benz_The_best_or_Nothing.csv', index=False )<set_options>
params = {
'alpha': np.logspace(-4, -2, base=10, num=50),
}
model_lassoreg_cv = GridSearchCV(model_lassoreg, params, iid=False, cv=5)
 model_lassoreg_cv.fit(x_train, y_train)
 model_lassoreg_cv_ypredict = model_lassoreg_cv.predict(x_validate)
 model_lassoreg_cv_rocaucscore = roc_auc_score(y_validate, model_lassoreg_cv_ypredict)
 model_lassoreg_cv_cvscores = cross_val_score(model_lassoreg_cv, x, y, cv=20, scoring='roc_auc')
 print('lasso regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_lassoreg_cv_rocaucscore, model_lassoreg_cv_cvscores.mean() , 2 * model_lassoreg_cv_cvscores.std()))
print(' best parameters: %s' %model_lassoreg_cv.best_params_ )
plt.rcParams['font.size'] = 18
%matplotlib inline
N_FOLDS = 5
MAX_EVALS = 5<load_from_csv>
model_ridgereg = Ridge(alpha=35)
 model_ridgereg.fit(x_train, y_train)
 model_ridgereg_ypredict = model_ridgereg.predict(x_validate)
 model_ridgereg_rocaucscore = roc_auc_score(y_validate, model_ridgereg_ypredict)
 model_ridgereg_cvscores = cross_val_score(model_ridgereg, x, y, cv=20, scoring='roc_auc')
 print('ridge regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_ridgereg_rocaucscore, model_ridgereg_cvscores.mean() , 2 * model_ridgereg_cvscores.std()))
features = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 features = features.sample(n = 16000, random_state = 42)
 features = features.select_dtypes('number')
 labels = np.array(features['TARGET'].astype(np.int32)).reshape(( -1,))
features = features.drop(columns = ['TARGET', 'SK_ID_CURR'])
 train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 6000, random_state = 42)
 print('Train shape: ', train_features.shape)
 print('Test shape: ', test_features.shape)
 train_features.head()<create_dataframe>
params = {'alpha': np.logspace(-4, 4, base=10, num=50)}
model_ridgereg_cv = GridSearchCV(model_ridgereg, params, iid=False, cv=5)
 model_ridgereg_cv.fit(x_train, y_train)
 model_ridgereg_cv_ypredict = model_ridgereg_cv.predict(x_validate)
 model_ridgereg_cv_rocaucscore = roc_auc_score(y_validate, model_ridgereg_cv_ypredict)
 model_ridgereg_cv_cvscores = cross_val_score(model_ridgereg_cv, x, y, cv=20, scoring='roc_auc')
 print('ridge regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_ridgereg_cv_rocaucscore, model_ridgereg_cv_cvscores.mean() , 2 * model_ridgereg_cv_cvscores.std()))
print(' best parameters: %s' %model_ridgereg_cv.best_params_ )
model = lgb.LGBMClassifier(random_state=50)
 train_set = lgb.Dataset(train_features, label = train_labels)
 test_set = lgb.Dataset(test_features, label = test_labels )<find_best_params>
model_elasticnetreg = ElasticNet(alpha=0.01, l1_ratio=0.9)
 model_elasticnetreg.fit(x_train, y_train)
 model_elasticnetreg_ypredict = model_elasticnetreg.predict(x_validate)
 model_elasticnetreg_rocaucscore = roc_auc_score(y_validate, model_elasticnetreg_ypredict)
 model_elasticnetreg_cvscores = cross_val_score(model_elasticnetreg, x, y, cv=20, scoring='roc_auc')
 print('elastic net regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_elasticnetreg_rocaucscore, model_elasticnetreg_cvscores.mean() , 2 * model_elasticnetreg_cvscores.std()))
hyperparameters = model.get_params()
del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 10000, nfold = N_FOLDS, metrics = 'auc',
early_stopping_rounds = 100, verbose_eval = False, seed = 42)
 best = cv_results['auc-mean'][-1]
best_std = cv_results['auc-stdv'][-1]
print('The maximium ROC AUC in cross validation was {:.5f} with std of {:.5f}.'.format(best, best_std))
print('The ideal number of iterations was {}.'.format(len(cv_results['auc-mean'])) )<compute_train_metric>
params = {'alpha': np.logspace(-4, -2, base=10, num=10),
'l1_ratio': np.linspace(0.1, 0.9, num=5),
}
model_elasticnetreg_cv = GridSearchCV(model_elasticnetreg, params, iid=False, cv=5)
 model_elasticnetreg_cv.fit(x_train, y_train)
 model_elasticnetreg_cv_ypredict = model_elasticnetreg_cv.predict(x_validate)
 model_elasticnetreg_cv_rocaucscore = roc_auc_score(y_validate, model_elasticnetreg_cv_ypredict)
 model_elasticnetreg_cv_cvscores = cross_val_score(model_elasticnetreg_cv, x, y, cv=20, scoring='roc_auc')
 print('elastic net regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_elasticnetreg_cv_rocaucscore, model_elasticnetreg_cv_cvscores.mean() , 2 * model_elasticnetreg_cv_cvscores.std()))
print(' best parameters: %s' %model_elasticnetreg_cv.best_params_ )
model.n_estimators = len(cv_results['auc-mean'])
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
baseline_auc = roc_auc_score(test_labels, preds)
 print('The baseline model scores {:.5f} ROC AUC on the test set.'.format(baseline_auc))<train_model>
model_kernelridgereg = KernelRidge(alpha=0.0001, kernel='polynomial', degree=4)
 model_kernelridgereg.fit(x_train, y_train)
 model_kernelridgereg_ypredict = model_kernelridgereg.predict(x_validate)
 model_kernelridgereg_rocaucscore = roc_auc_score(y_validate, model_kernelridgereg_ypredict)
 model_kernelridgereg_cvscores = cross_val_score(model_kernelridgereg, x, y, cv=20, scoring='roc_auc')
 print('kernel ridge regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_kernelridgereg_rocaucscore, model_kernelridgereg_cvscores.mean() , 2 * model_kernelridgereg_cvscores.std()))
def objective(hyperparameters):

 global ITERATION
ITERATION += 1
if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
subsample = hyperparameters['boosting_type'].get('subsample', 1.0)
 hyperparameters['boosting_type'] = hyperparameters['boosting_type']['boosting_type']
hyperparameters['subsample'] = subsample
for parameter_name in ['num_leaves', 'subsample_for_bin', 'min_child_samples']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 start = timer()
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 10000, nfold = N_FOLDS,
early_stopping_rounds = 100, metrics = 'auc', seed = 50)
 run_time = timer() - start
best_score = cv_results['auc-mean'][-1]
loss = 1 - best_score
n_estimators = len(cv_results['auc-mean'])
 hyperparameters['n_estimators'] = n_estimators
of_connection = open(OUT_FILE, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, hyperparameters, ITERATION, run_time, best_score])
 of_connection.close()
return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'train_time': run_time, 'status': STATUS_OK}<import_modules>
params = {'alpha': np.logspace(-4, -2, base=10, num=10),
'degree': [1, 2, 3, 4, 5],
}
model_kernelridgereg_cv = GridSearchCV(model_kernelridgereg, params, iid=False, cv=5)
 model_kernelridgereg_cv.fit(x_train, y_train)
 model_kernelridgereg_cv_ypredict = model_kernelridgereg_cv.predict(x_validate)
 model_kernelridgereg_cv_rocaucscore = roc_auc_score(y_validate, model_kernelridgereg_cv_ypredict)
 model_kernelridgereg_cv_cvscores = cross_val_score(model_kernelridgereg_cv, x, y, cv=20, scoring='roc_auc')
 print('kernel ridge regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_kernelridgereg_cv_rocaucscore, model_kernelridgereg_cv_cvscores.mean() , 2 * model_kernelridgereg_cv_cvscores.std()))
print(' best parameters: %s' %model_kernelridgereg_cv.best_params_ )
from hyperopt import hp
from hyperopt.pyll.stochastic import sample<set_options>
model_treereg = DecisionTreeRegressor(splitter='best', min_samples_split=5)
 model_treereg.fit(x_train, y_train)
 model_treereg_ypredict = model_treereg.predict(x_validate)
 model_treereg_rocaucscore = roc_auc_score(y_validate, model_treereg_ypredict)
 model_treereg_cvscores = cross_val_score(model_treereg, x, y, cv=20, scoring='roc_auc')
 print('decision tree regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_treereg_rocaucscore, model_treereg_cvscores.mean() , 2 * model_treereg_cvscores.std()))
learning_rate = {'learning_rate': hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) }<init_hyperparams>
model_forestreg = RandomForestRegressor(n_estimators=100, min_samples_split=3, random_state=58)
 model_forestreg.fit(x_train, y_train)
 model_forestreg_ypredict = model_forestreg.predict(x_validate)
 model_forestreg_rocaucscore = roc_auc_score(y_validate, model_forestreg_ypredict)
 model_forestreg_cvscores = cross_val_score(model_forestreg, x, y, cv=20, scoring='roc_auc')
 print('random forest regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_forestreg_rocaucscore, model_forestreg_cvscores.mean() , 2 * model_forestreg_cvscores.std()))
boosting_type = {'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample': 1.0}])}
hyperparams = sample(boosting_type)
 hyperparams<init_hyperparams>
model_code =
model_data = {
'N': 250,
'N2': 19750,
'K': 300,
'y': df_data.loc[df_data['datatype_training'] == 1, 'target'],
'X': df_data[df_data['datatype_training'] == 1].drop(['id', 'target', 'datatype_training'], axis=1),
'X_test': df_data[df_data['datatype_training'] == 0].drop(['id', 'target', 'datatype_training'], axis=1),
}
model_stan = pystan.StanModel(model_code=model_code)
 model_stan_fitted = model_stan.sampling(data=model_data, seed=58 )
subsample = hyperparams['boosting_type'].get('subsample', 1.0)
 hyperparams['boosting_type'] = hyperparams['boosting_type']['boosting_type']
hyperparams['subsample'] = subsample
hyperparams<init_hyperparams>
x_test = df_data[df_data['datatype_training'] == 0]
y_test = pd.DataFrame(np.mean(model_stan_fitted.extract(permuted=True)['y_pred'], axis=0), columns=['target'], index=df_data.loc[df_data['datatype_training'] == 0, 'id'] )
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt', 'subsample': hp.uniform('gdbt_subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('dart_subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample': 1.0}]),
'num_leaves': hp.quniform('num_leaves', 20, 150, 1),
'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.5)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000),
'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0),
'is_unbalance': hp.choice('is_unbalance', [True, False]),
}<prepare_output>
out = pd.DataFrame({'id': y_test.index, 'target': y_test['target']})
 out.to_csv('submission.csv', index=False )