Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<correct_missing_values>
plt.rcParams['font.size'] = 18
%matplotlib inline
N_FOLDS = 5
MAX_EVALS = 5
df.replace('?',np.NaN,inplace = True )<categorify>
features = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 features = features.sample(n = 16000, random_state = 42)
 features = features.select_dtypes('number')
 labels = np.array(features['TARGET'].astype(np.int32)).reshape(( -1,))
features = features.drop(columns = ['TARGET', 'SK_ID_CURR'])
 train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 6000, random_state = 42)
 print('Train shape: ', train_features.shape)
 print('Test shape: ', test_features.shape)
 train_features.head()
le = LabelEncoder()<categorify>
model = lgb.LGBMClassifier(random_state=50)
 train_set = lgb.Dataset(train_features, label = train_labels)
 test_set = lgb.Dataset(test_features, label = test_labels )
df['Col189'] = le.fit_transform(df['Col189'])
 df['Col190'] = df['Col190'].replace({'sacc1':1, 'sacc2':2, 'sacc4':3, 'sacc5':4})
 df['Col191'] = df['Col191'].replace({'time1':1, 'time2':2, 'time3':3})
 df['Col192'] = df['Col192'].replace({'p1':1,'p2':2,'p3':3,'p4':4,'p5':5,'p6':6,'p7':7,'p8':8,'p9':9,'p10':10})
 df['Col192'].fillna(df['Col192'].value_counts().idxmax() ,inplace = True)
 df['Col193'].fillna(df['Col193'].value_counts().idxmax() ,inplace = True )<count_values>
hyperparameters = model.get_params()
del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 10000, nfold = N_FOLDS, metrics = 'auc',
early_stopping_rounds = 100, verbose_eval = False, seed = 42)
 best = cv_results['auc-mean'][-1]
best_std = cv_results['auc-stdv'][-1]
print('The maximium ROC AUC in cross validation was {:.5f} with std of {:.5f}.'.format(best, best_std))
print('The ideal number of iterations was {}.'.format(len(cv_results['auc-mean'])) )
df['Col194'].fillna(df['Col194'].value_counts().idxmax() ,inplace = True)
 df['Col194'] = df['Col194'].replace({'ab':1, 'ac':2, 'ad':3})
 df['Col195'].fillna(df['Col195'].value_counts().idxmax() ,inplace = True)
 df['Col195'] = df['Col195'].replace({'Jb1':1, 'Jb2':2, 'Jb3':3, 'Jb4':4})
 df['Col196'].fillna(df['Col196'].value_counts().idxmax() ,inplace = True)
 df['Col196'] = df['Col196'].replace({'H1':1, 'H2':2, 'H3':3})
 df['Col197'] = df['Col197'].replace({'me':'ME', 'sm':'SM', 'M.E.':'ME','la':'LA'})
 df['Col197'].fillna(df['Col197'].value_counts().idxmax() ,inplace = True)
 df['Col197'] = df['Col197'].replace({'SM':1, 'ME':2, 'LA':3, 'XL':4})
 df = pd.get_dummies(data=df,columns = ['Col193','Col196','Col191'] )<define_variables>
model.n_estimators = len(cv_results['auc-mean'])
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
baseline_auc = roc_auc_score(test_labels, preds)
 print('The baseline model scores {:.5f} ROC AUC on the test set.'.format(baseline_auc))
cols = [col for col in df if col != 'Class'] + ['Class']<drop_column>
def objective(hyperparameters):

 global ITERATION
ITERATION += 1
if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
subsample = hyperparameters['boosting_type'].get('subsample', 1.0)
 hyperparameters['boosting_type'] = hyperparameters['boosting_type']['boosting_type']
hyperparameters['subsample'] = subsample
for parameter_name in ['num_leaves', 'subsample_for_bin', 'min_child_samples']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 start = timer()
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 10000, nfold = N_FOLDS,
early_stopping_rounds = 100, metrics = 'auc', seed = 50)
 run_time = timer() - start
best_score = cv_results['auc-mean'][-1]
loss = 1 - best_score
n_estimators = len(cv_results['auc-mean'])
 hyperparameters['n_estimators'] = n_estimators
of_connection = open(OUT_FILE, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, hyperparameters, ITERATION, run_time, best_score])
 of_connection.close()
return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'train_time': run_time, 'status': STATUS_OK}
df = df[cols]<data_type_conversions>
from hyperopt import hp
from hyperopt.pyll.stochastic import sample
df[obj_cols] = df[obj_cols].astype(float )<create_dataframe>
learning_rate = {'learning_rate': hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) }
data = df.copy()<prepare_x_and_y>
boosting_type = {'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample': 1.0}])}
hyperparams = sample(boosting_type)
 hyperparams
X = data.drop(['ID','Class'],axis=1)
 y = data['Class']<correct_missing_values>
subsample = hyperparams['boosting_type'].get('subsample', 1.0)
 hyperparams['boosting_type'] = hyperparams['boosting_type']['boosting_type']
hyperparams['subsample'] = subsample
hyperparams
X.fillna(X.mean() ,inplace = True )<create_dataframe>
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt', 'subsample': hp.uniform('gdbt_subsample', 0.5, 1)},
{'boosting_type': 'dart', 'subsample': hp.uniform('dart_subsample', 0.5, 1)},
{'boosting_type': 'goss', 'subsample': 1.0}]),
'num_leaves': hp.quniform('num_leaves', 20, 150, 1),
'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.5)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000, 20000),
'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0),
'is_unbalance': hp.choice('is_unbalance', [True, False]),
}
X_data = X<normalization>
x = sample(space)
 subsample = x['boosting_type'].get('subsample', 1.0)
 x['boosting_type'] = x['boosting_type']['boosting_type']
x['subsample'] = subsample
x
X = StandardScaler().fit_transform(X )<prepare_output>
x = sample(space)
 subsample = x['boosting_type'].get('subsample', 1.0)
 x['boosting_type'] = x['boosting_type']['boosting_type']
x['subsample'] = subsample
x
X = pd.DataFrame(X,columns = X_data.columns)
 X.head()<drop_column>
OUT_FILE = 'bayes_test.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
results = objective(sample(space))
print('The cross validation loss = {:.5f}.'.format(results['loss']))
print('The optimal number of estimators was {}.'.format(results['hyperparameters']['n_estimators']))
<prepare_x_and_y>
tpe_algorithm = tpe.suggest
X2 = X_data<prepare_x_and_y>
trials = Trials()
X2 = StandardScaler().fit_transform(X2)
 X2 = normalize(X2,norm='l1')
 X2 = pd.DataFrame(X2,columns = X_data.columns )<import_modules>
OUT_FILE = 'bayes_test.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score']
writer.writerow(headers)
 of_connection.close()
from sklearn.decomposition import PCA<create_dataframe>
from hyperopt import fmin
pca = PCA(n_components=70)
 principalComponents = pca.fit_transform(X2)
 features = range(pca.n_components_)
 plt.bar(features, pca.explained_variance_ratio_, color='blue')
 plt.xlabel('PCA features')
 plt.ylabel(range(10))
plt.xticks(features)
 PCA_components = pd.DataFrame(principalComponents )<categorify>
global ITERATION
ITERATION = 0
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,
max_evals = MAX_EVALS)
 best
def check_acc(old_y,new_y,k,mp):
for i in range(1,k+1):
mx = -1;
mxInd = 1;
for j in range(1,6):
count = 0
num_count = 0
for k in range(len(new_y)) :
if(old_y[k] == j):
num_count += 1
if(old_y[k]==j and new_y[k]==i):
count += 1
frac = count/num_count
if(mx<frac):
mx = frac
mxInd = j
mp.append(mxInd)
<count_unique_values>
trials_dict = sorted(trials.results, key = lambda x: x['loss'])
 trials_dict[:1]
for k in range(5,20):
kmeans = KMeans(n_clusters = k,random_state=100 ).fit(principalComponents)
 y2 = kmeans.labels_
y2 = y2 + 1
mp = []
mp = []
check_acc(y[:1300],y2[:1300],k,mp)
 edited_y = []
for i in range(len(y2)) :
for j in range(1,k+1):
if(y2[i]==j):
edited_y.append(mp[j-1])
 print(k)
 accuracy_score(edited_y[:1300],y[:1300])
 unique_elements, counts_elements = np.unique(edited_y[1300:], return_counts=True)
 print(np.asarray(( unique_elements, counts_elements)) )<define_variables>
results = pd.read_csv(OUT_FILE )
k = 16<train_model>
def evaluate(results, name):

 new_results = results.copy()
new_results['hyperparameters'] = new_results['hyperparameters'].map(ast.literal_eval)
 new_results = new_results.sort_values('score', ascending = False ).reset_index(drop = True)
 print('The highest cross validation score from {} was {:.5f} found on iteration {}.'.format(name, new_results.loc[0, 'score'], new_results.loc[0, 'iteration']))
hyperparameters = new_results.loc[0, 'hyperparameters']
model = lgb.LGBMClassifier(**hyperparameters)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('ROC AUC from {} on test data = {:.5f}.'.format(name, roc_auc_score(test_labels, preds)))
 hyp_df = pd.DataFrame(columns = list(new_results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(new_results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]),
ignore_index = True)
 hyp_df['iteration'] = new_results['iteration']
hyp_df['score'] = new_results['score']
return hyp_df
kmeans = KMeans(n_clusters = k, random_state=100 ).fit(principalComponents )<define_variables>
bayes_results = evaluate(results, name = 'Bayesian')
 bayes_results
y2 = kmeans.labels_
y2 = y2 + 1<count_unique_values>
MAX_EVALS = 10
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,
max_evals = MAX_EVALS )
unique_elements, counts_elements = np.unique(y2[:1300], return_counts=True)
 print("Frequency of unique values of the said array:")
 print(np.asarray(( unique_elements, counts_elements)) )<count_unique_values>
bayes_results = pd.read_csv('.. /input/home-credit-model-tuning/bayesian_trials_1000.csv' ).sort_values('score', ascending = False ).reset_index()
random_results = pd.read_csv('.. /input/home-credit-model-tuning/random_search_trials_1000.csv' ).sort_values('score', ascending = False ).reset_index()
random_results['loss'] = 1 - random_results['score']
bayes_params = evaluate(bayes_results, name = 'Bayesian')
 random_params = evaluate(random_results, name = 'random' )
unique_elements, counts_elements = np.unique(y[:1300], return_counts=True)
 print("Frequency of unique values of the said array:")
 print(np.asarray(( unique_elements, counts_elements)) )<compute_test_metric>
scores = pd.DataFrame({'ROC AUC': random_params['score'], 'iteration': random_params['iteration'], 'search': 'Random'})
 scores = scores.append(pd.DataFrame({'ROC AUC': bayes_params['score'], 'iteration': bayes_params['iteration'], 'search': 'Bayesian'}))
scores['ROC AUC'] = scores['ROC AUC'].astype(np.float32)
 scores['iteration'] = scores['iteration'].astype(np.int32)
 scores.head()
mp = []
check_acc(y[:1300],y2[:1300],k,mp )<define_variables>
best_random_params = random_params.iloc[random_params['score'].idxmax() , :].copy()
best_bayes_params = bayes_params.iloc[bayes_params['score'].idxmax() , :].copy()
edited_y = []
for i in range(len(y2)) :
for j in range(1,k+1):
if(y2[i]==j):
edited_y.append(mp[j-1] )<compute_test_metric>
train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv')
 test_ids = test['SK_ID_CURR']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
accuracy_score(edited_y[:1300],y[:1300] )<count_unique_values>
random_results['hyperparameters'] = random_results['hyperparameters'].map(ast.literal_eval)
 bayes_results['hyperparameters'] = bayes_results['hyperparameters'].map(ast.literal_eval )
unique_elements, counts_elements = np.unique(edited_y[1300:], return_counts=True)
 print("Frequency of unique values of the said array:")
 print(np.asarray(( unique_elements, counts_elements)) )<data_type_conversions>
train_set = lgb.Dataset(train, label = train_labels)
 hyperparameters = dict(**random_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS)
 print('The cross validation score on the full dataset for Random Search= {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )
ids = df['ID']
ids = ids[1300:]
ids = ids.tolist()<save_to_csv>
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_random_search.csv', index = False )
submission = pd.concat([pd.Series(ids),pd.Series(edited_y[1300:])],axis=1)
 submission.columns = ['ID','Class']
submission.to_csv('submission.csv', index=False)
 submission.head()<find_best_params>
hyperparameters = dict(**bayes_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS)
 print('The cross validation score on the full dataset for Bayesian optimization = {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )
<concatenate><EOS> 
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_bayesian_optimization.csv', index = False )