Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<categorify>
plt.rcParams['font.size'] = 18
plt.style.use('fivethirtyeight')
 %matplotlib inline
df_train['Initial'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don', 'Dona'],
['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr', 'Mr'],inplace=True)
 df_test['Initial'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don', 'Dona'],
['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr', 'Mr'],inplace=True )<feature_engineering>
random = pd.read_csv('.. /input/home-credit-model-tuning/random_search_simple.csv' ).sort_values('score', ascending = False ).reset_index()
opt = pd.read_csv('.. /input/home-credit-model-tuning/bayesian_trials_simple.csv' ).sort_values('score', ascending = False ).reset_index()
print('Best score from random search: {:.5f} found on iteration: {}.'.format(random.loc[0, 'score'], random.loc[0, 'iteration']))
print('Best score from bayesian optimization: {:.5f} found on iteration: {}.'.format(opt.loc[0, 'score'], opt.loc[0, 'iteration']))
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Mr'),'Age'] = 33
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Mrs'),'Age'] = 36
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Master'),'Age'] = 5
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Miss'),'Age'] = 22
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Other'),'Age'] = 46
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Mr'),'Age'] = 33
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Mrs'),'Age'] = 36
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Master'),'Age'] = 5
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Miss'),'Age'] = 22
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Other'),'Age'] = 46<data_type_conversions>
keys = []
for key, value in ast.literal_eval(random.loc[0, 'hyperparameters'] ).items() :
print(f'{key}: {value}')
 keys.append(key )
df_train['Embarked'].fillna('S', inplace=True )<feature_engineering>
for key in keys:
print('{}: {}'.format(key, ast.literal_eval(opt.loc[0, 'hyperparameters'])[key]))
df_train['Age_cat'] = 0
df_train.loc[df_train['Age'] < 20, 'Age_cat'] = 0
df_train.loc[(20 <= df_train['Age'])&(df_train['Age'] < 26), 'Age_cat'] = 1
df_train.loc[(26 <= df_train['Age'])&(df_train['Age'] < 33), 'Age_cat'] = 2
df_train.loc[(33 <= df_train['Age'])&(df_train['Age'] < 39), 'Age_cat'] = 3
df_train.loc[(39 <= df_train['Age']), 'Age_cat'] = 4
df_test['Age_cat'] = 0
df_test.loc[df_test['Age'] < 20, 'Age_cat'] = 0
df_test.loc[(20 <= df_test['Age'])&(df_test['Age'] < 26), 'Age_cat'] = 1
df_test.loc[(26 <= df_test['Age'])&(df_test['Age'] < 33), 'Age_cat'] = 2
df_test.loc[(33 <= df_test['Age'])&(df_test['Age'] < 39), 'Age_cat'] = 3
df_test.loc[(39 <= df_test['Age']), 'Age_cat'] = 4<feature_engineering>
random['set'] = 'random'
scores = random[['score', 'iteration', 'set']]
opt['set'] = 'opt'
scores = scores.append(opt[['set', 'iteration', 'score']], sort = True)
 scores.head()
<count_values>
scores.groupby('set')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )
print(df_train['Age_cat'].value_counts() )<categorify>
random_fit = np.polyfit(random['iteration'], random['score'], 1)
 print('Random search slope: {:.8f}'.format(random_fit[0]))
df_train['Initial'] = df_train['Initial'].map({'Master': 0, 'Miss': 1, 'Mr': 2, 'Mrs': 3, 'Other': 4})
 df_test['Initial'] = df_test['Initial'].map({'Master': 0, 'Miss': 1, 'Mr': 2, 'Mrs': 3, 'Other': 4} )<categorify>
opt_fit = np.polyfit(opt['iteration'], opt['score'], 1)
 print('opt search slope: {:.8f}'.format(opt_fit[0]))
df_train['Embarked'] = df_train['Embarked'].map({'C': 0, 'Q': 1, 'S': 2})
 df_test['Embarked'] = df_test['Embarked'].map({'C': 0, 'Q': 1, 'S': 2})
<categorify>
opt_fit[0] / random_fit[0]
df_train['Sex'] = df_train['Sex'].map({'female': 0, 'male': 1})
 df_test['Sex'] = df_test['Sex'].map({'female': 0, 'male': 1})
<categorify>
print('After 10,000 iterations, the random score is: {:.5f}.'.format(
 random_fit[0] * 1e5 + random_fit[1]))
df_train = pd.get_dummies(df_train, columns=['Age_cat'], prefix='Age_cat')
 df_test = pd.get_dummies(df_test, columns=['Age_cat'], prefix='Age_cat' )<categorify>
print('After 10,000 iterations, the bayesian score is: {:.5f}.'.format(
 opt_fit[0] * 1e5 + opt_fit[1]))
df_train = pd.get_dummies(df_train, columns=['Initial'], prefix='Initial')
 df_test = pd.get_dummies(df_test, columns=['Initial'], prefix='Initial' )<categorify>
def process(results):

 results = results.copy()
results['hyperparameters'] = results['hyperparameters'].map(ast.literal_eval)
 results = results.sort_values('score', ascending = False ).reset_index(drop = True)
 hyp_df = pd.DataFrame(columns = list(results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]),
ignore_index = True, sort= True)
 hyp_df['iteration'] = results['iteration']
hyp_df['score'] = results['score']
return hyp_df
df_train = pd.get_dummies(df_train, columns=['Embarked'], prefix='Embarked')
 df_test = pd.get_dummies(df_test, columns=['Embarked'], prefix='Embarked' )<drop_column>
random_hyp = process(random)
 opt_hyp = process(opt)
 random_hyp.head()
df_train.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True)
 df_test.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True)
<import_modules>
param_grid = {
'is_unbalance': [True, False],
'boosting_type': ['gbdt', 'goss', 'dart'],
'num_leaves': list(range(20, 150)) ,
'learning_rate': list(np.logspace(np.log10(0.005), np.log10(0.5), base = 10, num = 1000)) ,
'subsample_for_bin': list(range(20000, 300000, 20000)) ,
'min_child_samples': list(range(20, 500, 5)) ,
'reg_alpha': list(np.linspace(0, 1)) ,
'reg_lambda': list(np.linspace(0, 1)) ,
'colsample_bytree': list(np.linspace(0.6, 1, 10)) ,
'subsample': list(np.linspace(0.5, 1, 100))
}
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.model_selection import train_test_split<prepare_x_and_y>
best_random_hyp = random_hyp.loc[0, :]
best_opt_hyp = opt_hyp.loc[0, :]
X_train = df_train.drop('Survived', axis=1 ).values
target_label = df_train['Survived'].values
X_test = df_test.values<split>
random_hyp.groupby('boosting_type')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )
X_tr, X_vld, y_tr, y_vld = train_test_split(X_train, target_label, test_size=0.34, random_state=900 )<import_modules>
opt_hyp.groupby('boosting_type')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )
from sklearn.metrics import accuracy_score<train_model>
random_hyp.groupby('is_unbalance')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )
model = GradientBoostingClassifier()
model.fit(X_tr,y_tr)
 pred = model.predict(X_vld )<compute_test_metric>
opt_hyp.groupby('is_unbalance')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )
print('총 {}명 중 {:.2f}% 정확도로 생존을 맞춤'.format(y_vld.shape[0], 100 * metrics.accuracy_score(pred, y_vld)) )<load_from_csv>
random_hyp['set'] = 'Random Search'
opt_hyp['set'] = 'Bayesian'
hyp = random_hyp.append(opt_hyp, ignore_index = True, sort = True)
 hyp.head()
submission = pd.read_csv('.. /input/sample_submission.csv' )<predict_on_test>
plt.rcParams['axes.labelpad'] = 12
prediction = model.predict(X_test)
 submission['Survived'] = prediction<save_to_csv>
random_hyp['n_estimators'] = random_hyp['n_estimators'].astype(np.int32)
 random_hyp.corr() ['score']
submission.to_csv('./submission.csv', index=False )<set_options>
opt_hyp['n_estimators'] = opt_hyp['n_estimators'].astype(np.int32)
 opt_hyp.corr() ['score']
sns.set_style('whitegrid')
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 %matplotlib inline 
<load_from_csv>
hyp = hyp.drop(columns = ['metric', 'set', 'verbose'])
 hyp['n_estimators'] = hyp['n_estimators'].astype(np.int32)
 hyp['min_child_samples'] = hyp['min_child_samples'].astype(np.int32)
 hyp['num_leaves'] = hyp['num_leaves'].astype(np.int32)
 hyp['subsample_for_bin'] = hyp['subsample_for_bin'].astype(np.int32)
 hyp = pd.get_dummies(hyp)
 train_labels = hyp.pop('score')
 train = np.array(hyp.copy() )
train = pd.read_csv('/kaggle/input/eval-lab-2-f464/train.csv')
 train.drop('id',axis = 1,inplace = True)
 print('Train rows:', train.shape[0] )<normalization>
lr = LinearRegression()
lr.fit(train, train_labels )
train = shuffle(train )<count_values>
import lightgbm as lgb
train['class'].value_counts()<prepare_x_and_y>
train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 print('Full Training Features Shape: ', train.shape)
 test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv')
 print('Full Testing Features Shape: ', test.shape )
Y_class = train['class']
train = train.drop('class', axis=1 )<split>
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_ids = list(test['SK_ID_CURR'])
 test = test.drop(columns = ['SK_ID_CURR'] )
Y = Y_class.to_numpy()
X = train.to_numpy()
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.1,random_state=42)
 print(X_train.shape,y_train.shape)
 print(X_test.shape,y_test.shape )<train_on_grid>
random_best = ast.literal_eval(random.loc[0, 'hyperparameters'])
 rmodel = lgb.LGBMClassifier(**random_best)
 rmodel.fit(train, train_labels )
X_resampled, y_resampled = SMOTE().fit_resample(X, Y)
 print(sorted(Counter(y_resampled ).items())) 
<choose_model_class>
rpreds = rmodel.predict_proba(test)[:, 1]
rsub = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': rpreds})
 rsub.to_csv('submission_random_search.csv', index = False )
RANDOM_SEED = 42
clf1 = KNeighborsClassifier()
clf2 = RandomForestClassifier(random_state=RANDOM_SEED)
 clf3 = GaussianNB()
clf4 = ExtraTreesClassifier(random_state=RANDOM_SEED)
 clf5 = XGBClassifier(random_state=RANDOM_SEED)
 clf6 = LogisticRegression(random_state=RANDOM_SEED)
 sclf = Classifier(classifiers=[clf2, clf3, clf4, clf5, clf6], use_probas=True,meta_classifier=clf1,random_state=RANDOM_SEED)
 print('3-fold cross validation:
')
 for clf, label in zip([clf1, clf2, clf3, clf4, clf5, clf6, sclf],
['KNN',
'Random Forest',
'GaussianNB',
'ExtraTreesClassifier',
'XGBClassifier',
'LogisticRegression',
'Classifier']):
scores = model_selection.cross_val_score(clf, X, Y,
cv=3, scoring='accuracy')
 print("Accuracy: %0.2f(+/- %0.2f)[%s]"
%(scores.mean() , scores.std() , label))<train_model>
bayes_best = ast.literal_eval(opt.loc[0, 'hyperparameters'])
 bmodel = lgb.LGBMClassifier(**bayes_best)
 bmodel.fit(train, train_labels )
clf1.fit(X,Y)
 clf2.fit(X,Y)
 clf3.fit(X,Y)
 clf4.fit(X,Y)
 sclf.fit(X,Y)
 clf5.fit(X,Y)
 clf6.fit(X,Y )<load_from_csv>
bpreds = bmodel.predict_proba(test)[:, 1]
bsub = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': bpreds})
 bsub.to_csv('submission_bayesian_optimization.csv', index = False )
test = pd.read_csv('/kaggle/input/eval-lab-2-f464/test.csv')
 print('Data columns:', test.columns.drop(['id'] ).shape[0] )<drop_column>
random_fi = pd.DataFrame({'feature': features, 'importance': rmodel.feature_importances_})
 bayes_fi = pd.DataFrame({'feature': features, 'importance': bmodel.feature_importances_} )
testId = test['id']
label = test.drop('id',axis=1)
<data_type_conversions>
random.loc[0, 'hyperparameters']