Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<feature_engineering>
pd.set_option('display.max_columns', None)
 %matplotlib inline
color = sns.color_palette()
warnings.filterwarnings("ignore" )
temp = data.apply(lambda x: np.array(list(x[1])) [np.array(list(x[2])).astype(bool)], axis=1)
 temp = temp.apply(pd.Series ).stack().reset_index(level=1, drop=True)
 freq_occured = data.P_Seq.apply(lambda x: pd.Series(list(x)).value_counts() ).sum()
freq_bound = temp.value_counts()
freq = pd.merge(
 pd.DataFrame(freq_occured ).reset_index().rename({"index": "Peptide", 0: "Occured"}, axis=1),
pd.DataFrame(freq_bound ).reset_index().rename({"index": "Peptide", 0: "Bound"}, axis=1),
on='Peptide' 
)
 freq['Percent'] = freq['Bound'] / freq['Occured'] * 100
freq.head()<prepare_output>
application_train = pd.read_csv('.. /input/application_train.csv')
 application_test= pd.read_csv('.. /input/application_test.csv')
 bureau = pd.read_csv('.. /input/bureau.csv')
 bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 POS_CASH_balance = pd.read_csv('.. /input/POS_CASH_balance.csv')
 credit_card_balance = pd.read_csv('.. /input/credit_card_balance.csv')
 previous_application = pd.read_csv('.. /input/previous_application.csv')
 installments_payments = pd.read_csv('.. /input/installments_payments.csv' )
sub = test['P_Seq'].apply(list ).explode().reset_index()
sub = sub.rename({'index': 'Seq_No', "P_Seq": 'Peptide'}, axis=1)
 sub['Id'] = sub.index
sub = sub.iloc[:, [-1, 0, 1]]
sub.head()<data_type_conversions>
application_train = pd.read_csv('.. /input/application_train.csv')
 application_test= pd.read_csv('.. /input/application_test.csv' )
sub["Expected"] = sub.groupby("Seq_No")['Id'].transform(lambda x:(np.random.random(len(x)) < 0.35 ).astype(int))
sub.head()<save_to_csv>
def feature_type_split(data, special_list=[]):
cat_list = []
dis_num_list = []
num_list = []
for i in data.columns.tolist() :
if data[i].dtype == 'object':
cat_list.append(i)
 elif data[i].nunique() < 25:
dis_num_list.append(i)
 elif i in special_list:
dis_num_list.append(i)
 else:
num_list.append(i)
 return cat_list, dis_num_list, num_list
cat_list, dis_num_list, num_list = feature_type_split(application_train, special_list=['AMT_REQ_CREDIT_BUREAU_YEAR'] )
sub[['Id', 'Expected']].to_csv("Naive_submission.csv", index=False )<categorify>
application_train['TERM'] = application_train.AMT_CREDIT / application_train.AMT_ANNUITY
application_test['TERM'] = application_test.AMT_CREDIT / application_test.AMT_ANNUITY
mapper = dict(zip(freq.Peptide, freq.Percent))
print(mapper )<data_type_conversions>
application_train['OVER_EXPECT_CREDIT'] =(application_train.AMT_CREDIT > application_train.AMT_GOODS_PRICE ).map({False:0, True:1})
 application_test['OVER_EXPECT_CREDIT'] =(application_test.AMT_CREDIT > application_test.AMT_GOODS_PRICE ).map({False:0, True:1} )
sub['Expected'] =(
 sub.Peptide.map(mapper)/ 100
> sub.groupby('Peptide')['Id'].transform(lambda x: np.random.random(len(x)))
 ).astype(int )<categorify>
application_train['MEAN_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_train['TOTAL_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].sum(skipna=True, axis=1)
 application_test['MEAN_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_test['TOTAL_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].sum(skipna=True, axis=1 )
temp = np.random.choice(list(mapper.keys()))
mapper[temp] / 100, sub.loc[sub.Peptide == temp, 'Expected'].mean()<save_to_csv>
application_train['FLAG_DOCUMENT_TOTAL'] = application_train.iloc[:, 96:116].sum(axis=1)
 application_test['FLAG_DOCUMENT_TOTAL'] = application_test.iloc[:, 96:116].sum(axis=1 )
sub[['Id', 'Expected']].to_csv("Peptide_Based_Fprediction.csv", index=False )<categorify>
application_train['AMT_REQ_CREDIT_BUREAU_TOTAL'] = application_train.iloc[:, 116:122].sum(axis=1)
 application_test['AMT_REQ_CREDIT_BUREAU_TOTAL'] = application_test.iloc[:, 116:122].sum(axis=1 )
Y = data.Target.apply(list ).explode().values.astype(int)
 y_hat_naive =(np.random.random(len(Y)) < 0.35 ).astype(int)
 y_hat_freq_based =(
 data.P_Seq.apply(list ).explode().reset_index(drop=True ).map(mapper)/ 100
> 
(data.P_Seq.apply(list ).explode().to_frame().reset_index().groupby("P_Seq")
.transform(lambda x: np.random.random(len(x)))['index'])
 ).astype(int)
 print("Our estimates on TRAINING DATA:")
 print("
\tNaive model accuracy score: {:12.2f}
\tNaive model ROC score: {:17.2f}" 
.format(
 accuracy_score(Y, y_hat_naive),
roc_auc_score(Y, y_hat_naive)) 
)
 print("
\tNuanced Naive model accuracy score: {:.2f}
\tNuanced Naive model ROC score: {:9.2f}" 
.format(
 accuracy_score(Y, y_hat_freq_based),
roc_auc_score(Y, y_hat_freq_based))
)<data_type_conversions>
application_train['BIRTH_EMPLOTED_INTERVEL'] = application_train.DAYS_EMPLOYED - application_train.DAYS_BIRTH
application_test['BIRTH_EMPLOTED_INTERVEL'] = application_test.DAYS_EMPLOYED - application_test.DAYS_BIRTH
flat_data = pd.DataFrame({
"P_Seq": data.P_Seq.apply(list ).explode() ,
"Target": data.Target.apply(list ).explode().astype(int)
 })
 flat_data.head()<choose_model_class>
application_train['BIRTH_REGISTRATION_INTERVEL'] = application_train.DAYS_REGISTRATION - application_train.DAYS_BIRTH
application_test['BIRTH_REGISTRATION_INTERVEL'] = application_test.DAYS_REGISTRATION - application_test.DAYS_BIRTH
tf.keras.backend.clear_session()
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Input(shape=(len(peptide_mapper),)))
 model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=tf.keras.metrics.AUC())
 model.summary()<train_model>
application_train['MEAN_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_train['TOTAL_BUILDING_SCORE_AVG'] = application_train.iloc[:, 44:58].sum(skipna=True, axis=1)
 application_test['MEAN_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].mean(skipna=True, axis=1)
 application_test['TOTAL_BUILDING_SCORE_AVG'] = application_test.iloc[:, 44:58].sum(skipna=True, axis=1)
 application_train['INCOME_PER_FAMILY_MEMBER'] = application_train.AMT_INCOME_TOTAL / application_train.CNT_FAM_MEMBERS
application_test['INCOME_PER_FAMILY_MEMBER'] = application_test.AMT_INCOME_TOTAL / application_test.CNT_FAM_MEMBERS
application_train['SEASON_REMAINING'] = application_train.AMT_INCOME_TOTAL/4 - application_train.AMT_ANNUITY
application_test['SEASON_REMAINING'] = application_test.AMT_INCOME_TOTAL/4 - application_test.AMT_ANNUITY
application_train['RATIO_INCOME_GOODS'] = application_train.AMT_INCOME_TOTAL - application_train.AMT_GOODS_PRICE
application_test['RATIO_INCOME_GOODS'] = application_test.AMT_INCOME_TOTAL - application_test.AMT_GOODS_PRICE
application_train['CHILDREN_RATIO'] = application_train['CNT_CHILDREN'] / application_train['CNT_FAM_MEMBERS']
application_test['CHILDREN_RATIO'] = application_test['CNT_CHILDREN'] / application_test['CNT_FAM_MEMBERS']
def label_encoder(input_df, encoder_dict=None):

 categorical_feats = input_df.columns[input_df.dtypes == 'object']
for feat in categorical_feats:
encoder = LabelEncoder()
input_df[feat] = encoder.fit_transform(input_df[feat].fillna('NULL'))
return input_df, categorical_feats.tolist() , encoder_dict
application_train, categorical_feats, encoder_dict = label_encoder(application_train)
 application_test, categorical_feats, encoder_dict = label_encoder(application_test )
hist = model.fit(
 tf.one_hot(flat_data.P_Seq.map(peptide_mapper), depth=20),
flat_data.Target,
validation_split=0.2,
callbacks=tf.keras.callbacks.EarlyStopping(patience=10),
epochs=100,
verbose=0)
 print("Train Best ROC_AUC Score: {:.2f}".format(hist.history['auc'][-1]))
print("Val Best ROC_AUC Score: {:6.2f}".format(hist.history['val_auc'][-1]))
pd.DataFrame(hist.history ).iloc[:, [1, 3]].plot(figsize=(20, 5), title='Model Performance');<predict_on_test>
import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
sub['Expected'] =(
 model.predict(tf.one_hot(
 sub.Peptide.map(peptide_mapper),
depth=len(peptide_mapper)))
 )<save_to_csv>
X = application_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 y = application_train.TARGET
X_pred = application_test.drop(['SK_ID_CURR'], axis=1 )
sub[['Id', 'Expected']].to_csv("Linear_model_one_hot.csv", index=False )<categorify>
folds = StratifiedKFold(n_splits=5,random_state=6)
 oof_preds = np.zeros(X.shape[0])
 sub_preds = np.zeros(X_pred.shape[0] )
flat_data['Percent'] = flat_data.P_Seq.map(mapper ).astype(float)/ 100
flat_data = flat_data.merge(data.iloc[:, 3:7], right_index=True, left_index=True)
 flat_data['Position'] = flat_data.groupby(flat_data.index)['P_Seq'].transform(lambda x: np.arange(len(x)) / len(x))
flat_data.head()<feature_engineering>
start = time.time()
valid_score = 0
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(X, y)) :
trn_x, trn_y = X.iloc[trn_idx], y[trn_idx]
val_x, val_y = X.iloc[val_idx], y[val_idx]
train_data = lgb.Dataset(data=trn_x, label=trn_y,categorical_feature=categorical_feats)
 valid_data = lgb.Dataset(data=val_x, label=val_y)
 param = {'application':'binary','num_iterations':4000, 'learning_rate':0.05, 'num_leaves':24, 'feature_fraction':0.8, 'bagging_fraction':0.9,
'lambda_l1':0.1, 'lambda_l2':0.1, 'min_split_gain':0.01, 'early_stopping_round':100, 'max_depth':7, 'min_child_weight':40, 'metric':'auc'}
lgb_es_model = lgb.train(param, train_data, valid_sets=[train_data, valid_data], verbose_eval=100, categorical_feature=categorical_feats)
 oof_preds[val_idx] = lgb_es_model.predict(val_x, num_iteration=lgb_es_model.best_iteration)
 sub_preds += lgb_es_model.predict(X_pred, num_iteration=lgb_es_model.best_iteration)/ folds.n_splits
print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 valid_score += roc_auc_score(val_y, oof_preds[val_idx])
 print('valid score:', str(round(valid_score/folds.n_splits,4)))
 end = time.time()
print('training time:', str(round(( end - start)/60)) , 'mins' )
<categorify><EOS> 
application_test= pd.read_csv('.. /input/application_test.csv')
 output = pd.DataFrame({'SK_ID_CURR':application_test.SK_ID_CURR, 'TARGET': sub_preds})
 output.to_csv('only_application_pred.csv', index=False )