Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<compute_train_metric>
%matplotlib inline
for f in all_features:
scores = cross_validation_for_roc_auc(sgd, train[[f]] , train['stroke'])
 print(f, np.mean(scores))<define_variables>
data = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 prev = pd.read_csv('.. /input/previous_application.csv')
 buro = pd.read_csv('.. /input/bureau.csv')
 buro_balance = pd.read_csv('.. /input/bureau_balance.csv')
 credit_card = pd.read_csv('.. /input/credit_card_balance.csv')
 POS_CASH = pd.read_csv('.. /input/POS_CASH_balance.csv')
 payments = pd.read_csv('.. /input/installments_payments.csv' )
features_sgd=['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level',
'work_type_target_enc', 'work_type_is_Self-employed', 'work_type_is_children']<compute_train_metric>
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold
for f in all_features:
scores = cross_validation_for_roc_auc(lgb, train[[f]] , train['stroke'])
 print(f, np.mean(scores))<define_variables>
import xgboost as xgb
from xgboost import XGBClassifier
import gc
features_lgb = ['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level',
'work_type_target_enc', 'bmi', 'work_type_is_Self-employed', 'work_type_is_children']<compute_train_metric>
from sklearn.preprocessing import MinMaxScaler, LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix
from sklearn.feature_selection import VarianceThreshold
for f in all_features:
scores = cross_validation_for_roc_auc(rndf, train[[f]] , train['stroke'])
 print(f, np.mean(scores))<define_variables>
from sklearn.model_selection import GridSearchCV
from skopt import BayesSearchCV
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
features_rndf = ['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level',
'work_type_target_enc','bmi', 'work_type_is_Self-employed', 'work_type_is_children']<compute_train_metric>
def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best ROC-AUC: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
clf_name = bayes_cv_tuner.estimator.__class__.__name__
all_models.to_csv(clf_name+"_cv_results.csv" )
sgd = SGDClassifier(max_iter=2000, loss='log', penalty = 'elasticnet')
 scores = cross_validation_for_roc_auc(sgd, train[features_sgd] , train['stroke'])
 np.mean(scores )<compute_train_metric>
y = data['TARGET']
del data['TARGET']
lgb = LGBMClassifier(n_estimators=45,max_depth=3)
 scores = cross_validation_for_roc_auc(lgb, train[features_lgb] , train['stroke'])
 np.mean(scores )<compute_train_metric>
categorical_feature = [col for col in data.columns if data[col].dtype == 'object']
rndf = RandomForestClassifier(n_estimators=530, max_depth=6)
 scores = cross_validation_for_roc_auc(rndf, train[features_rndf] , train['stroke'])
 np.mean(scores )<save_to_csv>
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("Your selected dataframe has " + str(df.shape[1])+ " columns.
 "
"There are " + str(mis_val_table_ren_columns.shape[0])+
" columns that have missing values.")
 return mis_val_table_ren_columns
lgb.fit(train[features_rndf], train['stroke'])
 Y_pred = pd.DataFrame(lgb.predict_proba(test[features_rndf])) [1]
submission = pd.DataFrame({
"id": test["id"],
"stroke": Y_pred
})
 submission.to_csv(".. /working/submit.csv", index=False)
 submission.sample(10 )<define_variables>
missing_values = missing_values_table(data)
 missing_values.head(20 )
c_features = [
"LATITUDE",
"LONGITUDE",
"AYB",
"CMPLX_NUM",
"EYB",
"YR_RMDL",
"LIVING_GBA",
"LANDAREA",
"BATHRM",
"HF_BATHRM",
"HEAT",
"AC",
"ROOMS",
"BEDRM",
"FIREPLACES",
"SALEDATE",
"QUALIFIED",
"USECODE",
"SALE_NUM",
] 
<define_variables>
data.dtypes.value_counts()
r_features = [
"LATITUDE",
"LONGITUDE",
"AYB",
"EYB",
"YR_RMDL",
"STORIES",
"GBA",
"LANDAREA",
"BATHRM",
"HF_BATHRM",
"HEAT",
"AC",
"ROOMS",
"BEDRM",
"FIREPLACES",
"KITCHENS",
"STYLE",
"STRUCT",
"GRADE",
"CNDTN",
"INTWALL",
"EXTWALL",
"ROOF",
"NUM_UNITS",
"SALEDATE",
"QUALIFIED",
"USECODE",
"SALE_NUM",
] 
<load_from_csv>
data.select_dtypes('object' ).nunique()
pd.set_option("max_columns", 100)
 train = pd.read_csv(".. /input/property-price-prediction-challenge/train_set.csv", low_memory=False)
 test = pd.read_csv(".. /input/property-price-prediction-challenge/test_set.csv", low_memory=False )<split>
one_hot_df = pd.concat([data, test] )
def split_by_source(X):

 is_condominium =(X["SOURCE"] == "Condominium")
 X_c = X.loc[is_condominium, :]
X_r = X.loc[~is_condominium, :]
return X_c, X_r
def preprocess_c(df, c_features):

 df = _convert_date_to_int(df)
 df = _fill_na(df)
 obj_cols = df.select_dtypes("object" ).columns.to_list()
obj_cols += ["CMPLX_NUM"]
df = _convert_object_to_category(df, obj_cols)
 if "PRICE" in df:
X, y = _split_Xy(df, c_features, y="PRICE")
 y = _preprocess_y(y)
 return X, y
else:
X = df.loc[:, c_features]
return X
def preprocess_r(df, r_features):

 df = _convert_date_to_int(df)
 df = _fill_na(df)
 obj_cols = df.select_dtypes("object" ).columns.to_list()
df = _convert_object_to_category(df, obj_cols)
 if "PRICE" in df:
X, y = _split_Xy(df, r_features, y="PRICE")
 y = _preprocess_y(y)
 return X, y
else:
X = df.loc[:, r_features]
return X
def _convert_date_to_int(X):

 X.loc[X["SALEDATE"].isnull() , "SALEDATE"] = X["SALEDATE"].mode() [0]
X.loc[:, "SALEDATE"] = pd.to_datetime(X["SALEDATE"] ).astype(np.int64)/ 10**9
return X
def _fill_na(X):

 for f in X.columns:
if X.dtypes[f] == 'object':
X.loc[:,f] = X[f].fillna("None")
 else:
X.loc[:,f] = X[f].fillna(-1)
 return X
def _convert_object_to_category(X, obj_cols):

 for col in obj_cols:
X.loc[:, col] = X[col].astype("category")
 return X
def _split_Xy(df, features, y="PRICE"):

 y = df.loc[:, y]
X = df.loc[:, features]
return X, y
def _preprocess_y(y):
y = np.log1p(y)
 return y<prepare_x_and_y>
one_hot_df = pd.get_dummies(one_hot_df, columns=categorical_feature )
train_c, train_r = split_by_source(train)
 test_c, test_r = split_by_source(test)
 X_train_c, y_train_c = preprocess_c(train_c, c_features)
 X_test_c = preprocess_c(test_c, c_features)
 X_train_r, y_train_r = preprocess_r(train_r, r_features)
 X_test_r = preprocess_r(test_r, r_features)
<define_search_space>
data = one_hot_df.iloc[:data.shape[0],:]
test = one_hot_df.iloc[data.shape[0]:,]
%%time
parameter_range = {
"n_estimators": [1500],
"learning_rate": [0.05],
"min_child_samples": [3, 5],
"max_depth": [5, 7],
"num_leaves": [2**5, 2**7-50],
"colsample_bytree": [0.5, 0.7],
"lambda_l1": [0.0, 0.1, 0.5],
"lambda_l2": [0.0, 0.1, 0.5],
"n_jobs": [-1],
"random_state": [42],
}
cv_c = GridSearchCV(estimator = lgb.LGBMRegressor() ,
param_grid = parameter_range,
cv = 3, verbose = 1,
scoring = 'neg_mean_squared_error')
 cv_c.fit(X_train_c, y_train_c)
 print("best_score:")
 print(cv_c.best_score_)
 print("best_params:")
 print(cv_c.best_params_)
 cv_r = GridSearchCV(estimator = lgb.LGBMRegressor() ,
param_grid = parameter_range,
cv = 3, verbose = 1,
scoring = 'neg_mean_squared_error')
 cv_r.fit(X_train_r, y_train_r)
 print("best_score:")
 print(cv_r.best_score_)
 print("best_params:")
 print(cv_r.best_params_)
<init_hyperparams>
buro_grouped_size = buro_balance.groupby('SK_ID_BUREAU')['MONTHS_BALANCE'].size()
buro_grouped_max = buro_balance.groupby('SK_ID_BUREAU')['MONTHS_BALANCE'].max()
buro_grouped_min = buro_balance.groupby('SK_ID_BUREAU')['MONTHS_BALANCE'].min()
<train_model>
buro_counts = buro_balance.groupby('SK_ID_BUREAU')['STATUS'].value_counts(normalize = False )
%%time
es_c = cv_c.best_estimator_
X_train_c_, X_eval_c, y_train_c_, y_eval_c = train_test_split(
 X_train_c, y_train_c, test_size=0.1, random_state=42)
 es_c.n_estimators = 100_000
es_c.fit(X_train_c_, y_train_c_,
eval_metric="rmse",
eval_set=[(X_eval_c, y_eval_c)],
early_stopping_rounds=100,
verbose=False)
 print("best scores:", dict(es_c.best_score_["valid_0"]))
print("best n_estimators:", es_c.best_iteration_)
 es_r = cv_r.best_estimator_
X_train_r_, X_eval_r, y_train_r_, y_eval_r = train_test_split(
 X_train_r, y_train_r, test_size=0.1, random_state=42)
 es_r.n_estimators = 100_000
es_r.fit(X_train_r_, y_train_r_,
eval_metric="rmse",
eval_set=[(X_eval_r, y_eval_r)],
early_stopping_rounds=100,
verbose=False)
 print("best scores:", dict(es_r.best_score_["valid_0"]))
print("best n_estimators:", es_r.best_iteration_)
<predict_on_test>
buro_counts_unstacked.columns = ['STATUS_0', 'STATUS_1','STATUS_2','STATUS_3','STATUS_4','STATUS_5','STATUS_C','STATUS_X']
buro_counts_unstacked['MONTHS_COUNT'] = buro_grouped_size
buro_counts_unstacked['MONTHS_MIN'] = buro_grouped_min
buro_counts_unstacked['MONTHS_MAX'] = buro_grouped_max
def predict_and_add_index(model, X):

 y_pred = model.predict(X)
 y_pred = np.expm1(y_pred ).astype(int)
 return pd.Series(y_pred, index=X.index)
 y_pred_c = predict_and_add_index(es_c, X_eval_c)
 y_pred_r = predict_and_add_index(es_r, X_eval_r)
 y_pred = pd.concat([y_pred_r, y_pred_c] ).sort_index()
y_true = pd.concat([y_eval_r, y_eval_c] ).sort_index()
y_true = np.expm1(y_true ).astype(int )<compute_test_metric>
buro = buro.join(buro_counts_unstacked, how ='left', on ='SK_ID_BUREAU' )
def rmsle(y_true, y_pred):
error =(np.log(y_true)- np.log(y_pred))
return np.sqrt(np.mean(error**2))
print(f"RMSLE: {rmsle(y_true, y_pred):,.3f}" )<data_type_conversions>
prev_cat_features = [pcol for pcol in prev.columns if prev[pcol].dtype == 'object']
eval_df = pd.concat([X_eval_r, X_eval_c] ).sort_index()
eval_df["y_pred"] = y_pred
eval_df["y_true"] = y_true
eval_df["abs_error"] =(eval_df["y_true"] - eval_df["y_pred"] ).abs().astype(int )<train_on_grid>
prev = pd.get_dummies(prev, columns=prev_cat_features )
%%time
model_c = lgb.LGBMRegressor(**cv_c.best_params_)
 model_c.n_estimators = es_c.best_iteration_
model_c.fit(X_train_c, y_train_c)
 model_r = lgb.LGBMRegressor(**cv_r.best_params_)
 model_r.n_estimators = es_r.best_iteration_
model_r.fit(X_train_r, y_train_r)
<compute_train_metric>
avg_prev = prev.groupby('SK_ID_CURR' ).mean()
%%time
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
scoring = {
"r2": "r2",
"RMSE":make_scorer(rmse)
 }
print("
scores = cross_validate(model_c, X_train_c, y_train_c, cv = 10, scoring = scoring, return_train_score=False)
 for key,value in scores.items() :
print("{}:{:.3g}+/-{:.3g}".format(key, value.mean() , value.std()))
print("
scores = cross_validate(model_r, X_train_r, y_train_r, cv = 10, scoring = scoring, return_train_score=False)
 for key,value in scores.items() :
print("{}:{:.3g}+/-{:.3g}".format(key, value.mean() , value.std())) 
<predict_on_test>
cnt_prev = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
y_pred_c = predict_and_add_index(model_c, X_test_c)
 y_pred_r = predict_and_add_index(model_r, X_test_r)
 y_pred = pd.concat([y_pred_r, y_pred_c] ).sort_index()<save_to_csv>
avg_prev['nb_app'] = cnt_prev['SK_ID_PREV']
output = pd.DataFrame({
"Id": test["Id"],
"PRICE": pd.concat([y_pred_r, y_pred_c] ).sort_index()
})
 output.to_csv("output_ml_only.csv", index=False)
 output.head()<sort_values>
del avg_prev['SK_ID_PREV']
eval_df.sort_values("abs_error" ).loc[:, cols].tail(10 )<count_values>
buro_cat_features = [bcol for bcol in buro.columns if buro[bcol].dtype == 'object']
buro_cat_features
eval_df.query("CMPLX_NUM == 1657 and SALEDATE == 1176163200")["y_true"].value_counts()<train_model>
buro = pd.get_dummies(buro, columns=buro_cat_features )
class RuleBasedRegressor:

 def __init__(self, y="PRICE", min_y=10_000_000):
self.y = y
self.min_y = min_y
def fit(self, X, y):
self.features_ = X.columns
self.data = X.copy()
if max(y)< 1000:
self.data[self.y] = np.expm1(y)
 else:
self.data[self.y] = y
is_expensive =(self.data[self.y] > self.min_y ).to_numpy()
is_condominium =(self.data["CMPLX_NUM"] != 0 ).to_numpy()
self.data = self.data.loc[is_expensive & is_condominium, :].drop_duplicates().reset_index(drop=True)
 def predict(self, X):

 n = X.shape[0]
predicts = np.zeros(n)
 for _, row in self.data.iterrows() :
is_all_equal = np.array([True]*n)
 for col in self.features_:
is_all_equal &=(X[col].to_numpy() == row[col])
 predicts[is_all_equal] = row[self.y]
return predicts 
<train_on_grid>
avg_buro = buro.groupby('SK_ID_CURR' ).mean()
reg = RuleBasedRegressor()
X_train = pd.concat([X_train_r, X_train_c] ).sort_index()
y_train = pd.concat([y_train_r, y_train_c] ).sort_index()
X_test = pd.concat([X_test_r, X_test_c] ).sort_index()
features = ["CMPLX_NUM", "SALEDATE"]
reg.fit(X_train[features], y_train)
 y_rule = reg.predict(X_test)
 is_predicted =(y_rule > 0 )<data_type_conversions>
avg_buro['buro_count'] = buro[['SK_ID_BUREAU', 'SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
del avg_buro['SK_ID_BUREAU']
y_pred = output["PRICE"].to_numpy()
y_pred[is_predicted]<data_type_conversions>
le = LabelEncoder()
POS_CASH['NAME_CONTRACT_STATUS'] = le.fit_transform(POS_CASH['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
POS_CASH['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
POS_CASH['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
POS_CASH.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True )
y_rule[is_predicted].astype(int )<data_type_conversions>
credit_card_cat_features = [c_col for c_col in credit_card.columns if credit_card[c_col].dtype == 'object']
credit_card_cat_features
y_pred[is_predicted] = y_rule[is_predicted].astype(int )<save_to_csv>
credit_card['NAME_CONTRACT_STATUS'] = le.fit_transform(credit_card['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
credit_card['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
credit_card['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
credit_card.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True )
output["PRICE"] = y_pred
output.to_csv("output_with_rule.csv", index=False)
 output.head()<load_from_csv>
avg_payments = payments.groupby('SK_ID_CURR' ).mean()
avg_payments2 = payments.groupby('SK_ID_CURR' ).max()
avg_payments3 = payments.groupby('SK_ID_CURR' ).min()
del avg_payments['SK_ID_PREV']
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
data = data.merge(right=avg_prev.reset_index() , how='left', on ='SK_ID_CURR' )
X_train = train.iloc[:, 2:-1].values
Y_train = train.iloc[:, 21].values
X_test = test.iloc[:, 2:].values<concatenate>
test = test.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR' )
X_all = np.concatenate(( X_train, X_test), axis=0)
<correct_missing_values>
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR' )
imputer = Imputer(missing_values = 'NaN', strategy = 'most_frequent', axis = 0)
 imputer = imputer.fit(X_all)
 X_all = imputer.transform(X_all )<split>
data = data.merge(POS_CASH.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(POS_CASH.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR' )
X_train = X_all[0:900,:]
X_test = X_all[900:1340,:]<prepare_x_and_y>
data = data.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR' )
X = X_train
Y = Y_train<feature_engineering>
data = data.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR' )
sc = QuantileTransformer()
X = sc.fit_transform(X)
 X_test = sc.transform(X_test )<normalization>
data = data.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR' )
sc = AdditiveChi2Sampler()
X = sc.fit_transform(X)
 X_test = sc.transform(X_test )<train_model>
data = data.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR' )
clf = AdaBoostClassifier(n_estimators = 60 , learning_rate = 0.3)
 clf.fit(X, Y)
 Y_pred = clf.predict(X)
 accuracy_score(Y, Y_pred )<predict_on_test>
test = test[test.columns[data.isnull().mean() < 0.80]]
data = data[data.columns[data.isnull().mean() < 0.80]]
Y_test_pred = clf.predict(X_test )<save_to_csv>
gc.enable()
folds = KFold(n_splits=5, shuffle=True, random_state=546789)
 oof_preds = np.zeros(data.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in data.columns if f not in ['SK_ID_CURR']]
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(data)) :
trn_x, trn_y = data[feats].iloc[trn_idx], y.iloc[trn_idx]
val_x, val_y = data[feats].iloc[val_idx], y.iloc[val_idx]
clf = LGBMClassifier(
 n_estimators=10000,
learning_rate=0.02,
num_leaves=255,
colsample_bytree=0.81,
subsample=0.82,
max_depth = 8,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_sum_hessian_in_leaf = 150,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=200 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()
cols = { 'PlayerID': [i+901 for i in range(440)] , 'TARGET_5Yrs': Y_test_pred }
submission = pd.DataFrame(cols)
 print(submission)
 submission.to_csv("submission.csv", index=False )<import_modules>
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
<load_from_csv><EOS> 
test['TARGET'] = sub_preds
test[['SK_ID_CURR', 'TARGET']].to_csv('home_risk_1submission.csv', index=False )