Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<predict_on_test>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import gc
import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from hyperopt import hp, fmin, tpe, Trials
from hyperopt.pyll.base import scope
from tqdm import tqdm
preds = clf.predict_proba(apps_test.drop(['SK_ID_CURR', 'TARGET'], axis=1)) [:, 1 ]<save_to_csv>
path = '.. /input/tabular-playground-series-jan-2021/'
train = pd.read_csv(path+'train.csv')
 train.set_index('id',drop=True,inplace=True)
 train.drop(284103,inplace=True )
app_test['TARGET'] = preds
app_test[['SK_ID_CURR', 'TARGET']].to_csv('apps_baseline_02.csv', index=False )<feature_engineering>
features = [col for col in train.columns if 'cont' in col]
label = 'target'
def get_apps_processed(apps):
apps['APPS_EXT_SOURCE_MEAN'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps['APPS_EXT_SOURCE_STD'].fillna(apps['APPS_EXT_SOURCE_STD'].mean())
 apps['APPS_ANNUITY_CREDIT_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_CREDIT']
apps['APPS_GOODS_CREDIT_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_CREDIT']
apps['APPS_ANNUITY_INCOME_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_INCOME_TOTAL']
apps['APPS_CREDIT_INCOME_RATIO'] = apps['AMT_CREDIT']/apps['AMT_INCOME_TOTAL']
apps['APPS_GOODS_INCOME_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_INCOME_TOTAL']
apps['APPS_CNT_FAM_INCOME_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['CNT_FAM_MEMBERS']
apps['APPS_EMPLOYED_BIRTH_RATIO'] = apps['DAYS_EMPLOYED']/apps['DAYS_BIRTH']
apps['APPS_INCOME_EMPLOYED_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_EMPLOYED']
apps['APPS_INCOME_BIRTH_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_BIRTH']
apps['APPS_CAR_BIRTH_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_BIRTH']
apps['APPS_CAR_EMPLOYED_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_EMPLOYED']
return apps<load_from_csv>
fig.clear()
plt.close(fig )
prev = pd.read_csv('.. /input/home-credit-default-risk/previous_application.csv')
 print(prev.shape, apps.shape )<groupby>
X_train,X_valid, y_train,y_valid = train_test_split(train[features],train[label],test_size=0.2)
 d_tr = xgb.DMatrix(X_train, y_train)
 d_val = xgb.DMatrix(X_valid,y_valid )
prev.groupby('SK_ID_CURR')['SK_ID_CURR'].count().mean()<merge>
params_base = {'objective': 'reg:squarederror',
'tree_method': 'gpu_hist',
'random_state': 0}
base_model = xgb.train(params = params_base,
dtrain = d_tr,
num_boost_round = 1000,
evals = [(d_val,'eval')],
early_stopping_rounds=10,
verbose_eval = 20)
 y_pred_base = base_model.predict(d_val)
 base_score = mean_squared_error(y_valid, y_pred_base,squared=False)
 print(base_score )
app_prev_target = prev.merge(app_train[['SK_ID_CURR', 'TARGET']], on='SK_ID_CURR', how='left')
 app_prev_target.shape<feature_engineering>
def score(params):
ps = {'learning_rate': params['learning_rate'],
'max_depth': params['max_depth'],
'gamma': params['gamma'],
'min_child_weight': params['min_child_weight'],
'subsample': params['subsample'],
'colsample_bytree': params['colsample_bytree'],
'verbosity': 1,
'objective': 'reg:squarederror',
'eval_metric': 'rmse',
'tree_method': 'gpu_hist',
'random_state': 27,
}
model = xgb.train(ps,d_tr, params['n_round'], [(d_val, 'eval')], early_stopping_rounds=10, verbose_eval = False)
 y_pred = model.predict(d_val)
 score = mean_squared_error(y_valid, y_pred,squared=False)
 return score
def get_prev_processed(prev):
prev['PREV_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']
prev['PREV_GOODS_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_GOODS_PRICE']
prev['PREV_CREDIT_APPL_RATIO'] = prev['AMT_CREDIT']/prev['AMT_APPLICATION']
prev['PREV_ANNUITY_APPL_RATIO'] = prev['AMT_ANNUITY']/prev['AMT_APPLICATION']
prev['PREV_GOODS_APPL_RATIO'] = prev['AMT_GOODS_PRICE']/prev['AMT_APPLICATION']
prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['PREV_DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']
all_pay = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']
prev['PREV_INTERESTS_RATE'] =(all_pay/prev['AMT_CREDIT'] - 1)/prev['CNT_PAYMENT']
return prev<groupby>
param_space = {'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),
'n_round': scope.int(hp.quniform('n_round', 200, 3000, 100)) ,
'max_depth': scope.int(hp.quniform('max_depth', 5, 16, 1)) ,
'gamma': hp.uniform('gamma', 0, 10),
'min_child_weight': hp.uniform('min_child_weight', 0, 10),
'subsample': hp.uniform('subsample', 0.1, 1),
'colsample_bytree': hp.uniform('colsample_bytree', 0.1, 1)
 }
def get_prev_amt_agg(prev):
agg_dict = {
'SK_ID_CURR':['count'],
'AMT_CREDIT':['mean', 'max', 'sum'],
'AMT_ANNUITY':['mean', 'max', 'sum'],
'AMT_APPLICATION':['mean', 'max', 'sum'],
'AMT_DOWN_PAYMENT':['mean', 'max', 'sum'],
'AMT_GOODS_PRICE':['mean', 'max', 'sum'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'PREV_CREDIT_DIFF':['mean', 'max', 'sum'],
'PREV_CREDIT_APPL_RATIO':['mean', 'max'],
'PREV_GOODS_DIFF':['mean', 'max', 'sum'],
'PREV_GOODS_APPL_RATIO':['mean', 'max'],
'PREV_DAYS_LAST_DUE_DIFF':['mean', 'max', 'sum'],
'PREV_INTERESTS_RATE':['mean', 'max']
}
prev_group = prev.groupby('SK_ID_CURR')
 prev_amt_agg = prev_group.agg(agg_dict)
 prev_amt_agg.columns = ["PREV_"+ "_".join(x ).upper() for x in prev_amt_agg.columns.ravel() ]
prev_amt_agg = prev_amt_agg.reset_index()
return prev_amt_agg<groupby>
%time
trials = Trials()
hopt = fmin(fn = score,
space = param_space,
algo = tpe.suggest,
max_evals = 1000,
trials = trials,
)
def get_prev_refused_appr_agg(prev):
prev_refused_appr_group = prev[prev['NAME_CONTRACT_STATUS'].isin(['Approved', 'Refused'])].groupby([ 'SK_ID_CURR', 'NAME_CONTRACT_STATUS'])
 prev_refused_appr_agg = prev_refused_appr_group['SK_ID_CURR'].count().unstack()
prev_refused_appr_agg.columns = ['PREV_APPROVED_COUNT', 'PREV_REFUSED_COUNT' ]
prev_refused_appr_agg = prev_refused_appr_agg.fillna(0)
 prev_refused_appr_agg = prev_refused_appr_agg.reset_index()
return prev_refused_appr_agg<merge>
params_best = hopt
params_best['max_depth'] = int(hopt['max_depth'])
 n_rounds_best = int(hopt['n_round'])
 del params_best['n_round']
print(params_best)
 print(n_rounds_best )
def get_prev_agg(prev):
prev = get_prev_processed(prev)
 prev_amt_agg = get_prev_amt_agg(prev)
 prev_refused_appr_agg = get_prev_refused_appr_agg(prev)
 prev_agg = prev_amt_agg.merge(prev_refused_appr_agg, on='SK_ID_CURR', how='left')
 prev_agg['PREV_REFUSED_RATIO'] = prev_agg['PREV_REFUSED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg['PREV_APPROVED_RATIO'] = prev_agg['PREV_APPROVED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg = prev_agg.drop(['PREV_REFUSED_COUNT', 'PREV_APPROVED_COUNT'], axis=1)
 return prev_agg<merge>
%time
params_best['tree_method'] = 'gpu_hist'
d = xgb.DMatrix(train[features], train[label])
 xgb_final = xgb.train(params_best,d,n_rounds_best )
def get_apps_all_with_prev_agg(apps, prev):
apps_all = get_apps_processed(apps)
 prev_agg = get_prev_agg(prev)
 print('prev_agg shape:', prev_agg.shape)
 print('apps_all before merge shape:', apps_all.shape)
 apps_all = apps_all.merge(prev_agg, on='SK_ID_CURR', how='left')
 print('apps_all after merge with prev_agg shape:', apps_all.shape)
 return apps_all<categorify>
y_pred_final = xgb_final.predict(d)
 score_final = np.sqrt(mean_squared_error(train[label], y_pred_final))
print(score_final )
def get_apps_all_encoded(apps_all):
object_columns = apps_all.dtypes[apps_all.dtypes == 'object'].index.tolist()
for column in object_columns:
apps_all[column] = pd.factorize(apps_all[column])[0]
return apps_all<split>
test = pd.read_csv(path + 'test.csv')
 test.set_index('id',drop=True,inplace=True)
 d_tst = xgb.DMatrix(test[features])
def get_apps_all_train_test(apps_all):
apps_all_train = apps_all[~apps_all['TARGET'].isnull() ]
apps_all_test = apps_all[apps_all['TARGET'].isnull() ]
apps_all_test = apps_all_test.drop('TARGET', axis=1)
 return apps_all_train, apps_all_test<split>
models = []
for seed in range(0,10):
params_best['seed'] = seed
xgb_final = xgb.train(params_best,d,num_boost_round = n_rounds_best)
 models.append(xgb_final)
 xgb_pred = xgb_final.predict(d_tst )
def train_apps_all(apps_all_train):
ftr_app = apps_all_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 target_app = apps_all_train['TARGET']
train_x, valid_x, train_y, valid_y = train_test_split(ftr_app, target_app, test_size=0.3, random_state=2020)
 print('train shape:', train_x.shape, 'valid shape:', valid_x.shape)
 clf = LGBMClassifier(
 n_jobs=-1,
n_estimators=1000,
learning_rate=0.02,
num_leaves=32,
subsample=0.8,
max_depth=12,
silent=-1,
verbose=-1 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], eval_metric= 'auc', verbose= 100,
early_stopping_rounds= 100)
 return clf<load_from_csv>
ids = test.index
output = pd.DataFrame({'id': ids,
'target': xgb_pred})
 output.to_csv('submission.csv', index=False )