Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<choose_model_class>
warnings.filterwarnings("ignore" )
def XGB_CV(
 max_depth,
gamma,
min_child_weight,
scale_pos_weight,
reg_alpha,
reg_lambda 
):
global GINIbest
n_splits = 5
n_estimators = 200
folds = StratifiedKFold(n_splits=n_splits, random_state=1001)
 xgb_evals = np.zeros(( n_estimators, n_splits))
oof = np.empty(len(trn_df))
sub_preds = np.zeros(len(sub_df))
increase = True
np.random.seed(0)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(target, target)) :
trn_dat, trn_tgt = trn_df.iloc[trn_idx], target.iloc[trn_idx]
val_dat, val_tgt = trn_df.iloc[val_idx], target.iloc[val_idx]
clf = XGBClassifier(n_estimators=n_estimators,
max_depth=int(max_depth),
objective="binary:logistic",
learning_rate=0.1,
max_delta_step=1,
subsample=0.8,
colsample_bytree=0.8,
gamma=gamma,
reg_alpha=reg_alpha,
reg_lambda=reg_lambda,
scale_pos_weight=scale_pos_weight,
min_child_weight=min_child_weight,
nthread=4)
 if increase:
pos = pd.Series(trn_tgt == 1)
 trn_dat = pd.concat([trn_dat, trn_dat.loc[pos]], axis=0)
 trn_tgt = pd.concat([trn_tgt, trn_tgt.loc[pos]], axis=0)
 idx = np.arange(len(trn_dat))
np.random.shuffle(idx)
 trn_dat = trn_dat.iloc[idx]
trn_tgt = trn_tgt.iloc[idx]
clf.fit(trn_dat, trn_tgt,
eval_set=[(trn_dat, trn_tgt),(val_dat, val_tgt)],
eval_metric=gini_xgb,
early_stopping_rounds=None,
verbose=False)
 xgb_evals[:, fold_] = clf.evals_result_["validation_1"]["gini"]
best_round = np.argsort(xgb_evals[:, fold_])[::-1][0]
mean_eval = np.mean(xgb_evals, axis=1)
 std_eval = np.std(xgb_evals, axis=1)
 best_round = np.argsort(mean_eval)[::-1][0]
print(' Stopped after %d iterations with val-gini = %.6f +- %.6f' %(best_round, mean_eval[best_round], std_eval[best_round]))
if(mean_eval[best_round] > GINIbest):
GINIbest = mean_eval[best_round]
return mean_eval[best_round]<load_from_csv>
%%time
train_df = pd.read_csv(os.path.join(PATH,'train.csv'), dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )
GINIbest = -1.
 ce = CE(strategy='random_projection', verbose=True)
 start_time = timer(None)
 train_loader = pd.read_csv('.. /input/train.csv', dtype={'target': np.int8, 'id': np.int32})
 train = train_loader.drop(['target', 'id'], axis=1)
 print('
Shape of raw train data:', train.shape)
 col_to_drop = train.columns[train.columns.str.startswith('ps_calc_')]
train.drop(col_to_drop, axis=1, inplace=True)
 target = train_loader['target']
train_ids = train_loader['id'].values
test_loader = pd.read_csv('.. /input/test.csv', dtype={'id': np.int32})
 test = test_loader.drop(['id'], axis=1)
 print(' Shape of raw test data:', test.shape)
 test.drop(col_to_drop, axis=1, inplace=True)
 test_ids = test_loader['id'].values
col_to_embed = train.columns[train.columns.str.endswith('_cat')].astype(str ).tolist()
embed_train = train[col_to_embed].astype(np.str)
 embed_test = test[col_to_embed].astype(np.str)
 train.drop(col_to_embed, axis=1, inplace=True)
 test.drop(col_to_embed, axis=1, inplace=True)
 print('
Learning random projections - this will take less time than entity embedding...')
 Learning entity embedding - this will take a while...')
 ce.fit(embed_train, target)
 embed_enc_train = ce.transform(embed_train)
 embed_enc_test = ce.transform(embed_test)
 trn_df = pd.concat(( train, embed_enc_train), axis=1)
 sub_df = pd.concat(( test, embed_enc_test), axis=1)
 print('
Shape of processed train data:', trn_df.shape)
 print(' Shape of processed test data:', sub_df.shape)
 timer(start_time )<choose_model_class>
print("Train: rows:{} cols:{}".format(train_df.shape[0], train_df.shape[1]))
XGB_BO = BayesianOptimization(XGB_CV, {
'max_depth':(2, 6.99),
'gamma':(0.1, 5),
'min_child_weight':(0, 5),
'scale_pos_weight':(1, 5),
'reg_alpha':(0, 10),
'reg_lambda':(1, 10),
})
<save_to_csv>
rows = 150000
segments = int(np.floor(train_df.shape[0] / rows))
print("Number of segments: ", segments )
print('-'*126)
 print('
Final Results')
 print(' Maximum XGBOOST value: %f' % XGB_BO.res['max']['max_val'])
 print(' Best XGBOOST parameters: ', XGB_BO.res['max']['max_params'])
 grid_file = 'Bayes-gini-5fold-XGB-target-enc-run-04-v1-grid.csv'
print(' Saving grid search parameters to %s' % grid_file)
 XGB_BO.points_to_csv(grid_file )<init_hyperparams>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
max_depth = int(XGB_BO.res['max']['max_params']['max_depth'])
 gamma = XGB_BO.res['max']['max_params']['gamma']
min_child_weight = XGB_BO.res['max']['max_params']['min_child_weight']
scale_pos_weight = XGB_BO.res['max']['max_params']['scale_pos_weight']
reg_alpha = XGB_BO.res['max']['max_params']['reg_alpha']
reg_lambda = XGB_BO.res['max']['max_params']['reg_lambda']
start_time = timer(None)
 print('
Making final prediction - this will take a while...')
 n_splits = 5
n_estimators = 800
folds = StratifiedKFold(n_splits=n_splits, random_state=1001)
 imp_df = np.zeros(( len(trn_df.columns), n_splits))
xgb_evals = np.zeros(( n_estimators, n_splits))
oof = np.empty(len(trn_df))
sub_preds = np.zeros(len(sub_df))
increase = True
np.random.seed(0)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(target, target)) :
trn_dat, trn_tgt = trn_df.iloc[trn_idx], target.iloc[trn_idx]
val_dat, val_tgt = trn_df.iloc[val_idx], target.iloc[val_idx]
clf = XGBClassifier(n_estimators=n_estimators,
max_depth=max_depth,
objective="binary:logistic",
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.8,
max_delta_step=1,
gamma=gamma,
min_child_weight=min_child_weight,
reg_alpha=reg_alpha,
reg_lambda=reg_lambda,
scale_pos_weight=scale_pos_weight,
nthread=4)
 if increase:
pos = pd.Series(trn_tgt == 1)
 trn_dat = pd.concat([trn_dat, trn_dat.loc[pos]], axis=0)
 trn_tgt = pd.concat([trn_tgt, trn_tgt.loc[pos]], axis=0)
 idx = np.arange(len(trn_dat))
np.random.shuffle(idx)
 trn_dat = trn_dat.iloc[idx]
trn_tgt = trn_tgt.iloc[idx]
clf.fit(trn_dat, trn_tgt,
eval_set=[(trn_dat, trn_tgt),(val_dat, val_tgt)],
eval_metric=gini_xgb,
early_stopping_rounds=None,
verbose=False)
 imp_df[:, fold_] = clf.feature_importances_
xgb_evals[:, fold_] = clf.evals_result_["validation_1"]["gini"]
best_round = np.argsort(xgb_evals[:, fold_])[::-1][0]
oof[val_idx] = clf.predict_proba(val_dat, ntree_limit=best_round)[:, 1]
sub_preds += clf.predict_proba(sub_df, ntree_limit=best_round)[:, 1] / n_splits
print("Fold %2d : %.6f @%4d / best score is %.6f @%4d"
%(fold_ + 1,
eval_gini(val_tgt, oof[val_idx]),
n_estimators,
xgb_evals[best_round, fold_],
best_round))
print("Full OOF score : %.6f" % eval_gini(target, oof))
mean_eval = np.mean(xgb_evals, axis=1)
 std_eval = np.std(xgb_evals, axis=1)
 best_round = np.argsort(mean_eval)[::-1][0]
print("Best mean score : %.6f + %.6f @%4d"
%(mean_eval[best_round], std_eval[best_round], best_round))
best_gini = round(mean_eval[best_round], 6)
 importances = sorted([(trn_df.columns[i], imp)for i, imp in enumerate(imp_df.mean(axis=1)) ],
key=lambda x: x[1])
 for f, imp in importances[::-1]:
print("%-34s : %10.4f" %(f, imp))
timer(start_time)
 final_df = pd.DataFrame(test_ids, columns=['id'])
 final_df['target'] = sub_preds
now = datetime.now()
sub_file = 'submission_5fold-xgb-upsampling-target-enc-01_' + str(best_gini)+ '_' + str(now.strftime('%Y-%m-%d-%H-%M')) + '.csv'
print('
Writing submission: %s' % sub_file)
 final_df.to_csv(sub_file, index=False, float_format="%.9f" )<set_options>
train_X = pd.DataFrame(index=range(segments), dtype=np.float64)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])
 total_mean = train_df['acoustic_data'].mean()
total_std = train_df['acoustic_data'].std()
total_max = train_df['acoustic_data'].max()
total_min = train_df['acoustic_data'].min()
total_sum = train_df['acoustic_data'].sum()
total_abs_sum = np.abs(train_df['acoustic_data'] ).sum()
warnings.simplefilter(action='ignore', category=FutureWarning )<categorify>
def create_features(seg_id, seg, X):
xc = pd.Series(seg['acoustic_data'].values)
 zc = np.fft.fft(xc)
 X.loc[seg_id, 'mean'] = xc.mean()
X.loc[seg_id, 'std'] = xc.std()
X.loc[seg_id, 'max'] = xc.max()
X.loc[seg_id, 'min'] = xc.min()
realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'Rmean'] = realFFT.mean()
X.loc[seg_id, 'Rstd'] = realFFT.std()
X.loc[seg_id, 'Rmax'] = realFFT.max()
X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imean'] = imagFFT.mean()
X.loc[seg_id, 'Istd'] = imagFFT.std()
X.loc[seg_id, 'Imax'] = imagFFT.max()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()
X.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()
X.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()
X.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()
X.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()
X.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(xc))
X.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero(( np.diff(xc)/ xc[:-1])) [0])
 X.loc[seg_id, 'abs_max'] = np.abs(xc ).max()
X.loc[seg_id, 'abs_min'] = np.abs(xc ).min()
X.loc[seg_id, 'std_first_50000'] = xc[:50000].std()
X.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()
X.loc[seg_id, 'std_first_10000'] = xc[:10000].std()
X.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()
X.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()
X.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()
X.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()
X.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()
X.loc[seg_id, 'min_first_50000'] = xc[:50000].min()
X.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()
X.loc[seg_id, 'min_first_10000'] = xc[:10000].min()
X.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()
X.loc[seg_id, 'max_first_50000'] = xc[:50000].max()
X.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()
X.loc[seg_id, 'max_first_10000'] = xc[:10000].max()
X.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()
X.loc[seg_id, 'max_to_min'] = xc.max() / np.abs(xc.min())
 X.loc[seg_id, 'max_to_min_diff'] = xc.max() - np.abs(xc.min())
 X.loc[seg_id, 'count_big'] = len(xc[np.abs(xc)> 500])
 X.loc[seg_id, 'sum'] = xc.sum()
X.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero(( np.diff(xc[:50000])/ xc[:50000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero(( np.diff(xc[-50000:])/ xc[-50000:][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero(( np.diff(xc[:10000])/ xc[:10000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero(( np.diff(xc[-10000:])/ xc[-10000:][:-1])) [0])
 X.loc[seg_id, 'q95'] = np.quantile(xc, 0.95)
 X.loc[seg_id, 'q99'] = np.quantile(xc, 0.99)
 X.loc[seg_id, 'q05'] = np.quantile(xc, 0.05)
 X.loc[seg_id, 'q01'] = np.quantile(xc, 0.01)
 X.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(xc), 0.95)
 X.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(xc), 0.99)
 X.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(xc), 0.05)
 X.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(xc), 0.01)
 X.loc[seg_id, 'trend'] = add_trend_feature(xc)
 X.loc[seg_id, 'abs_trend'] = add_trend_feature(xc, abs_values=True)
 X.loc[seg_id, 'abs_mean'] = np.abs(xc ).mean()
X.loc[seg_id, 'abs_std'] = np.abs(xc ).std()
X.loc[seg_id, 'mad'] = xc.mad()
X.loc[seg_id, 'kurt'] = xc.kurtosis()
X.loc[seg_id, 'skew'] = xc.skew()
X.loc[seg_id, 'med'] = xc.median()
X.loc[seg_id, 'Hilbert_mean'] = np.abs(hilbert(xc)).mean()
X.loc[seg_id, 'Hann_window_mean'] =(convolve(xc, hann(150), mode='same')/ sum(hann(150)) ).mean()
X.loc[seg_id, 'classic_sta_lta1_mean'] = classic_sta_lta(xc, 500, 10000 ).mean()
X.loc[seg_id, 'classic_sta_lta2_mean'] = classic_sta_lta(xc, 5000, 100000 ).mean()
X.loc[seg_id, 'classic_sta_lta3_mean'] = classic_sta_lta(xc, 3333, 6666 ).mean()
X.loc[seg_id, 'classic_sta_lta4_mean'] = classic_sta_lta(xc, 10000, 25000 ).mean()
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_1500_mean'] = xc.rolling(window=1500 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_3000_mean'] = xc.rolling(window=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_6000_mean'] = xc.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(xc, span=300 ).mean() ).mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(xc, span=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(xc, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(xc, [75, 25]))
X.loc[seg_id, 'q999'] = np.quantile(xc,0.999)
 X.loc[seg_id, 'q001'] = np.quantile(xc,0.001)
 X.loc[seg_id, 'ave10'] = stats.trim_mean(xc, 0.1)
 for windows in [10, 100, 1000]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns = categorical_columns, dummy_na = nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
def label_encoder(df):
le = LabelEncoder()
le_count = 0
for col in df:
if df[col].dtype == 'object':
if len(list(df[col].unique())) <= 2:
le.fit(df[col])
 df[col] = le.transform(df[col])
 le_count += 1
print('%d columns were label encoded.' % le_count)
 return df<load_from_csv>
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*rows:seg_id*rows+rows]
create_features(seg_id, seg, train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]
def application_train_test(num_rows = None, nan_as_category = False):
df = pd.read_csv(".. /input/home-credit-default-risk/application_train.csv", nrows = num_rows)
 test_df = pd.read_csv(".. /input/home-credit-default-risk/application_test.csv", nrows = num_rows)
 df = df.append(test_df ).reset_index()
del df["index"]
df = df[df['CODE_GENDER'] != 'XNA']
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 df['APP_NEW_AGE'] = df['DAYS_BIRTH'] /(- 365.25)
 df['APP_NEW_DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['APP_NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['APP_NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['APP_NEW_ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['APP_NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
df["APP_NEW_CREDIT_GOODS_PRICE_RATIO"] = df["AMT_CREDIT"] / df["AMT_GOODS_PRICE"]
df["APP_NEW_CREDIT_DOWNPAYMENT"] = df["AMT_GOODS_PRICE"] - df["AMT_CREDIT"]
df.loc[(df.AMT_INCOME_TOTAL < 30000),'APP_NEW_INCOME_BAND'] = 1
df.loc[(df.AMT_INCOME_TOTAL >= 30000)&(df.AMT_INCOME_TOTAL < 65000),'APP_NEW_INCOME_BAND'] = 2
df.loc[(df.AMT_INCOME_TOTAL >= 65000)&(df.AMT_INCOME_TOTAL < 95000),'APP_NEW_INCOME_BAND'] = 3
df.loc[(df.AMT_INCOME_TOTAL >= 95000)&(df.AMT_INCOME_TOTAL < 130000),'APP_NEW_INCOME_BAND'] = 4
df.loc[(df.AMT_INCOME_TOTAL >= 130000)&(df.AMT_INCOME_TOTAL < 160000),'APP_NEW_INCOME_BAND'] = 5
df.loc[(df.AMT_INCOME_TOTAL >= 160000)&(df.AMT_INCOME_TOTAL < 190000),'APP_NEW_INCOME_BAND'] = 6
df.loc[(df.AMT_INCOME_TOTAL >= 190000)&(df.AMT_INCOME_TOTAL < 220000),'APP_NEW_INCOME_BAND'] = 7
df.loc[(df.AMT_INCOME_TOTAL >= 220000)&(df.AMT_INCOME_TOTAL < 275000),'APP_NEW_INCOME_BAND'] = 8
df.loc[(df.AMT_INCOME_TOTAL >= 275000)&(df.AMT_INCOME_TOTAL < 325000),'APP_NEW_INCOME_BAND'] = 9
df.loc[(df.AMT_INCOME_TOTAL >= 325000),'APP_NEW_INCOME_BAND'] = 10
APP_NEW_AGE_CAT = pd.Series(["Young", "Adult 1","Adult 2","Adult 3", "Adult 4"], dtype = "object")
 df["APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT
df.loc[(df["APP_NEW_AGE"] > 20.0)&(df["APP_NEW_AGE"] <= 30.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[0]
df.loc[(df["APP_NEW_AGE"] > 30.0)&(df["APP_NEW_AGE"] <= 40.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[1]
df.loc[(df["APP_NEW_AGE"] > 40.0)&(df["APP_NEW_AGE"] <= 50.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[2]
df.loc[(df["APP_NEW_AGE"] > 50.0)&(df["APP_NEW_AGE"] <= 60.0), "APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[3]
df.loc[df["APP_NEW_AGE"] > 60 ,"APP_NEW_AGE_CAT"] = APP_NEW_AGE_CAT[4]
df["APP_NEW_AGE_DAYS_EMP"] = df["DAYS_EMPLOYED"] /(- 365.25)
 df["APP_NEW_AGE_WORK_PERCENT"] =(df["APP_NEW_AGE_DAYS_EMP"] / df['APP_NEW_AGE'])* 100
df['APP_NEW_CREDIT_INCOME_PERCENT'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
df['APP_NEW_ANNUITY_INCOME_PERCENT'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']
df['APP_NEW_DAYS_EMPLOYED_PERCENT'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['APP_NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['APP_NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['APP_NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
df['APP_NEW_AMT_PAY_YEAR'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['APP_NEW_AGE_PAYOFF'] = df['APP_NEW_AGE'] + df['APP_NEW_AMT_PAY_YEAR']
df['APP_NEW_AMT_DIFF_CREDIT_GOODS'] = df['AMT_CREDIT'] - df['AMT_GOODS_PRICE']
df['APP_NEW_AMT_CREDIT_GOODS_PERC'] =(( df['AMT_GOODS_PRICE'] / df['AMT_CREDIT'])* 100)
 df['APP_NEW_CNT_ADULT'] = df['CNT_FAM_MEMBERS'] - df['CNT_CHILDREN']
df['APP_NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 df['APP_NEW_EXT_SOURCES_MAX'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].max(axis=1)
 df['APP_NEW_EXT_SOURCES_SUM'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].sum(axis=1)
 df = label_encoder(df)
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 del test_df
gc.collect()
return df<load_from_csv>
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns )
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/home-credit-default-risk/bureau.csv', nrows = num_rows)
 bureau_balance = pd.read_csv('.. /input/home-credit-default-risk/bureau_balance.csv', nrows = num_rows)
 def _status_to_int(status):
if status in ['X', 'C']:
return 0
if pd.isnull(status):
return np.nan
return int(status)
 bureau_balance['NEW_BUREAU_BALANCE_DPD_LEVEL'] = bureau_balance['STATUS'].apply(_status_to_int)
 bureau_balance['NEW_BUREAU_BALANCE_STATUS_UNKNOWN'] =(bureau_balance['STATUS'] == 'X' ).astype(int)
 bureau_balance["MONTHS_BALANCE"] =(-1*bureau_balance["MONTHS_BALANCE"])+1
bb_aggregations = {'MONTHS_BALANCE': ["max"],
'NEW_BUREAU_BALANCE_DPD_LEVEL':['sum', 'mean', 'max', 'std', 'skew'],
'NEW_BUREAU_BALANCE_STATUS_UNKNOWN':['sum', 'mean']}
bb_agg = bureau_balance.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)
 del bureau_balance, bb_agg
gc.collect()
bureau.drop(["AMT_CREDIT_SUM_LIMIT","AMT_CREDIT_SUM_OVERDUE","CREDIT_DAY_OVERDUE","AMT_CREDIT_SUM_OVERDUE"],axis=1,inplace=True)
 bureau['BUREAU_CREDIT_TYPE_CONSUMER'] =(bureau['CREDIT_TYPE'] == 'Consumer credit' ).astype(int)
 bureau['BUREAU_CREDIT_TYPE_CAR'] =(bureau['CREDIT_TYPE'] == 'Car loan' ).astype(int)
 bureau['BUREAU_CREDIT_TYPE_MORTGAGE'] =(bureau['CREDIT_TYPE'] == 'Mortgage' ).astype(int)
 bureau['BUREAU_CREDIT_TYPE_CREDIT_CARD'] =(bureau['CREDIT_TYPE'] == 'Credit card' ).astype(int)
 bureau['BUREAU_CREDIT_TYPE_OTHER'] =(~(bureau['CREDIT_TYPE'].isin(['Consumer credit', 'Car loan', 'Mortgage', 'Credit card'])) ).astype(int)
 bureau['BUREAU_UNUSUAL_CURRENCY'] =(~(bureau['CREDIT_CURRENCY'] == 'currency 1')).astype(int)
 bureau['NEW_PAYMENT_RATE_SUM'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_SUM']
bureau['NEW_PAYMENT_RATE_SUM_DEBT'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_SUM_DEBT']
bureau['NEW_PAYMENT_RATE_AMT_CREDIT_MAX_OVERDUE'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_MAX_OVERDUE']
bureau.drop(["CREDIT_TYPE","CREDIT_CURRENCY"],axis=1,inplace=True)
 bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-42056.001)&(bureau["DAYS_CREDIT_ENDDATE"]<=-1773.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 1
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-1773.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-1273.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 2
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-1273.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-777.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 3
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-777.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-474.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 4
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-474.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-268.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 5
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-268.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-133.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 6
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-133.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-38.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 7
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["DAYS_CREDIT_ENDDATE"]>-38.0)&(bureau["DAYS_CREDIT_ENDDATE"]<=-1.0),"BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE"] = 8
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-42023.001)&(bureau["DAYS_ENDDATE_FACT"]<=-2145.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 1
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-2145.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1598.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 2
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1598.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1191.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 3
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1191.0)&(bureau["DAYS_ENDDATE_FACT"]<=-902.) ,"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 4
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-902.) &(bureau["DAYS_ENDDATE_FACT"]<=-656.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 5
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-656.0)&(bureau["DAYS_ENDDATE_FACT"]<=-439.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 6
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-439.0)&(bureau["DAYS_ENDDATE_FACT"]<=-229.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 7
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]<bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-229.0)&(bureau["DAYS_ENDDATE_FACT"]<=0.0),"BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE"] = 8
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-2876.001)&(bureau["DAYS_ENDDATE_FACT"]<=-2046.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 1
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-2046.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1501.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 2
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1501.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1182.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 3
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1182.0)&(bureau["DAYS_ENDDATE_FACT"]<=-1012.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 4
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-1012.0)&(bureau["DAYS_ENDDATE_FACT"]<=-878.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 5
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-878.0)&(bureau["DAYS_ENDDATE_FACT"]<=-615.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 6
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-615.0)&(bureau["DAYS_ENDDATE_FACT"]<=-307.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 7
bureau.loc[(bureau["CREDIT_ACTIVE"]=="Closed")&(bureau["DAYS_ENDDATE_FACT"]>bureau["DAYS_CREDIT_ENDDATE"])&(bureau["DAYS_ENDDATE_FACT"]>-307.0)&(bureau["DAYS_ENDDATE_FACT"]<=0.0),"BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE"] = 8
bureau['BUREAU_NEW_DAYS_CREDIT_SCORE'] = pd.qcut(bureau['DAYS_CREDIT'],10,labels=[10.0,9.0,8.0,7.0,6.0,5.0,4.0,3.0,2.0,1.0])
 bureau.loc[(bureau["CREDIT_ACTIVE"]=="Active")&(bureau["BUREAU_NEW_DAYS_CREDIT_SCORE"] == 10),"BUREAU_NEW_LAST_ACTIVE_DAYS_CREDIT"] = 1
num_aggregations = {
"DAYS_CREDIT": ['min', 'max', 'mean', 'var', 'sum'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'DAYS_ENDDATE_FACT': ['mean', 'sum'],
'AMT_ANNUITY': ['min', 'max', 'mean', 'var', 'sum'],
'CNT_CREDIT_PROLONG': ['sum','std'],
'MONTHS_BALANCE_MAX': ['max'],
"NEW_BUREAU_BALANCE_DPD_LEVEL_SUM" :['max',"sum"],
"NEW_BUREAU_BALANCE_DPD_LEVEL_MEAN" :['max',"sum","mean"],
"NEW_BUREAU_BALANCE_DPD_LEVEL_MAX" :['max',"sum"],
"NEW_BUREAU_BALANCE_DPD_LEVEL_STD" :['max',"sum","std"],
"NEW_BUREAU_BALANCE_DPD_LEVEL_SKEW" :['max',"sum","skew"],
"NEW_BUREAU_BALANCE_STATUS_UNKNOWN_SUM" :['max',"sum"],
"NEW_BUREAU_BALANCE_STATUS_UNKNOWN_MEAN" :['max',"sum","mean"],
'BUREAU_CREDIT_TYPE_CONSUMER': ['mean', 'sum'],
'BUREAU_CREDIT_TYPE_CAR': ['mean', 'sum'],
'BUREAU_CREDIT_TYPE_MORTGAGE': ['mean', 'sum'],
'BUREAU_CREDIT_TYPE_CREDIT_CARD': ['mean', 'sum'],
'BUREAU_CREDIT_TYPE_OTHER': ['mean', 'sum'],
'BUREAU_UNUSUAL_CURRENCY': ['mean', 'sum'],
'NEW_PAYMENT_RATE_SUM':['max',"mean","sum"],
'NEW_PAYMENT_RATE_SUM_DEBT':['max',"mean","sum"],
'NEW_PAYMENT_RATE_AMT_CREDIT_MAX_OVERDUE':['max',"mean","sum",'min'],
'BUREAU_NEW_ACTIVE_EARLY_PAYERS_SCORE':['mean','sum','max','min'],
'BUREAU_NEW_CLOSED_EARLY_PAYERS_SCORE':['mean','sum','max','min'],
'BUREAU_NEW_CLOSED_LATE_PAYERS_SCORE':['mean','sum','max','min'],
}
bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index )
def previous_applications(num_rows = None, nan_as_category = True):
df = pd.read_csv(".. /input/home-credit-default-risk/previous_application.csv", nrows = num_rows)
 df.replace(365243,np.nan,inplace = True)
 df.replace("XNA",np.nan,inplace = True)
 df['NEW_RETURN_DAY'] = df['DAYS_DECISION'] + df['CNT_PAYMENT'] * 30
df['NEW_DAYS_TERMINATION_diff'] = df['DAYS_TERMINATION'] - df['NEW_RETURN_DAY']
df['NEW_AMT_DOWN_PAYMENT_rate'] = df['AMT_DOWN_PAYMENT'] /(df['AMT_CREDIT'] + 0.01)
 df['NEW_AMT_SPEND_TO_PRODUCT'] = df['AMT_GOODS_PRICE'] / df['AMT_CREDIT']
df['NEW_DAYS_DUE']=df['DAYS_FIRST_DUE'] - df['DAYS_LAST_DUE_1ST_VERSION']
df['NEW_APP_CREDIT_PERC'] = df['AMT_APPLICATION'] / df['AMT_CREDIT']
df["NAME_PAYMENT_TYPE"].replace(["Non-cash from your account","Cashless from the account of the employer"],np.nan,inplace=True)
 a = ["Channel of corporate sales","Car dealer"]
df["CHANNEL_TYPE"].replace(a,"Others_Type",inplace=True)
 b = ['Family', 'Spouse, partner', 'Children', 'Other_B', 'Other_A', 'Group of people']
df["NAME_TYPE_SUITE"] = df["NAME_TYPE_SUITE"].replace(b, 'not_alone')
 df["WEEKDAY_APPR_PROCESS_START"] = df["WEEKDAY_APPR_PROCESS_START"].replace(['MONDAY','TUESDAY', 'WEDNESDAY','THURSDAY','FRIDAY'], 'WEEK_DAY')
 df["WEEKDAY_APPR_PROCESS_START"] = df["WEEKDAY_APPR_PROCESS_START"].replace(['SATURDAY', 'SUNDAY'], 'WEEKEND')
 a = ['Auto technology', 'Jewelry', 'MLM partners', 'Tourism']
df["NAME_SELLER_INDUSTRY"] = df["NAME_SELLER_INDUSTRY"].replace(a, 'Other_Ind')
 a = ['Auto Accessories', 'Jewelry', 'Homewares', 'Medical Supplies', 'Vehicles', 'Sport and Leisure','Gardening', 'Other', 'Office Appliances', 'Tourism', 'Medicine', 'Direct Sales', 'Fitness', 'Additional Service','Education', 'Weapon', 'Insurance', 'House Construction', 'Animals']
df["NAME_GOODS_CATEGORY"] = df["NAME_GOODS_CATEGORY"].replace(a, 'Other_Cat')
 a = ['Buying a used car','Building a house or an annex','Everyday expenses','Medicine','Payments on other loans','Education','Journey', 'Purchase of electronic equipment','Buying a new car','Wedding / gift / holiday','Buying a home','Car repairs','Furniture','Buying a holiday home / land', 'Business development','Gasification / water supply','Buying a garage','Hobby','Money for a third person','Refusal to name the goal','Urgent needs','Other']
df['NAME_CASH_LOAN_PURPOSE']= df['NAME_CASH_LOAN_PURPOSE'].replace(a,'Others')
 df["NAME_PORTFOLIO"].replace("cars",np.nan,inplace=True)
 a = [8,9,10,11,12,13,14,15,16,17]
df["HOUR_APPR_PROCESS_START"] = df["HOUR_APPR_PROCESS_START"].replace(a, 'Working_Hours')
 b = [18,19,20,21,22,23,0,1,2,3,4,5,6,7]
df["HOUR_APPR_PROCESS_START"] = df["HOUR_APPR_PROCESS_START"].replace(b, 'Off_Hours')
 drops = ["RATE_INTEREST_PRIMARY","RATE_INTEREST_PRIVILEGED","FLAG_LAST_APPL_PER_CONTRACT","NFLAG_LAST_APPL_IN_DAY","NAME_PRODUCT_TYPE","SELLERPLACE_AREA"]
df.drop(drops,inplace=True,axis=1)
 df["NFLAG_INSURED_ON_APPROVAL"] = df["NFLAG_INSURED_ON_APPROVAL"].astype("object")
 cat_features = list(df.select_dtypes(['object'] ).columns)
 df = pd.get_dummies(df, columns= cat_features, dummy_na= True,drop_first=True)
 agg1 = {'SK_ID_CURR': ['size'],
'AMT_ANNUITY': ['max', 'min', 'mean','std', 'sum'],
'AMT_APPLICATION':['max', 'min', 'mean','std', 'sum'],
'AMT_CREDIT':['max', 'min', 'mean','std', 'sum'],
'AMT_DOWN_PAYMENT': ['max', 'min', 'mean','std', 'sum'],
'AMT_GOODS_PRICE': ['max', 'min', 'mean','std', 'sum'],
'RATE_DOWN_PAYMENT': ['max', 'min', 'mean','std'],
'DAYS_DECISION': ['max', 'min', 'mean', 'sum'],
'CNT_PAYMENT': ['max', 'min', 'mean','std', 'sum'],
'DAYS_FIRST_DRAWING': ['max','mean', 'sum'],
'DAYS_FIRST_DUE': ['max', 'min', 'mean', 'sum'],
'DAYS_LAST_DUE_1ST_VERSION': ['max', 'min', 'mean', 'sum'],
'DAYS_LAST_DUE': ['max', 'min', 'mean', 'sum'],
'DAYS_TERMINATION': ['max', 'min', 'mean','std', 'sum'],
'NEW_RETURN_DAY': ['max', 'min', 'mean','std', 'sum'],
'NEW_DAYS_TERMINATION_diff': ['max', 'mean','std', 'sum'],
'NEW_AMT_DOWN_PAYMENT_rate': ['max','mean','std'],
'NEW_AMT_SPEND_TO_PRODUCT': ['max', 'mean','std', 'sum'],
'NEW_APP_CREDIT_PERC': ['max', 'min', 'mean'],
'NAME_CONTRACT_TYPE_Consumer loans': ['max', 'min','sum'],
'NAME_CONTRACT_TYPE_Revolving loans': ['max', 'min','sum'],
'NAME_CONTRACT_TYPE_nan': ['max', 'min','sum'],
'WEEKDAY_APPR_PROCESS_START_WEEK_DAY': ['max', 'min', 'sum'],
'WEEKDAY_APPR_PROCESS_START_nan': ['max', 'sum'],
'HOUR_APPR_PROCESS_START_Working_Hours': ['max', 'min', 'sum'],
'HOUR_APPR_PROCESS_START_nan': ['max','sum'],
'NAME_CASH_LOAN_PURPOSE_Repairs': ['max', 'min', 'sum'],
'NAME_CASH_LOAN_PURPOSE_XAP': ['max', 'sum'],
'NAME_CASH_LOAN_PURPOSE_nan': ['max', 'sum'],
'NAME_CONTRACT_STATUS_Canceled': ['max','sum'],
'NAME_CONTRACT_STATUS_Refused': ['max','sum'],
'NAME_CONTRACT_STATUS_Unused offer': ['max','sum'],
'NAME_CONTRACT_STATUS_nan': ['max', 'sum'],
'NAME_PAYMENT_TYPE_nan': ['max', 'sum'],
'CODE_REJECT_REASON_HC': ['max', 'min','sum'],
'CODE_REJECT_REASON_LIMIT': ['max', 'min','sum'],
'CODE_REJECT_REASON_SCO': ['max', 'min','sum'],
'CODE_REJECT_REASON_SCOFR': ['max', 'sum'],
'CODE_REJECT_REASON_VERIF': ['max', 'sum'],
'CODE_REJECT_REASON_XAP': ['max', 'sum'],
'CODE_REJECT_REASON_nan': ['max','sum'],
'NAME_TYPE_SUITE_not_alone': ['max', 'min','sum'],
'NAME_TYPE_SUITE_nan': ['max', 'min', 'sum'],
'NAME_CLIENT_TYPE_Refreshed': ['max', 'min','sum'],
'NAME_CLIENT_TYPE_Repeater': ['max', 'min', 'sum'],
'NAME_CLIENT_TYPE_nan': ['max', 'min','sum'],
'NAME_GOODS_CATEGORY_Clothing and Accessories': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_Computers': ['max', 'min','sum'],
'NAME_GOODS_CATEGORY_Construction Materials': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_Consumer Electronics': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_Furniture': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_Mobile': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_Other_Cat': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_Photo / Cinema Equipment': ['max', 'min', 'sum'],
'NAME_GOODS_CATEGORY_nan': ['max', 'min', 'sum'],
'NAME_PORTFOLIO_Cars': ['max', 'min', 'sum'],
'NAME_PORTFOLIO_Cash': ['max', 'min', 'sum'],
'NAME_PORTFOLIO_POS': ['max', 'min','sum'],
'NAME_PORTFOLIO_nan': ['max', 'min', 'sum'],
'CHANNEL_TYPE_Contact center': ['max', 'min', 'sum'],
'CHANNEL_TYPE_Country-wide': ['max', 'min', 'sum'],
'CHANNEL_TYPE_Credit and cash offices': ['max', 'min', 'sum'],
'CHANNEL_TYPE_Others_Type': ['max', 'min', 'sum'],
'CHANNEL_TYPE_Regional / Local': ['max', 'min','sum'],
'CHANNEL_TYPE_Stone': ['max', 'min','sum'],
'CHANNEL_TYPE_nan': ['max', 'min', 'sum'],
'NAME_SELLER_INDUSTRY_Connectivity': ['max', 'min','sum'],
'NAME_SELLER_INDUSTRY_Construction': ['max', 'min', 'sum'],
'NAME_SELLER_INDUSTRY_Consumer electronics': ['max', 'min', 'sum'],
'NAME_SELLER_INDUSTRY_Furniture': ['max', 'min', 'sum'],
'NAME_SELLER_INDUSTRY_Industry': ['max', 'min', 'sum'],
'NAME_SELLER_INDUSTRY_Other_Ind': ['max', 'min','sum'],
'NAME_SELLER_INDUSTRY_nan': ['max', 'min','sum'],
'NAME_YIELD_GROUP_low_action': ['max', 'min', 'sum'],
'NAME_YIELD_GROUP_low_normal': ['max', 'min', 'sum'],
'NAME_YIELD_GROUP_middle': ['max', 'min','sum'],
'NAME_YIELD_GROUP_nan': ['max', 'min','sum'],
'PRODUCT_COMBINATION_Card X-Sell': ['max', 'min', 'sum'],
'PRODUCT_COMBINATION_Cash': ['max', 'min', 'sum'],
'PRODUCT_COMBINATION_Cash Street: high': ['max', 'min', 'sum'],
'PRODUCT_COMBINATION_Cash Street: low': ['max', 'min','sum'],
'PRODUCT_COMBINATION_Cash Street: middle': ['max', 'min','sum'],
'PRODUCT_COMBINATION_Cash X-Sell: high': ['max', 'min','sum'],
'PRODUCT_COMBINATION_Cash X-Sell: low': ['max', 'min','sum'],
'PRODUCT_COMBINATION_Cash X-Sell: middle': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS household with interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS household without interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS industry with interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS industry without interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS mobile with interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS mobile without interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS other with interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_POS others without interest': ['max', 'min','sum'],
'PRODUCT_COMBINATION_nan': ['max', 'min','sum'],
'NFLAG_INSURED_ON_APPROVAL_1.0': ['max', 'min','sum'],
'NFLAG_INSURED_ON_APPROVAL_nan': ['max', 'min','sum']}
df = df.groupby(['SK_ID_CURR'] ).agg(agg1)
 df.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in df.columns.tolist() ])
 return df<load_from_csv>
for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 create_features(seg_id, seg, test_X )
def pos_cash(num_rows = None, nan_as_category = True):
df=pd.read_csv('.. /input/home-credit-default-risk/POS_CASH_balance.csv',nrows = num_rows)
 df['NEW_ADJOURNMENT']=df['SK_DPD']-df['SK_DPD_DEF']
b = ["Demand","Returned to the store","Approved","Amortized debt","Canceled","XNA"]
df["NAME_CONTRACT_STATUS"].replace(b, 'Others',inplace=True)
 cat_features = list(df.select_dtypes(['object'] ).columns)
 df = pd.get_dummies(df, columns= cat_features, dummy_na= True)
 agg={
'MONTHS_BALANCE': ['max'],
'SK_DPD': ['max', 'mean',"std"],
'SK_DPD_DEF': ['max', 'mean',"std"],
'CNT_INSTALMENT':['min','mean','max'],
'CNT_INSTALMENT_FUTURE':['mean','min','max'],
'SK_ID_CURR':['max','size'],
'NEW_ADJOURNMENT':['max','mean',"std"],
'NAME_CONTRACT_STATUS_Active':['sum'],
'NAME_CONTRACT_STATUS_Completed':['sum'],
'NAME_CONTRACT_STATUS_Signed':['sum'],
'NAME_CONTRACT_STATUS_Others':['sum']
}
pos_agg = df.groupby(['SK_ID_PREV'] ).agg(agg)
 pos_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg["NEW_PAID_MONTH"] = pos_agg["CNT_INSTALMENT_MAX"] - pos_agg["CNT_INSTALMENT_FUTURE_MIN"]
agg2={
"MONTHS_BALANCE_MAX":["max","mean"],
"SK_DPD_MAX":["max","mean"],
"SK_DPD_MEAN":["max","mean"],
"SK_DPD_STD":["max","mean","min","std"],
"SK_DPD_DEF_MAX":["max","mean"],
"SK_DPD_DEF_MEAN":["max","mean"],
"SK_DPD_DEF_STD":["max","mean"],
"CNT_INSTALMENT_MIN":["max","mean","min"],
"CNT_INSTALMENT_MEAN":["max","mean"],
"CNT_INSTALMENT_MAX":["max","mean"],
"CNT_INSTALMENT_FUTURE_MEAN":["max","mean"],
"CNT_INSTALMENT_FUTURE_MIN":["max","mean"],
"CNT_INSTALMENT_FUTURE_MAX":["max","mean"],
"SK_ID_CURR_MAX":["max"],
"SK_ID_CURR_SIZE":["max","min"],
"NEW_ADJOURNMENT_MAX":["max","mean"],
"NEW_ADJOURNMENT_MEAN":["max","mean"],
"NEW_ADJOURNMENT_STD":["max","mean","min"],
"NAME_CONTRACT_STATUS_Active_SUM":["max","sum"],
'NAME_CONTRACT_STATUS_Signed_SUM':["max","sum"],
'NAME_CONTRACT_STATUS_Completed_SUM':["max","sum"],
'NAME_CONTRACT_STATUS_Others_SUM':["max","sum"]
}
pos_agg2 = pos_agg.groupby(["SK_ID_CURR_MAX"] ).agg(agg2)
 pos_agg2.index.names = ['SK_ID_CURR']
pos_agg2.columns = pd.Index(["POS" + "_" + e[0] + "_" + e[1].upper() for e in pos_agg2.columns.tolist() ])
 del df
gc.collect()
return pos_agg2<load_from_csv>
scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns )
def installments_payments(num_rows=None, nan_as_category = True):
pd.options.mode.chained_assignment = None
df = pd.read_csv(".. /input/home-credit-default-risk/installments_payments.csv", nrows = num_rows)
 df["NEW_DELAY"] = df["DAYS_INSTALMENT"] - df["DAYS_ENTRY_PAYMENT"]
df['NEW_FLAG_DELAY'] = df['NEW_DELAY'].apply(lambda x : 1 if x < 0 else 0)
 df['NEW_RATIO_DELAY'] = df[['SK_ID_PREV','NEW_FLAG_DELAY']].groupby('SK_ID_PREV')['NEW_FLAG_DELAY'].transform(lambda x : x.sum() / x.count())
 df["NEW_PAYMENT_DIFF"] = df["AMT_INSTALMENT"] - df["AMT_PAYMENT"]
df["IP_NEW_PAYMENT_INSTALMENT_DIFF"] = df["AMT_PAYMENT"] - df["AMT_INSTALMENT"]
df["NUM_INSTALMENT_VERSION"] = df["NUM_INSTALMENT_VERSION"].astype("object")
 df[(df["NUM_INSTALMENT_VERSION"] != 1)&(df["NUM_INSTALMENT_VERSION"] != 0)&(df["NUM_INSTALMENT_VERSION"] != 2)&(df["NUM_INSTALMENT_VERSION"] != 3)]['NUM_INSTALMENT_VERSION'] = 4
cat_features = list(df.select_dtypes(['object'] ).columns)
 df = pd.get_dummies(df, columns= cat_features,drop_first=True)
 agg1 = {'SK_ID_CURR': ['count','max'],
'NEW_DELAY': ['max', 'min', 'mean','std', 'sum'],
'NUM_INSTALMENT_NUMBER':['min','max'],
'DAYS_INSTALMENT':['max','min','std'],
'NEW_PAYMENT_DIFF': ['max', 'mean', 'std', 'min','sum'],
'IP_NEW_PAYMENT_INSTALMENT_DIFF': ['max', 'mean', 'std', 'min','sum'],
'AMT_INSTALMENT': ['max', 'mean', 'sum', 'min', 'std'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum', 'std'],
'DAYS_ENTRY_PAYMENT': ['max', 'min', 'std'],
"NUM_INSTALMENT_VERSION_1.0":["sum"],
"NUM_INSTALMENT_VERSION_2.0":["sum"],
"NUM_INSTALMENT_VERSION_3.0":["sum"],
"NUM_INSTALMENT_VERSION_4.0":["sum"]
}
Installments_agg = df.groupby(['SK_ID_PREV'] ).agg(agg1)
 Installments_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in Installments_agg.columns.tolist() ])
 del df
gc.collect()
return Installments_agg<load_from_csv>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = scaled_train_X.columns.values
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/home-credit-default-risk/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)
 cc['number_of_instalments'] = cc.groupby(
 by=['SK_ID_CURR'])['CNT_INSTALMENT_MATURE_CUM'].agg('max' ).reset_index() [
'CNT_INSTALMENT_MATURE_CUM']
cc['AMT_DRAWINGS_ATM_CURRENT'][cc['AMT_DRAWINGS_ATM_CURRENT'] < 0] = np.nan
cc['AMT_DRAWINGS_CURRENT'][cc['AMT_DRAWINGS_CURRENT'] < 0] = np.nan
cc_agg = cc.groupby('SK_ID_CURR' ).agg(['min', 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg<merge>
params = {'num_leaves': 51,
'min_data_in_leaf': 10,
'objective':'regression',
'max_depth': -1,
'learning_rate': 0.001,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 1,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": -1,
"random_state": 42}
def main(debug = False):
num_rows = 10000 if debug else None
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Run LightGBM with kfold"):
feat_importance = kfold_lightgbm(df, num_folds= 10, stratified= False, debug= debug)
 if __name__ == "__main__":
submission_file_name = "submission.csv"
with timer("Full model run"):
main()<set_options>
oof = np.zeros(len(scaled_train_X))
predictions = np.zeros(len(scaled_test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(scaled_train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = scaled_train_X.iloc[trn_idx], scaled_train_X.iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 20000, n_jobs = -1)
 model.fit(X_tr,
y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)],
eval_metric='mae',
verbose=1000,
early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val, num_iteration=model.best_iteration_)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(scaled_test_X, num_iteration=model.best_iteration_)/ folds.n_splits
<load_from_csv><EOS> 
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )