Kaggle competition: Exam for Students20200129
<SOS> metric: rmsle Kaggle data source: exam-for-students20200129<data_type_conversions>
import gc
import os
from pathlib import Path
import sys
import collections
import pandas as pd
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm
import joblib
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold
from sklearn.pipeline import FeatureUnion, Pipeline, make_pipeline
from sklearn.preprocessing import OneHotEncoder, QuantileTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import SelectFromModel
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics import mean_squared_error
import category_encoders as ce
import lightgbm as lgbm
def fillna_numeric(fillna_target_col, fillna_by='median', val_min=None, val_max=None, add_flg=False):
print('fillna_numeric('{}', fillna_by={})'.format(fillna_target_col, fillna_by), '+flg' if add_flg else '')
 global df_train, df_test, cols_quantitative
isnull_flgs_train = df_train[fillna_target_col].isnull()
isnull_flgs_test = df_test[fillna_target_col].isnull()
if(type(fillna_by)is int)or(type(fillna_by)is float):
df_train.loc[isnull_flgs_train, fillna_target_col] = fillna_by
df_test.loc[isnull_flgs_test, fillna_target_col] = fillna_by
if fillna_by == 'random':
fill_by_train = random.choices(df_train.loc[~isnull_flgs_train, fillna_target_col].tolist() , k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(df_test.loc[~isnull_flgs_test, fillna_target_col].tolist() , k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'minmax':
val_btwn_minmax_flgs = df_train[fillna_target_col].between(val_min, val_max)
 val_btwn_minmax = df_train.loc[val_btwn_minmax_flgs, fillna_target_col].tolist()
fill_by_train = random.choices(val_btwn_minmax, k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(val_btwn_minmax, k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'median':
median = df_train[fillna_target_col].median()
df_train.loc[isnull_flgs_train, fillna_target_col] = median
df_test.loc[isnull_flgs_test, fillna_target_col] = median
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
cols_quantitative.append(col_isnull_flg)
 return<feature_engineering>
%matplotlib inline
plt.rcParams["figure.figsize"] =(15, 5)
 pd.options.display.max_columns = 50
def log_transformation(transform_target_col, remove_outlier=False, add_flg=False):
print('log_transformation('{}')'.format(transform_target_col), '+remove_outlier' if remove_outlier else '', '+flg' if add_flg else '')
 global df_train, df_test, cols_flg
df_train[transform_target_col] = df_train[transform_target_col].apply(np.log1p)
 df_test[transform_target_col] = df_test[transform_target_col].apply(np.log1p)
 if remove_outlier:
p01 = df_train[transform_target_col].quantile(0.01)
 p99 = df_train[transform_target_col].quantile(0.99)
 isoutlier_flgs_train =(( df_train[transform_target_col] < p01)|(df_train[transform_target_col] > p99))
isoutlier_flgs_test =(( df_test[transform_target_col] < p01)|(df_test[transform_target_col] > p99))
df_train[transform_target_col] = df_train[transform_target_col].clip(p01, p99)
 df_test[transform_target_col] = df_test[transform_target_col].clip(p01, p99)
 if add_flg:
col_isoutlier_flg = 'FLG:{}_isoutlier'.format(transform_target_col)
 df_train[col_isoutlier_flg] = 0
df_train.loc[isoutlier_flgs_train, col_isoutlier_flg] = 1
df_test[col_isoutlier_flg] = 0
df_test.loc[isoutlier_flgs_test, col_isoutlier_flg] = 1
update_cols(add_cols_flg=[col_isoutlier_flg])
 return<define_search_space>
VERSION = "1.6.6"
def binning_values(cut_target_col, interval):
print('cut_values('{}', interval={})'.format(cut_target_col, interval))
global df_train, df_test
if int(str(interval*1000 ).split('.')[1])!= 0:
print(' > error: interval is too small')
 return
a = 1
for i in range(3):
if interval*a < 1:
a *= 10
def cut(x):
return(x*a-(( x*a)%(interval*a)))/a
df_train[cut_target_col] = df_train[cut_target_col].map(cut)
 df_test[cut_target_col] = df_test[cut_target_col].map(cut)
 return<categorify>
IS_KAGGLE = "KAGGLE_URL_BASE" in os.environ
print(f"IS_KAGGLE: {IS_KAGGLE}")
 USE_GPU = "NVIDIA_VISIBLE_DEVICES" in os.environ
print(f"USE_GPU: {USE_GPU}")
 USE_CACHE = False
def rank_gauss_transformation(n_split=5):
print('rank_gauss_transformation(n_split={})'.format(n_split))
global df_train, df_test, cols_quantitative
def split_list(l, n):
for idx in range(0, len(l), n):
yield l[idx:idx + n]
for cols in list(split_list(cols_quantitative, n_split)) :
df_all = pd.concat([df_train[cols], df_test[cols]], axis=0)
 df_all[cols] = quantile_transform(df_all[cols], n_quantiles=100, random_state=0, output_distribution='normal')
 df_train_transformed = df_all.iloc[:df_train.shape[0], :]
df_test_transformed = df_all.iloc[df_train.shape[0]:, :]
del df_all
df_train = pd.concat([df_train.drop(columns=cols), df_train_transformed], axis=1)
 df_test = pd.concat([df_test.drop(columns=cols), df_test_transformed], axis=1)
 del df_train_transformed, df_test_transformed
return<data_type_conversions>
train = pd.read_csv(".. /input/exam-for-students20200129/train.csv", index_col=0, na_values="" ).pipe(reduce_mem_usage)
 X_test = pd.read_csv(".. /input/exam-for-students20200129/test.csv", index_col=0, na_values="" ).pipe(reduce_mem_usage )
def fillna_text(fillna_target_col, add_flg=False):
print('fillna_text('{}')'.format(fillna_target_col), '+flg' if add_flg else '')
 global df_train, df_test, cols_flg
df_train[fillna_target_col] = df_train[fillna_target_col].fillna('
df_test[fillna_target_col] = df_test[fillna_target_col].fillna('
isnull_flgs_train =(df_train[fillna_target_col] == '
isnull_flgs_test =(df_test[fillna_target_col] == '
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
update_cols(add_cols_flg=[col_isnull_flg])
 return<data_type_conversions>
df_country = pd.read_csv(".. /input/exam-for-students20200129/country_info.csv", decimal="," ).pipe(reduce_mem_usage )
def fillna_numeric(fillna_target_col, fillna_by='median', val_min=None, val_max=None, add_flg=False):
print('fillna_numeric('{}', fillna_by={})'.format(fillna_target_col, fillna_by), '+flg' if add_flg else '')
 global df_train, df_test, cols_flg
isnull_flgs_train = df_train[fillna_target_col].isnull()
isnull_flgs_test = df_test[fillna_target_col].isnull()
if(type(fillna_by)is int)or(type(fillna_by)is float):
df_train.loc[isnull_flgs_train, fillna_target_col] = fillna_by
df_test.loc[isnull_flgs_test, fillna_target_col] = fillna_by
if fillna_by == 'random':
fill_by_train = random.choices(df_train.loc[~isnull_flgs_train, fillna_target_col].tolist() , k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(df_test.loc[~isnull_flgs_test, fillna_target_col].tolist() , k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'minmax':
val_btwn_minmax_flgs = df_train[fillna_target_col].between(val_min, val_max)
 val_btwn_minmax = df_train.loc[val_btwn_minmax_flgs, fillna_target_col].tolist()
fill_by_train = random.choices(val_btwn_minmax, k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(val_btwn_minmax, k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'median':
df_train.loc[isnull_flgs_train, fillna_target_col] = df_train[fillna_target_col].median()
df_test.loc[isnull_flgs_test, fillna_target_col] = df_test[fillna_target_col].median()
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
update_cols(add_cols_flg=[col_isnull_flg])
 return<drop_column>
df_country["GDP($ per capita)"] = df_country["GDP($ per capita)"].astype(np.float32 )
def update_cols(add_cols_qualitative=[], add_cols_quantitative=[], add_cols_flg=[], add_cols_unnecessary=[], show_detail=False):
global df_train, df_test, cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary
if show_detail:
print(' - update_cols(add_cols_qualitative={}, add_cols_quantitative={}, add_cols_flg={}, add_cols_unnecessary={})'.format(add_cols_qualitative, add_cols_quantitative, add_cols_flg, add_cols_unnecessary))
if add_cols_qualitative != []:
cols_qualitative = sorted(list(set(cols_qualitative)| set(add_cols_qualitative)))
 if add_cols_quantitative != []:
cols_quantitative = sorted(list(set(cols_quantitative)| set(add_cols_quantitative)))
 if add_cols_flg != []:
cols_flg = sorted(list(set(cols_flg)| set(add_cols_flg)))
 if add_cols_unnecessary != []:
cols_unnecessary = sorted(list(set(cols_unnecessary)| set(add_cols_unnecessary)))
 cols_qualitative = [col for col in cols_qualitative if col not in cols_unnecessary]
cols_quantitative = [col for col in cols_quantitative if col not in cols_unnecessary]
cols_flg = [col for col in cols_flg if col not in cols_unnecessary]
if show_detail:
print(' - no use : {}'.format(cols_unnecessary))
print(' > category: {}'.format(cols_qualitative))
print(' > numeric : {}'.format(cols_quantitative))
print(' > flag : {}'.format(cols_flg))
return<compute_train_metric>
train = train.reset_index().merge(df_country, how="left", on="Country" ).set_index("Respondent")
 X_test = X_test.reset_index().merge(df_country, how="left", on="Country" ).set_index("Respondent" )
def tfidf_pca(extraction_target_col, n_dim_tfidf=500, n_dim_pca=10):
print('tfidf_pca('{}', n_dim_tfidf={}, n_dim_pca={})'.format(extraction_target_col, n_dim_tfidf, n_dim_pca))
global df_train, df_test
global tfidf_pca_title_train, tfidf_pca_title_test, tfidf_pca_emp_title_train, tfidf_pca_emp_title_test
extraction_begin = time.time()
if extraction_target_col == 'title':
if tfidf_pca_title_train is not None:
print(' - already done, loading cache.')
 df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_title_train.columns.tolist())
 else:
print(' - run TfidfVectorizer')
 tfidfv = TfidfVectorizer(max_features=n_dim_tfidf, use_idf=True, analyzer='word', ngram_range=(1, 2))
tfidfv.fit_transform(df_train[extraction_target_col])
 f_text_train = tfidfv.transform(df_train[extraction_target_col] ).todense()
f_text_test = tfidfv.transform(df_test[extraction_target_col] ).todense()
del tfidfv
print(' - run PCA')
 pca = PCA(n_components=n_dim_pca)
 pca.fit(f_text_train)
 f_text_train = pca.transform(f_text_train)
 f_text_test = pca.transform(f_text_test)
 del pca
cols_tfidf_pca = ['{}_tfidf_pca_{}'.format(extraction_target_col, i)for i in range(n_dim_pca)]
tfidf_pca_title_train = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_train)
 tfidf_pca_title_test = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_test)
 tfidf_pca_title_train = tfidf_pca_title_train.reset_index(drop=True)
 tfidf_pca_title_test = tfidf_pca_title_test.reset_index(drop=True)
 del f_text_train, f_text_test
df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_title_train.columns.tolist())
 else:
if tfidf_pca_emp_title_train is not None:
print(' - already done, loading cache.')
 df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_emp_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_emp_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_emp_title_train.columns.tolist())
 else:
print(' - run TfidfVectorizer')
 tfidfv = TfidfVectorizer(max_features=n_dim_tfidf, use_idf=True, analyzer='word', ngram_range=(1, 2))
tfidfv.fit_transform(df_train[extraction_target_col])
 f_text_train = tfidfv.transform(df_train[extraction_target_col] ).todense()
f_text_test = tfidfv.transform(df_test[extraction_target_col] ).todense()
del tfidfv
print(' - run PCA')
 pca = PCA(n_components=n_dim_pca)
 pca.fit(f_text_train)
 f_text_train = pca.transform(f_text_train)
 f_text_test = pca.transform(f_text_test)
 del pca
cols_tfidf_pca = ['{}_tfidf_pca_{}'.format(extraction_target_col, i)for i in range(n_dim_pca)]
tfidf_pca_emp_title_train = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_train)
 tfidf_pca_emp_title_test = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_test)
 tfidf_pca_emp_title_train = tfidf_pca_emp_title_train.reset_index(drop=True)
 tfidf_pca_emp_title_test = tfidf_pca_emp_title_test.reset_index(drop=True)
 del f_text_train, f_text_test
df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_emp_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_emp_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_emp_title_train.columns.tolist())
 extraction_elapsed_time = time.time() -extraction_begin
extraction_elapsed_sec = int(extraction_elapsed_time%60)
 extraction_elapsed_min = int(( extraction_elapsed_time-extraction_elapsed_sec)/60)
 print(' > elapsed time: {}:{}'.format(extraction_elapsed_min, str(extraction_elapsed_sec ).rjust(2, '0')))
 return<categorify>
X_train = train.drop(columns="ConvertedSalary")
 y_train = np.log1p(train.ConvertedSalary )
def year2decade(convert_target_col, remove_original=True):
print('year2decade({}, remove_original={})'.format(convert_target_col, remove_original))
global df_train, df_test, cols_qualitative, cols_quantitative
col_decade_year = '{}_decade_year'.format(convert_target_col)
 df_train[col_decade_year] = df_train[convert_target_col].dt.year.astype(int)
 df_train[col_decade_year] -= df_train[col_decade_year] % 10
df_test[col_decade_year] = df_test[convert_target_col].dt.year.astype(int)
 df_test[col_decade_year] -= df_test[col_decade_year] % 10
if remove_original:
update_cols(add_cols_quantitative=[col_decade_year], add_cols_unnecessary=[convert_target_col])
 else:
update_cols(add_cols_quantitative=[col_decade_year])
 return<compute_train_metric>
split_cols = [
"DevType",
"CommunicationTools",
"FrameworkWorkedWith",
"AdsActions",
"ErgonomicDevices",
"Gender",
"SexualOrientation",
"RaceEthnicity"
]
def adversarial_validation_lgb(rate_sampling=1.0, n_folds=3, plot_feature_importance=True, ignore_target_enc=True, ignore_txt=True):
print('adversarial_validation_lgb(rate_sampling={})'.format(rate_sampling))
global df_train, df_test
cols_to_use = get_cols_to_use()
if ignore_target_enc:
cols_to_use = [col for col in cols_to_use if 'TARGET' not in col]
if ignore_txt:
cols_to_use = [col for col in cols_to_use if 'TXT' not in col]
df_all = pd.concat([df_train[cols_to_use], df_test[cols_to_use]])
 df_all['test'] = np.concatenate([np.zeros(len(df_train)) , np.ones(len(df_test)) ])
 X_all = df_all.drop(columns=['test'] ).values
y_all = df_all['test'].values
df_subset = df_all.sample(n=int(len(df_all)*rate_sampling))
X_subset = df_subset.drop(columns=['test'] ).values
y_subset = df_subset['test'].values
scores = []
f_importances = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_subset, y_subset)) :
fold_begin = time.time()
_X_train, _y_train = X_subset[train_ix], y_subset[train_ix]
_X_val, _y_val = X_subset[val_ix], y_subset[val_ix]
lgb_train = lgb.Dataset(_X_train, _y_train)
 lgb_eval = lgb.Dataset(_X_val, _y_val, reference=lgb_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval, verbose_eval=False)
 _y_pred = model.predict(_X_val, num_iteration=model.best_iteration)
 _score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 print(' > Average CV Score is {:.5f}'.format(np.mean(scores)))
 lgb_train = lgb.Dataset(X_all, y_all)
 model = lgb.train(lgbm_params, lgb_train)
 df_all['test-like'] = model.predict(X_all, num_iteration=model.best_iteration)
 df_train['test-like'] = df_all.loc[df_all['test'] == 0, 'test-like']
update_cols(add_cols_unnecessary=['test-like'])
 del df_all
if plot_feature_importance:
f_importance = model.feature_importance(importance_type='gain')
 df_importance = pd.DataFrame(f_importance, index=cols_to_use, columns=['importance'])
 df_importance = df_importance.sort_values('importance')
 plt.figure()
df_importance.plot.barh(figsize=(8, 20), title='Feature Importances of Adversarial Validation')
 return<categorify>
def flatten(l):
for el in l:
if isinstance(el, collections.abc.Iterable)and not isinstance(el,(str, bytes)) :
yield from flatten(el)
 else:
yield el
def count_encoding(encoding_target_cols, remove_original=True):
print('count_encoding({}, remove_original={})'.format(encoding_target_cols, remove_original))
global df_train, df_test
for col in encoding_target_cols:
col_encoded = 'COUNT:{}'.format(col)
 freq = df_train[col].value_counts()
df_train[col_encoded] = df_train[col].map(freq)
 df_train.loc[df_train[col].isnull() , col_encoded] = len(df_train[col].isnull())
 df_test[col_encoded] = df_test[col].map(freq)
 df_test.loc[df_test[col].isnull() , col_encoded] = len(df_test[col].isnull())
 if remove_original:
update_cols(add_cols_quantitative=[col_encoded], add_cols_unnecessary=[col])
 else:
update_cols(add_cols_quantitative=[col_encoded])
 return<categorify>
def split_text(df):
for col in split_cols:
categories = list(set(flatten(X_train[col].str.split(";" ).tolist())))
 categories = [i for i in categories if str(i)!= 'nan']
for category in categories:
df[f"{col}_{category}"] = df[col].str.contains(category ).astype(np.float32)
 df[f"count_{col}"] = df[col].str.count(";")
 return df
def target_encoding(encoding_target_cols, remove_original=True):
print('target_encoding({}, remove_original={})'.format(encoding_target_cols, remove_original))
global df_train, df_test
X_temp = df_train[encoding_target_cols+[COL_TARGET]]
y_train = df_train[COL_TARGET]
for col in encoding_target_cols:
col_encoded = 'TARGET:{}'.format(col)
 target_mean_all = X_temp.groupby(col)[COL_TARGET].mean()
df_test[col_encoded] = df_test[col].map(target_mean_all)
 skf = StratifiedKFold(n_splits=5, random_state=81, shuffle=True)
 encoded_train = pd.Series(np.zeros(len(df_train)) , index=df_train.index)
 for train_ix, val_ix in skf.split(X_temp, y_train):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
target_mean = X_train_.groupby([col])[COL_TARGET].mean()
encoded_train.iloc[val_ix] = X_val[col].map(target_mean)
 df_train[col_encoded] = encoded_train
if remove_original:
update_cols(add_cols_quantitative=[col_encoded], add_cols_unnecessary=[col])
 else:
update_cols(add_cols_quantitative=[col_encoded])
 return<categorify>
X_train = split_text(X_train)
 X_test = split_text(X_test )
def onehot_encoding(encoding_target_cols):
print('onehot_encoding({})'.format(encoding_target_cols))
global df_train, df_test
for col in encoding_target_cols:
ohe = ce.OneHotEncoder(cols=[col], handle_unknown='ignore', use_cat_names=True)
 ohe.fit(df_train[col])
 df_ohe_train = ohe.transform(df_train[col])
 df_ohe_test = ohe.transform(df_test[col])
 df_train = pd.concat([df_train, df_ohe_train], axis=1)
 df_test = pd.concat([df_test, df_ohe_test], axis=1)
 update_cols(add_cols_quantitative=df_ohe_train.columns.tolist() , add_cols_unnecessary=[col])
 return<init_hyperparams>
X_train.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in X_train.columns]
X_test.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in X_test.columns]
def pertition_holdout(rate_holdout=0.2, use_test_like_records=True, use_recent_records=False):
print('pertition_holdout(rate_holdout={}, use_test_like_records={}, use_recent_records={})'.format(rate_holdout, use_test_like_records, use_recent_records))
global df_train, df_test, df_holdout, cols_qualitative, cols_quantitative
df_train = df_train.reset_index(drop=True)
 n_holdout = len(df_train)*rate_holdout
true_rate = len(df_train[df_train[COL_TARGET] == 1])/len(df_train)
 holdout_ix = None
if use_test_like_records:
df_tmp = df_train[['test-like', COL_TARGET]].copy()
df_tmp = df_tmp.sort_values('test-like', ascending=False)
 ix_true = df_tmp[df_tmp[COL_TARGET] == 1][:int(n_holdout*true_rate)].index.tolist()
ix_false = df_tmp[df_tmp[COL_TARGET] == 0][:int(n_holdout*(1-true_rate)) ].index.tolist()
holdout_ix = ix_true + ix_false
del df_tmp
elif use_recent_records:
holdout_ix = df_train[df_train['issue_d'] >= dt.datetime(2015, 1, 1)].index.tolist()
else:
ix_true = df_train[df_train[COL_TARGET] == 1].sample(n=int(n_holdout*true_rate)).index.tolist()
ix_false = df_train[df_train[COL_TARGET] == 0].sample(n=int(n_holdout*(1-true_rate)) ).index.tolist()
holdout_ix = ix_true + ix_false
df_holdout = df_train.iloc[holdout_ix].copy()
df_holdout = df_holdout.reset_index(drop=True)
 df_train = df_train.drop(index=holdout_ix)
 df_train = df_train.reset_index(drop=True)
 return<prepare_x_and_y>
class BaseTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
return self
def get_feature_names(self):
pass
class KFoldTargetEncoder(BaseTransformer):

 def __init__(self, cols=None,
n_splits=5, random_state=24, shuffle=True, **kwargs):
super().__init__()
if cols is None:
cols = []
self.cols = cols
self.enc = ce.TargetEncoder(cols=cols, return_df=False, **kwargs)
 self.n_splits = n_splits
self.random_state = random_state
self.shuffle = shuffle
self.kwargs = kwargs
self.train_idx = None
self.y = None
def fit(self, X, y=None):
self.enc.fit(X[self.cols], y)
 self.train_idx = X.index
self.y = y
return self
def transform(self, X):
if self.train_idx is X.index:
df = pd.DataFrame(index=X.index, columns=self.cols, dtype=float)
 skf = KFold(n_splits=self.n_splits, random_state=self.random_state, shuffle=self.shuffle)
 for train_idx, val_idx in skf.split(X, self.y):
enc = ce.TargetEncoder(cols=self.cols, return_df=False, **self.kwargs)
 X_train, y_train = X.loc[X.index[train_idx], self.cols], self.y.iloc[train_idx]
X_val = X.loc[X.index[val_idx], self.cols]
enc.fit(X_train, y_train)
 df.loc[df.index[val_idx], self.cols] = enc.transform(X_val)
 return df
return self.enc.transform(X[self.cols])
 def get_feature_names(self):
return self.cols
def cross_validation_lgb(rate_sampling=1.0, n_folds=5, plot_feature_importance=True, print_topworst_features=True, n_top=10):
print('
cross_validation_lgb(rate_sampling={})'.format(rate_sampling))
global df_train, df_test
global score_cv_avr
cols_to_use = get_cols_to_use()
df_train_subset = df_train.sample(n=int(len(df_train)*rate_sampling))
X_train = df_train_subset[cols_to_use].values
y_train = df_train_subset[COL_TARGET].values
X_test = df_test[cols_to_use].values
del df_train_subset
scores = []
f_importances = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
fold_begin = time.time()
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
lgb_train = lgb.Dataset(_X_train, _y_train)
 lgb_eval = lgb.Dataset(_X_val, _y_val, reference=lgb_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval, verbose_eval=False)
 _y_pred = model.predict(_X_val, num_iteration=model.best_iteration)
 _score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 if plot_feature_importance or print_topworst_features:
f_importances.append(model.feature_importance(importance_type='gain'))
elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 score_cv_avr = np.mean(scores)
 print(' > Average CV Score is {:.5f}'.format(score_cv_avr))
if plot_feature_importance:
mean_of_feature_importances = np.array(f_importances ).mean(axis=0)
 df_mean_importance_of_cv = pd.DataFrame(mean_of_feature_importances, index=cols_to_use, columns=['importance'])
 df_mean_importance_of_cv = df_mean_importance_of_cv.sort_values('importance')
 plt.figure()
df_mean_importance_of_cv.plot.barh(figsize=(8, 20), title='Feature Importances of Cross Validation')
 del df_mean_importance_of_cv
if print_topworst_features:
common_features_top_n = None
common_features_worst_n = None
for i in range(n_folds):
sr_importance_tmp = pd.Series(f_importances[i], index=cols_to_use)
 sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 features_top_n = sr_importance_tmp[:n_top].index
features_worst_n = sr_importance_tmp[::-1][:n_top].index
if common_features_top_n is None:
common_features_top_n = features_top_n
common_features_worst_n = features_worst_n
else:
common_features_top_n = list(set(common_features_top_n)& set(features_top_n))
common_features_worst_n = list(set(common_features_worst_n)& set(features_worst_n))
print('
Print top and worst features(all)')
 print(' - top {} features commonly in each fold:'.format(n_top))
print(' > {}'.format(common_features_top_n))
print(' - worst {} features commonly in each fold:'.format(n_top))
print(' > {}'.format(common_features_worst_n))
common_features_top_n = None
common_features_worst_n = None
for i in range(n_folds):
sr_importance_tmp = pd.Series(f_importances[i], index=cols_to_use)
 cols_except_target_enc = [col for col in cols_to_use if 'TARGET' not in col]
sr_importance_tmp = sr_importance_tmp[cols_except_target_enc]
sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 features_top_n = sr_importance_tmp[:n_top].index
features_worst_n = sr_importance_tmp[::-1][:n_top].index
if common_features_top_n is None:
common_features_top_n = features_top_n
common_features_worst_n = features_worst_n
else:
common_features_top_n = list(set(common_features_top_n)& set(features_top_n))
common_features_worst_n = list(set(common_features_worst_n)& set(features_worst_n))
print('
Print top and worst features(ignore target enc)')
 print(' - top {} features commonly in each fold(ignore target enc):'.format(n_top))
print(' > {}'.format(common_features_top_n))
print(' - worst {} features commonly in each fold(ignore target enc):'.format(n_top))
print(' > {}'.format(common_features_worst_n))
return<prepare_x_and_y>
feature_union1 = FeatureUnion([ 
("te", KFoldTargetEncoder(
 cols=object_cols,
smoothing=.8 
)) ,
], n_jobs=None, verbose=True )
def cross_validation_xgb(rate_sampling=1.0, n_folds=5):
print('
cross_validation_xgb(rate_sampling={})'.format(rate_sampling))
global df_train, df_test
global score_cv_avr
cols_to_use = get_cols_to_use()
df_train_subset = df_train.sample(n=int(len(df_train)*rate_sampling))
X_train = df_train_subset[cols_to_use].values
y_train = df_train_subset[COL_TARGET].values
X_test = df_test[cols_to_use].values
del df_train_subset
scores = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
fold_begin = time.time()
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
D_train = xgb.DMatrix(_X_train, label=_y_train)
 D_val = xgb.DMatrix(_X_val, label=_y_val)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, verbose_eval=False)
 _y_pred = model.predict(D_val, ntree_limit=model.best_ntree_limit)
 _score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 score_cv_avr = np.mean(scores)
 print(' > Average CV Score is {:.5f}'.format(score_cv_avr))
return<prepare_x_and_y>
X_train1 = feature_union1.fit_transform(X_train, y_train)
 X_test1 = feature_union1.transform(X_test )
def cross_validation(clf_name, rate_sampling=1.0, n_folds=5):
print('
cross_validation(clf_name='{}', rate_sampling={})'.format(clf_name, rate_sampling))
global df_train, df_test
global score_cv_avr
cols_to_use = get_cols_to_use()
df_train_subset = df_train.sample(n=int(len(df_train)*rate_sampling))
X_train = df_train_subset[cols_to_use].values
y_train = df_train_subset[COL_TARGET].values
X_test = df_test[cols_to_use].values
del df_train_subset
scores = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
fold_begin = time.time()
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = Sequential()
clf.add(Dense(32, activation='relu'))
clf.add(Dense(16, activation='relu'))
clf.add(Dense(1, activation='sigmoid'))
clf.compile(loss='binary_crossentropy', optimizer='adam', metrics=[AUC() ])
 _y_pred = None
if clf_name == 'NeuralNetwork':
clf.fit(_X_train, _y_train, epochs=5)
 _y_pred = clf.predict(_X_val)[:, 0]
else:
clf.fit(_X_train, _y_train)
 _y_pred = clf.predict_proba(_X_val)[:,1]
_score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 score_cv_avr = np.mean(scores)
 print(' > Average CV Score is {:.5f}'.format(score_cv_avr))
return<prepare_x_and_y>
oe = ce.OrdinalEncoder(cols=object_cols )
def holdout_validation_lgb(plot_feature_importance=True, print_topworst_features=True, print_topworst_features_by_conditions=True):
print('
holdout_validation_lgb() ')
 global df_train, df_test, df_holdout
global score_holdout
validation_begin = time.time()
cols_to_use = get_cols_to_use()
X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_val = df_holdout[cols_to_use].values
y_val = df_holdout[COL_TARGET].values
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval, verbose_eval=False)
 y_pred = model.predict(X_val, num_iteration=model.best_iteration)
 score_holdout = roc_auc_score(y_val, y_pred)
 f_importance = model.feature_importance(importance_type='gain')
 elapsed_time = time.time() -validation_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - Score of Holdout is {:.5f}({}:{})'.format(score_holdout, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 if plot_feature_importance:
df_mean_importance_of_cv = pd.DataFrame(f_importance, index=cols_to_use, columns=['importance'])
 df_mean_importance_of_cv = df_mean_importance_of_cv.sort_values('importance')
 plt.figure()
df_mean_importance_of_cv.plot.barh(figsize=(8, 20), title='Feature Importances of Holdout Validation')
 del df_mean_importance_of_cv
if print_topworst_features:
sr_importance_tmp = pd.Series(f_importance, index=cols_to_use)
 sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 print('
Features sorted by importance(all): {}'.format(sr_importance_tmp.index.tolist()))
if print_topworst_features_by_conditions:
conds = ['INTRCTN', 'DIFF', 'DIV']
for cond in conds:
sr_importance_tmp = pd.Series(f_importance, index=cols_to_use)
 sr_importance_tmp = sr_importance_tmp[sr_importance_tmp.index.str.startswith(cond)]
sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 print('
Features sorted by importance(including {}): {}'.format(cond, sr_importance_tmp.index.tolist()))
return<prepare_x_and_y>
X_train = oe.fit_transform(X_train, y_train)
 X_test = oe.transform(X_test )
def holdout_validation_xgb() :
print('
holdout_validation_xgb() ')
 global df_train, df_test, df_holdout
global score_holdout
validation_begin = time.time()
cols_to_use = get_cols_to_use()
X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_val = df_holdout[cols_to_use].values
y_val = df_holdout[COL_TARGET].values
D_train = xgb.DMatrix(X_train, label=y_train)
 D_val = xgb.DMatrix(X_val, label=y_val)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, verbose_eval=False)
 y_pred = model.predict(D_val, ntree_limit=model.best_ntree_limit)
 score_holdout = roc_auc_score(y_val, y_pred)
 elapsed_time = time.time() -validation_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - Score of Holdout is {:.5f}({}:{})'.format(score_holdout, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 return<prepare_x_and_y>
X_train = np.hstack([X_train, X_train1])
 X_test = np.hstack([X_test, X_test1] )
def holdout_validation(clf_name, plot_feature_importance=False):
print('
holdout_validation(clf_name='{}')'.format(clf_name))
global df_train, df_test, df_holdout
global score_holdout
validation_begin = time.time()
cols_to_use = get_cols_to_use()
X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_val = df_holdout[cols_to_use].values
y_val = df_holdout[COL_TARGET].values
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = Sequential()
clf.add(Dense(32, activation='relu'))
clf.add(Dense(16, activation='relu'))
clf.add(Dense(1, activation='sigmoid'))
clf.compile(loss='binary_crossentropy', optimizer='adam', metrics=[AUC() ])
 y_pred = None
if clf_name == 'NeuralNetwork':
clf.fit(X_train, y_train, epochs=5)
 y_pred = clf.predict(X_val)[:, 0]
else:
clf.fit(X_train, y_train)
 y_pred = clf.predict_proba(X_val)[:,1]
score_holdout = roc_auc_score(y_val, y_pred)
 if plot_feature_importance and(clf_name == 'LogisticRegression'):
sr_importance = pd.Series(clf.coef_[0], index=cols_to_use)
 sr_importance = sr_importance.abs()
sr_importance = sr_importance.sort_values()
plt.figure()
sr_importance.plot.barh(figsize=(8, 20), title='Importance of Logistic Regression Features')
 print(' - ranking of important features: {}'.format(sr_importance.index.tolist() [::-1]))
elapsed_time = time.time() -validation_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - Score of Holdout is {:.5f}({}:{})'.format(score_holdout, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 return<prepare_x_and_y>
models = []
seeds = [114]
for seed in seeds:
params = {
"objective": "regression",
"learning_rate":.02,
"tree_learner": "data",
"device_type": "cpu",
"num_leaves": 128,
"seed": seed,
"colsample_bytree":.8,
"max_depth": 7,
"subsample":.9,
"metric": ["rmse"]
}
skf = KFold(n_splits=5, random_state=seed, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(skf.split(X_train, y_train)) :
X_train_, y_train_ = X_train[train_ix,], y_train.values[train_ix]
X_val_, y_val_ = X_train[test_ix,], y_train.values[test_ix]
train_data = lgbm.Dataset(
 data=X_train_,
label=y_train_,
feature_name=feature_names 
)
 val_data = lgbm.Dataset(
 data=X_val_,
label=y_val_,
reference=train_data,
feature_name=feature_names 
)
 model = lgbm.train(params=params, train_set=train_data, num_boost_round=9999, valid_sets=[val_data],
early_stopping_rounds=200, verbose_eval=100)
 models.append(model)
 y_pred_ = model.predict(X_val_)
 score = mean_squared_error(y_val_, y_pred_)
 print('CV Score of Fold_%d is %f' %(i, score))
def modeling_and_prediction_lgb(cross_validation_ensemble=False, n_folds=5):
print('modeling_and_prediction_lgb(cross_validation_ensemble={})'.format(cross_validation_ensemble))
global df_train, df_test, df_holdout
global y_pred_final
cols_to_use = get_cols_to_use()
df_train = pd.concat([df_train, df_holdout])
 X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_test = df_test[cols_to_use].values
if cross_validation_ensemble:
y_preds_sum = [0 for i in range(len(X_test)) ]
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
lgb_train = lgb.Dataset(_X_train, _y_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_train, verbose_eval=False)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum,
model.predict(X_test, num_iteration=model.best_iteration)) ]
y_pred_final = [pr/n_folds for pr in y_preds_sum]
else:
lgb_train = lgb.Dataset(X_train, y_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_train, verbose_eval=False)
 y_pred_final = model.predict(X_test, num_iteration=model.best_iteration)
 return<prepare_x_and_y>
joblib.dump(models, model_dir / f"models-lgbm-{VERSION}.joblib" )
def modeling_and_prediction_xgb(cross_validation_ensemble=False, n_folds=5):
print('modeling_and_prediction_xgb(cross_validation_ensemble={})'.format(cross_validation_ensemble))
global df_train, df_test, df_holdout
global y_pred_final
cols_to_use = get_cols_to_use()
df_train = pd.concat([df_train, df_holdout])
 X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_test = df_test[cols_to_use].values
D_test = xgb.DMatrix(X_test)
 if cross_validation_ensemble:
y_preds_sum = [0 for i in range(len(X_test)) ]
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
D_train = xgb.DMatrix(_X_train, label=_y_train)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, verbose_eval=False)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum,
model.predict(D_test, ntree_limit=model.best_ntree_limit)) ]
y_pred_final = y_preds_sum / n_folds
else:
D_train = xgb.DMatrix(X_train, label=y_train)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, valid_sets=D_train, verbose_eval=False)
 y_pred_final = model.predict(D_test, ntree_limit=model.best_ntree_limit)
 return<prepare_x_and_y>
for i, model in enumerate(models):
if i == 0:
y_preds = model.predict(X_test)
 else:
y_preds = np.vstack(( y_preds, model.predict(X_test)) )
def modeling_and_prediction(clf_name, cross_validation_ensemble=False, n_folds=5):
print('modeling_and_prediction(clf_name='{}', cross_validation_ensemble={})'.format(clf_name, cross_validation_ensemble))
global df_train, df_test, df_holdout
global y_pred_final
cols_to_use = get_cols_to_use()
df_train = pd.concat([df_train, df_holdout])
 X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_test = df_test[cols_to_use].values
if cross_validation_ensemble:
y_preds_sum = [0 for i in range(len(X_test)) ]
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = Sequential()
clf.add(Dense(32, activation='relu'))
clf.add(Dense(16, activation='relu'))
clf.add(Dense(1, activation='sigmoid'))
clf.compile(loss='binary_crossentropy', optimizer='adam', metrics=[AUC() ])
 _y_pred = None
if clf_name == 'NeuralNetwork':
clf.fit(_X_train, _y_train, epochs=5)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum, clf.predict(X_test)[:, 0])]
else:
clf.fit(_X_train, _y_train)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum, clf.predict_proba(X_test)[:,1])]
y_pred_final = [pr/n_folds for pr in y_preds_sum]
else:
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = MLPClassifier()
clf.fit(X_train, y_train)
 y_pred_final = clf.predict_proba(X_test)[:,1]
return 
<feature_engineering>
y_preds = np.expm1(y_preds )
cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary = [], [], [], []
print('
')
 notebook_begin = time.time()
load_merged_df(cache=True)
 COL_ID = 'ID'
COL_TARGET = 'loan_condition'
print('
')
 print('Adding n_nan')
 df_train['n_nan'] = df_train.isnull().sum(axis=1)
 df_test['n_nan'] = df_test.isnull().sum(axis=1)
 object2date('issue_d')
 object2date('earliest_cr_line')
 extract_recent_data(oldest_year=2014)
 cols_unnecessary.append('issue_d')
 ordinal2int('grade')
 ordinal2int('sub_grade')
 emplength2int()
object2str('title')
 object2str('emp_title')
 preprocess_text('title')
 preprocess_words_of_text('title')
 zipcode2str()
print('
')
 load_cols_by_dtype()
print('
')
 fillna_text('title', add_flg=False)
 fillna_text('emp_title', add_flg=False)
 fillna_numeric('dti', fillna_by=999.0)
 fillna_numeric('mths_since_last_delinq', fillna_by='minmax', val_min=0, val_max=85, add_flg=True)
 fillna_numeric('mths_since_last_record', fillna_by='random', add_flg=True)
 fillna_numeric('revol_util', fillna_by='minmax', val_min=0, val_max=100, add_flg=True)
 fillna_numeric('mths_since_last_major_derog', fillna_by='minmax', val_min=0, val_max=90, add_flg=True)
 fillna_numeric('inq_last_6mths', fillna_by='median')
 fillna_numeric('EXTER:TotalWages', fillna_by='random', add_flg=True)
 print('
')
 log_transformation('loan_amnt', remove_outlier=False, add_flg=True)
 log_transformation('installment', remove_outlier=False, add_flg=True)
 log_transformation('annual_inc', remove_outlier=True, add_flg=True)
 log_transformation('revol_bal', remove_outlier=True, add_flg=True)
 log_transformation('tot_coll_amt', remove_outlier=True, add_flg=True)
 log_transformation('tot_cur_bal', remove_outlier=True, add_flg=True)
 log_transformation('EXTER:TotalWages', remove_outlier=False)
 print('
')
 tfidf_pca('emp_title', n_dim_tfidf=500, n_dim_pca=10)
 update_cols(add_cols_unnecessary=['title', 'emp_title'])
 year2decade('earliest_cr_line', remove_original=True)
 ordinal2int('earliest_cr_line_decade_year')
 print('
')
 update_cols(add_cols_unnecessary=['zip_code'])
 onehot_encoding(cols_qualitative)
 print('
')
 print('Scaling with Standard Scaler')
 for col in cols_quantitative:
scaler = StandardScaler()
scaler.fit(df_train[[col]])
 df_train[col] = scaler.transform(df_train[[col]])
 df_test[col] = scaler.transform(df_test[[col]])
 print('Copying Dataframe to df_linear')
 cols_to_use = get_cols_to_use()
df_train_linear = df_train[cols_to_use].copy()
df_test_linear = df_test[cols_to_use].copy()
print('number of features: {}'.format(len(get_cols_to_use())))
 print('
')
 print('number of features: {}'.format(len(get_cols_to_use())))
 pertition_holdout(rate_holdout=0.2, use_test_like_records=False, use_recent_records=True)
 cross_validation(clf_name='LogisticRegression', rate_sampling=1.0)
 print('
')
 holdout_validation(clf_name='LogisticRegression', plot_feature_importance=True)
 print('
')
 modeling_and_prediction(clf_name='LogisticRegression', cross_validation_ensemble=True, n_folds=5)
 print('Appending model predictions to y_preds_list')
 y_preds_list_for_ensemble.append(y_pred_final)
 cvscore_list_for_ensemble.append(score_cv_avr)
 holdoutscore_list_for_ensemble.append(score_holdout)
 print('
')
 print('number of features: {}'.format(len(get_cols_to_use())))
 print('Print scores')
 print(' > Avr of CV: {:.5f}'.format(score_cv_avr))
print(' > Holdout : {:.5f}'.format(score_holdout))
print('Calculating total elapsed time')
 elapsed_time = time.time() -notebook_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' > total elapsed time...{}:{}'.format(elapsed_min, str(elapsed_sec ).rjust(2, '0')))
<feature_engineering>
y_pred = y_preds.mean(axis=0 )
cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary = [], [], [], []
print('
')
 notebook_begin = time.time()
load_merged_df(cache=True)
 COL_ID = 'ID'
COL_TARGET = 'loan_condition'
print('
')
 print('Adding n_nan')
 df_train['n_nan'] = df_train.isnull().sum(axis=1)
 df_test['n_nan'] = df_test.isnull().sum(axis=1)
 object2date('issue_d')
 object2date('earliest_cr_line')
 extract_recent_data(oldest_year=2014)
 cols_unnecessary.append('issue_d')
 ordinal2int('grade')
 ordinal2int('sub_grade')
 emplength2int()
object2str('title')
 object2str('emp_title')
 preprocess_text('title')
 preprocess_words_of_text('title')
 zipcode2str()
print('
')
 load_cols_by_dtype()
print('
')
 log_transformation('loan_amnt', remove_outlier=False)
 log_transformation('installment', remove_outlier=False)
 log_transformation('annual_inc', remove_outlier=True, add_flg=True)
 log_transformation('revol_bal', remove_outlier=True, add_flg=True)
 log_transformation('tot_coll_amt', remove_outlier=True, add_flg=True)
 log_transformation('tot_cur_bal', remove_outlier=True, add_flg=True)
 log_transformation('EXTER:TotalWages', remove_outlier=False)
 print('
')
 fillna_text('title', add_flg=False)
 fillna_text('emp_title', add_flg=False)
 fillna_numeric('dti', fillna_by=-999)
 fillna_numeric('mths_since_last_delinq', fillna_by=-999)
 fillna_numeric('mths_since_last_record', fillna_by=-999)
 fillna_numeric('revol_util', fillna_by=-999)
 fillna_numeric('mths_since_last_major_derog', fillna_by=-999)
 fillna_numeric('inq_last_6mths', fillna_by='median')
 fillna_numeric('EXTER:TotalWages', fillna_by=-999)
 print('
')
 tfidf_pca_kmeans_cluster('emp_title', n_dim_tfidf=500, n_dim_pca=-1, n_cluster=100)
 tfidf_pca_logreg_prediction('emp_title', n_dim_tfidf=500, n_dim_pca=-1)
 update_cols(add_cols_unnecessary=['emp_title'])
 year2decade('earliest_cr_line', remove_original=True)
 ordinal2int('earliest_cr_line_decade_year')
 record_clustering_kmeans(n_cluster=100)
 print('
')
 cols_qualitative_all = cols_qualitative.copy()
count_encoding(cols_qualitative_all, remove_original=True)
 count_encoding(cols_qualitative_all, remove_original=False)
 target_encoding(cols_qualitative_all, remove_original=True)
 del cols_qualitative_all
cols_quantitative_few_unique = ['acc_now_delinq', 'collections_12_mths_ex_med',
'earliest_cr_line_decade_year', 'emp_length', 'grade',
'inq_last_6mths', 'n_nan']
target_encoding(cols_quantitative_few_unique, remove_original=False)
 for col in cols_quantitative:
if 'TARGET' in col:
df_train[col] = df_train[col].fillna(df_train[col].median())
 df_test[col] = df_test[col].fillna(df_test[col].median())
 print('
')
 generate_interaction_terms(['sub_grade', 'grade', 'TXT:logreg_prediction_on_emp_title', 'loan_amnt', 'dti',
'COUNT:home_ownership', 'tot_cur_bal', 'open_acc', 'installment', 'emp_length'],
n_conbinations=2)
 print(' > {} interaction features added'.format(len([col for col in cols_quantitative if(col not in cols_unnecessary)and('INTRCTN' in col)])))
 generate_difference_terms(['sub_grade', 'grade', 'TXT:logreg_prediction_on_emp_title', 'loan_amnt', 'dti',
'COUNT:home_ownership', 'tot_cur_bal', 'open_acc', 'installment', 'emp_length'])
 print(' > {} difference features added'.format(len([col for col in cols_quantitative if(col not in cols_unnecessary)and('DIFF' in col)])))
 generate_division_terms([['EXTER:State & Local Spending', 'annual_inc'], ['loan_amnt', 'annual_inc'],
['installment', 'annual_inc'], ['tot_coll_amt', 'annual_inc'], ['tot_cur_bal', 'annual_inc'],
['loan_amnt', 'revol_bal'], ['installment', 'revol_bal'], ['tot_coll_amt', 'revol_bal'],
['tot_cur_bal', 'revol_bal'], ['loan_amnt', 'tot_cur_bal'], ['installment', 'tot_cur_bal'],
['tot_coll_amt', 'tot_cur_bal']])
 print(' > {} division features added'.format(len([col for col in cols_quantitative if(col not in cols_unnecessary)and('DIV' in col)])))
 cols_unnecessary_tmp = ['INTRCTN:COUNT:home_ownership(times)emp_length', 'INTRCTN:emp_length(times)grade',
'DIFF:COUNT:home_ownership(minus)emp_length', 'DIFF:dti(minus)loan_amnt',
'INTRCTN:COUNT:home_ownership(times)grade',
'DIFF:TXT:logreg_prediction_on_emp_title(minus)dti', 'DIFF:grade(minus)open_acc',
'DIFF:COUNT:home_ownership(minus)grade', 'DIFF:emp_length(minus)grade',
'DIFF:grade(minus)sub_grade', 'INTRCTN:grade(times)sub_grade']
cols_unnecessary_tmp += [col for col in cols_quantitative if 'DIFF' in col]
update_cols(add_cols_unnecessary=cols_unnecessary_tmp)
 print('
')
 pertition_holdout(rate_holdout=0.2, use_test_like_records=False, use_recent_records=True)
 cross_validation(clf_name='HistGradientBoosting', rate_sampling=1.0, n_folds=5)
 print('
')
 holdout_validation(clf_name='HistGradientBoosting')
 print('
')
 modeling_and_prediction(clf_name='HistGradientBoosting', cross_validation_ensemble=True, n_folds=5)
 print('Appending model predictions to y_preds_list')
 y_preds_list_for_ensemble.append(y_pred_final)
 cvscore_list_for_ensemble.append(score_cv_avr)
 holdoutscore_list_for_ensemble.append(score_holdout)
 print('
')
 print('lgbm_params: {}'.format(lgbm_params))
print('number of features: {}'.format(len(get_cols_to_use())))
 print('Print scores')
 print(' > Avr of CV: {:.5f}'.format(score_cv_avr))
print(' > Holdout : {:.5f}'.format(score_holdout))
print('Calculating total elapsed time')
 elapsed_time = time.time() -notebook_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' > total elapsed time...{}:{}'.format(elapsed_min, str(elapsed_sec ).rjust(2, '0')))
<feature_engineering>
submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission.ConvertedSalary = y_pred
submission.to_csv(model_dir / f'submission-{VERSION}.csv' )
<create_dataframe><EOS> 
feature_importances["mean"] = feature_importances.mean(axis=1 )