Kaggle competition: Home Credit Default Risk
automl = TabularUtilizedAutoML(task = task,
verbose=2,
timeout = TIMEOUT,
general_params = {'nested_cv': False, 'use_algos': [['linear_l2', 'lgb', 'lgb_tuned', 'cb', 'cb_tuned']]},
reader_params = {'cv': N_FOLDS, 'random_state': RANDOM_STATE},
tuning_params = {'max_tuning_iter': 20},
lgb_params = {'default_params': {'num_threads': N_THREADS, }},
cb_params = {'default_params': {'thread_count': N_THREADS, }} 
)
 oof_pred = automl.fit_predict(train, roles = roles)
<compute_test_metric>
%matplotlib inline
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 100)
 pd.set_option('display.max_columns', 200 )
print(roc_auc_score(train[TARGET_NAME].values.ravel() , oof_pred.data.ravel()))<compute_test_metric>
app_train = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 app_test = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv' )
thres =.5
print(classification_report(train[TARGET_NAME].values.ravel() ,(oof_pred.data.ravel() > thres ).astype(int), digits=6))<predict_on_test>
print(app_train.isnull().sum())
 print("결측치 있는 컴럼 개수: ",sum(app_train.isnull().sum() !=0))
test_pred = automl.predict(test )<feature_engineering>
app_train['TARGET'].value_counts()
submission_df['target'] = test_pred.data.ravel()
submission_df.head()<save_to_csv>
columns = ['AMT_INCOME_TOTAL','AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE', 'DAYS_BIRTH', 'DAYS_EMPLOYED', 'DAYS_ID_PUBLISH',
'DAYS_REGISTRATION', 'DAYS_LAST_PHONE_CHANGE', 'CNT_FAM_MEMBERS', 'REGION_RATING_CLIENT', 'EXT_SOURCE_1',
'EXT_SOURCE_2', 'EXT_SOURCE_3', 'AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_WEEK',
'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR']
show_hist_by_target(app_train, columns )
submission_df.to_csv('submission.csv', index=False )<import_modules>
cond_1 =(app_train['TARGET'] == 1)
 cond_0 =(app_train['TARGET'] == 0)
 print('CODE_GENDER
')
 print(app_train['CODE_GENDER'].value_counts() /app_train.shape[0])
 print('
연체인 경우
',app_train[cond_1]['CODE_GENDER'].value_counts() /app_train[cond_1].shape[0])
 print('
연체가 아닌 경우
',app_train[cond_0]['CODE_GENDER'].value_counts() /app_train[cond_0].shape[0] )
import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_auc_score
from multiprocessing import cpu_count
from tqdm.notebook import tqdm
from cairosvg import svg2png
from PIL import Image
from io import BytesIO
import gc
import os
import sys<define_variables>
app_train['DAYS_EMPLOYED'].value_counts()
VERSION = 'V1E'
NUM_BOOST_ROUND = 5000
VERBOSE_EVAL = 10
METRICS = ['auc']
N_ROWS = 99271300
def get_index_np() :
return np.arange(N_ROWS )<load_pretrained>
app_train['DAYS_EMPLOYED'] = app_train['DAYS_EMPLOYED'].replace(365243, np.nan)
 app_train['DAYS_EMPLOYED'].value_counts(dropna=False )
FEATURES = np.load(f'/kaggle/input/riiid-training-and-prediction-using-a-state-data/train_features_{VERSION}.npz', allow_pickle=True )<define_variables>
app_train['CODE_GENDER'].value_counts()
given_features = [
'prior_question_elapsed_time',
]
deduced_features = [
'mean_user_accuracy',
'answered_correctly_user',
'answered_user',
'mean_content_accuracy',
'part',
'hmean_user_content_accuracy',
'attempt',
]
features = given_features + deduced_features
target = 'answered_correctly'
categorical_feature = ['part', 'tags', 'tags_label', 'prior_question_had_explanation']
categorical_feature_idxs = []
for v in categorical_feature:
try:
categorical_feature_idxs.append(features.index(v))
except:
pass<data_type_conversions>
apps = pd.concat([app_train, app_test])
 print(apps.shape )
def get_train_val_idxs(TRAIN_SIZE, VAL_SIZE):
train_idxs = []
val_idxs = []
NEW_USER_FRAC = 1/4
np.random.seed(42)
 df = pd.DataFrame(index=get_index_np())
 for col in ['user_id']:
df[col] = FEATURES[col]
df['index'] = df.index.values.astype(np.uint32)
 user_id_index = df.groupby('user_id')['index'].apply(np.array)
 for indices in user_id_index.sample(user_id_index.size, random_state=42):
if len(train_idxs)> TRAIN_SIZE:
break
if len(val_idxs)< VAL_SIZE:
if np.random.rand() < NEW_USER_FRAC:
val_idxs += list(indices)
 else:
offset = np.random.randint(0, indices.size)
 train_idxs += list(indices[:offset])
 val_idxs += list(indices[offset:])
 else:
train_idxs += list(indices)
 return train_idxs, val_idxs
train_idxs, val_idxs = get_train_val_idxs(int(50e6), 2.5e6)
 print(f'len train_idxs: {len(train_idxs)}, len validation_idxs: {len(val_idxs)}' )<prepare_x_and_y>
apps['APPS_EXT_SOURCE_MEAN'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1 )
def make_x_y(FEATURES, train_idxs, val_idxs):
X_train = np.ndarray(shape=(len(train_idxs), len(features)) , dtype=np.float32)
 X_val = np.ndarray(shape=(len(val_idxs), len(features)) , dtype=np.float32)
 for idx, feature in enumerate(tqdm(features)) :
X_train[:,idx] = FEATURES[feature][train_idxs].astype(np.float32)
 X_val[:,idx] = FEATURES[feature][val_idxs].astype(np.float32)
 y_train = FEATURES[target][train_idxs].astype(np.int8)
 y_val = FEATURES[target][val_idxs].astype(np.int8)
 return X_train, y_train, X_val, y_val
X_train, y_train, X_val, y_val = make_x_y(FEATURES, train_idxs, val_idxs )<create_dataframe>
apps['APPS_EXT_SOURCE_STD'] = apps['APPS_EXT_SOURCE_STD'].fillna(apps['APPS_EXT_SOURCE_STD'].mean() )
pd.DataFrame(X_train[:10], columns=features )<create_dataframe>
apps['APPS_ANNUITY_CREDIT_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_CREDIT']
apps['APPS_GOODS_CREDIT_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_CREDIT']
apps['APPS_CREDIT_GOODS_DIFF'] = apps['AMT_CREDIT'] - apps['AMT_GOODS_PRICE']
train_data = lgb.Dataset(
 data = X_train,
label = y_train,
categorical_feature = None, 
)
 val_data = lgb.Dataset(
 data = X_val,
label = y_val,
categorical_feature = None,
)<drop_column>
apps['APPS_ANNUITY_INCOME_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_INCOME_TOTAL']
apps['APPS_CREDIT_INCOME_RATIO'] = apps['AMT_CREDIT']/apps['AMT_INCOME_TOTAL']
apps['APPS_GOODS_INCOME_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_INCOME_TOTAL']
apps['APPS_CNT_FAM_INCOME_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['CNT_FAM_MEMBERS']
del X_train, y_train, X_val, y_val
gc.collect()<init_hyperparams>
apps['APPS_EMPLOYED_BIRTH_RATIO'] = apps['DAYS_EMPLOYED']/apps['DAYS_BIRTH']
apps['APPS_INCOME_EMPLOYED_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_EMPLOYED']
apps['APPS_INCOME_BIRTH_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_BIRTH']
apps['APPS_CAR_BIRTH_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_BIRTH']
apps['APPS_CAR_EMPLOYED_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_EMPLOYED']
lgbm_params = {
'objective': 'binary',
'metric': METRICS,
}<train_model>
object_columns = apps.dtypes[apps.dtypes=='object'].index.tolist()
for column in object_columns:
apps[column] = pd.factorize(apps[column])[0]
%%time
def train() :
evals_result = {}
model = lgb.train(
 params = lgbm_params,
train_set = train_data,
valid_sets = [val_data],
num_boost_round = NUM_BOOST_ROUND,
verbose_eval = VERBOSE_EVAL,
evals_result = evals_result,
early_stopping_rounds = 10,
categorical_feature = categorical_feature_idxs,
feature_name = features, 
)
 model.save_model(f'model_{VERSION}_{NUM_BOOST_ROUND}.lgb')
 return model, evals_result
model, evals_result = train()<save_to_csv>
ftr_app = apps_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 target_app = app_train['TARGET']
train_x, valid_x, train_y, valid_y = train_test_split(ftr_app, target_app, test_size=0.3, random_state=2020)
 train_x.shape, valid_x.shape
def show_feature_importances(model, importance_type, max_num_features=10**10):
feature_importances = pd.DataFrame()
feature_importances['feature'] = features
feature_importances['value'] = pd.DataFrame(model.feature_importance(importance_type))
feature_importances = feature_importances.sort_values(by='value', ascending=False)
 feature_importances.to_csv(f'feature_importances_{importance_type}.csv')
 feature_importances = feature_importances[:max_num_features]
plt.figure(figsize=(20, 8))
plt.xlim([0, feature_importances.value.max() *1.1])
 plt.title(f'Feature {importance_type}', fontsize=18);
sns.barplot(data=feature_importances, x='value', y='feature', palette='rocket');
for idx, v in enumerate(feature_importances.value):
plt.text(v, idx, " {:.2e}".format(v))
show_feature_importances(model, 'gain')
 show_feature_importances(model, 'split' )<drop_column>
clf = LGBMClassifier(
 n_jobs=-1,
n_estimators=1000,
learning_rate=0.02,
num_leaves=32,
subsample=0.8,
max_depth=12,
silent=-1,
verbose=-1
)
del train_data
gc.collect()<create_dataframe>
clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], eval_metric= 'auc', verbose= 100,
early_stopping_rounds= 100 )
def get_features_questions_df() :
features_df = pd.DataFrame(index=get_index_np())
 for col in tqdm(['content_id', 'part', 'tags', 'tags_label', 'mean_content_accuracy']):
features_df[col] = FEATURES[col]
features_questions_df = features_df.groupby('content_id')[[
'content_id',
'part',
'tags',
'tags_label',
'mean_content_accuracy',
]].first().reset_index(drop=True ).sort_values('content_id')
 return features_questions_df
features_questions_df = get_features_questions_df()
print(f'features_questions_df, rows: {features_questions_df.shape[0]}')
 display(features_questions_df.head() )<groupby>
preds = clf.predict_proba(apps_test.drop(['SK_ID_CURR', 'TARGET'], axis=1)) [:, 1 ]
def get_state() :
features_df = pd.DataFrame(index=get_index_np())
 for col in tqdm(['user_id', 'content_id', 'answered_correctly']):
features_df[col] = FEATURES[col]
mean_user_accuracy = features_df.groupby('user_id')['answered_correctly'].mean().values
answered_correctly_user = features_df.groupby('user_id')['answered_correctly'].sum().values
answered_user = features_df.groupby('user_id')['answered_correctly'].count().values
state = dict()
for user_id in features_df['user_id'].unique() :
state[user_id] = {}
total = len(state.keys())
 user_content = features_df.groupby('user_id')['content_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = features_df.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array ).values
user_attempts -= 1
for user_id, content, attempt in tqdm(zip(state.keys() , user_content, user_attempts),total=total):
state[user_id]['user_content_attempts'] = dict(zip(content, attempt))
del user_content, user_attempts
gc.collect()
for idx, user_id in enumerate(state.keys()):
state[user_id]['mean_user_accuracy'] = mean_user_accuracy[idx]
state[user_id]['answered_correctly_user'] = answered_correctly_user[idx]
state[user_id]['answered_user'] = answered_user[idx]
return state
state = get_state()
print('Example of the state for user 2746, attempt counting starts at 0 as the pandas cumcount function is used to create the attempt feature')
 display(state[2746] )<feature_engineering>
app_test['TARGET'] = preds
app_test[['SK_ID_CURR', 'TARGET']].to_csv('apps_baseline_02.csv', index=False )
def get_user_data(state, test_df):
attempt, mean_user_accuracy, answered_correctly_user, answered_user = [], [], [], []
for idx,(user_id, content_id)in test_df[['user_id', 'content_id']].iterrows() :
if user_id in state:
if content_id in state[user_id]['user_content_attempts']:
state[user_id]['user_content_attempts'][content_id] = min(4, state[user_id]['user_content_attempts'][content_id] + 1)
 else:
state[user_id]['user_content_attempts'][content_id] = 0
else:
dict_keys = ['mean_user_accuracy', 'answered_correctly_user', 'answered_user', 'user_content_attempts']
dict_default_vals = [0.680, 0, 0, dict(zip([content_id],[0])) ]
state[user_id] = dict(zip(dict_keys, dict_default_vals))
attempt.append(state[user_id]['user_content_attempts'][content_id])
 mean_user_accuracy.append(state[user_id]['mean_user_accuracy'])
 answered_correctly_user.append(state[user_id]['answered_correctly_user'])
 answered_user.append(state[user_id]['answered_user'])
 return attempt, mean_user_accuracy, answered_correctly_user, answered_user<feature_engineering>
def get_apps_processed(apps):
apps['APPS_EXT_SOURCE_MEAN'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps['APPS_EXT_SOURCE_STD'].fillna(apps['APPS_EXT_SOURCE_STD'].mean())
 apps['APPS_ANNUITY_CREDIT_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_CREDIT']
apps['APPS_GOODS_CREDIT_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_CREDIT']
apps['APPS_ANNUITY_INCOME_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_INCOME_TOTAL']
apps['APPS_CREDIT_INCOME_RATIO'] = apps['AMT_CREDIT']/apps['AMT_INCOME_TOTAL']
apps['APPS_GOODS_INCOME_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_INCOME_TOTAL']
apps['APPS_CNT_FAM_INCOME_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['CNT_FAM_MEMBERS']
apps['APPS_EMPLOYED_BIRTH_RATIO'] = apps['DAYS_EMPLOYED']/apps['DAYS_BIRTH']
apps['APPS_INCOME_EMPLOYED_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_EMPLOYED']
apps['APPS_INCOME_BIRTH_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_BIRTH']
apps['APPS_CAR_BIRTH_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_BIRTH']
apps['APPS_CAR_EMPLOYED_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_EMPLOYED']
return apps
def update_user_data(state, features_questions_df, prev_test_df):
for user_id, content_id, answered_correctly in prev_test_df[['user_id', 'content_id', 'answered_correctly']].values:
state[user_id]['answered_correctly_user'] += answered_correctly
state[user_id]['answered_user'] += 1
state[user_id]['mean_user_accuracy'] = state[user_id]['answered_correctly_user'] / state[user_id]['answered_user']<split>
prev = pd.read_csv('.. /input/home-credit-default-risk/previous_application.csv')
 print(prev.shape, apps.shape )
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
prev.groupby('SK_ID_CURR')['SK_ID_CURR'].count().mean()
prev_test_df = None
mean_attempt_acc_factor = FEATURES['mean_attempt_acc_factor']
for idx,(test_df, _)in tqdm(enumerate(iter_test)) :
if prev_test_df is not None:
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 update_user_data(state, features_questions_df, prev_test_df.loc[prev_test_df['content_type_id'] == 0])
 if idx is 1:
display(test_df)
 display(prev_test_df)
 attempt, mean_user_accuracy, answered_correctly_user, answered_user = get_user_data(state, test_df)
 test_df['attempt'] = attempt
test_df['mean_user_accuracy'] = mean_user_accuracy
test_df['answered_correctly_user'] = answered_correctly_user
test_df['answered_user'] = answered_user
test_df = features_questions_df.merge(test_df, how='right', on='content_id')
 test_df['prior_question_elapsed_time'].fillna(23916, inplace=True)
 test_df['hmean_user_content_accuracy'] = 2 *(
(test_df['mean_user_accuracy'] * test_df['mean_content_accuracy'])/ 
(test_df['mean_user_accuracy'] + test_df['mean_content_accuracy'])
)
 test_df['answered_correctly'] = model.predict(test_df[features])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 prev_test_df = test_df.copy()<load_from_csv>
app_prev_target = prev.merge(app_train[['SK_ID_CURR', 'TARGET']], on='SK_ID_CURR', how='left')
 app_prev_target.shape
submission = pd.read_csv('./submission.csv' )<set_options>
def get_prev_processed(prev):
prev['PREV_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']
prev['PREV_GOODS_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_GOODS_PRICE']
prev['PREV_CREDIT_APPL_RATIO'] = prev['AMT_CREDIT']/prev['AMT_APPLICATION']
prev['PREV_ANNUITY_APPL_RATIO'] = prev['AMT_ANNUITY']/prev['AMT_APPLICATION']
prev['PREV_GOODS_APPL_RATIO'] = prev['AMT_GOODS_PRICE']/prev['AMT_APPLICATION']
prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['PREV_DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']
all_pay = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']
prev['PREV_INTERESTS_RATE'] =(all_pay/prev['AMT_CREDIT'] - 1)/prev['CNT_PAYMENT']
return prev
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
def get_prev_amt_agg(prev):
agg_dict = {
'SK_ID_CURR':['count'],
'AMT_CREDIT':['mean', 'max', 'sum'],
'AMT_ANNUITY':['mean', 'max', 'sum'],
'AMT_APPLICATION':['mean', 'max', 'sum'],
'AMT_DOWN_PAYMENT':['mean', 'max', 'sum'],
'AMT_GOODS_PRICE':['mean', 'max', 'sum'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'PREV_CREDIT_DIFF':['mean', 'max', 'sum'],
'PREV_CREDIT_APPL_RATIO':['mean', 'max'],
'PREV_GOODS_DIFF':['mean', 'max', 'sum'],
'PREV_GOODS_APPL_RATIO':['mean', 'max'],
'PREV_DAYS_LAST_DUE_DIFF':['mean', 'max', 'sum'],
'PREV_INTERESTS_RATE':['mean', 'max']
}
prev_group = prev.groupby('SK_ID_CURR')
 prev_amt_agg = prev_group.agg(agg_dict)
 prev_amt_agg.columns = ["PREV_"+ "_".join(x ).upper() for x in prev_amt_agg.columns.ravel() ]
prev_amt_agg = prev_amt_agg.reset_index()
return prev_amt_agg
import numpy as np
import pandas as pd
from datetime import datetime
from collections import Counter
import json
import os
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout
from tensorflow.keras.preprocessing.image import ImageDataGenerator<define_variables>
def get_prev_refused_appr_agg(prev):
prev_refused_appr_group = prev[prev['NAME_CONTRACT_STATUS'].isin(['Approved', 'Refused'])].groupby([ 'SK_ID_CURR', 'NAME_CONTRACT_STATUS'])
 prev_refused_appr_agg = prev_refused_appr_group['SK_ID_CURR'].count().unstack()
prev_refused_appr_agg.columns = ['PREV_APPROVED_COUNT', 'PREV_REFUSED_COUNT' ]
prev_refused_appr_agg = prev_refused_appr_agg.fillna(0)
 prev_refused_appr_agg = prev_refused_appr_agg.reset_index()
return prev_refused_appr_agg
dim = 256
train_files = []
test_files = []
country_file = ''
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
path = os.path.join(dirname, filename)
 if 'train' in path:
train_files.append(path)
 elif 'test' in path:
test_files.append(path)
 elif 'json' in path:
country_file = path<feature_engineering>
def get_prev_agg(prev):
prev = get_prev_processed(prev)
 prev_amt_agg = get_prev_amt_agg(prev)
 prev_refused_appr_agg = get_prev_refused_appr_agg(prev)
 prev_agg = prev_amt_agg.merge(prev_refused_appr_agg, on='SK_ID_CURR', how='left')
 prev_agg['PREV_REFUSED_RATIO'] = prev_agg['PREV_REFUSED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg['PREV_APPROVED_RATIO'] = prev_agg['PREV_APPROVED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg = prev_agg.drop(['PREV_REFUSED_COUNT', 'PREV_APPROVED_COUNT'], axis=1)
 return prev_agg
with open(country_file)as json_file:
tmp = json.load(json_file)
 country = {}
for key in tmp.keys() :
country[int(key)] = tmp[key].split(',')[-1]<create_dataframe>
def get_apps_all_with_prev_agg(apps, prev):
apps_all = get_apps_processed(apps)
 prev_agg = get_prev_agg(prev)
 print('prev_agg shape:', prev_agg.shape)
 print('apps_all before merge shape:', apps_all.shape)
 apps_all = apps_all.merge(prev_agg, on='SK_ID_CURR', how='left')
 print('apps_all after merge with prev_agg shape:', apps_all.shape)
 return apps_all
train_set = []
for f in train_files:
idx = int(f.split('/')[6])
 train_set.append([f, country[idx]])
 train_set = pd.DataFrame(train_set, columns=['Image','Country'])
 test_set = []
for f in test_files:
test_set.append(f)
 test_set = pd.DataFrame(test_set, columns=['Image'] )<choose_model_class>
def get_apps_all_encoded(apps_all):
object_columns = apps_all.dtypes[apps_all.dtypes == 'object'].index.tolist()
for column in object_columns:
apps_all[column] = pd.factorize(apps_all[column])[0]
return apps_all
train_data_gen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True )<create_dataframe>
def get_apps_all_train_test(apps_all):
apps_all_train = apps_all[~apps_all['TARGET'].isnull() ]
apps_all_test = apps_all[apps_all['TARGET'].isnull() ]
apps_all_test = apps_all_test.drop('TARGET', axis=1)
 return apps_all_train, apps_all_test
train_generator = train_data_gen.flow_from_dataframe(
 dataframe = train_set,
directory="",
x_col="Image",
y_col="Country",
class_mode="categorical",
target_size=(dim,dim),
batch_size=32 )<count_values>
def train_apps_all(apps_all_train):
ftr_app = apps_all_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 target_app = apps_all_train['TARGET']
train_x, valid_x, train_y, valid_y = train_test_split(ftr_app, target_app, test_size=0.3, random_state=2020)
 print('train shape:', train_x.shape, 'valid shape:', valid_x.shape)
 clf = LGBMClassifier(
 n_jobs=-1,
n_estimators=1000,
learning_rate=0.02,
num_leaves=32,
subsample=0.8,
max_depth=12,
silent=-1,
verbose=-1 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], eval_metric= 'auc', verbose= 100,
early_stopping_rounds= 100)
 return clf
num_classes = len(Counter(train_generator.classes ).keys() )<choose_model_class>
def get_dataset() :
app_train = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 app_test = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv')
 apps = pd.concat([app_train, app_test])
 prev = pd.read_csv('.. /input/home-credit-default-risk/previous_application.csv')
 return apps, prev
classifier = Sequential()
classifier.add(Conv2D(filters = 32, kernel_size =(3,3), activation = 'relu', input_shape =(dim,dim,3)))
 classifier.add(MaxPool2D(pool_size =(2,2)))
 classifier.add(Conv2D(64,(3,3),activation = 'relu'))
classifier.add(Conv2D(64,(3,3),activation = 'relu'))
classifier.add(MaxPool2D(pool_size =(2,2)))
 classifier.add(Conv2D(128,(3,3),activation = 'relu'))
classifier.add(Conv2D(128,(3,3),activation = 'relu'))
classifier.add(MaxPool2D(pool_size =(2,2)))
 classifier.add(Flatten())
 classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = num_classes , activation = 'softmax'))
classifier.compile(
 optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy','accuracy'] 
)
 classifier.summary()<train_model>
apps, prev = get_dataset()
apps_all = get_apps_all_with_prev_agg(apps, prev)
 apps_all = get_apps_all_encoded(apps_all)
 apps_all_train, apps_all_test = get_apps_all_train_test(apps_all)
 clf = train_apps_all(apps_all_train )
classifier.fit_generator(train_generator, epochs = 100, steps_per_epoch = 70 )<find_best_params>
preds = clf.predict_proba(apps_all_test.drop(['SK_ID_CURR'], axis=1)) [:, 1 ]
apps_all_test['TARGET'] = preds
apps_all_test[['SK_ID_CURR', 'TARGET']].to_csv('prev_baseline_03.csv', index=False )
classes = train_generator.class_indices
print(classes )<predict_on_test>
bureau = pd.read_csv('.. /input/home-credit-default-risk/bureau.csv')
 bureau_bal = pd.read_csv('.. /input/home-credit-default-risk/bureau_balance.csv' )
Y_pred = []
for idx in range(test_set.shape[0]):
img = image.load_img(path=test_set['Image'][idx],target_size=(dim,dim,3))
img = image.img_to_array(img)
 test_img = img.reshape(( 1,dim,dim,3))
img_class = classifier.predict_classes(test_img)
 prediction = img_class[0]
Y_pred.append(prediction )<find_best_params>
bureau_app = bureau.merge(app_train[['SK_ID_CURR', 'TARGET']], left_on='SK_ID_CURR', right_on='SK_ID_CURR', how='inner')
 bureau_app.shape
prediction_classes = [ inverted_classes.get(item,item)for item in Y_pred ]
print(prediction_classes )<save_to_csv>
num_columns = bureau_app.dtypes[bureau_app.dtypes != 'object'].index.tolist()
num_columns = [column for column in num_columns if column not in['SK_ID_BUREAU', 'SK_ID_CURR', 'TARGET']]
num_columns
predictions = []
for idx in range(test_set.shape[0]):
predictions.append([test_set['Image'][idx].split('/')[6].split('.')[0],prediction_classes[idx]])
 predictions = pd.DataFrame(predictions, columns=['ID','Country'])
 predictions['ID'] = predictions['ID'].astype(int)
 predictions.sort_values(by=['ID'], inplace=True)
 predictions.to_csv(datetime.now().strftime("gallivanters_%Y%m%d_%H%M%S.csv"), index=False)
 predictions.head(20 )<load_from_csv>
object_columns = bureau.dtypes[bureau.dtypes=='object'].index.tolist()
object_columns
pd.read_csv?<load_from_csv>
show_category_by_target(bureau_app, object_columns )
fn = '.. /input/male-daan-schnell-mal-klassifizieren/train.csv'
df = pd.read_csv(fn)
 df.head(5 )<load_from_csv>
def get_bureau_processed(bureau):
bureau['BUREAU_ENDDATE_FACT_DIFF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
bureau['BUREAU_CREDIT_FACT_DIFF'] = bureau['DAYS_CREDIT'] - bureau['DAYS_ENDDATE_FACT']
bureau['BUREAU_CREDIT_ENDDATE_DIFF'] = bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE']
bureau['BUREAU_CREDIT_DEBT_RATIO']=bureau['AMT_CREDIT_SUM_DEBT']/bureau['AMT_CREDIT_SUM']
bureau['BUREAU_CREDIT_DEBT_DIFF'] = bureau['AMT_CREDIT_SUM_DEBT'] - bureau['AMT_CREDIT_SUM']
bureau['BUREAU_IS_DPD'] = bureau['CREDIT_DAY_OVERDUE'].apply(lambda x: 1 if x > 0 else 0)
 bureau['BUREAU_IS_DPD_OVER120'] = bureau['CREDIT_DAY_OVERDUE'].apply(lambda x: 1 if x >120 else 0)
 return bureau
df = pd.read_csv(fn,index_col='Id')
 df.head()<create_dataframe>
def get_bureau_day_amt_agg(bureau):
bureau_agg_dict = {
'SK_ID_BUREAU':['count'],
'DAYS_CREDIT':['min', 'max', 'mean'],
'CREDIT_DAY_OVERDUE':['min', 'max', 'mean'],
'DAYS_CREDIT_ENDDATE':['min', 'max', 'mean'],
'DAYS_ENDDATE_FACT':['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean', 'sum'],
'BUREAU_ENDDATE_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_ENDDATE_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_RATIO':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_DIFF':['min', 'max', 'mean'],
'BUREAU_IS_DPD':['mean', 'sum'],
'BUREAU_IS_DPD_OVER120':['mean', 'sum']
}
bureau_grp = bureau.groupby('SK_ID_CURR')
 bureau_day_amt_agg = bureau_grp.agg(bureau_agg_dict)
 bureau_day_amt_agg.columns = ['BUREAU_'+('_' ).join(column ).upper() for column in bureau_day_amt_agg.columns.ravel() ]
bureau_day_amt_agg = bureau_day_amt_agg.reset_index()
print('bureau_day_amt_agg shape:', bureau_day_amt_agg.shape)
 return bureau_day_amt_agg
data = df.values
data<prepare_x_and_y>
def get_bureau_active_agg(bureau):
cond_active = bureau['CREDIT_ACTIVE'] == 'Active'
bureau_active_grp = bureau[cond_active].groupby(['SK_ID_CURR'])
 bureau_agg_dict = {
'SK_ID_BUREAU':['count'],
'DAYS_CREDIT':['min', 'max', 'mean'],
'CREDIT_DAY_OVERDUE':['min', 'max', 'mean'],
'DAYS_CREDIT_ENDDATE':['min', 'max', 'mean'],
'DAYS_ENDDATE_FACT':['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean', 'sum'],
'BUREAU_ENDDATE_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_ENDDATE_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_RATIO':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_DIFF':['min', 'max', 'mean'],
'BUREAU_IS_DPD':['mean', 'sum'],
'BUREAU_IS_DPD_OVER120':['mean', 'sum']
}
bureau_active_agg = bureau_active_grp.agg(bureau_agg_dict)
 bureau_active_agg.columns = ['BUREAU_ACT_'+('_' ).join(column ).upper() for column in bureau_active_agg.columns.ravel() ]
bureau_active_agg = bureau_active_agg.reset_index()
print('bureau_active_agg shape:', bureau_active_agg.shape)
 return bureau_active_agg
nTrain=5000
Xtrain = data[:nTrain,:-1]
ytrain = data[:nTrain,-1]
Xtest = data[nTrain:,:-1]
ytest = data[nTrain:,-1]
Xtrain.shape,ytrain.shape,Xtest.shape,ytest.shape<data_type_conversions>
def get_bureau_bal_agg(bureau, bureau_bal):
bureau_bal = bureau_bal.merge(bureau[['SK_ID_CURR', 'SK_ID_BUREAU']], on='SK_ID_BUREAU', how='left')
 bureau_bal['BUREAU_BAL_IS_DPD'] = bureau_bal['STATUS'].apply(lambda x: 1 if x in['1','2','3','4','5'] else 0)
 bureau_bal['BUREAU_BAL_IS_DPD_OVER120'] = bureau_bal['STATUS'].apply(lambda x: 1 if x =='5' else 0)
 bureau_bal_grp = bureau_bal.groupby('SK_ID_CURR')
 bureau_bal_agg_dict = {
'SK_ID_CURR':['count'],
'MONTHS_BALANCE':['min', 'max', 'mean'],
'BUREAU_BAL_IS_DPD':['mean', 'sum'],
'BUREAU_BAL_IS_DPD_OVER120':['mean', 'sum']
}
bureau_bal_agg = bureau_bal_grp.agg(bureau_bal_agg_dict)
 bureau_bal_agg.columns = [ 'BUREAU_BAL_'+('_' ).join(column ).upper() for column in bureau_bal_agg.columns.ravel() ]
bureau_bal_agg = bureau_bal_agg.reset_index()
print('bureau_bal_agg shape:', bureau_bal_agg.shape)
 return bureau_bal_agg
ytrain = ytrain.astype('int')
 ytest = ytest.astype('int')
 ytrain, ytrain.dtype<choose_model_class>
def get_bureau_agg(bureau, bureau_bal):
bureau = get_bureau_processed(bureau)
 bureau_day_amt_agg = get_bureau_day_amt_agg(bureau)
 bureau_active_agg = get_bureau_active_agg(bureau)
 bureau_bal_agg = get_bureau_bal_agg(bureau, bureau_bal)
 bureau_agg = bureau_day_amt_agg.merge(bureau_active_agg, on='SK_ID_CURR', how='left')
 bureau_agg = bureau_agg.merge(bureau_bal_agg, on='SK_ID_CURR', how='left')
 print('bureau_agg shape:', bureau_agg.shape)
 return bureau_agg
k=7<define_variables>
def get_apps_all_with_prev_bureau_agg(apps, prev, bureau, bureau_bal):
apps_all = get_apps_processed(apps)
 prev_agg = get_prev_agg(prev)
 bureau_agg = get_bureau_agg(bureau, bureau_bal)
 print('prev_agg shape:', prev_agg.shape)
 print('bueau_agg shape:', bureau_agg.shape)
 print('apps_all before merge shape:', apps_all.shape)
 apps_all = apps_all.merge(prev_agg, on='SK_ID_CURR', how='left')
 apps_all = apps_all.merge(bureau_agg, on='SK_ID_CURR', how='left')
 print('apps_all after merge with prev_agg, bureau_agg shape:', apps_all.shape)
 return apps_all
zufälliger_Index = np.random.randint(low=0,high=len(ytest))
zufälliger_Index<split>
apps_all = get_apps_all_with_prev_bureau_agg(apps, prev, bureau, bureau_bal )
Testzeile = Xtest[zufälliger_Index,:]
Testlabel = ytest[zufälliger_Index]
Testzeile,Testlabel<compute_test_metric>
apps_all = get_apps_all_encoded(apps_all)
 apps_all_train, apps_all_test = get_apps_all_train_test(apps_all )
distanz = np.sqrt(((Xtrain - Testzeile)**2 ).sum(axis=1))<sort_values>
clf = train_apps_all(apps_all_train )
<sort_values><EOS> 
preds = clf.predict_proba(apps_all_test.drop(['SK_ID_CURR'], axis=1)) [:, 1 ]
apps_all_test['TARGET'] = preds
apps_all_test[['SK_ID_CURR', 'TARGET']].to_csv('bureau_baseline_04.csv', index=False )