Kaggle competition: Home Credit Default Risk
<define_search_space><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<train_model>
import pandas as pd
import numpy as np
learn.fit_one_cycle(5, slice(lr))<train_model>
def return_size(df):

 return round(sys.getsizeof(df)/ 1e9, 2)
 def convert_types(df):
print(f'Original size of data: {return_size(df)} gb.')
 for c in df:
if df[c].dtype == 'object':
df[c] = df[c].astype('category')
 print(f'New size of data: {return_size(df)} gb.')
 return df
learn.fit_one_cycle(5, slice(lr))<train_model>
app_train = pd.read_csv('.. /input/application_train.csv' ).replace({365243: np.nan})
 app_test = pd.read_csv('.. /input/application_test.csv' ).replace({365243: np.nan})
 bureau = pd.read_csv('.. /input/bureau.csv' ).replace({365243: np.nan})
 bureau_balance = pd.read_csv('.. /input/bureau_balance.csv' ).replace({365243: np.nan})
 app_test['TARGET'] = np.nan
app = app_train.append(app_test, ignore_index = True, sort = True)
 app = convert_types(app)
 bureau = convert_types(bureau)
 bureau_balance = convert_types(bureau_balance)
 gc.enable()
del app_train, app_test
gc.collect()
learn.fit_one_cycle(5, slice(lr))<save_model>
def agg_numeric(df, parent_var, df_name):

 for col in df:
if col != parent_var and 'SK_ID' in col:
df = df.drop(columns = col)
 parent_ids = df[parent_var].copy()
numeric_df = df.select_dtypes('number' ).copy()
numeric_df[parent_var] = parent_ids
agg = numeric_df.groupby(parent_var ).agg(['count', 'mean', 'max', 'min', 'sum'])
 columns = []
for var in agg.columns.levels[0]:
if var != parent_var:
for stat in agg.columns.levels[1]:
columns.append('%s_%s_%s' %(df_name, var, stat))
agg.columns = columns
_, idx = np.unique(agg, axis = 1, return_index=True)
 agg = agg.iloc[:, idx]
return agg
learn.save("model-1" )<define_search_space>
def agg_categorical(df, parent_var, df_name):

 categorical = pd.get_dummies(df.select_dtypes('category'))
categorical[parent_var] = df[parent_var]
categorical = categorical.groupby(parent_var ).agg(['sum', 'count', 'mean'])
 column_names = []
for var in categorical.columns.levels[0]:
for stat in ['sum', 'count', 'mean']:
column_names.append('%s_%s_%s' %(df_name, var, stat))
categorical.columns = column_names
_, idx = np.unique(categorical, axis = 1, return_index = True)
 categorical = categorical.iloc[:, idx]
return categorical
lr_a = 1e-4<train_model>
def agg_child(df, parent_var, df_name):

 df_agg = agg_numeric(df, parent_var, df_name)
 df_agg_cat = agg_categorical(df, parent_var, df_name)
 df_info = df_agg.merge(df_agg_cat, on = parent_var, how = 'outer')
 _, idx = np.unique(df_info, axis = 1, return_index = True)
 df_info = df_info.iloc[:, idx]
gc.enable()
del df_agg, df_agg_cat
gc.collect()
return df_info
learn.fit_one_cycle(5,slice(lr_a,lr/5)) 
<save_model>
def agg_grandchild(df, parent_df, parent_var, grandparent_var, df_name):

 parent_df = parent_df[[parent_var, grandparent_var]].copy().set_index(parent_var)
 df_agg = agg_numeric(df, parent_var, '%s_LOAN' % df_name)
 df_agg = df_agg.merge(parent_df,
on = parent_var, how = 'left')
 df_agg_client = agg_numeric(df_agg, grandparent_var, '%s_CLIENT' % df_name)
 if any(df.dtypes == 'category'):
df_agg_cat = agg_categorical(df, parent_var, '%s_LOAN' % df_name)
 df_agg_cat = df_agg_cat.merge(parent_df,
on = parent_var, how = 'left')
 df_agg_cat_client = agg_numeric(df_agg_cat, grandparent_var, '%s_CLIENT' % df_name)
 df_info = df_agg_client.merge(df_agg_cat_client, on = grandparent_var, how = 'outer')
 gc.enable()
del df_agg, df_agg_client, df_agg_cat, df_agg_cat_client
gc.collect()
else:
df_info = df_agg_client.copy()
gc.enable()
del df_agg, df_agg_client
gc.collect()
_, idx = np.unique(df_info, axis = 1, return_index=True)
 df_info = df_info.iloc[:, idx]
return df_info
learn.save("model-2" )<train_model>
app['LOAN_RATE'] = app['AMT_ANNUITY'] / app['AMT_CREDIT']
app['CREDIT_INCOME_RATIO'] = app['AMT_CREDIT'] / app['AMT_INCOME_TOTAL']
app['EMPLOYED_BIRTH_RATIO'] = app['DAYS_EMPLOYED'] / app['DAYS_BIRTH']
app['EXT_SOURCE_SUM'] = app[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].sum(axis = 1)
 app['EXT_SOURCE_MEAN'] = app[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis = 1)
 app['AMT_REQ_SUM'] = app[[x for x in app.columns if 'AMT_REQ_' in x]].sum(axis = 1 )
learn.fit_one_cycle(5,slice(1e-5,lr/5)) 
<save_model>
bureau['LOAN_RATE'] = bureau['AMT_ANNUITY'] / bureau['AMT_CREDIT_SUM']
learn.save("model-3" )<train_model>
bureau_info = agg_child(bureau, 'SK_ID_CURR', 'BUREAU')
 bureau_info.head()
learn.fit_one_cycle(5,slice(1e-5,lr/5))<load_pretrained>
bureau_balance['PAST_DUE'] = bureau_balance['STATUS'].isin(['1', '2', '3', '4', '5'])
 bureau_balance['ON_TIME'] = bureau_balance['STATUS'] == '0'
learn.load("model-3" )<predict_on_test>
bureau_balance_info = agg_grandchild(bureau_balance, bureau, 'SK_ID_BUREAU', 'SK_ID_CURR', 'BB')
 del bureau_balance, bureau
bureau_balance_info.head()
test_probs, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_preds = [data.classes[pred] for pred in np.argmax(test_probs.numpy() , axis=-1)]<save_to_csv>
app = app.set_index('SK_ID_CURR')
 app = app.merge(bureau_info, on = 'SK_ID_CURR', how = 'left')
 del bureau_info
app.shape
sub_df.predicted_class = test_preds
sub_df.to_csv("submission.csv", index=False)
<save_to_csv>
app = app.merge(bureau_balance_info, on = 'SK_ID_CURR', how = 'left')
 del bureau_balance_info
app.shape
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index = False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(sub_df )<define_variables>
previous = pd.read_csv('.. /input/previous_application.csv' ).replace({365243: np.nan})
 previous = convert_types(previous)
 previous['LOAN_RATE'] = previous['AMT_ANNUITY'] / previous['AMT_CREDIT']
previous["AMT_DIFFERENCE"] = previous['AMT_CREDIT'] - previous['AMT_APPLICATION']
FileLink("/tmp/model/export.pkl")
<find_best_params>
app = app.merge(previous_info, on = 'SK_ID_CURR', how = 'left')
 del previous_info
app.shape
lr_find(learn )<save_model>
installments = pd.read_csv('.. /input/installments_payments.csv' ).replace({365243: np.nan})
 installments = convert_types(installments)
 installments['LATE'] = installments['DAYS_ENTRY_PAYMENT'] > installments['DAYS_INSTALMENT']
installments['LOW_PAYMENT'] = installments['AMT_PAYMENT'] < installments['AMT_INSTALMENT']
learn.save("model-2" )<load_pretrained>
app = app.merge(installments_info, on = 'SK_ID_CURR', how = 'left')
 del installments_info
app.shape
learn.load("model-2" )<load_pretrained>
cash = pd.read_csv('.. /input/POS_CASH_balance.csv' ).replace({365243: np.nan})
 cash = convert_types(cash)
 cash['LATE_PAYMENT'] = cash['SK_DPD'] > 0.0
cash['INSTALLMENTS_PAID'] = cash['CNT_INSTALMENT'] - cash['CNT_INSTALMENT_FUTURE']
data_1= ImageDataBunch.from_folder(train_path,valid_pct = 0.2,test = test_path,ds_tfms = get_transforms() ,size = 128 ).normalize(imagenet_stats )<feature_engineering>
cash_info = agg_grandchild(cash, previous, 'SK_ID_PREV', 'SK_ID_CURR', 'CASH')
 del cash
cash_info.shape
learn.data=data_1
learn = learn.to_fp16()<find_best_params>
app = app.merge(cash_info, on = 'SK_ID_CURR', how = 'left')
 del cash_info
app.shape
learn.freeze()
learn.lr_find()
learn.recorder.plot()<train_model>
credit = pd.read_csv('.. /input/credit_card_balance.csv' ).replace({365243: np.nan})
 credit = convert_types(credit)
 credit['OVER_LIMIT'] = credit['AMT_BALANCE'] > credit['AMT_CREDIT_LIMIT_ACTUAL']
credit['BALANCE_CLEARED'] = credit['AMT_BALANCE'] == 0.0
credit['LOW_PAYMENT'] = credit['AMT_PAYMENT_CURRENT'] < credit['AMT_INST_MIN_REGULARITY']
credit['LATE'] = credit['SK_DPD'] > 0.0
learn.fit_one_cycle(5,1e-3 )<find_best_params>
credit_info = agg_grandchild(credit, previous, 'SK_ID_PREV', 'SK_ID_CURR', 'CC')
 del credit, previous
credit_info.shape
learn.lr_find() 
<train_model>
gc.collect()
gc.enable()
learn.fit_one_cycle(2,1e-6 )<choose_model_class>
time.sleep(600)
 app = app.merge(credit_info, on = 'SK_ID_CURR', how = 'left')
 del credit_info
app.shape
interp = ClassificationInterpretation.from_learner(learn)
<save_model>
print('After manual feature engineering, there are {} features.'.format(app.shape[1] - 2))
learn.save("model1" )<find_best_params>
gc.enable()
gc.collect()
lr_find(learn )<load_from_csv>
app.to_csv('clean_manual_features.csv', chunksize = 100 )
df = pd.read_csv("submission.csv" )<save_to_csv>
app.reset_index(inplace = True)
 train, test = app[app['TARGET'].notnull() ].copy() , app[app['TARGET'].isnull() ].copy()
gc.enable()
del app
gc.collect()
test_probs, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_preds = [data.classes[pred] for pred in np.argmax(test_probs.numpy() , axis=-1)]
sub_df.predicted_class = test_preds
sub_df.to_csv("submission.csv", index=False)
 sub_df.head()<save_to_csv>
params = {'is_unbalance': True,
'n_estimators': 2673,
'num_leaves': 77,
'learning_rate': 0.00764,
'min_child_samples': 460,
'boosting_type': 'gbdt',
'subsample_for_bin': 240000,
'reg_lambda': 0.20,
'reg_alpha': 0.88,
'subsample': 0.95,
'colsample_bytree': 0.7}
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(df )<import_modules>
train_labels = np.array(train.pop('TARGET')).reshape(( -1,))
test_ids = list(test.pop('SK_ID_CURR'))
test = test.drop(columns = ['TARGET'])
 train = train.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
learn.export('/tmp/model/learn.pkl')
<import_modules>
model = lgb.LGBMClassifier(**params)
 model.fit(train, train_labels )
from fastai.vision import *
from fastai.metrics import error_rate<create_dataframe>
preds = model.predict_proba(test)[:, 1]
submission = pd.DataFrame({'SK_ID_CURR': test_ids,
'TARGET': preds})
 submission['SK_ID_CURR'] = submission['SK_ID_CURR'].astype(int)
 submission['TARGET'] = submission['TARGET'].astype(float)
 submission.to_csv('submission_manual.csv', index = False )
train_arr = []
for file in glob.glob(".. /input/train/train/*/*"):
train_arr.append({"name": file, "label": file.split("/")[-2]})
 df = pd.DataFrame(train_arr )<load_from_csv>
features = list(train.columns)
 fi = pd.DataFrame({'feature': features,
'importance': model.feature_importances_} )