Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
%matplotlib inline
default_dir = ".. /input/home-credit-default-risk/"<data_type_conversions>
train_data = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test_data = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
def get_balance_data() :
pos_dtype = {
'SK_ID_PREV':np.uint32, 'SK_ID_CURR':np.uint32, 'MONTHS_BALANCE':np.int32, 'SK_DPD':np.int32,
'SK_DPD_DEF':np.int32, 'CNT_INSTALMENT':np.float32,'CNT_INSTALMENT_FUTURE':np.float32
}
install_dtype = {
'SK_ID_PREV':np.uint32, 'SK_ID_CURR':np.uint32, 'NUM_INSTALMENT_NUMBER':np.int32, 'NUM_INSTALMENT_VERSION':np.float32,
'DAYS_INSTALMENT':np.float32, 'DAYS_ENTRY_PAYMENT':np.float32, 'AMT_INSTALMENT':np.float32, 'AMT_PAYMENT':np.float32
}
card_dtype = {
'SK_ID_PREV':np.uint32, 'SK_ID_CURR':np.uint32, 'MONTHS_BALANCE':np.int16,
'AMT_CREDIT_LIMIT_ACTUAL':np.int32, 'CNT_DRAWINGS_CURRENT':np.int32, 'SK_DPD':np.int32,'SK_DPD_DEF':np.int32,
'AMT_BALANCE':np.float32, 'AMT_DRAWINGS_ATM_CURRENT':np.float32, 'AMT_DRAWINGS_CURRENT':np.float32,
'AMT_DRAWINGS_OTHER_CURRENT':np.float32, 'AMT_DRAWINGS_POS_CURRENT':np.float32, 'AMT_INST_MIN_REGULARITY':np.float32,
'AMT_PAYMENT_CURRENT':np.float32, 'AMT_PAYMENT_TOTAL_CURRENT':np.float32, 'AMT_RECEIVABLE_PRINCIPAL':np.float32,
'AMT_RECIVABLE':np.float32, 'AMT_TOTAL_RECEIVABLE':np.float32, 'CNT_DRAWINGS_ATM_CURRENT':np.float32,
'CNT_DRAWINGS_OTHER_CURRENT':np.float32, 'CNT_DRAWINGS_POS_CURRENT':np.float32, 'CNT_INSTALMENT_MATURE_CUM':np.float32
}
pos_bal = pd.read_csv(os.path.join(default_dir,'POS_CASH_balance.csv'), dtype=pos_dtype)
 install = pd.read_csv(os.path.join(default_dir,'installments_payments.csv'), dtype=install_dtype)
 card_bal = pd.read_csv(os.path.join(default_dir, 'credit_card_balance.csv'), dtype=card_dtype)
 return pos_bal, install, card_bal
pos_bal, install, card_bal = get_balance_data()<import_modules>
X = train_data.drop(["label"],axis = 1 ).values
Y = train_data["label"].values
from sklearn.model_selection import train_test_split
from lightgbm import LGBMClassifier<feature_engineering>
X = X.reshape([42000,28,28,1])
 Y = Y.reshape([42000,1] )
def get_apps_processed(apps):

 apps['APPS_EXT_SOURCE_MEAN'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis = 1)
 apps['APPS_EXT_SOURCE_STD'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps['APPS_EXT_SOURCE_STD'].fillna(apps['APPS_EXT_SOURCE_STD'].mean())
 apps['APPS_ANNUITY_CREDIT_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_CREDIT']
apps['APPS_GOODS_CREDIT_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_CREDIT']
apps['APPS_ANNUITY_INCOME_RATIO'] = apps['AMT_ANNUITY']/apps['AMT_INCOME_TOTAL']
apps['APPS_CREDIT_INCOME_RATIO'] = apps['AMT_CREDIT']/apps['AMT_INCOME_TOTAL']
apps['APPS_GOODS_INCOME_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_INCOME_TOTAL']
apps['APPS_CNT_FAM_INCOME_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['CNT_FAM_MEMBERS']
apps['APPS_EMPLOYED_BIRTH_RATIO'] = apps['DAYS_EMPLOYED']/apps['DAYS_BIRTH']
apps['APPS_INCOME_EMPLOYED_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_EMPLOYED']
apps['APPS_INCOME_BIRTH_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_BIRTH']
apps['APPS_CAR_BIRTH_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_BIRTH']
apps['APPS_CAR_EMPLOYED_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_EMPLOYED']
return apps
def get_prev_processed(prev):

 prev['PREV_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']
prev['PREV_GOODS_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_GOODS_PRICE']
prev['PREV_CREDIT_APPL_RATIO'] = prev['AMT_CREDIT']/prev['AMT_APPLICATION']
prev['PREV_GOODS_APPL_RATIO'] = prev['AMT_GOODS_PRICE']/prev['AMT_APPLICATION']
prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['PREV_DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']
all_pay = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']
prev['PREV_INTERESTS_RATE'] =(all_pay/prev['AMT_CREDIT'] - 1)/prev['CNT_PAYMENT']
return prev
def get_prev_amt_agg(prev):

 agg_dict = {
'SK_ID_CURR':['count'],
'AMT_CREDIT':['mean', 'max', 'sum'],
'AMT_ANNUITY':['mean', 'max', 'sum'],
'AMT_APPLICATION':['mean', 'max', 'sum'],
'AMT_DOWN_PAYMENT':['mean', 'max', 'sum'],
'AMT_GOODS_PRICE':['mean', 'max', 'sum'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'PREV_CREDIT_DIFF':['mean', 'max', 'sum'],
'PREV_CREDIT_APPL_RATIO':['mean', 'max'],
'PREV_GOODS_DIFF':['mean', 'max', 'sum'],
'PREV_GOODS_APPL_RATIO':['mean', 'max'],
'PREV_DAYS_LAST_DUE_DIFF':['mean', 'max', 'sum'],
'PREV_INTERESTS_RATE':['mean', 'max']
}
prev_group = prev.groupby('SK_ID_CURR')
 prev_amt_agg = prev_group.agg(agg_dict)
 prev_amt_agg.columns = ["PREV_"+ "_".join(x ).upper() for x in prev_amt_agg.columns.ravel() ]
return prev_amt_agg
def get_prev_refused_appr_agg(prev):

 prev_refused_appr_group = prev[prev['NAME_CONTRACT_STATUS'].isin(['Approved', 'Refused'])].groupby([ 'SK_ID_CURR', 'NAME_CONTRACT_STATUS'])
 prev_refused_appr_agg = prev_refused_appr_group['SK_ID_CURR'].count().unstack()
prev_refused_appr_agg.columns = ['PREV_APPROVED_COUNT', 'PREV_REFUSED_COUNT' ]
prev_refused_appr_agg = prev_refused_appr_agg.fillna(0)
 return prev_refused_appr_agg
def get_prev_days365_agg(prev):

 cond_days365 = prev['DAYS_DECISION'] > -365
prev_days365_group = prev[cond_days365].groupby('SK_ID_CURR')
 agg_dict = {
'SK_ID_CURR':['count'],
'AMT_CREDIT':['mean', 'max', 'sum'],
'AMT_ANNUITY':['mean', 'max', 'sum'],
'AMT_APPLICATION':['mean', 'max', 'sum'],
'AMT_DOWN_PAYMENT':['mean', 'max', 'sum'],
'AMT_GOODS_PRICE':['mean', 'max', 'sum'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'PREV_CREDIT_DIFF':['mean', 'max', 'sum'],
'PREV_CREDIT_APPL_RATIO':['mean', 'max'],
'PREV_GOODS_DIFF':['mean', 'max', 'sum'],
'PREV_GOODS_APPL_RATIO':['mean', 'max'],
'PREV_DAYS_LAST_DUE_DIFF':['mean', 'max', 'sum'],
'PREV_INTERESTS_RATE':['mean', 'max']
}
prev_days365_agg = prev_days365_group.agg(agg_dict)
 prev_days365_agg.columns = ["PREV_D365_"+ "_".join(x ).upper() for x in prev_days365_agg.columns.ravel() ]
return prev_days365_agg
def get_prev_agg(prev):
prev = get_prev_processed(prev)
 prev_amt_agg = get_prev_amt_agg(prev)
 prev_refused_appr_agg = get_prev_refused_appr_agg(prev)
 prev_days365_agg = get_prev_days365_agg(prev)
 prev_agg = prev_amt_agg.merge(prev_refused_appr_agg, on='SK_ID_CURR', how='left')
 prev_agg = prev_agg.merge(prev_days365_agg, on='SK_ID_CURR', how='left')
 prev_agg['PREV_REFUSED_RATIO'] = prev_agg['PREV_REFUSED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg['PREV_APPROVED_RATIO'] = prev_agg['PREV_APPROVED_COUNT']/prev_agg['PREV_SK_ID_CURR_COUNT']
prev_agg = prev_agg.drop(['PREV_REFUSED_COUNT', 'PREV_APPROVED_COUNT'], axis=1)
 return prev_agg
def get_bureau_processed(bureau):
bureau['BUREAU_ENDDATE_FACT_DIFF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
bureau['BUREAU_CREDIT_FACT_DIFF'] = bureau['DAYS_CREDIT'] - bureau['DAYS_ENDDATE_FACT']
bureau['BUREAU_CREDIT_ENDDATE_DIFF'] = bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE']
bureau['BUREAU_CREDIT_DEBT_RATIO']=bureau['AMT_CREDIT_SUM_DEBT']/bureau['AMT_CREDIT_SUM']
bureau['BUREAU_CREDIT_DEBT_DIFF'] = bureau['AMT_CREDIT_SUM_DEBT'] - bureau['AMT_CREDIT_SUM']
bureau['BUREAU_IS_DPD'] = bureau['CREDIT_DAY_OVERDUE'].apply(lambda x: 1 if x > 0 else 0)
 bureau['BUREAU_IS_DPD_OVER120'] = bureau['CREDIT_DAY_OVERDUE'].apply(lambda x: 1 if x >120 else 0)
 return bureau
def get_bureau_day_amt_agg(bureau):

 bureau_agg_dict = {
'SK_ID_BUREAU':['count'],
'DAYS_CREDIT':['min', 'max', 'mean'],
'CREDIT_DAY_OVERDUE':['min', 'max', 'mean'],
'DAYS_CREDIT_ENDDATE':['min', 'max', 'mean'],
'DAYS_ENDDATE_FACT':['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean', 'sum'],
'BUREAU_ENDDATE_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_ENDDATE_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_RATIO':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_DIFF':['min', 'max', 'mean'],
'BUREAU_IS_DPD':['mean', 'sum'],
'BUREAU_IS_DPD_OVER120':['mean', 'sum']
}
bureau_grp = bureau.groupby('SK_ID_CURR')
 bureau_day_amt_agg = bureau_grp.agg(bureau_agg_dict)
 bureau_day_amt_agg.columns = ['BUREAU_'+('_' ).join(column ).upper() for column in bureau_day_amt_agg.columns.ravel() ]
bureau_day_amt_agg = bureau_day_amt_agg.reset_index()
return bureau_day_amt_agg
def get_bureau_active_agg(bureau):

 cond_active = bureau['CREDIT_ACTIVE'] == 'Active'
bureau_active_grp = bureau[cond_active].groupby(['SK_ID_CURR'])
 bureau_agg_dict = {
'SK_ID_BUREAU':['count'],
'DAYS_CREDIT':['min', 'max', 'mean'],
'CREDIT_DAY_OVERDUE':['min', 'max', 'mean'],
'DAYS_CREDIT_ENDDATE':['min', 'max', 'mean'],
'DAYS_ENDDATE_FACT':['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean', 'sum'],
'BUREAU_ENDDATE_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_ENDDATE_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_RATIO':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_DIFF':['min', 'max', 'mean'],
'BUREAU_IS_DPD':['mean', 'sum'],
'BUREAU_IS_DPD_OVER120':['mean', 'sum']
}
bureau_active_agg = bureau_active_grp.agg(bureau_agg_dict)
 bureau_active_agg.columns = ['BUREAU_ACT_'+('_' ).join(column ).upper() for column in bureau_active_agg.columns.ravel() ]
bureau_active_agg = bureau_active_agg.reset_index()
return bureau_active_agg
def get_bureau_days750_agg(bureau):
cond_days750 = bureau['DAYS_CREDIT'] > -750
bureau_days750_group = bureau[cond_days750].groupby('SK_ID_CURR')
 bureau_agg_dict = {
'SK_ID_BUREAU':['count'],
'DAYS_CREDIT':['min', 'max', 'mean'],
'CREDIT_DAY_OVERDUE':['min', 'max', 'mean'],
'DAYS_CREDIT_ENDDATE':['min', 'max', 'mean'],
'DAYS_ENDDATE_FACT':['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean', 'sum'],
'BUREAU_ENDDATE_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_FACT_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_ENDDATE_DIFF':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_RATIO':['min', 'max', 'mean'],
'BUREAU_CREDIT_DEBT_DIFF':['min', 'max', 'mean'],
'BUREAU_IS_DPD':['mean', 'sum'],
'BUREAU_IS_DPD_OVER120':['mean', 'sum']
}
bureau_days750_agg = bureau_days750_group.agg(bureau_agg_dict)
 bureau_days750_agg.columns = ['BUREAU_ACT_'+('_' ).join(column ).upper() for column in bureau_days750_agg.columns.ravel() ]
bureau_days750_agg = bureau_days750_agg.reset_index()
return bureau_days750_agg
def get_bureau_bal_agg(bureau, bureau_bal):
bureau_bal = bureau_bal.merge(bureau[['SK_ID_CURR', 'SK_ID_BUREAU']], on='SK_ID_BUREAU', how='left')
 bureau_bal['BUREAU_BAL_IS_DPD'] = bureau_bal['STATUS'].apply(lambda x: 1 if x in['1','2','3','4','5'] else 0)
 bureau_bal['BUREAU_BAL_IS_DPD_OVER120'] = bureau_bal['STATUS'].apply(lambda x: 1 if x =='5' else 0)
 bureau_bal_grp = bureau_bal.groupby('SK_ID_CURR')
 bureau_bal_agg_dict = {
'SK_ID_CURR':['count'],
'MONTHS_BALANCE':['min', 'max', 'mean'],
'BUREAU_BAL_IS_DPD':['mean', 'sum'],
'BUREAU_BAL_IS_DPD_OVER120':['mean', 'sum']
}
bureau_bal_agg = bureau_bal_grp.agg(bureau_bal_agg_dict)
 bureau_bal_agg.columns = [ 'BUREAU_BAL_'+('_' ).join(column ).upper() for column in bureau_bal_agg.columns.ravel() ]
bureau_bal_agg = bureau_bal_agg.reset_index()
return bureau_bal_agg
def get_bureau_agg(bureau, bureau_bal):
bureau = get_bureau_processed(bureau)
 bureau_day_amt_agg = get_bureau_day_amt_agg(bureau)
 bureau_active_agg = get_bureau_active_agg(bureau)
 bureau_days750_agg = get_bureau_days750_agg(bureau)
 bureau_bal_agg = get_bureau_bal_agg(bureau, bureau_bal)
 bureau_agg = bureau_day_amt_agg.merge(bureau_active_agg, on='SK_ID_CURR', how='left')
 bureau_agg['BUREAU_ACT_IS_DPD_RATIO'] = bureau_agg['BUREAU_ACT_BUREAU_IS_DPD_SUM']/bureau_agg['BUREAU_SK_ID_BUREAU_COUNT']
bureau_agg['BUREAU_ACT_IS_DPD_OVER120_RATIO'] = bureau_agg['BUREAU_ACT_BUREAU_IS_DPD_OVER120_SUM']/bureau_agg['BUREAU_SK_ID_BUREAU_COUNT']
bureau_agg = bureau_agg.merge(bureau_bal_agg, on='SK_ID_CURR', how='left')
 bureau_agg = bureau_agg.merge(bureau_days750_agg, on='SK_ID_CURR', how='left')
 return bureau_agg
def get_apps_all_with_prev_agg(apps, prev):
apps_all = get_apps_processed(apps)
 prev_agg = get_prev_agg(prev)
 print('prev_agg shape:', prev_agg.shape)
 print('apps_all before merge shape:', apps_all.shape)
 apps_all = apps_all.merge(prev_agg, on='SK_ID_CURR', how='left')
 print('apps_all after merge with prev_agg shape:', apps_all.shape)
 return apps_all
def get_apps_all_encoded(apps_all):
object_columns = apps_all.dtypes[apps_all.dtypes == 'object'].index.tolist()
for column in object_columns:
apps_all[column] = pd.factorize(apps_all[column])[0]
return apps_all
def get_apps_all_train_test(apps_all):
apps_all_train = apps_all[~apps_all['TARGET'].isnull() ]
apps_all_test = apps_all[apps_all['TARGET'].isnull() ]
apps_all_test = apps_all_test.drop('TARGET', axis=1)
 return apps_all_train, apps_all_test
def train_apps_all(apps_all_train):
ftr_app = apps_all_train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
 target_app = apps_all_train['TARGET']
train_x, valid_x, train_y, valid_y = train_test_split(ftr_app, target_app, test_size=0.3, random_state=2020)
 print('train shape:', train_x.shape, 'valid shape:', valid_x.shape)
 clf = LGBMClassifier(
 nthread=4,
n_estimators=2000,
learning_rate=0.02,
max_depth = 11,
num_leaves=58,
colsample_bytree=0.613,
subsample=0.708,
max_bin=407,
reg_alpha=3.564,
reg_lambda=4.930,
min_child_weight= 6,
min_child_samples=165,
silent=-1,
verbose=-1, 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], eval_metric= 'auc', verbose= 100,
early_stopping_rounds= 200)
 return clf<define_variables>
Y = to_categorical(Y, num_classes = 10 )
def get_pos_bal_agg(pos_bal):
cond_over_0 = pos_bal['SK_DPD'] > 0
cond_100 =(pos_bal['SK_DPD'] < 100)&(pos_bal['SK_DPD'] > 0)
 cond_over_100 =(pos_bal['SK_DPD'] >= 100)
 pos_bal['POS_IS_DPD'] = pos_bal['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
 pos_bal['POS_IS_DPD_UNDER_120'] = pos_bal['SK_DPD'].apply(lambda x:1 if(x > 0)&(x <120)else 0)
 pos_bal['POS_IS_DPD_OVER_120'] = pos_bal['SK_DPD'].apply(lambda x:1 if x >= 120 else 0)
 pos_bal_grp = pos_bal.groupby('SK_ID_CURR')
 pos_bal_agg_dict = {
'SK_ID_CURR':['count'],
'MONTHS_BALANCE':['min', 'mean', 'max'],
'SK_DPD':['min', 'max', 'mean', 'sum'],
'CNT_INSTALMENT':['min', 'max', 'mean', 'sum'],
'CNT_INSTALMENT_FUTURE':['min', 'max', 'mean', 'sum'],
'POS_IS_DPD':['mean', 'sum'],
'POS_IS_DPD_UNDER_120':['mean', 'sum'],
'POS_IS_DPD_OVER_120':['mean', 'sum']
}
pos_bal_agg = pos_bal_grp.agg(pos_bal_agg_dict)
 pos_bal_agg.columns = [('POS_')+('_' ).join(column ).upper() for column in pos_bal_agg.columns.ravel() ]
cond_months = pos_bal['MONTHS_BALANCE'] > -20
pos_bal_m20_grp = pos_bal[cond_months].groupby('SK_ID_CURR')
 pos_bal_m20_agg_dict = {
'SK_ID_CURR':['count'],
'MONTHS_BALANCE':['min', 'mean', 'max'],
'SK_DPD':['min', 'max', 'mean', 'sum'],
'CNT_INSTALMENT':['min', 'max', 'mean', 'sum'],
'CNT_INSTALMENT_FUTURE':['min', 'max', 'mean', 'sum'],
'POS_IS_DPD':['mean', 'sum'],
'POS_IS_DPD_UNDER_120':['mean', 'sum'],
'POS_IS_DPD_OVER_120':['mean', 'sum']
}
pos_bal_m20_agg = pos_bal_m20_grp.agg(pos_bal_m20_agg_dict)
 pos_bal_m20_agg.columns = [('POS_M20')+('_' ).join(column ).upper() for column in pos_bal_m20_agg.columns.ravel() ]
pos_bal_agg = pos_bal_agg.merge(pos_bal_m20_agg, on='SK_ID_CURR', how='left')
 pos_bal_agg = pos_bal_agg.reset_index()
return pos_bal_agg
def get_install_agg(install):
install['AMT_DIFF'] = install['AMT_INSTALMENT'] - install['AMT_PAYMENT']
install['AMT_RATIO'] =(install['AMT_PAYMENT'] +1)/(install['AMT_INSTALMENT'] + 1)
 install['SK_DPD'] = install['DAYS_ENTRY_PAYMENT'] - install['DAYS_INSTALMENT']
install['INS_IS_DPD'] = install['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
 install['INS_IS_DPD_UNDER_120'] = install['SK_DPD'].apply(lambda x:1 if(x > 0)&(x <120)else 0)
 install['INS_IS_DPD_OVER_120'] = install['SK_DPD'].apply(lambda x:1 if x >= 120 else 0)
 install_grp = install.groupby('SK_ID_CURR')
 install_agg_dict = {
'SK_ID_CURR':['count'],
'NUM_INSTALMENT_VERSION':['nunique'],
'DAYS_ENTRY_PAYMENT':['mean', 'max', 'sum'],
'DAYS_INSTALMENT':['mean', 'max', 'sum'],
'AMT_INSTALMENT':['mean', 'max', 'sum'],
'AMT_PAYMENT':['mean', 'max','sum'],
'AMT_DIFF':['mean','min', 'max','sum'],
'AMT_RATIO':['mean', 'max'],
'SK_DPD':['mean', 'min', 'max'],
'INS_IS_DPD':['mean', 'sum'],
'INS_IS_DPD_UNDER_120':['mean', 'sum'],
'INS_IS_DPD_OVER_120':['mean', 'sum']
}
install_agg = install_grp.agg(install_agg_dict)
 install_agg.columns = ['INS_'+('_' ).join(column ).upper() for column in install_agg.columns.ravel() ]
cond_day = install['DAYS_ENTRY_PAYMENT'] >= -365
install_d365_grp = install[cond_day].groupby('SK_ID_CURR')
 install_d365_agg_dict = {
'SK_ID_CURR':['count'],
'NUM_INSTALMENT_VERSION':['nunique'],
'DAYS_ENTRY_PAYMENT':['mean', 'max', 'sum'],
'DAYS_INSTALMENT':['mean', 'max', 'sum'],
'AMT_INSTALMENT':['mean', 'max', 'sum'],
'AMT_PAYMENT':['mean', 'max','sum'],
'AMT_DIFF':['mean','min', 'max','sum'],
'AMT_RATIO':['mean', 'max'],
'SK_DPD':['mean', 'min', 'max'],
'INS_IS_DPD':['mean', 'sum'],
'INS_IS_DPD_UNDER_120':['mean', 'sum'],
'INS_IS_DPD_OVER_120':['mean', 'sum']
}
install_d365_agg = install_d365_grp.agg(install_d365_agg_dict)
 install_d365_agg.columns = ['INS_D365'+('_' ).join(column ).upper() for column in install_d365_agg.columns.ravel() ]
install_agg = install_agg.merge(install_d365_agg, on='SK_ID_CURR', how='left')
 install_agg = install_agg.reset_index()
return install_agg
def get_card_bal_agg(card_bal):
card_bal['BALANCE_LIMIT_RATIO'] = card_bal['AMT_BALANCE']/card_bal['AMT_CREDIT_LIMIT_ACTUAL']
card_bal['DRAWING_LIMIT_RATIO'] = card_bal['AMT_DRAWINGS_CURRENT'] / card_bal['AMT_CREDIT_LIMIT_ACTUAL']
card_bal['CARD_IS_DPD'] = card_bal['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)
 card_bal['CARD_IS_DPD_UNDER_120'] = card_bal['SK_DPD'].apply(lambda x:1 if(x > 0)&(x <120)else 0)
 card_bal['CARD_IS_DPD_OVER_120'] = card_bal['SK_DPD'].apply(lambda x:1 if x >= 120 else 0)
 card_bal_grp = card_bal.groupby('SK_ID_CURR')
 card_bal_agg_dict = {
'SK_ID_CURR':['count'],
'AMT_BALANCE':['max'],
'AMT_CREDIT_LIMIT_ACTUAL':['max'],
'AMT_DRAWINGS_ATM_CURRENT': ['max', 'sum'],
'AMT_DRAWINGS_CURRENT': ['max', 'sum'],
'AMT_DRAWINGS_POS_CURRENT': ['max', 'sum'],
'AMT_INST_MIN_REGULARITY': ['max', 'mean'],
'AMT_PAYMENT_TOTAL_CURRENT': ['max','sum'],
'AMT_TOTAL_RECEIVABLE': ['max', 'mean'],
'CNT_DRAWINGS_ATM_CURRENT': ['max','sum'],
'CNT_DRAWINGS_CURRENT': ['max', 'mean', 'sum'],
'CNT_DRAWINGS_POS_CURRENT': ['mean'],
'SK_DPD': ['mean', 'max', 'sum'],
'BALANCE_LIMIT_RATIO':['min','max'],
'DRAWING_LIMIT_RATIO':['min', 'max'],
'CARD_IS_DPD':['mean', 'sum'],
'CARD_IS_DPD_UNDER_120':['mean', 'sum'],
'CARD_IS_DPD_OVER_120':['mean', 'sum']
}
card_bal_agg = card_bal_grp.agg(card_bal_agg_dict)
 card_bal_agg.columns = ['CARD_'+('_' ).join(column ).upper() for column in card_bal_agg.columns.ravel() ]
card_bal_agg = card_bal_agg.reset_index()
cond_month = card_bal.MONTHS_BALANCE >= -3
card_bal_m3_grp = card_bal[cond_month].groupby('SK_ID_CURR')
 card_bal_m3_agg = card_bal_m3_grp.agg(card_bal_agg_dict)
 card_bal_m3_agg.columns = ['CARD_M3'+('_' ).join(column ).upper() for column in card_bal_m3_agg.columns.ravel() ]
card_bal_agg = card_bal_agg.merge(card_bal_m3_agg, on='SK_ID_CURR', how='left')
 card_bal_agg = card_bal_agg.reset_index()
return card_bal_agg<categorify>
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size = 0.1, random_state = 14 )
def get_apps_all_with_all_agg(apps, prev, bureau, bureau_bal, pos_bal, install, card_bal):

 apps_all = get_apps_processed(apps)
 prev_agg = get_prev_agg(prev)
 bureau_agg = get_bureau_agg(bureau, bureau_bal)
 pos_bal_agg = get_pos_bal_agg(pos_bal)
 install_agg = get_install_agg(install)
 card_bal_agg = get_card_bal_agg(card_bal)
 print('prev_agg shape:', prev_agg.shape, 'bureau_agg shape:', bureau_agg.shape)
 print('pos_bal_agg shape:', pos_bal_agg.shape, 'install_agg shape:', install_agg.shape, 'card_bal_agg shape:', card_bal_agg.shape)
 print('apps_all before merge shape:', apps_all.shape)
 apps_all = apps_all.merge(prev_agg, on='SK_ID_CURR', how='left')
 apps_all = apps_all.merge(bureau_agg, on='SK_ID_CURR', how='left')
 apps_all = apps_all.merge(pos_bal_agg, on='SK_ID_CURR', how='left')
 apps_all = apps_all.merge(install_agg, on='SK_ID_CURR', how='left')
 apps_all = apps_all.merge(card_bal_agg, on='SK_ID_CURR', how='left')
 print('apps_all after merge with all shape:', apps_all.shape)
 return apps_all<load_from_csv>
x_train = x_train/255
x_test = x_test/255
def get_dataset() :

 app_train = pd.read_csv(os.path.join(default_dir,'application_train.csv'))
app_test = pd.read_csv(os.path.join(default_dir,'application_test.csv'))
apps = pd.concat([app_train, app_test])
 prev = pd.read_csv(os.path.join(default_dir,'previous_application.csv'))
bureau = pd.read_csv(os.path.join(default_dir,'bureau.csv'))
bureau_bal = pd.read_csv(os.path.join(default_dir,'bureau_balance.csv'))
pos_bal, install, card_bal = get_balance_data()
return apps, prev, bureau, bureau_bal, pos_bal, install, card_bal<create_dataframe>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding = 'same', activation='relu', input_shape=(28, 28, 1)) ,
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(64,(3,3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(128,(3,3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(128,(3,3), padding = 'same', activation='relu'),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(10, activation='softmax')
 ] )
apps, prev, bureau, bureau_bal, pos_bal, install, card_bal = get_dataset()<categorify>
optimizer = Adam(learning_rate = 0.001, beta_1 = 0.9, beta_2 = 0.999)
 model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['accuracy'] )
apps_all = get_apps_all_with_all_agg(apps, prev, bureau, bureau_bal, pos_bal, install, card_bal)
 apps_all = get_apps_all_encoded(apps_all)
 apps_all_train, apps_all_test = get_apps_all_train_test(apps_all)
 clf = train_apps_all(apps_all_train )<save_to_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.6,
min_lr=0.00001 )
output_dir = ".. /output/kaggle/working/"
preds = clf.predict_proba(apps_all_test.drop(['SK_ID_CURR'], axis=1)) [:, 1 ]
apps_all_test['TARGET'] = preds
apps_all_test[['SK_ID_CURR', 'TARGET']]<save_to_csv>
batch_size = 64
epochs = 30
apps_all_test[['SK_ID_CURR', 'TARGET']].to_csv('submission.csv', index=False )<import_modules>
train_datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range = 0.1,
horizontal_flip=False,
vertical_flip=False 
)
 train_datagen.fit(x_train )
from lightgbm import plot_importance<load_from_csv>
history = model.fit(
 train_datagen.flow(x_train,y_train,batch_size = batch_size),
validation_data =(x_test,y_test),
batch_size = batch_size,
steps_per_epoch = x_train.shape[0]//batch_size,
epochs = epochs,
verbose = 1,
callbacks=[learning_rate_reduction]
)
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
model.evaluate(x_test,y_test )
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()<define_variables>
test_pred = model.predict(data)
 test_pred = np.argmax(test_pred,axis=1)
 print(test_pred.shape )
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )<define_variables>
sample_submission = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
 sample_submission
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )<save_to_csv>
index = sample_submission.ImageId
data = {'ImageId' : index,'Label': test_pred}
df = pd.DataFrame(data)
 df.head
<concatenate><EOS> 
df.to_csv('submission.csv', index=False )