Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<split>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.filterwarnings("ignore" )
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=2018)
 pred_test_full = 0
params = {
'boosting': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'learning_rate': 0.01,
'num_leaves': 25,
'max_depth': 8,
'colsample_bytree': 0.3,
'seed': 101
}
res = []
idx = 0
for dev_index, val_index in kf.split(train, train['TARGET'].values):
dev, valid = train.loc[dev_index,:], train.loc[val_index,:]
dtrain = lgb.Dataset(dev[predictors].values, label=dev['TARGET'].values,
feature_name=predictors,
categorical_feature=categorical 
)
 dvalid = lgb.Dataset(valid[predictors].values, label=valid['TARGET'].values,
feature_name=predictors,
categorical_feature=categorical 
)
 print("Training the model...")
 lgb_model = lgb.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train','valid'],
num_boost_round= 30000,
early_stopping_rounds=500,
verbose_eval=100,
feval=None)
 oof = pd.DataFrame()
oof['id'] = valid['SK_ID_CURR'].values
oof['target'] = valid['TARGET'].values
oof['preds'] = lgb_model.predict(valid[predictors],num_iteration=lgb_model.best_iteration)
 res.append(oof)
 pred_test_full += lgb_model.predict(test[predictors],num_iteration=lgb_model.best_iteration)
 sub = pd.read_csv(".. /input/sample_submission.csv")
 sub['TARGET'] = pred_test_full/NFOLDS
sub.to_csv("sub_lgb.csv", index=False)
 res = pd.concat(res, ignore_index=True)
 res.to_csv("oof_lgb.csv", index=False)
 print(roc_auc_score(res['target'], res['preds'])) 
<merge>
data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 print(data.shape)
 data.head()
tmp = install.groupby(['SK_ID_PREV','NUM_INSTALMENT_NUMBER'])['DAYS_INSTALMENT'].count().reset_index()
tmp = tmp[tmp['DAYS_INSTALMENT'] > 1]
tmp.columns = ['SK_ID_PREV','NUM_INSTALMENT_NUMBER','count_dup']
install = install.merge(tmp, on = ['SK_ID_PREV','NUM_INSTALMENT_NUMBER'], how='left')
 tmp = install[install['count_dup'] > 1]<drop_column>
data_explore = data.copy()
data_explore = data_explore.drop(columns="Id", axis=1 )
install.drop(['count_dup_x','count_dup_y'], axis=1, inplace = True )<sort_values>
nulls = data_explore.isna().sum()
nulls[nulls>0]
tmp.sort_values(by=['SK_ID_PREV','NUM_INSTALMENT_NUMBER','DAYS_ENTRY_PAYMENT'] )<count_values>
na_cols = ["Alley", "BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1", "BsmtFinType2", "GarageType", "GarageFinish", "GarageCond", "GarageQual"]
data_explore[na_cols] = data_explore[na_cols].fillna("NA" )
tmp['SK_ID_CURR'].value_counts()<load_from_csv>
data_explore["Alley"].value_counts()
def get_combined_dataset() :
application_train = pd.read_csv('.. /input/application_train.csv')
 application_test = pd.read_csv('.. /input/application_test.csv')
 application=application_train.append(application_test, ignore_index=True,sort=False)
 application.set_index('SK_ID_CURR')
 return(application)
 def get_application_dataset() :
df = get_combined_dataset()
filteredColList =['NAME_TYPE_SUITE','NAME_INCOME_TYPE','NAME_EDUCATION_TYPE','NAME_FAMILY_STATUS','NAME_HOUSING_TYPE','OCCUPATION_TYPE',
'WEEKDAY_APPR_PROCESS_START','ORGANIZATION_TYPE','FONDKAPREMONT_MODE']
df = df[[x for x in list(df)if x not in filteredColList]]
oheCols = ['NAME_CONTRACT_TYPE','CODE_GENDER','FLAG_OWN_CAR','FLAG_OWN_REALTY','HOUSETYPE_MODE','WALLSMATERIAL_MODE','EMERGENCYSTATE_MODE']
df.loc[df.CODE_GENDER == 'XNA' ,'CODE_GENDER'] = 'F'
df.loc[(df.DAYS_EMPLOYED > 0),'DAYS_EMPLOYED'] = np.nan
df.loc[(df.REGION_RATING_CLIENT_W_CITY < 0),'REGION_RATING_CLIENT_W_CITY'] = np.nan
df.loc[(df.OBS_30_CNT_SOCIAL_CIRCLE > 10),'OBS_30_CNT_SOCIAL_CIRCLE'] = 10
df.loc[(df.DEF_30_CNT_SOCIAL_CIRCLE > 10),'DEF_30_CNT_SOCIAL_CIRCLE'] = 10
df.loc[(df.OBS_60_CNT_SOCIAL_CIRCLE > 10),'OBS_60_CNT_SOCIAL_CIRCLE'] = 10
df.loc[(df.DEF_60_CNT_SOCIAL_CIRCLE > 10),'DEF_60_CNT_SOCIAL_CIRCLE'] = 10
df.loc[(df.AMT_REQ_CREDIT_BUREAU_QRT > 10),'AMT_REQ_CREDIT_BUREAU_QRT'] = 10
df = pd.get_dummies(df,columns=oheCols)
 df['NEW_INCOME2Credit']=df['AMT_CREDIT']/df['AMT_INCOME_TOTAL']
df['NEW_Credit2ANNUITY']=df['AMT_ANNUITY']/df['AMT_CREDIT']
df['NEW_INCOME2ANNUITY']=df['AMT_ANNUITY']/df['AMT_INCOME_TOTAL']
df['NEW_DAYS_EMPLOYED2DAYS_BIRTH'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_AMT_INCOME_TOTAL2CNT_FAM_MEMBERS'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['NEW_CREDIT2GOODS'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] /(1 + df['CNT_CHILDREN'])
 df['NEW_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 df['NEW_SCORES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 df['NEW_SCORES_STD'] = df['NEW_SCORES_STD'].fillna(df['NEW_SCORES_STD'].mean())
 df['NEW_OWN_CAR_AGE2DAYS_BIRTH'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['NEW_OWN_CAR_AGE2DAYS_EMPLOYED'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['NEW_DAYS_LAST_PHONE_CHANGE2DAYS_BIRTH'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['NEW_DAYS_LAST_PHONE_CHANGE2DAYS_EMPLOYED'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
return(df)
 def transform_application(df):
logTransformation = ['AMT_INCOME_TOTAL','AMT_CREDIT','AMT_ANNUITY','AMT_GOODS_PRICE','NEW_AMT_INCOME_TOTAL2CNT_FAM_MEMBERS','NEW_INC_PER_CHLD']
df[logTransformation] = df[logTransformation].apply(lambda x : np.log(x+1),axis=1)
 sqrtTransformation = ['DAYS_BIRTH','DAYS_EMPLOYED','DAYS_REGISTRATION','DAYS_ID_PUBLISH','OWN_CAR_AGE','DAYS_LAST_PHONE_CHANGE','NEW_INCOME2Credit']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<load_from_csv>
num_imputer = SimpleImputer(strategy="mean")
 cat_imputer = SimpleImputer(strategy="most_frequent" )
def bureau_balance() :
df = pd.read_csv('.. /input/bureau_balance.csv')
 df1 = df.groupby(['SK_ID_BUREAU'] ).agg(
 {'MONTHS_BALANCE': min,
})
 df2 = df.groupby(['SK_ID_BUREAU'] ).agg(
 {'MONTHS_BALANCE': max,
} ).reset_index()
df2 = pd.merge(df2,df,on=['SK_ID_BUREAU','MONTHS_BALANCE'],how='inner')
 df2 = pd.crosstab(df2['SK_ID_BUREAU'], df2['STATUS'])
 df = pd.merge(df1,df2,on=['SK_ID_BUREAU'],how='left' ).reset_index()
df.columns = ['SK_ID_BUREAU','MONTHS_BALANCE','BB_S_0','BB_S_1','BB_S_2','BB_S_3','BB_S_4','BB_S_5','BB_S_C','BB_S_X']
return(df)
 def get_bureau_dataset() :
b = pd.read_csv('.. /input/bureau.csv')
 bb = bureau_balance()
df = pd.merge(b,bb,on='SK_ID_BUREAU',how='left')
 df.loc[(df.DAYS_CREDIT_ENDDATE < 0)|(df.DAYS_CREDIT_ENDDATE > 5000),'DAYS_CREDIT_ENDDATE'] = np.nan
df.loc[(df.DAYS_ENDDATE_FACT < -5000),'DAYS_ENDDATE_FACT'] = np.nan
df.loc[(df.AMT_CREDIT_MAX_OVERDUE > 40000),'AMT_CREDIT_MAX_OVERDUE'] = 40000
df.loc[(df.DAYS_CREDIT_UPDATE < -3000),'DAYS_CREDIT_UPDATE'] = np.nan
df.loc[(df.AMT_CREDIT_SUM_DEBT < 0),'AMT_CREDIT_SUM_DEBT'] = np.nan
df.loc[(df.AMT_CREDIT_SUM_LIMIT < 0),'AMT_CREDIT_SUM_LIMIT'] = np.nan
All = df.groupby(['SK_ID_CURR'] ).agg(
 {'DAYS_CREDIT': [min, max],
'CREDIT_DAY_OVERDUE':max,
'DAYS_CREDIT_ENDDATE':max,
'DAYS_ENDDATE_FACT':[min,max],
'AMT_CREDIT_MAX_OVERDUE':max,
'CNT_CREDIT_PROLONG':max,
'AMT_CREDIT_SUM':max,
'AMT_CREDIT_SUM_DEBT':max,
'AMT_CREDIT_SUM_LIMIT':max,
'DAYS_CREDIT_UPDATE':min,
'AMT_ANNUITY':max,
'MONTHS_BALANCE':min,
'BB_S_0':sum,
'BB_S_1':sum,
'BB_S_2':sum,
'BB_S_3':sum,
'BB_S_4':sum,
'BB_S_5':sum,
'BB_S_C':sum,
'BB_S_X':sum
})
 All.columns = ["_all_".join(x)for x in All.columns.ravel() ]
Active = df.query('CREDIT_ACTIVE == "Active"' ).groupby(['SK_ID_CURR'] ).agg(
 {'CREDIT_DAY_OVERDUE':max,
'AMT_CREDIT_MAX_OVERDUE': max,
'CNT_CREDIT_PROLONG':[max,sum],
'AMT_CREDIT_SUM':sum,
'AMT_CREDIT_SUM_DEBT':sum,
'AMT_CREDIT_SUM_LIMIT':sum,
'AMT_CREDIT_SUM_OVERDUE':sum,
'DAYS_CREDIT_UPDATE':min,
'AMT_ANNUITY':sum,
'MONTHS_BALANCE':min,
'BB_S_0':sum,
'BB_S_1':sum,
'BB_S_2':sum,
'BB_S_3':sum,
'BB_S_4':sum,
'BB_S_5':sum,
'BB_S_C':sum,
'BB_S_X':sum
})
 Active.columns = ["_act_".join(x)for x in Active.columns.ravel() ]
CREDIT_ACTIVE_ctab = pd.crosstab(df['SK_ID_CURR'], df['CREDIT_ACTIVE'] ).rename_axis(None, axis=1)
 dfs = [All,Active,CREDIT_ACTIVE_ctab]
df_final = reduce(lambda left,right: pd.merge(left,right,on='SK_ID_CURR',how='outer'), dfs)
 df_final.reset_index(inplace=True)
 return(df_final)
 CREDIT_ACTIVE_ctab = pd.crosstab(df['SK_ID_CURR'], df['CREDIT_ACTIVE'] ).rename_axis(None, axis=1)
 dfs = [All,Active,CREDIT_ACTIVE_ctab]
df_final = reduce(lambda left,right: pd.merge(left,right,on='SK_ID_CURR',how='outer'), dfs)
 df_final.reset_index(inplace=True)
 return(df_final)
 def bureau_newFeature(df):
df['AMT_CREDIT_SUM_sum2AMT_CREDIT_SUM_DEBT_sum'] = df['AMT_CREDIT_SUM_DEBT_act_sum']/df['AMT_CREDIT_SUM_act_sum']
df['AMT_CREDIT_SUM_sum2AMT_ANNUITY_sum'] = df['AMT_CREDIT_SUM_act_sum']/df['AMT_ANNUITY_act_sum']
df['AMT_CREDIT_SUM_DEBT_sum2AMT_ANNUITY_sum'] = df['AMT_CREDIT_SUM_DEBT_act_sum']/df['AMT_ANNUITY_act_sum']
df.replace([np.inf, -np.inf], np.nan,inplace=True)
 df.loc[df.AMT_CREDIT_SUM_sum2AMT_CREDIT_SUM_DEBT_sum>2,'AMT_CREDIT_SUM_sum2AMT_CREDIT_SUM_DEBT_sum'] = np.nan
df.loc[df.AMT_CREDIT_SUM_sum2AMT_ANNUITY_sum>120,'AMT_CREDIT_SUM_sum2AMT_ANNUITY_sum'] = np.nan
df.loc[df.AMT_CREDIT_SUM_DEBT_sum2AMT_ANNUITY_sum>80,'AMT_CREDIT_SUM_DEBT_sum2AMT_ANNUITY_sum'] = np.nan
return(df)
 def transform_bureau(df):
logTransformation = ['CREDIT_DAY_OVERDUE_all_max','AMT_CREDIT_MAX_OVERDUE_all_max','AMT_CREDIT_SUM_all_max','AMT_CREDIT_SUM_DEBT_all_max',
'AMT_CREDIT_SUM_LIMIT_all_max','AMT_ANNUITY_all_max','AMT_CREDIT_MAX_OVERDUE_act_max','AMT_CREDIT_SUM_act_sum',
'AMT_CREDIT_SUM_DEBT_act_sum','AMT_CREDIT_SUM_LIMIT_act_sum','AMT_CREDIT_SUM_OVERDUE_act_sum','AMT_ANNUITY_act_sum']
df[logTransformation] = df[logTransformation].apply(lambda x : np.log(x+1),axis=1)
 sartLogTransformation = ['CREDIT_DAY_OVERDUE_act_max','DAYS_CREDIT_UPDATE_act_min']
df[sartLogTransformation] = df[sartLogTransformation].apply(lambda x : np.sqrt(np.log(np.abs(x+1))),axis=1)
 sqrtTransformation = ['DAYS_CREDIT_all_min','DAYS_CREDIT_all_max','DAYS_CREDIT_ENDDATE_all_max','DAYS_ENDDATE_FACT_all_min','DAYS_ENDDATE_FACT_all_max',
'DAYS_CREDIT_UPDATE_all_min','MONTHS_BALANCE_all_min','MONTHS_BALANCE_act_min']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<feature_engineering>
num_nans = ['LotFrontage', 'MasVnrArea', 'GarageYrBlt']
cat_nans = ['MasVnrType', 'Electrical', 'FireplaceQu']
data_explore[num_nans] = num_imputer.fit_transform(data[num_nans])
 data_explore[cat_nans] = cat_imputer.fit_transform(data[cat_nans] )
def get_previous_application() :
df = pd.read_csv('.. /input/previous_application.csv')
 df.loc[df.DAYS_FIRST_DRAWING >0,'DAYS_FIRST_DRAWING'] = np.nan
df.loc[df.DAYS_FIRST_DUE >0,'DAYS_FIRST_DUE'] = np.nan
df.loc[df.DAYS_LAST_DUE_1ST_VERSION >2000,'DAYS_LAST_DUE_1ST_VERSION'] = np.nan
df.loc[df.DAYS_LAST_DUE >3000,'DAYS_LAST_DUE'] = np.nan
df.loc[df.DAYS_TERMINATION >3000,'DAYS_TERMINATION'] = np.nan
dff = df.query('FLAG_LAST_APPL_PER_CONTRACT == "Y" and NFLAG_LAST_APPL_IN_DAY == 1')
 NAME_CONTRACT_STATUS_ctab = pd.crosstab(df['SK_ID_CURR'], df['NAME_CONTRACT_STATUS'])
 df['AMT_ANNUITY2AMT_CREDIT'] = df['AMT_ANNUITY']/df['AMT_CREDIT']
df['AMT_APPLICATION2AMT_CREDIT'] = df['AMT_APPLICATION']/df['AMT_CREDIT']
df['AMT_GOODS_PRICE2AMT_CREDIT'] = df['AMT_GOODS_PRICE']/df['AMT_CREDIT']
NAME_CONTRACT_STATUS_ctab = pd.crosstab(df['SK_ID_CURR'], df['NAME_CONTRACT_STATUS'])
 df_grouped = df.query('NAME_CONTRACT_STATUS != "Refused" and FLAG_LAST_APPL_PER_CONTRACT == "Y" and NFLAG_LAST_APPL_IN_DAY == 1')\ 
.groupby(['SK_ID_CURR'])\ 
.agg(
 {'AMT_ANNUITY':max,
'AMT_APPLICATION':max,
'AMT_CREDIT':max,
'AMT_DOWN_PAYMENT':max,
'AMT_GOODS_PRICE':max,
'RATE_DOWN_PAYMENT':[min, max,'mean'],
'RATE_INTEREST_PRIMARY':[min, max,'mean'],
'RATE_INTEREST_PRIVILEGED':[min, max,'mean'],
'DAYS_DECISION':[min, max,'mean'],
'CNT_PAYMENT':[min, max,'mean'],
'DAYS_FIRST_DRAWING':min,
'DAYS_FIRST_DUE':[min, max],
'DAYS_LAST_DUE_1ST_VERSION':[min, max],
'DAYS_LAST_DUE':[min, max],
'DAYS_TERMINATION':[min, max],
'NFLAG_INSURED_ON_APPROVAL':sum
})
 df_final = pd.merge(df_grouped,NAME_CONTRACT_STATUS_ctab,on='SK_ID_CURR',how='outer')
 df_final.reset_index(inplace=True)
 df_final.columns = ['SK_ID_CURR','AMT_ANNUITY_max','AMT_APPLICATION_max','AMT_CREDIT_max','AMT_DOWN_PAYMENT_max','AMT_GOODS_PRICE_max',
'RATE_DOWN_PAYMENT_min','RATE_DOWN_PAYMENT_max','RATE_INTEREST_PRIMARY_min','RATE_INTEREST_PRIMARY_max',
'RATE_INTEREST_PRIVILEGED_min','RATE_INTEREST_PRIVILEGED_max','DAYS_DECISION_min','DAYS_DECISION_max','CNT_PAYMENT_min',
'CNT_PAYMENT_max','DAYS_FIRST_DRAWING_min','DAYS_FIRST_DUE_min','DAYS_FIRST_DUE_max',
'DAYS_LAST_DUE_1ST_VERSION_min','DAYS_LAST_DUE_1ST_VERSION_max','DAYS_LAST_DUE_min','DAYS_LAST_DUE_max','DAYS_TERMINATION_min',
'DAYS_TERMINATION_max','NFLAG_INSURED_ON_APPROVAL_sum','Approved','Canceled','Refused','Unused_offer']
df_final.head()
return(df_final)
 def transform_previous_application(df):
logTransformation = ['AMT_ANNUITY_max','AMT_APPLICATION_max','AMT_CREDIT_max', 'AMT_DOWN_PAYMENT_max','AMT_GOODS_PRICE_max']
df[logTransformation] = df[logTransformation].apply(lambda x : np.log(x+1),axis=1)
 sqrtTransformation = ['DAYS_DECISION_min','DAYS_DECISION_max','DAYS_FIRST_DRAWING_min','DAYS_FIRST_DUE_min','DAYS_FIRST_DUE_max','DAYS_LAST_DUE_min',
'DAYS_LAST_DUE_max','DAYS_TERMINATION_min','DAYS_TERMINATION_max']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<feature_engineering>
nulls = data_explore.isna().sum()
nan_cols = nulls[nulls>0].index
nan_cols
def get_previous_application() :
df = pd.read_csv('.. /input/previous_application.csv')
 df.loc[df.DAYS_FIRST_DRAWING >0,'DAYS_FIRST_DRAWING'] = np.nan
df.loc[df.DAYS_FIRST_DUE >0,'DAYS_FIRST_DUE'] = np.nan
df.loc[df.DAYS_LAST_DUE_1ST_VERSION >2000,'DAYS_LAST_DUE_1ST_VERSION'] = np.nan
df.loc[df.DAYS_LAST_DUE >3000,'DAYS_LAST_DUE'] = np.nan
df.loc[df.DAYS_TERMINATION >3000,'DAYS_TERMINATION'] = np.nan
df['AMT_ANNUITY2AMT_CREDIT'] = df['AMT_ANNUITY']/df['AMT_CREDIT']
df['AMT_APPLICATION2AMT_CREDIT'] = df['AMT_APPLICATION']/df['AMT_CREDIT']
df['AMT_GOODS_PRICE2AMT_CREDIT'] = df['AMT_GOODS_PRICE']/df['AMT_CREDIT']
NAME_CONTRACT_STATUS_ctab = pd.crosstab(df['SK_ID_CURR'], df['NAME_CONTRACT_STATUS'])
 df_grouped = df.query('NAME_CONTRACT_STATUS != "Refused" and FLAG_LAST_APPL_PER_CONTRACT == "Y" and NFLAG_LAST_APPL_IN_DAY == 1')\ 
.groupby(['SK_ID_CURR'])\ 
.agg(
 {'AMT_ANNUITY':max,
'AMT_APPLICATION':max,
'AMT_CREDIT':max,
'AMT_DOWN_PAYMENT':max,
'AMT_GOODS_PRICE':max,
'RATE_DOWN_PAYMENT':[min, max,'mean'],
'RATE_INTEREST_PRIMARY':[min, max,'mean'],
'RATE_INTEREST_PRIVILEGED':[min, max,'mean'],
'DAYS_DECISION':[min, max,'mean'],
'CNT_PAYMENT':[min, max,'mean'],
'DAYS_FIRST_DRAWING':min,
'DAYS_FIRST_DUE':[min, max],
'DAYS_LAST_DUE_1ST_VERSION':[min, max],
'DAYS_LAST_DUE':[min, max],
'DAYS_TERMINATION':[min, max],
'NFLAG_INSURED_ON_APPROVAL':sum
})
 df_final = pd.merge(df_grouped,NAME_CONTRACT_STATUS_ctab,on='SK_ID_CURR',how='outer')
 df_final.reset_index(inplace=True)
 return(df_final)
 def transform_previous_application(df):
logTransformation = ['AMT_ANNUITY_max','AMT_APPLICATION_max','AMT_CREDIT_max', 'AMT_DOWN_PAYMENT_max','AMT_GOODS_PRICE_max']
df[logTransformation] = df[logTransformation].apply(lambda x : np.log(x+1),axis=1)
 sqrtTransformation = ['DAYS_DECISION_min','DAYS_DECISION_max','DAYS_FIRST_DRAWING_min','DAYS_FIRST_DUE_min','DAYS_FIRST_DUE_max','DAYS_LAST_DUE_min',
'DAYS_LAST_DUE_max','DAYS_TERMINATION_min','DAYS_TERMINATION_max']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<load_from_csv>
data_explore['MSSubClass'] = data_explore['MSSubClass'].astype(str)
 cat_attrs = []
num_attrs = []
columns = list(data_explore.columns)
 for col in columns:
if data_explore[col].dtype=='O':
cat_attrs.append(col)
 else:
num_attrs.append(col )
def get_POS_CASH_balance() :
POS_CASH_balance = pd.read_csv('.. /input/POS_CASH_balance.csv')
 Closed_Loans = POS_CASH_balance[POS_CASH_balance['SK_ID_PREV'].isin(POS_CASH_balance.query('NAME_CONTRACT_STATUS == "Completed"' ).SK_ID_PREV)]
Active_Loans = POS_CASH_balance[~POS_CASH_balance['SK_ID_PREV'].isin(POS_CASH_balance.query('NAME_CONTRACT_STATUS == "Active" and MONTHS_BALANCE == -1' ).SK_ID_PREV)]
Active = Active_Loans.groupby(['SK_ID_CURR'] ).agg(
 { 'MONTHS_BALANCE':min,
'CNT_INSTALMENT':[min,max],
'CNT_INSTALMENT_FUTURE':[min,max]
})
 Closed = Closed_Loans.groupby(['SK_ID_CURR'] ).agg(
 { 'MONTHS_BALANCE':[min,max],
'CNT_INSTALMENT':max
})
 NAME_CONTRACT_STATUS = POS_CASH_balance.query('(NAME_CONTRACT_STATUS == "Completed")or(NAME_CONTRACT_STATUS == "Active" and MONTHS_BALANCE == -1)')[['SK_ID_PREV','SK_ID_CURR','NAME_CONTRACT_STATUS']].drop_duplicates()
NAME_CONTRACT_STATUS_ctab = pd.crosstab(NAME_CONTRACT_STATUS['SK_ID_CURR'], NAME_CONTRACT_STATUS['NAME_CONTRACT_STATUS'])
 dfs = [NAME_CONTRACT_STATUS_ctab,Active,Closed]
df_final = reduce(lambda left,right: pd.merge(left,right,on='SK_ID_CURR',how='outer'), dfs)
 df_final.reset_index(inplace=True)
 df_final.columns = ['SK_ID_CURR','Active','Completed','MONTHS_BALANCE_A_min','CNT_INSTALMENT_A_min','CNT_INSTALMENT_A_max','CNT_INSTALMENT_FUTURE_A_min',
'CNT_INSTALMENT_FUTURE_max','MONTHS_BALANCE_C_min','MONTHS_BALANCE_C_max','CNT_INSTALMENT_C_max']
return(df_final)
 def transform_POS_CASH_balance(df):
sqrtTransformation = ['CNT_INSTALMENT_A_min','CNT_INSTALMENT_A_max','CNT_INSTALMENT_FUTURE_A_min','CNT_INSTALMENT_FUTURE_max','CNT_INSTALMENT_C_max']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<load_from_csv>
Q1 = data_explore.quantile(0.25)
 Q3 = data_explore.quantile(0.75)
 IQR = Q3 - Q1
outliers =(( data_explore <(Q1 - 1.5 * IQR)) |(data_explore >(Q3 + 1.5 * IQR)) ).sum()
outliers[outliers>0]
def get_installment_payments() :
instalment_payments = pd.read_csv('.. /input/installments_payments.csv')
 instalment_payments['MONTH']=(instalment_payments['DAYS_INSTALMENT']/30 ).astype(int)
 Active = instalment_payments.query('MONTH == -1' ).groupby('SK_ID_CURR' ).agg({
'NUM_INSTALMENT_VERSION':max,
'NUM_INSTALMENT_NUMBER':max,
'AMT_INSTALMENT':sum,
'AMT_PAYMENT':sum
})
 Closed = instalment_payments.groupby('SK_ID_CURR' ).agg({
'NUM_INSTALMENT_VERSION':max,
'NUM_INSTALMENT_NUMBER':max,
'DAYS_INSTALMENT':min,
'AMT_INSTALMENT':[max,min]
})
 df_final = pd.merge(Active,Closed,on='SK_ID_CURR',how='outer')
 df_final.reset_index(inplace=True)
 df_final.columns=['SK_ID_CURR','NUM_INSTALMENT_VERSION_A_max','NUM_INSTALMENT_NUMBER_A_max','AMT_INSTALMENT_A_sum','AMT_PAYMENT_A_sum',
'NUM_INSTALMENT_VERSION_C_max','NUM_INSTALMENT_NUMBER_C_max','DAYS_INSTALMENT_C_min','AMT_INSTALMENT_C_max','AMT_INSTALMENT_c_min']
return(df_final)
 def transform_installment_payments(df):
logTransformation = ['AMT_INSTALMENT_A_sum','AMT_PAYMENT_A_sum','AMT_INSTALMENT_C_max','AMT_INSTALMENT_c_min']
df[logTransformation] = df[logTransformation].apply(lambda x : np.log(x+1),axis=1)
 sqrtTransformation = ['NUM_INSTALMENT_VERSION_A_max','NUM_INSTALMENT_NUMBER_A_max','NUM_INSTALMENT_VERSION_C_max','NUM_INSTALMENT_NUMBER_C_max',
'DAYS_INSTALMENT_C_min']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<load_from_csv>
corr_matrix['SalePrice'].sort_values(ascending=False )
def get_credit_card_balance() :
df = pd.read_csv('.. /input/credit_card_balance.csv')
 dfa = df.query('NAME_CONTRACT_STATUS == "Active"' ).groupby(['SK_ID_CURR','MONTHS_BALANCE'] ).agg({
'AMT_BALANCE':sum,
'AMT_CREDIT_LIMIT_ACTUAL':sum,
'AMT_DRAWINGS_ATM_CURRENT':sum,
'AMT_DRAWINGS_CURRENT':sum,
'AMT_DRAWINGS_OTHER_CURRENT':sum,
'AMT_DRAWINGS_POS_CURRENT':sum,
'AMT_INST_MIN_REGULARITY':sum,
'AMT_PAYMENT_CURRENT':sum,
'AMT_PAYMENT_TOTAL_CURRENT':sum,
'AMT_RECEIVABLE_PRINCIPAL':sum,
'AMT_RECIVABLE':sum,
'AMT_TOTAL_RECEIVABLE':sum,
'CNT_DRAWINGS_ATM_CURRENT':sum,
'CNT_DRAWINGS_CURRENT':sum,
'CNT_DRAWINGS_POS_CURRENT':sum,
'CNT_DRAWINGS_OTHER_CURRENT':sum,
'CNT_INSTALMENT_MATURE_CUM':sum,
'SK_DPD':max,
'SK_DPD_DEF':max
})
 dfa = dfa.groupby(['SK_ID_CURR'] ).agg(['mean',max,min,'std'])
 dfa.columns = ["_".join(x)for x in dfa.columns.ravel() ]
dfp = pd.pivot_table(df.query('NAME_CONTRACT_STATUS == ["Active","Completed","Demand"]' ).groupby(['SK_ID_CURR','SK_ID_PREV'] ).agg({'MONTHS_BALANCE':max} ).merge(df[['SK_ID_CURR','SK_ID_PREV','MONTHS_BALANCE','NAME_CONTRACT_STATUS']],on=['SK_ID_CURR','SK_ID_PREV','MONTHS_BALANCE']),
values='SK_ID_PREV',index=['SK_ID_CURR'],columns=['NAME_CONTRACT_STATUS'],aggfunc=np.size ).reset_index().rename_axis(None, axis=1)
 dfp.fillna(0,inplace=True)
 dfcc = dfa.merge(dfp,on='SK_ID_CURR')
 dfcc[dfcc < 0] = 0
return(dfcc)
 def transform_credit_card_balance(df):
logTransformation = ['AMT_BALANCE_mean','AMT_BALANCE_max','AMT_BALANCE_min','AMT_BALANCE_std',
'AMT_CREDIT_LIMIT_ACTUAL_mean','AMT_CREDIT_LIMIT_ACTUAL_max','AMT_CREDIT_LIMIT_ACTUAL_min','AMT_CREDIT_LIMIT_ACTUAL_std',
'AMT_DRAWINGS_ATM_CURRENT_mean','AMT_DRAWINGS_ATM_CURRENT_max','AMT_DRAWINGS_ATM_CURRENT_min','AMT_DRAWINGS_ATM_CURRENT_std',
'AMT_DRAWINGS_CURRENT_mean','AMT_DRAWINGS_CURRENT_max','AMT_DRAWINGS_CURRENT_min','AMT_DRAWINGS_CURRENT_std',
'AMT_DRAWINGS_OTHER_CURRENT_mean','AMT_DRAWINGS_OTHER_CURRENT_max','AMT_DRAWINGS_OTHER_CURRENT_std','AMT_DRAWINGS_POS_CURRENT_mean',
'AMT_DRAWINGS_POS_CURRENT_max','AMT_DRAWINGS_POS_CURRENT_min','AMT_DRAWINGS_POS_CURRENT_std','AMT_INST_MIN_REGULARITY_mean',
'AMT_INST_MIN_REGULARITY_max','AMT_INST_MIN_REGULARITY_min','AMT_INST_MIN_REGULARITY_std','AMT_PAYMENT_CURRENT_mean',
'AMT_PAYMENT_CURRENT_max','AMT_PAYMENT_CURRENT_min','AMT_PAYMENT_CURRENT_std','AMT_PAYMENT_TOTAL_CURRENT_mean','AMT_PAYMENT_TOTAL_CURRENT_max',
'AMT_PAYMENT_TOTAL_CURRENT_min','AMT_PAYMENT_TOTAL_CURRENT_std','AMT_RECEIVABLE_PRINCIPAL_mean','AMT_RECEIVABLE_PRINCIPAL_max','AMT_RECEIVABLE_PRINCIPAL_min',
'AMT_RECEIVABLE_PRINCIPAL_std','AMT_RECIVABLE_mean','AMT_RECIVABLE_max','AMT_RECIVABLE_min','AMT_RECIVABLE_std','AMT_TOTAL_RECEIVABLE_mean',
'AMT_TOTAL_RECEIVABLE_max','AMT_TOTAL_RECEIVABLE_min','AMT_TOTAL_RECEIVABLE_std']
df[logTransformation] = df[logTransformation].apply(lambda x : np.log(x+1),axis=1)
 sqrtTransformation = ['CNT_DRAWINGS_ATM_CURRENT_mean','CNT_DRAWINGS_ATM_CURRENT_max','CNT_DRAWINGS_ATM_CURRENT_min',
'CNT_DRAWINGS_ATM_CURRENT_std','CNT_DRAWINGS_CURRENT_mean','CNT_DRAWINGS_CURRENT_max','CNT_DRAWINGS_CURRENT_min',
'CNT_DRAWINGS_CURRENT_std','CNT_DRAWINGS_POS_CURRENT_mean','CNT_DRAWINGS_POS_CURRENT_max','CNT_DRAWINGS_POS_CURRENT_min',
'CNT_DRAWINGS_POS_CURRENT_std','CNT_INSTALMENT_MATURE_CUM_mean','CNT_INSTALMENT_MATURE_CUM_max','CNT_INSTALMENT_MATURE_CUM_min',
'CNT_INSTALMENT_MATURE_CUM_std','SK_DPD_mean','SK_DPD_std','SK_DPD_DEF_mean','SK_DPD_DEF_max','SK_DPD_DEF_std']
df[sqrtTransformation] = df[sqrtTransformation].apply(lambda x: np.sqrt(np.abs(x)) ,axis=1)
 return(df )<merge>
data_explore['GarageCars'].value_counts()
def getFinalDataSet() :
application = get_application_dataset()
application = transform_application(get_application_dataset())
 bureau = transform_bureau(get_bureau_dataset())
 previous_application = get_previous_application()
POS_CASH_balance = transform_POS_CASH_balance(get_POS_CASH_balance())
 installment_payments = transform_installment_payments(get_installment_payments())
 credit_card_balance = transform_credit_card_balance(get_credit_card_balance())
 dfs = [application, bureau, previous_application, POS_CASH_balance, installment_payments, credit_card_balance]
df = reduce(lambda left,right: pd.merge(left,right,on='SK_ID_CURR',how='left'), dfs)
 return(df )<normalization>
X = data.drop(columns=['SalePrice'], axis=1)
 y = data['SalePrice'].copy()
def scaleNfillna(df):
df.replace([np.inf, -np.inf], np.nan,inplace=True)
 df.fillna(0,inplace=True)
 scaler = MinMaxScaler()
df = scaler.fit_transform(df)
 return(df )<import_modules>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 y_log_train = np.log(y_train)
 y_log_test = np.log(y_test )
from keras.models import Sequential, Model
from keras.layers import Input, Dense, Dropout, BatchNormalization
from sklearn.model_selection import train_test_split
from sklearn import metrics<define_search_model>
na_cols = ["Alley", "BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1", "BsmtFinType2", "GarageType", "GarageFinish", "GarageCond", "GarageQual", "PoolQC", "Fence", "MiscFeature"]
cat_attrs = [cat for cat in cat_attrs if not cat in na_cols]
num_attrs.remove('SalePrice' )
def kfold_lightgbm(df, num_folds, stratified = False, debug= False):
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['SK_ID_CURR','TARGET']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y= train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1, 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 200)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
if not debug:
test_df['TARGET'] = sub_preds
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
display_importances(feature_importance_df)
 return(test_df[['SK_ID_CURR', 'TARGET']])
 def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:40].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances01.png' )<choose_model_class>
from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PowerTransformer, OneHotEncoder
def ANN(X_train,y_train,X_test,y_test,L_dim,num_epochs = 2):
ann = Sequential()
ann.add(Dense(L_dim[0], input_dim=X_train.shape[1], activation='relu'))
ann.add(BatchNormalization())
 ann.add(Dropout(0.2))
ann.add(Dense(L_dim[1], activation='relu'))
ann.add(BatchNormalization())
 ann.add(Dropout(0.2))
ann.add(Dense(L_dim[2], activation='relu'))
ann.add(BatchNormalization())
 ann.add(Dropout(0.2))
ann.add(Dense(L_dim[3], activation='relu'))
ann.add(BatchNormalization())
 ann.add(Dropout(0.2))
ann.add(Dense(L_dim[4], activation='relu'))
ann.add(BatchNormalization())
 ann.add(Dense(1, activation='sigmoid'))
ann.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
 ann.summary()
ann.fit(X_train,
y_train,
epochs=num_epochs,
batch_size=32,
validation_data=(X_test,y_test),
shuffle=True,
verbose=1)
 y_pred = ann.predict(X_test)
 cm = metrics.confusion_matrix(y_test, y_pred > 0.5)
 print(cm)
 fpr, tpr, thresholds = metrics.roc_curve(y_test+1, y_pred, pos_label=2)
 print(metrics.auc(fpr, tpr))
return(ann )<categorify>
num_pipeline = Pipeline([('imputer', SimpleImputer(strategy="mean")) , 
('transformer', PowerTransformer(method='yeo-johnson', standardize=True)) ])
 cat_pipeline_1 = Pipeline([('cat_na_fill', SimpleImputer(strategy="constant", fill_value='NA')) , 
('encoder', OneHotEncoder(handle_unknown='ignore')) ])
 cat_pipeline_2 = Pipeline([('cat_nan_fill', SimpleImputer(strategy="most_frequent")) , 
('encoder', OneHotEncoder(handle_unknown='ignore')) ] )
def AE(X):
input_data = Input(shape=(X.shape[1],))
encoded = Dense(128, activation='relu' )(input_data)
 encoded = BatchNormalization()(encoded)
 encoded = Dense(32, activation='relu' )(encoded)
 encoded = BatchNormalization()(encoded)
 encoded = Dense(16, activation='relu' )(encoded)
 encoded = BatchNormalization(name='encoded_layer' )(encoded)
 decoded = Dense(32, activation='relu' )(encoded)
 decoded = BatchNormalization()(decoded)
 decoded = Dense(64, activation='relu' )(decoded)
 decoded = BatchNormalization()(decoded)
 decoded = Dense(X.shape[1], activation='linear' )(encoded)
 autoencoder = Model(input_data, decoded)
 autoencoder.compile(optimizer='adadelta', loss='mean_squared_error')
 autoencoder.fit(X, X,epochs=10, batch_size=32,shuffle=True)
 return(autoencoder )<save_to_csv>
pre_process = ColumnTransformer([('drop_id', 'drop', ['Id']), 
('cat_pipeline_1', cat_pipeline_1, na_cols), 
('cat_pipeline_2', cat_pipeline_2, cat_attrs), 
('num_pipeline', num_pipeline, num_attrs)], remainder='passthrough' )
def submitLGBM(debug=True):
df = getFinalDataSet()
submission = kfold_lightgbm(df, 4, stratified = True)
 if not debug:
print("writing the submission file")
 submission.to_csv('submission_1.csv', index=False )<split>
X_train_transformed = pre_process.fit_transform(X_train)
 X_test_transformed = pre_process.transform(X_test )
def ANN_prediction(df):
feats =[x for x in list(df)if x not in ['SK_ID_CURR','TARGET']]
df[feats] = scaleNfillna(df[feats])
 X = df.loc[df['TARGET'].notnull() ,feats].values
y = df[df['TARGET'].notnull() ].TARGET.values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=42)
 ros = RandomOverSampler(random_state=0, sampling_strategy=0.4)
 X_resampled, y_resampled = ros.fit_sample(X_train, y_train)
 L_dim =(128,64,32,16,8)
 ann = ANN(X_resampled, y_resampled,X_test,y_test,L_dim,5)
 annPred = pd.DataFrame()
annPred['SK_ID_CURR'] = df['SK_ID_CURR']
annPred['annPred'] = ann.predict(df[feats].values)
 return(annPred)
 def AE_prediction(df):
feats =[x for x in list(df)if x not in ['SK_ID_CURR','TARGET']]
df.loc[:,feats] = scaleNfillna(df.loc[:,feats])
 X = df[feats].values
ae = AE(X)
 intermediate_layer_model = Model(inputs=ae.input, outputs=ae.get_layer('encoded_layer' ).output)
 aePred = pd.DataFrame(columns=['SK_ID_CURR']+['ae'+str(x)for x in range(1,17)])
 aePred['SK_ID_CURR'] = df['SK_ID_CURR']
aePred.loc[:,'ae1':'ae16'] = intermediate_layer_model.predict(X)
 return(aePred)
 def submit() :
df = getFinalDataSet()
annPred = ANN_prediction(df)
 df = pd.merge(df,annPred,on='SK_ID_CURR',how='left')
 submission = kfold_lightgbm(df, 4, stratified = True)
 print("writing the submission file")
 submission.to_csv('submission_1.csv', index=False)
 def submitWeighted() :
df = getFinalDataSet()
annPred = ANN_prediction(df)
 lgbPred = kfold_lightgbm(df, 4, stratified = True)
 weighted = pd.merge(lgbPred,annPred,on='SK_ID_CURR',how='left')
 weighted['TARGET_avg'] =(0.8)*weighted['TARGET'] +(0.2)*weighted['annPred']
submission = weighted.loc[:,['SK_ID_CURR','TARGET_avg']]
submission.rename(columns={'TARGET_avg': 'TARGET'}, inplace=True)
 submission.to_csv('submission_1.csv', index=False )<find_best_model_class>
oh_na_cols = list(pre_process.transformers_[1][1]['encoder'].get_feature_names(na_cols))
oh_nan_cols = list(pre_process.transformers_[2][1]['encoder'].get_feature_names(cat_attrs))
feature_columns = oh_na_cols+oh_nan_cols + num_attrs
def ilo() :
df = getFinalDataSet()
feats =[x for x in list(df)if x not in ['SK_ID_CURR','TARGET']]
df[feats] = scaleNfillna(df[feats])
 X = df[feats].values
ae = AE(X)
 intermediate_layer_model = Model(inputs=ae.input, outputs=ae.get_layer('encoded_layer' ).output)
 X = intermediate_layer_model.predict(X[df["TARGET"].notnull() ])
 y = df[df['TARGET'].notnull() ].TARGET.values
kfold_lightgbm(X,y, 2, False, False )<load_from_csv>
from sklearn.model_selection import GridSearchCV, KFold
data = pd.read_csv(PATH+"/application_train.csv")
 test = pd.read_csv(PATH+"/application_test.csv")
 bureau = pd.read_csv(PATH+"/bureau.csv", nrows=50000)
 bureau_balance = pd.read_csv(PATH+"/bureau_balance.csv", nrows=50000)
 credit_card_balance = pd.read_csv(PATH+"/credit_card_balance.csv", nrows=50000)
 installments_payments = pd.read_csv(PATH+"/installments_payments.csv", nrows=50000)
 previous_application = pd.read_csv(PATH+"/previous_application.csv", nrows=50000)
 POS_CASH_balance = pd.read_csv(PATH+"/POS_CASH_balance.csv", nrows=50000 )<import_modules>
kf = KFold(n_splits=5, shuffle=True, random_state=42 )
import numpy as np<import_modules>
from sklearn.linear_model import ElasticNet
import pandas as pd<import_modules>
elastic_net_grid_param = [{'l1_ratio': list(np.linspace(0, 1, 10)) , 'alpha': [0.0001, 0.005, 0.001, 0.005, 0.01, 0.05, 0.1]}]
elastic_net_grid_search = GridSearchCV(ElasticNet(random_state=42), elastic_net_grid_param, cv=kf, scoring='neg_root_mean_squared_error', return_train_score=True, n_jobs=-1)
 elastic_net_grid_search.fit(X_train_transformed, y_log_train )
import pandas as pd<import_modules>
train_results=[]
train_results.append(['Elastic Net', elastic_net_grid_search.best_params_, -elastic_net_grid_search.best_score_])
 elastic_net_grid_search.best_params_, -elastic_net_grid_search.best_score_
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns<define_variables>
best_elastic_net_reg = elastic_net_grid_search.best_estimator_
best_elastic_net_reg
data['DAYS_EMPLOYED']<categorify>
from sklearn.svm import SVR
data['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True )<filter>
svr_grid_param = [{'C':list(np.linspace(0.1, 1, 10)) , 'epsilon':[0.01, 0.05, 0.1, 0.5, 1]}]
svr_grid_search = GridSearchCV(SVR(kernel="poly", degree=2), svr_grid_param, cv=kf, scoring="neg_root_mean_squared_error", return_train_score=True, n_jobs=-1)
 svr_grid_search.fit(X_train_transformed, y_log_train )
data['CODE_GENDER'].loc[data['CODE_GENDER']=='XNA']<define_variables>
train_results.append(['SVR', svr_grid_search.best_params_, -svr_grid_search.best_score_])
 svr_grid_search.best_params_, -svr_grid_search.best_score_
data['CODE_GENDER'].replace({'XNA': 'F'}, inplace=True )<filter>
best_svr_reg = svr_grid_search.best_estimator_
best_svr_reg
data['CODE_GENDER'].loc[data['CODE_GENDER']=='F']<categorify>
from sklearn.ensemble import RandomForestRegressor
data['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True )<feature_engineering>
from sklearn.ensemble import RandomForestRegressor
data['YEARS_BUILD_CREDIT'] = data['AMT_CREDIT']/data['YEARS_BUILD_AVG']<feature_engineering>
rf_grid_param = [{'max_features':[0.2, 0.4, 0.6, 'auto'], 'max_depth':[8, 12, 16, 20]}]
rf_grid_search = GridSearchCV(RandomForestRegressor(n_estimators=300, random_state=42, n_jobs=-1), rf_grid_param, cv=kf, scoring='neg_root_mean_squared_error', return_train_score=True, n_jobs=-1)
 rf_grid_search.fit(X_train_transformed, y_log_train )
data['Annuity_Income'] = data['AMT_ANNUITY']/data['AMT_INCOME_TOTAL']<feature_engineering>
train_results.append(['Random Forest', rf_grid_search.best_params_, -rf_grid_search.best_score_])
 rf_grid_search.best_params_, -rf_grid_search.best_score_
data['Income_Cred'] = data['AMT_CREDIT']/data['AMT_INCOME_TOTAL']<feature_engineering>
best_rf_reg = rf_grid_search.best_estimator_
best_rf_reg
data['EMP_AGE'] = data['DAYS_EMPLOYED']/data['DAYS_BIRTH']<feature_engineering>
from xgboost import XGBRegressor
data['Income_PP'] = data['AMT_INCOME_TOTAL']/data['CNT_FAM_MEMBERS']<feature_engineering>
xgb_grid_parm=[{'max_depth':[4, 6, 8, 12], 'subsample':[0.5, 0.75, 1.0]}]
xgb_grid_search = GridSearchCV(XGBRegressor(objective='reg:squarederror', n_estimators=300, learning_rate=0.1, random_state=42, n_jobs=-1), xgb_grid_parm, cv=kf, scoring="neg_root_mean_squared_error", return_train_score=True, n_jobs=-1)
 xgb_grid_search.fit(X_train_transformed, y_log_train )
data['CHILDREN_RATIO'] =(1 + data['CNT_CHILDREN'])/ data['CNT_FAM_MEMBERS']<feature_engineering>
train_results.append(['XGBoost', xgb_grid_search.best_params_, -xgb_grid_search.best_score_])
 xgb_grid_search.best_params_, -xgb_grid_search.best_score_
data['PAYMENTS'] = data['AMT_ANNUITY']/ data['AMT_CREDIT']<feature_engineering>
cvres = xgb_grid_search.cv_results_
for train_mean_score, test_mean_score, params in zip(cvres["mean_train_score"], cvres["mean_test_score"], cvres["params"]):
print(-train_mean_score, -test_mean_score, params )
data['NEW_CREDIT_TO_GOODS_RATIO'] = data['AMT_CREDIT'] / data['AMT_GOODS_PRICE']
data['GOODS_INCOME'] = data['AMT_GOODS_PRICE']/data['AMT_INCOME_TOTAL']<feature_engineering>
best_xgb_reg = xgb_grid_search.best_estimator_
best_xgb_reg
data['Ext_source_mult'] = data['EXT_SOURCE_1'] * data['EXT_SOURCE_2'] * data['EXT_SOURCE_3']
data['Ext_SOURCE_MEAN'] = data[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis = 1)
 data['Ext_SOURCE_SD'] = data[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis = 1 )<define_variables>
from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import ElasticNetCV
columns = ['Annuity_Income', 'Income_Cred', 'EMP_AGE', 'Income_PP'] 
<feature_engineering>
base_estimators = [('elastic_net', best_elastic_net_reg),('svr', best_svr_reg),('rf', best_rf_reg),('xgb', best_xgb_reg)]
stack_reg = StackingRegressor(estimators=base_estimators, final_estimator=ElasticNetCV(l1_ratio=[.1,.5,.7,.9,.95,.99, 1], random_state=42), cv=kf, passthrough=False, n_jobs=-1)
 stack_reg.fit(X_train_transformed, y_log_train )
test['CODE_GENDER'].replace({'XNA': 'F'}, inplace=True)
 test['YEARS_BUILD_CREDIT'] = test['AMT_CREDIT']/test['YEARS_BUILD_AVG']
test['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 test['Annuity_Income'] = test['AMT_ANNUITY']/test['AMT_INCOME_TOTAL']
test['Income_Cred'] = test['AMT_CREDIT']/test['AMT_INCOME_TOTAL']
test['EMP_AGE'] = test['DAYS_EMPLOYED']/test['DAYS_BIRTH']
test['Income_PP'] = test['AMT_INCOME_TOTAL']/test['CNT_FAM_MEMBERS']
test['CHILDREN_RATIO'] =(1 + test['CNT_CHILDREN'])/ test['CNT_FAM_MEMBERS']
test['PAYMENTS'] = test['AMT_ANNUITY']/ test['AMT_CREDIT']
test['NEW_CREDIT_TO_GOODS_RATIO'] = test['AMT_CREDIT'] / test['AMT_GOODS_PRICE']
test['GOODS_INCOME'] = test['AMT_GOODS_PRICE']/test['AMT_INCOME_TOTAL']
test['Ext_source_mult'] = test['EXT_SOURCE_1'] * test['EXT_SOURCE_2'] * test['EXT_SOURCE_3']
test['Ext_SOURCE_MEAN'] = test[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis = 1)
 test['Ext_SOURCE_SD'] = test[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis = 1 )<prepare_output>
stack_rmse_scores = cross_val_score(stack_reg, X_train_transformed, y_log_train, scoring='neg_root_mean_squared_error', cv=kf, n_jobs=-1)
 stack_rmse = np.round(np.mean(-stack_rmse_scores), 4)
 train_results.append(['Stacking', '', stack_rmse] )
bureau_new = bureau<groupby>
pd.set_option('display.max_colwidth', -1)
 train_models_df = pd.DataFrame(train_results, columns=['Model', 'Best Paramas', 'RMSLE'])
 train_models_df
group = bureau_new[['SK_ID_CURR', 'DAYS_CREDIT']].groupby('SK_ID_CURR')['DAYS_CREDIT'].count().reset_index().rename(index=str, columns={'DAYS_CREDIT': 'BUREAU_LOAN_COUNT'} )<merge>
results = dict()
best_models = [best_elastic_net_reg, best_svr_reg, best_rf_reg, best_xgb_reg, stack_reg]
model_names = []
model_rmse = []
for model in best_models:
test_rmse_scores = cross_val_score(model, X_test_transformed, y_log_test, scoring='neg_root_mean_squared_error', cv=kf, n_jobs=-1)
 test_rmse_scores = np.round(-test_rmse_scores,4)
 test_rmse = np.round(np.mean(test_rmse_scores),4)
 model_names.append(model.__class__.__name__)
 model_rmse.append(test_rmse )
bureau_new = bureau_new.merge(group, how = 'left', on = 'SK_ID_CURR' )<drop_column>
best_model = best_models[np.argmin(model_rmse)]
best_model
del group<count_unique_values>
y_train_pred = best_model.predict(X_train_transformed)
 y_test_pred = best_model.predict(X_test_transformed)
 y_train_pred = np.exp(y_train_pred)
 y_test_pred = np.exp(y_test_pred)
 predicted = np.concatenate([y_train_pred, y_test_pred], axis=0)
 obsereved = np.concatenate([y_train, y_test], axis=0 )
group = bureau_new[['SK_ID_CURR', 'CREDIT_TYPE']].groupby('SK_ID_CURR')['CREDIT_TYPE'].nunique().reset_index().rename(index=str, columns = {'CREDIT_TYPE': 'LOAN_TYPES_PER_CUST'} )<merge>
combine_data = pd.concat([X_train, X_test], axis=0)
 combine_data['SalePrice'] = obsereved
combine_data['Predicted_SalePrice'] = predicted
combine_data.shape
bureau_new = bureau_new.merge(group,on = ['SK_ID_CURR'], how = 'left')
 del group<feature_engineering>
final_model = Pipeline([('pre_process', pre_process), 
('best_model', best_model)])
 final_model.fit(X_train, y_log_train )
bureau_new["AVERAGE_LOAN_TYPE"] = bureau_new['BUREAU_LOAN_COUNT']/bureau_new['LOAN_TYPES_PER_CUST']<categorify>
test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )
replace = {'Active': 1, 'Closed':0, 'Sold': 1, 'Bad debt': 1}
bureau_new['CREDIT_ACTIVE'] = bureau_new['CREDIT_ACTIVE'].replace(replace )<groupby>
log_predictions = final_model.predict(test_data)
 predictions = np.exp(log_predictions )
gp = bureau_new.groupby('SK_ID_CURR')['CREDIT_ACTIVE'].mean().reset_index().rename(index=str, columns={'CREDIT_ACTIVE': 'ACTIVE_LOANS_PERCENTAGE'} )<merge>
test_predictions = pd.DataFrame(test_data['Id'])
 test_predictions['SalePrice'] = predictions.copy()
<drop_column><EOS> 
test_predictions.to_csv("./submission.csv", index=False )