Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<define_variables>
%matplotlib inline
filename = []
num = len(learn.data.test_ds)
 for i in range(num):
filename.append(str(learn.data.test_ds.items[i] ).split('/')[-1] )<load_from_csv>
train = import_data('.. /input/application_train.csv' )
submit = pd.DataFrame({'filename':filename,'category':category})
 test_csv = pd.read_csv('.. /input/shopee-product-detection-student/test.csv')
 submit = submit[submit['filename'].isin(test_csv['filename'])]
submit<save_to_csv>
test = import_data('.. /input/application_test.csv' )
submit.to_csv('submission.csv', header=True, index=False )<import_modules>
bureau = import_data('.. /input/bureau.csv' )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers.core import Dense,Activation,Dropout
from keras.layers import LSTM
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import seaborn as sns
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from xgboost import XGBClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier
from keras.callbacks import EarlyStopping
import math 
<load_from_csv>
bureau_balance = import_data('.. /input/bureau_balance.csv' )
df=pd.read_csv('.. /input/Gastric_Ulcer_Train.csv',delimiter=',')
 df.head(3)
<categorify>
previous_application = import_data('.. /input/previous_application.csv' )
df=pd.read_csv('.. /input/Gastric_Ulcer_Train.csv',delimiter=',')
 df.head(3)
 del df['ID']
del df['Age']
df=pd.concat([df,pd.get_dummies(df['Personality Type'], drop_first=True, prefix=1)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Gender'], drop_first=True, prefix=2)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Fixed Mealtimes'], drop_first=True, prefix=3)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Alcohol Consumption'], drop_first=True, prefix=4)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Vegetarian'], drop_first=True, prefix=5)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Smoker'], drop_first=True, prefix=6)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['CLO Test'], drop_first=True, prefix=7)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Coronary Disease'], drop_first=True, prefix=8)], axis=1,sort=False)
 df=df.drop(['Personality Type','Gender','Fixed Mealtimes','Alcohol Consumption','Vegetarian','Smoker','CLO Test','Coronary Disease'],axis=1)
 df.head()
df1=df
target=df['Gastric Ulcer']
sns.countplot(target)
 del df['Gastric Ulcer']
sc=StandardScaler()
sc.fit_transform(df)
 X_train,X_test,Y_train,Y_test=train_test_split(df,target,test_size=0.3)
<train_model>
credit_card_balance = import_data('.. /input/credit_card_balance.csv' )
model = XGBClassifier(objective="binary:logistic",n_estimators=500,max_depth=8,learning_rate=0.07)
 model.fit(X_train, Y_train)
 predict3=model.predict(X_test)
 c=0
for i in range(len(predict3)) :
if(predict3[i]==Y_test.iloc[i]):
c+=1
c3=(c/len(predict3)) *100
print('XGBoost Accuracy Score is',c3)
<train_model>
installments_payments = import_data('.. /input/installments_payments.csv' )
model= XGBClassifier(objective="binary:logistic",n_estimators=500,max_depth=8,learning_rate=0.07)
 model.fit(df,target )<categorify>
POS_CASH_balance = import_data('.. /input/POS_CASH_balance.csv' )
df=pd.read_csv('.. /input/Gastric_Ulcer_Test.csv',delimiter=',')
 del df['ID']
del df['Age']
df=pd.concat([df,pd.get_dummies(df['Personality Type'], drop_first=True, prefix=1)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Gender'], drop_first=True, prefix=2)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Fixed Mealtimes'], drop_first=True, prefix=3)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Alcohol Consumption'], drop_first=True, prefix=4)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Vegetarian'], drop_first=True, prefix=5)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Smoker'], drop_first=True, prefix=6)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['CLO Test'], drop_first=True, prefix=7)], axis=1,sort=False)
 df=pd.concat([df,pd.get_dummies(df['Coronary Disease'], drop_first=True, prefix=8)], axis=1,sort=False)
 df=df.drop(['Personality Type','Gender','Fixed Mealtimes','Alcohol Consumption','Vegetarian','Smoker','CLO Test','Coronary Disease'],axis=1)
 df.head()
sc=StandardScaler()
sc.fit_transform(df)
<save_to_csv>
train.TARGET.value_counts()
predict1=model.predict(df)
 iid=np.arange(0,200,1)+801
ind=np.arange(1,201,1)
 g=pd.DataFrame({'Id':iid,'Gastric Ulcer':predict1},index=ind)
 g.head()
g.to_csv("output.csv",index=False)
<save_to_csv>
sum(train.isna().sum())/(train.shape[0]*train.shape[1])*100
with open('test.csv', 'w')as f:
f.write('ImageId,Class
')
 for i in range(300):
f.write('{},{}
'.format(i,1))<load_from_csv>
train_NA =(np.round(( train.isna().sum() /len(train)) ,3)).sort_values(ascending = False)*100
train_NA = train_NA.sort_values(ascending = False)
 train_NA.head(10 )
df = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<concatenate>
train.dtypes.value_counts()
df = df.append(test )<data_type_conversions>
train.select_dtypes('category' ).apply(pd.Series.nunique, axis = 0 )
for col in ['regiao', 'estado', 'porte']:
df[col] = df[col].astype('category')
 df[col] = df[col].cat.codes<data_type_conversions>
data = train.append(test )
df['populacao'] = df['populacao'].str.replace(',','')
 df['populacao'] = df['populacao'].str.replace('\(\)','')
 df['populacao'] = df['populacao'].str.replace('\(1\)','')
 df['populacao'] = df['populacao'].str.replace('\(2\)','')
 df['populacao'] = df['populacao'].astype('float' )<data_type_conversions>
data.TARGET.isna().sum()
df['area'] = df['area'].str.replace(',','' ).astype('float')
 df['densidade_dem'] = df['densidade_dem'].str.replace(',','' ).astype('float' )<data_type_conversions>
sum(data.SK_ID_CURR[data.TARGET.isna() ] == test.SK_ID_CURR )
df['in_densidade_dem'] = df['densidade_dem'].isna().astype('int')
 df['densidade_dem'] = df['densidade_dem'].fillna(df['densidade_dem'].mean() )<data_type_conversions>
sum(data.SK_ID_CURR.isin(test.SK_ID_CURR)) == len(test )
df['in_participacao_transf_receita'] = df['participacao_transf_receita'].isna().astype('int')
 df['participacao_transf_receita'] = df['participacao_transf_receita'].fillna(df['participacao_transf_receita'].mean() )<data_type_conversions>
previous_loan_counts = bureau.groupby('SK_ID_CURR', as_index=False)['SK_ID_BUREAU'].count().rename(columns = {'SK_ID_BUREAU': 'PREVIOUS_LOANS_COUNT'})
 previous_loan_counts.head()
df['in_servidores'] = df['servidores'].isna().astype('int')
 df['servidores'] = df['servidores'].fillna(df['servidores'].mean() )<data_type_conversions>
data = data.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left' )
df['in_perc_pop_econ_ativa'] = df['perc_pop_econ_ativa'].isna().astype('int')
 df['perc_pop_econ_ativa'] = df['perc_pop_econ_ativa'].fillna(df['perc_pop_econ_ativa'].mean() )<data_type_conversions>
def extract_mean(x):
y = x.groupby('SK_ID_BUREAU', as_index=False ).mean().add_prefix('BUR_BAL_MEAN_')
 return y
df['in_gasto_pc_saude'] = df['gasto_pc_saude'].isna().astype('int')
 df['gasto_pc_saude'] = df['gasto_pc_saude'].fillna(df['gasto_pc_saude'].mean())
 df['in_hab_p_medico'] = df['hab_p_medico'].isna().astype('int')
 df['hab_p_medico'] = df['hab_p_medico'].fillna(df['hab_p_medico'].mean())
 df['in_exp_vida'] = df['exp_vida'].isna().astype('int')
 df['exp_vida'] = df['exp_vida'].fillna(df['exp_vida'].mean())
 df['in_gasto_pc_educacao'] = df['gasto_pc_educacao'].isna().astype('int')
 df['gasto_pc_educacao'] = df['gasto_pc_educacao'].fillna(df['gasto_pc_educacao'].mean())
 df['in_exp_anos_estudo'] = df['exp_anos_estudo'].isna().astype('int')
 df['exp_anos_estudo'] = df['exp_anos_estudo'].fillna(df['exp_anos_estudo'].mean() )<feature_engineering>
bureau_bal_mean = extract_mean(bureau_balance )
df['div_gasto'] = df['gasto_pc_saude'] / df['pib']
df['exp_pib'] = df['pib'] / df['exp_vida']
df['exp_pib_estudos'] = df['pib'] / df['exp_anos_estudo']<split>
bureau_bal_mean = bureau_bal_mean.rename(columns = {'BUR_BAL_MEAN_SK_ID_BUREAU' : 'SK_ID_BUREAU'} )
test = df[df['nota_mat'].isnull() ]
df = df[~df['nota_mat'].isnull() ]<split>
bureau = bureau.merge(bureau_bal_mean, on = 'SK_ID_BUREAU', how = 'left')
 bureau.drop('SK_ID_BUREAU', axis = 1, inplace = True )
train, valid = train_test_split(df, random_state=42)
 train.shape, valid.shape<define_variables>
def extract_mean(x):
y = x.groupby('SK_ID_CURR', as_index=False ).mean().add_prefix('PREV_BUR_MEAN_')
 return y
removed_cols = ['municipio', 'comissionados_por_servidor', 'nota_mat']
feats = [c for c in df.columns if c not in removed_cols]<import_modules>
bureau_mean_values = extract_mean(bureau)
 bureau_mean_values = bureau_mean_values.rename(columns = {'PREV_BUR_MEAN_SK_ID_CURR' : 'SK_ID_CURR'} )
from sklearn.ensemble import RandomForestClassifier<choose_model_class>
data = data.merge(bureau_mean_values, on = 'SK_ID_CURR', how = 'left' )
rf = RandomForestClassifier(n_estimators=200, min_samples_split=5, max_depth=4, random_state=42 )<train_model>
len(previous_application.SK_ID_CURR.isin(data.SK_ID_CURR)) == len(previous_application )
rf.fit(train[feats], train['nota_mat'] )<predict_on_test>
credit_card_balance.drop('SK_ID_CURR', axis = 1, inplace = True)
 installments_payments.drop('SK_ID_CURR', axis = 1, inplace = True)
 POS_CASH_balance.drop('SK_ID_CURR', axis = 1, inplace = True )
preds = rf.predict(valid[feats] )<compute_test_metric>
previous_application_counts = previous_application.groupby('SK_ID_CURR', as_index=False)['SK_ID_PREV'].count().rename(columns = {'SK_ID_PREV': 'PREVIOUS_APPLICATION_COUNT'})
 previous_application_counts.head()
accuracy_score(valid['nota_mat'], preds )<compute_train_metric>
data = data.merge(previous_application_counts, on = 'SK_ID_CURR', how = 'left' )
def cv(df, test, feats, y_name, k=10):
preds, score, fis = [], [], []
chunk = df.shape[0] // k
for i in range(k):
if i + 1 < k:
valid = df.iloc[i*chunk:(i+1)*chunk]
train = df.iloc[:i*chunk].append(df.iloc[(i+1)*chunk:])
 else:
valid = df.iloc[i*chunk:]
train = df.iloc[:i*chunk]
rf = RandomForestClassifier(random_state=42, n_jobs=-1, n_estimators=210, min_samples_leaf=5)
 rf.fit(train[feats], train[y_name])
 score.append(accuracy_score(valid[y_name], rf.predict(valid[feats])))
 preds.append(rf.predict(test[feats]))
fis.append(rf.feature_importances_)
 print(i, 'OK')
 return score, preds, fis 
<compute_test_metric>
def extract_mean(x):
y = x.groupby('SK_ID_PREV', as_index=False ).mean().add_prefix('CARD_MEAN_')
 return y
credit_card_balance_mean = extract_mean(credit_card_balance)
 credit_card_balance_mean = credit_card_balance_mean.rename(columns = {'CARD_MEAN_SK_ID_PREV' : 'SK_ID_PREV'} )
score, preds, fis = cv(df, test, feats, 'nota_mat' )<create_dataframe>
previous_application = previous_application.merge(credit_card_balance_mean, on = 'SK_ID_PREV', how = 'left' )
final = pd.DataFrame({'codigo_mun': test['codigo_mun'], 'nota_mat': preds[2]} )<save_to_csv>
def extract_mean(x):
y = x.groupby('SK_ID_PREV', as_index=False ).mean().add_prefix('INSTALL_MEAN_')
 return y
install_pay_mean = extract_mean(installments_payments)
 install_pay_mean = install_pay_mean.rename(columns = {'INSTALL_MEAN_SK_ID_PREV' : 'SK_ID_PREV'} )
final.to_csv('arquivopablo.csv', index=False )<load_from_csv>
previous_application = previous_application.merge(install_pay_mean, on = 'SK_ID_PREV', how = 'left' )
train = pd.read_csv('.. /input/train.csv', thousands=',')
 test = pd.read_csv('.. /input/test.csv', thousands=',' )<concatenate>
def extract_mean(x):
y = x.groupby('SK_ID_PREV', as_index=False ).mean().add_prefix('POS_MEAN_')
 return y
POS_mean = extract_mean(POS_CASH_balance)
 POS_mean = POS_mean.rename(columns = {'POS_MEAN_SK_ID_PREV' : 'SK_ID_PREV'} )
df = pd.concat([train, test], sort=False)
 df.head(10 ).T<filter>
previous_application = previous_application.merge(POS_mean, on = 'SK_ID_PREV', how = 'left' )
df[df['capital']==1]<feature_engineering>
def extract_mean(x):
y = x.groupby('SK_ID_CURR', as_index=False ).mean().add_prefix('PREV_APPL_MEAN_')
 return y
prev_appl_mean = extract_mean(previous_application)
 prev_appl_mean = prev_appl_mean.rename(columns = {'PREV_APPL_MEAN_SK_ID_CURR' : 'SK_ID_CURR'})
 prev_appl_mean = prev_appl_mean.drop('PREV_APPL_MEAN_SK_ID_PREV', axis = 1 )
df['populacao'] = df['populacao'].str.replace(',', '' ).str.replace('.', '')
 df['populacao'] = df['populacao'].apply(lambda x: re.sub(r'(\(\d\)) ', '', x))
df['populacao'] = df['populacao'].astype(float)
 df['comissionados_por_servidor'] = df['comissionados'] / df['servidores']
df['pib_pc'] = df['pib'] / df['populacao']
df['densidade_dem'] = df['populacao'] / df['area']<sort_values>
data = data.merge(prev_appl_mean, on = 'SK_ID_CURR', how = 'left' )
df = df.sort_values(['estado', 'municipio'] )<feature_engineering>
train1 = data[data['SK_ID_CURR'].isin(train.SK_ID_CURR)]
test1 = data[data.SK_ID_CURR.isin(test.SK_ID_CURR)]
test1.drop('TARGET', axis = 1, inplace = True )
cols = ['jornada_trabalho', 'anos_estudo_empreendedor', 'taxa_empreendedorismo']
df[cols] = df[cols].replace(0, np.NaN )<feature_engineering>
train1 = pd.get_dummies(train1)
 test1 = pd.get_dummies(test1 )
print(df.isna().sum())
 cols = df.columns[df.isna().any() ].tolist()
cols.remove('nota_mat')
 print(cols)
 for c in cols:
df[c] = df.groupby('estado')[c].transform(lambda x: x.fillna(x.median())) 
<feature_engineering>
print('Training Features shape with dummy variables: ', train1.shape)
 print('Testing Features shape with dummy variables: ', test1.shape )
df['educacao_pelo_pib'] = df['gasto_pc_educacao'] / df['pib_pc']
df['servidores_pc'] = df['servidores'] / df['populacao'] 
<categorify>
TARGET = train1.TARGET
train1.drop('TARGET', axis = 1, inplace = True)
 train1, test1 = train1.align(test1, join = 'inner', axis = 1 )
categoricas = ['regiao', 'porte']
for cat in categoricas:
dummies = pd.get_dummies(df[cat], prefix=cat)
 df = pd.concat([df, dummies], axis=1)
 cols = [c for c in df.columns if c not in categoricas]
df = df[cols]<feature_engineering>
imputer = Imputer(strategy = 'median' )
cols = ['populacao', 'area', 'pib', 'pib_pc', 'servidores', 'comissionados', 'educacao_pelo_pib', 'densidade_dem', 'gasto_pc_educacao', 'gasto_pc_saude', 'hab_p_medico']
for col in cols:
df[col] = np.log(df[col]+1)
 df['perc_pop_econ_ativa'] = df['perc_pop_econ_ativa'] / 100
cols = [c for c in df.columns if(( df[c].dtype==np.float64)or(df[c].dtype==np.int64)) ]
cols.remove('nota_mat')
 cols.remove('codigo_mun')
 cols.remove('comissionados_por_servidor')
 for c in ['populacao', 'area', 'pib', 'pib_pc', 'servidores', 'comissionados', 'perc_pop_econ_ativa',]:
cols.remove(c)
 for c in cols:
df[c] =(df[c] - df[c].min())/(df[c].max() - df[c].min())
 df.head()<split>
print('Missing values in train data: ', sum(train1.isnull().sum()))
print('Missing values in test data: ', sum(test1.isnull().sum()))
train, valid = train_test_split(train, random_state=42, test_size=0.2)
 train.shape, valid.shape<train_model>
imputer.fit(train1 )
params = {'max_depth': 5, 'min_samples_split': 2, 'n_estimators': 250, 'oob_score': False}
rf = RandomForestClassifier(random_state=42, n_jobs=-1, **params)
 rf.fit(train[feats], train['nota_mat'])
 preds = rf.predict(valid[feats] )<compute_test_metric>
imputed_train = imputer.transform(train1)
 imputed_test = imputer.transform(test1)
print(mean_squared_error(valid['nota_mat'], preds)**(1/2))
print(accuracy_score(valid['nota_mat'], preds))<save_to_csv>
train1[train1.isnull() ] = imputed_train
test1[test1.isnull() ] = imputed_test
preds = rf.predict(test[feats])
 test['nota_mat'] = preds
test[['codigo_mun', 'nota_mat']].to_csv('saida_forest.csv', index=False )<compute_test_metric>
sum(train1.isnull().sum() )
def cv(df, test, feats, y_name, k=5, oob_score=False):
preds, score, fis, acc = [], [], [], []
chunk = df.shape[0] // k
for i in range(k):
if i + 1 < k:
valid = df.iloc[i*chunk:(i+1)*chunk]
train = df.iloc[:i*chunk].append(df.iloc[(i+1)*chunk:])
 else:
valid = df.iloc[i*chunk:]
train = df.iloc[:i*chunk]
rf = RandomForestClassifier(random_state=42, n_jobs=-1, n_estimators=200, max_depth=5, oob_score=oob_score)
 rf.fit(train[feats], train[y_name])
 acc.append(accuracy_score(valid[y_name], rf.predict(valid[feats])))
 score.append(mean_squared_error(valid[y_name], rf.predict(valid[feats])) **(1/2))
preds.append(rf.predict(test[feats]))
fis.append(rf.feature_importances_)
 print(i, 'OK')
 return score, preds, fis, acc<compute_test_metric>
scaler = MinMaxScaler(feature_range =(0, 1))
score, preds, fis, acc = cv(train, test, feats, 'nota_mat' )<compute_test_metric>
scaler.fit(train1 )
score, acc<create_dataframe>
scaled_train = scaler.transform(train1)
 scaled_test = scaler.transform(test1 )
x = pd.DataFrame(preds )<feature_engineering>
train1 = pd.DataFrame(scaled_train, index=train1.index, columns=train1.columns)
 test1 = pd.DataFrame(scaled_test, index=test1.index, columns=test1.columns )
test['nota_mat'] = x.mode(axis=0 ).T<save_to_csv>
GLM = LogisticRegression(C = 0.0001)
 GLM.fit(train1, TARGET)
 GLM_pred = GLM.predict_proba(test1)[:, 1]
submission = test[['SK_ID_CURR']]
submission['TARGET'] = GLM_pred
test[['codigo_mun', 'nota_mat']].to_csv('saida_cross.csv', index=False )<predict_on_test>
submission.to_csv('GLM_baseline.csv', index = False )
clf = SVC(gamma='auto')
 clf.fit(train[feats], train['nota_mat'])
 preds = clf.predict(valid[feats] )<compute_test_metric>
X_train = train1
y_train = TARGET
X_test = test1
dtrain = xgb.DMatrix(X_train, y_train)
 dtest = xgb.DMatrix(X_test)
 params = {
'min_child_weight': 19.0,
'objective': 'binary:logistic',
'max_depth': 7,
'eta': 0.025,
'eval_metric': 'auc',
'max_delta_step': 1.8,
'colsample_bytree': 0.4,
'subsample': 0.8,
'gamma': 0.65
}
watchlist = [(dtrain, 'train')]
XGB_model = xgb.train(params, dtrain,
300,
watchlist,
verbose_eval=50)
 XGB_pred = XGB_model.predict(dtest )
print(mean_squared_error(valid['nota_mat'], preds)**(1/2))
print(accuracy_score(valid['nota_mat'], preds))<train_model>
submission_2 = test[['SK_ID_CURR']]
submission_2['TARGET'] = XGB_pred
lr = LogisticRegression(random_state=42, solver='lbfgs', multi_class='multinomial', max_iter=500)
 lr.fit(train[feats], train['nota_mat'])
 preds = lr.predict(valid[feats] )<compute_test_metric>
submission_2.to_csv('XGB_pred2.csv', index = False )