Kaggle competition: Home Credit Default Risk
X_train_sample, y_train_sample = select_sample(mult=50)
 f1_est = []
num_estimators = 10
while num_estimators <= 100:
rfclassifier = RandomForestClassifier(n_estimators = num_estimators, max_features = 10)
 rfclassifier.fit(X_train_sample, y_train_sample)
 train_f1_score, valid_f1_score = get_f1_score(rfclassifier, X_train_sample, X_valid, y_train_sample, y_valid)
 f1_est.append({'num_estimators': num_estimators, 'train_f1_score': train_f1_score, 'valid_f1_score': valid_f1_score})
 num_estimators+=10<load_from_disk>
cash = pd.read_csv('.. /input/POS_CASH_balance.csv')
 bureau_bal = pd.read_csv('.. /input/bureau_balance.csv')
 card = pd.read_csv('.. /input/credit_card_balance.csv')
 bureau = pd.read_csv('.. /input/bureau.csv')
 train = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 previous = pd.read_csv('.. /input/previous_application.csv')
 installment = pd.read_csv('.. /input/installments_payments.csv')
 print('Done!')
 print(cash.shape, bureau_bal.shape, card.shape, bureau.shape, train.shape, test.shape, previous.shape, installment.shape )
f1_est_df = pd.io.json.json_normalize(f1_est )<train_model>
na = train.isnull().sum() /len(train)
 na.sort_values(ascending=False ).head(5 )
X_train_sample, y_train_sample = select_sample(mult=50)
 f1_feat = []
max_features = 2
while max_features <= 20:
rfclassifier = RandomForestClassifier(n_estimators = 60, max_features = max_features)
 rfclassifier.fit(X_train_sample, y_train_sample)
 train_f1_score, valid_f1_score = get_f1_score(rfclassifier, X_train_sample, X_valid, y_train_sample, y_valid)
 f1_feat.append({'max_features': max_features, 'train_f1_score': train_f1_score, 'valid_f1_score': valid_f1_score})
 max_features+=1<load_from_disk>
train.loc[:,['SK_ID_CURR','TARGET','AMT_INCOME_TOTAL']].sort_values('AMT_INCOME_TOTAL', ascending=False ).head(10 )
f1_feat_df = pd.io.json.json_normalize(f1_feat )<train_model>
train['DAYS_EMPLOYED'].value_counts().head()
final_classifier = RandomForestClassifier(n_estimators = 60, max_features = 19)
 final_classifier.fit(X_train_sample, y_train_sample )<compute_test_metric>
bureau_bal.STATUS.value_counts()
get_f1_score(final_classifier, X_train_sample, X_valid, y_train_sample, y_valid )<split>
house_train = train.loc[:,'APARTMENTS_AVG':'EMERGENCYSTATE_MODE']
house_test = test.loc[:,'APARTMENTS_AVG':'EMERGENCYSTATE_MODE']
house_var = house_train.columns.tolist()
house_train = pd.get_dummies(house_train)
 house_test = pd.get_dummies(house_test)
 house_train.fillna(0, inplace=True)
 house_test.fillna(0, inplace=True)
 house_train.head()
X_test = X[m:]<predict_on_test>
pca = PCA(n_components=3)
 house_train2 = pca.fit_transform(house_train)
 house_test2 = pca.transform(house_test)
 house_train2.shape, house_test2.shape
y_test_preds = final_classifier.predict(X_test )<create_dataframe>
house_train2 = pd.DataFrame(house_train2, columns=['house_pc1','house_pc2','house_pc3'])
 house_test2 = pd.DataFrame(house_test, columns=['house_pc1','house_pc2','house_pc3'] )
submission = pd.DataFrame({'id': test_og.id, 'target': y_test_preds.astype(int)} )<save_to_csv>
train = pd.concat([train.drop(columns=house_var), house_train2], axis=1)
 test = pd.concat([test.drop(columns=house_var), house_test2], axis=1)
 train.shape, test.shape
submission.to_csv('submission_dk_201910192153.csv',index=False )<categorify>
train['CREDIT_INCOME_PERCENT'] = train['AMT_CREDIT']/train['AMT_INCOME_TOTAL']
train['ANNUITY_INCOME_PERCENT'] = train['AMT_ANNUITY']/train['AMT_INCOME_TOTAL']
train['DAYS_EMPLOYED_PERCENT'] = train['DAYS_EMPLOYED']/train['DAYS_BIRTH']
train['INCOME_PER_PERSON'] = train['AMT_INCOME_TOTAL'] / train['CNT_FAM_MEMBERS']
train['PAYMENT_RATE'] = train['AMT_ANNUITY'] / train['AMT_CREDIT']
test['CREDIT_INCOME_PERCENT'] = test['AMT_CREDIT']/test['AMT_INCOME_TOTAL']
test['ANNUITY_INCOME_PERCENT'] = test['AMT_ANNUITY']/test['AMT_INCOME_TOTAL']
test['DAYS_EMPLOYED_PERCENT'] = test['DAYS_EMPLOYED']/test['DAYS_BIRTH']
test['INCOME_PER_PERSON'] = test['AMT_INCOME_TOTAL'] / test['CNT_FAM_MEMBERS']
test['PAYMENT_RATE'] = test['AMT_ANNUITY'] / test['AMT_CREDIT']
test_sample.replace('na', np.nan, inplace = True )<data_type_conversions>
log_features = ['AMT_INCOME_TOTAL','AMT_CREDIT','AMT_ANNUITY','AMT_GOODS_PRICE', 'DAYS_EMPLOYED', 'DAYS_REGISTRATION']
days_features = ['DAYS_BIRTH', 'DAYS_EMPLOYED','DAYS_REGISTRATION', 'DAYS_ID_PUBLISH']
train[days_features]=-train[days_features]
test[days_features]=-test[days_features]
train[log_features]=np.log1p(train[log_features])
 test[log_features]=np.log1p(test[log_features] )
test_sample_nonna = test_sample.fillna(data.mean() )<normalization>
train['na_col'] = train.isnull().sum(axis=1)
 test['na_col'] = test.isnull().sum(axis=1 )
X_test_sample = scaler.transform(test_sample_nonna.iloc[:,1:] )<predict_on_test>
train = pd.get_dummies(train)
 test = pd.get_dummies(test)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
y_test_sample_preds = final_classifier.predict(X_test_sample )<load_pretrained>
safe = bureau.loc[(bureau.CREDIT_DAY_OVERDUE==0)&(bureau.CNT_CREDIT_PROLONG==0),['SK_ID_CURR','SK_ID_BUREAU']]
safe = safe.groupby('SK_ID_CURR' ).count().reset_index()
safe.columns = ['SK_ID_CURR','past_loan_np']
prob = bureau[['SK_ID_CURR','CREDIT_DAY_OVERDUE','CNT_CREDIT_PROLONG']].groupby('SK_ID_CURR' ).max()
prob.columns=['overdue_max','prolong_max']
prob=prob.reset_index()
loan_type=pd.get_dummies(bureau.CREDIT_TYPE)
 loan_type['SK_ID_CURR'] = bureau['SK_ID_CURR']
loan_type = loan_type.groupby('SK_ID_CURR' ).mean().reset_index()
data.mean().to_pickle('data_means.pkl' )<load_from_csv>
_ = bureau_bal.drop(columns=['STATUS'] ).groupby('SK_ID_BUREAU' ).max().reset_index()
final_status = _.merge(bureau_bal, on=['SK_ID_BUREAU','MONTHS_BALANCE'], how='left')
 final_status = pd.get_dummies(final_status ).drop(columns='MONTHS_BALANCE')
 final_status2 = bureau[['SK_ID_CURR','SK_ID_BUREAU']].merge(final_status, on='SK_ID_BUREAU', how='left')
 final_status2.STATUS_X.fillna(1, inplace=True)
 final_status2.fillna(0, inplace=True)
 final_status2 = final_status2.drop(columns='SK_ID_BUREAU' ).groupby('SK_ID_CURR' ).mean().reset_index()
final_status2 = final_status2.drop(columns=['STATUS_0','STATUS_C'] )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 df=test.append(train)
 feats = [c for c in df.columns if c not in ['codigo_mun', 'comissionados_por_servidor']] 
<data_type_conversions>
del bureau, bureau_bal
regx = re.compile('\(\d+\)')
 regx2 = re.compile('\,?\.?')
 df['populacao']=df['populacao'].str.replace(regx2,'' ).str.replace(regx, '' ).astype(dtype=np.int )<data_type_conversions>
cash_dpd = cash.loc[:,['SK_ID_CURR','SK_DPD_DEF']].groupby('SK_ID_CURR' ).agg(['max','median'] ).reset_index()
cash_dpd.columns = ['SK_ID_CURR','Cash_SK_DPD_DEF_max', 'Cash_SK_DPD_DEF_median']
card_dpd = card.loc[:,['SK_ID_CURR','SK_DPD_DEF']].groupby('SK_ID_CURR' ).agg(['max','median'] ).reset_index()
card_dpd.columns = ['SK_ID_CURR','Card_SK_DPD_DEF_max', 'Card_SK_DPD_DEF_median']
df['area']=df['area'].str.replace(',','' ).astype(float )<data_type_conversions>
del cash, card
df['densidade_dem']=df['densidade_dem'].str.replace(',','' ).astype(float)
 df['cat_porte'] = df['porte'].astype('category' ).cat.codes
df['cat_regiao'] = df['regiao'].astype('category' ).cat.codes
df['cat_estado'] = df['estado'].astype('category' ).cat.codes 
<define_variables>
p_good = previous.loc[(previous.NAME_CONTRACT_STATUS=='Approved')|(previous.NAME_CONTRACT_STATUS=='Unused offer'), ['SK_ID_CURR','AMT_CREDIT']]
p_bad = previous.loc[(previous.NAME_CONTRACT_STATUS=='Canceled')|(previous.NAME_CONTRACT_STATUS=='Refused'), ['SK_ID_CURR','AMT_CREDIT']]
p_good = p_good.dropna()
p_good2 = p_good.groupby('SK_ID_CURR' ).sum().reset_index()
p_good2.columns = ['SK_ID_CURR','good_credit']
p_bad2 = p_bad.groupby('SK_ID_CURR' ).sum().reset_index()
p_bad2.columns = ['SK_ID_CURR','bad_credit']
feats = [c for c in df.columns if c not in ['codigo_mun', 'comissionados_por_servidor','nota_mat', 'densidade_dem',
'participacao_transf_receita', 'servidores', 'perc_pop_econ_ativa',
'gasto_pc_saude', 'hab_p_medico',
'exp_anos_estudo', 'regiao', 'estado', 'porte', 'municipio']]<split>
installment['diff'] = installment.AMT_PAYMENT - installment.AMT_INSTALMENT
ins_comp = installment.dropna(subset=['AMT_PAYMENT'] ).loc[:,['SK_ID_CURR','AMT_INSTALMENT','AMT_PAYMENT','diff']].groupby('SK_ID_CURR' ).sum().reset_index()
ins_comp['repay']=ins_comp.AMT_PAYMENT / ins_comp.AMT_INSTALMENT
ins_comp = ins_comp.loc[ins_comp.AMT_INSTALMENT!=0,:]
train=df[~df.nota_mat.isnull() ]
test_submissao=df[df.nota_mat.isnull() ]<split>
del previous, installment
train_2, test = train_test_split(train, test_size=0.20, random_state=42)
 train_2, valid = train_test_split(train_2, test_size=0.20, random_state=42)
 rf = RandomForestClassifier(random_state=42, n_estimators=300, min_samples_split=5, max_depth=4)
<predict_on_test>
def merge_data(train, test, dfs):
for df in dfs:
train = train.merge(df, on='SK_ID_CURR', how='left')
 test = test.merge(df, on='SK_ID_CURR', how='left')
 return train, test
train, test = merge_data(train, test, [safe, prob, loan_type, final_status2, cash_dpd, card_dpd, p_good2, p_bad2, ins_comp] )
rf.fit(train_2[feats], train_2['nota_mat'])
 preds = rf.predict(valid[feats])
<save_to_csv>
def rm_collinear(train, test):
threshold = 0.9
corr_matrix = train.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > threshold)and column != 'SK_ID_CURR']
train = train.drop(columns = to_drop)
 test = test.drop(columns = to_drop)
 return train, test, to_drop
train, test, collin_feat = rm_collinear(train, test )
test_submissao['nota_mat']=rf.predict(test_submissao[feats])
 test_submissao[['codigo_mun','nota_mat']].to_csv('respostas.csv', index=False)
<compute_test_metric>
from sklearn.preprocessing import MinMaxScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.metrics import roc_auc_score, accuracy_score, confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.calibration import CalibratedClassifierCV
from lightgbm import LGBMClassifier
from time import time
accuracy_score(valid['nota_mat'], preds)
<compute_test_metric>
x_train = train.drop(columns = ['TARGET','SK_ID_CURR'])
 x_test = test.drop(columns=['SK_ID_CURR'])
 features = list(x_train.columns)
 scaler = MinMaxScaler(feature_range =(0, 1))
scaler.fit(x_train)
 x_train = scaler.transform(x_train)
 x_test = scaler.transform(x_test )
accuracy_score(test['nota_mat'], rf.predict(test[feats])) 
<compute_train_metric>
gb = GradientBoostingClassifier(n_estimators=30, learning_rate=0.1, min_samples_split=4, random_state=10, verbose=1)
 ab = AdaBoostClassifier(n_estimators=300, learning_rate=1, random_state=123)
 rf0 = RandomForestClassifier(n_estimators=49, max_depth = 5, random_state=100)
 lr = LogisticRegression(C=1, random_state=100)
 nn = MLPClassifier(hidden_layer_sizes=(32,), alpha=0.01, learning_rate_init=0.01, max_iter=100, batch_size=4082)
 dt = DecisionTreeClassifier(max_features=100)
 lgb = LGBMClassifier(n_estimators=2000, learning_rate=0.02, objective='binary', reg_lambda=0.1, num_leaves=34 )
rf=RandomForestRegressor(random_state=42, n_estimators=100)
 rf.fit(train_2[feats], train_2['nota_mat'])
 preds = rf.predict(valid[feats])
 mean_squared_error(valid['nota_mat'], preds)**(1/2)
<train_model>
x_train_x, x_train_v, y_train_x, y_train_v = train_test_split(x_train, train_labels, test_size=0.1, random_state=168)
 def run_model_try(ml):
start = time()
model = ml
model.fit(x_train_x, y_train_x)
 train_pred = model.predict_proba(x_train_x)[:,1]
v_pred = model.predict_proba(x_train_v)[:,1]
train_auc = roc_auc_score(y_train_x, train_pred)
 v_auc = roc_auc_score(y_train_v, v_pred)
 train_acc = accuracy_score(y_train_x, model.predict(x_train_x))
v_acc = accuracy_score(y_train_v, model.predict(x_train_v))
end = time()
print("Training and validation auc: %.4f, %.4f" %(train_auc, v_auc))
print("Training and validation accuracy: %.4f, %.4f" %(train_acc, v_acc))
print("Time used: %.2f" %(end-start))
return model
def run_model_real(ml):
model = ml
model.fit(x_train, train_labels)
 return model
dt=DecisionTreeRegressor(random_state=44)
 dt.fit(train_2[feats], train_2['nota_mat'])
 dt.predict(train_2[feats])
 data = tree.export_graphviz(dt, out_file=None, feature_names=feats, class_names=['nota_mat'],
filled=True, rounded=True, special_characters=True, max_depth=2 )<load_from_csv>
model = run_model_real(lgb)
 test_pred = model.predict_proba(x_test)[:,1]
<load_from_csv><EOS> 
submit = test[['SK_ID_CURR']]
submit['TARGET'] = test_pred
submit.to_csv('lgb0825.csv', index = False )