Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<import_modules>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
path1=".. /input/"
data_files=list(os.listdir(path1))
df_files=pd.DataFrame(data_files,columns=['File_Name'])
 df_files['Size_in_MB']=df_files.File_Name.apply(lambda x:round(os.stat(path1+x ).st_size/(1024*1024),2))
df_files
train = pd.read_csv('.. /input/smhrd12/train.csv')
 test = pd.read_csv('.. /input/smhrd12/test.csv' )<categorify>
def feature_summary(df_fa):
print('DataFrame shape')
 print('rows:',df_fa.shape[0])
 print('cols:',df_fa.shape[1])
 col_list=['Null','Unique_Count','Data_type','Max/Min','Mean','Median','Std','Skewness','Sample_values']
df=pd.DataFrame(index=df_fa.columns,columns=col_list)
 df['Null']=list([len(df_fa[col][df_fa[col].isnull() ])for i,col in enumerate(df_fa.columns)])
 df['Unique_Count']=list([len(df_fa[col].unique())for i,col in enumerate(df_fa.columns)])
 df['Data_type']=list([df_fa[col].dtype for i,col in enumerate(df_fa.columns)])
 for i,col in enumerate(df_fa.columns):
if 'float' in str(df_fa[col].dtype)or 'int' in str(df_fa[col].dtype):
df.at[col,'Max/Min']=str(round(df_fa[col].max() ,2)) +'/'+str(round(df_fa[col].min() ,2))
df.at[col,'Mean']=df_fa[col].mean()
df.at[col,'Median']=df_fa[col].median()
df.at[col,'Std']=df_fa[col].std()
df.at[col,'Skewness']=df_fa[col].skew()
df.at[col,'Sample_values']=list(df_fa[col].unique())
 return(df.fillna('-'))
def drop_corr_col(df_corr):
upper = df_corr.where(np.triu(np.ones(df_corr.shape),
k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.999)]
return(to_drop)
 def cnt_unique(df):
return(len(df.unique()))
train['sex']=pd.get_dummies(train['sex'],prefix='Sex')
 test['sex']=pd.get_dummies(test['sex'],prefix='Sex')
 encoder = LabelEncoder()
train['workclass'] = encoder.fit_transform(train['workclass'])
 test['workclass'] = encoder.transform(test['workclass'])
 encoder = LabelEncoder()
train['education'] = encoder.fit_transform(train['education'])
 test['education'] = encoder.transform(test['education'])
 encoder = LabelEncoder()
train['marital-status'] = encoder.fit_transform(train['marital-status'])
 test['marital-status'] = encoder.transform(test['marital-status'])
 encoder = LabelEncoder()
train['occupation'] = encoder.fit_transform(train['occupation'])
 test['occupation'] = encoder.transform(test['occupation'])
 encoder = LabelEncoder()
train['relationship'] = encoder.fit_transform(train['relationship'])
 test['relationship'] = encoder.transform(test['relationship'])
 encoder = LabelEncoder()
train['race'] = encoder.fit_transform(train['race'])
 test['race'] = encoder.transform(test['race'])
 encoder = LabelEncoder()
train['native-country'] = encoder.fit_transform(train['native-country'])
 test['native-country'] = encoder.transform(test['native-country'])
<feature_engineering>
%%time
train=pd.read_csv(path1+'application_train.csv')
 test=pd.read_csv(path1+'application_test.csv' )
train['rel+gain'] = train['relationship']+train['capital-gain']
train['loss-gain'] = train['capital-loss']-train['capital-gain']
train['rel+gain+edu'] = train['relationship']+train['capital-gain']+train['education-num']
train['rel+gain+edu+age'] = train['relationship']+train['capital-gain']+train['education-num']
train['(rel+gain+edu+age)*rel'] =(train['relationship']+train['capital-gain']+train['education-num']+train['capital-loss'])*train['relationship']<create_dataframe>
print('Train shape:',train.shape,'Test shape:',test.shape)
 df_comb=pd.concat([train.drop(['TARGET'],axis=1),test],ignore_index=True)
 df_comb_fs=feature_summary(df_comb)
 cat_features=df_comb_fs[df_comb_fs.Data_type=='object'].index
for col in cat_features:
df_comb[col]=df_comb[col].apply(lambda x: str(x ).replace(" ","_"))
print('categorical features',len(cat_features))
<prepare_x_and_y>
cat_train=pd.DataFrame()
for col in cat_features:
dummy=pd.get_dummies(df_comb[col],prefix='DUM_'+col)
 cat_train=pd.concat([cat_train,dummy],axis=1)
 display(cat_train.head())
 print('Newly created dummy columns:',cat_train.shape )
del train['no']
df2 = train
Y = df2['income']
del df2['income']
X = df2.iloc[:,:]<feature_engineering>
cat_train.drop(['DUM_CODE_GENDER_XNA','DUM_NAME_INCOME_TYPE_Businessman','DUM_NAME_INCOME_TYPE_Maternity_leave','DUM_NAME_INCOME_TYPE_Student',
'DUM_NAME_INCOME_TYPE_Unemployed','DUM_NAME_FAMILY_STATUS_Unknown','DUM_OCCUPATION_TYPE_Security_staff','DUM_FONDKAPREMONT_MODE_nan',
'DUM_OCCUPATION_TYPE_IT_staff','DUM_ORGANIZATION_TYPE_Police','DUM_ORGANIZATION_TYPE_Telecom','DUM_ORGANIZATION_TYPE_Business_Entity_Type_1'],
axis=1,inplace=True)
 print('Categorical feature count after dropping irrelevant features',cat_train.shape )
del test['no']
test['rel+gain'] = test['relationship']+test['capital-gain']
test['loss-gain'] = test['capital-loss']-test['capital-gain']
test['rel+gain+edu'] = test['relationship']+test['capital-gain']+test['education-num']
test['rel+gain+edu+age'] = test['relationship']+test['capital-gain']+test['education-num']
test['(rel+gain+edu+age)*rel'] =(test['relationship']+test['capital-gain']+test['education-num']+test['capital-loss'])*test['relationship']<find_best_model_class>
df_final['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 df_final['CALC_PERC_DAYS_EMPLOYED'] = df_final['DAYS_EMPLOYED'] / df_final['DAYS_BIRTH']
df_final['CALC_PERC_INCOME_CREDIT'] = df_final['AMT_INCOME_TOTAL'] /df_final['AMT_CREDIT']
df_final['CALC_INCOME_PER_PERSON'] = df_final['AMT_INCOME_TOTAL'] / df_final['CNT_FAM_MEMBERS']
df_final['CALC_ANNUITY_INCOME_PERC'] = df_final['AMT_ANNUITY'] / df_final['AMT_INCOME_TOTAL']
df_final['CALC_PAYMENT_RATE'] = df_final['AMT_ANNUITY'] / df_final['AMT_CREDIT']
df_final['CALC_GOODS_PRICE_PER']=df_final['AMT_GOODS_PRICE']/df_final['AMT_CREDIT']
df_final['CALC_PERC_CHILDREN']=df_final['CNT_CHILDREN']/df_final['CNT_FAM_MEMBERS']
df_final['CALC_RATIO_CAR_TO_BRITH']=df_final['OWN_CAR_AGE'] / df_final['DAYS_BIRTH']
df_final['CALC_RATIO_CAR_TO_EMPLOY'] = df_final['OWN_CAR_AGE'] / df_final['DAYS_EMPLOYED']
df_final['CALC_INCOME_PER_CHILD'] = df_final['AMT_INCOME_TOTAL'] /(1 + df_final['CNT_CHILDREN'])
 df_final['CALC_INCOME_PER_PERSON'] = df_final['AMT_INCOME_TOTAL'] / df_final['CNT_FAM_MEMBERS']
df_final['CALC_RATIO_PHONE_TO_BIRTH'] = df_final['DAYS_LAST_PHONE_CHANGE'] / df_final['DAYS_BIRTH']
df_final['CALC_RATIO_PHONE_TO_EMPLOY'] = df_final['DAYS_LAST_PHONE_CHANGE'] / df_final['DAYS_EMPLOYED']
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size = 0.2 , random_state=40)
 xtree = ExtraTreesClassifier(n_estimators=100, random_state=100)
 xtree.fit(X_train, Y_train)
 y_pred = xtree.predict(X_test)
 xtree_score = cross_val_score(xtree,X_train,Y_train,cv=5 ).mean()
print('Accuracy =', accuracy_score(Y_test, y_pred))
print("I : {}, 교차 검증 점수 : {:.3f}".format(0,xtree_score))
print("I : {}, 훈련 세트 정확도: {:.3f}".format(0,xtree.score(X_train, Y_train)))
 print("I : {}, 테스트 세트 정확도: {:.3f}".format(0,xtree.score(X_test, Y_test)))
 xtree_scoreT = cross_val_score(xtree,X_test,Y_test,cv=5 ).mean()
print("테스트 교차검증" ,xtree_scoreT )<find_best_model_class>
df_final.drop(['FLAG_MOBIL','FLAG_DOCUMENT_2','FLAG_DOCUMENT_4','FLAG_DOCUMENT_10','FLAG_DOCUMENT_12'],axis=1,inplace=True )
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size = 0.2 , random_state=40)
 rf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1)
 rf.fit(X_train, Y_train)
 y_pred = rf.predict(X_test)
 rf_score = cross_val_score(rf,X_train,Y_train,cv=5 ).mean()
print('Accuracy =', accuracy_score(Y_test, y_pred))
print("I : {}, 교차 검증 점수 : {:.3f}".format(0,rf_score))
print("I : {}, 훈련 세트 정확도: {:.3f}".format(0,rf.score(X_train, Y_train)))
 print("I : {}, 테스트 세트 정확도: {:.3f}".format(0,rf.score(X_test, Y_test)))
 rf_scoreT = cross_val_score(rf,X_test,Y_test,cv=5 ).mean()
print("테스트 교차검증" ,rf_scoreT )<find_best_model_class>
del dummy
gc.collect()
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size = 0.2 , random_state=40)
 rf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1)
 xtree = ExtraTreesClassifier(n_estimators=100, random_state=0)
 voting = VotingClassifier(
 estimators=[('rf', rf),('xtree', xtree)],
voting='hard')
 voting.fit(X_train, Y_train)
 vo_score = cross_val_score(voting,X_train,Y_train,cv=7 ).mean()
print(",교차 검증 점수 : {:.3f}".format(vo_score))
print(",훈련 세트 정확도: {:.3f}".format(voting.score(X_train, Y_train)))
 print(",테스트 세트 정확도: {:.3f}".format(voting.score(X_test, Y_test)))
 print('================================================')
<train_model>
%%time
bur=pd.read_csv(path1+'bureau.csv')
 print('bureau set reading complete...')
 bur_bal=pd.read_csv(path1+'bureau_balance.csv')
 print('bureau balance set reading complete...' )
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size = 0.2 , random_state=40)
 scaler = StandardScaler()
scaler.fit(X_train)
 scaler.fit(test)
 test_sc = scaler.transform(test)
 test_sc = pd.DataFrame(test_sc,columns=test.columns)
 X_train_sc = scaler.transform(X_train)
 X_test_sc = scaler.transform(X_test)
 X_train_sc = pd.DataFrame(X_train_sc,columns=X_train.columns)
 X_test_sc = pd.DataFrame(X_test_sc,columns=X_train.columns)
 gbrt = GradientBoostingClassifier(random_state=100,max_depth=5,learning_rate=0.12)
 gbrt.fit(X_train_sc, Y_train)
 y_pred = gbrt.predict(X_test_sc)
 gbrt_score = cross_val_score(gbrt,X_train_sc,Y_train,cv=5 ).mean()
print('Accuracy =', accuracy_score(Y_test, y_pred))
print("I : {}, 교차 검증 점수 : {:.3f}".format(0,gbrt_score))
print("I : {}, 훈련 세트 정확도: {:.3f}".format(0,gbrt.score(X_train_sc, Y_train)))
 print("I : {}, 테스트 세트 정확도: {:.3f}".format(0,gbrt.score(X_test_sc, Y_test)))
 gbrt_scoreT = cross_val_score(gbrt,X_test,Y_test,cv=5 ).mean()
print(gbrt_scoreT)
 gbrt_pred = gbrt.predict(test_sc)
<create_dataframe>
%%time
for col in ['CREDIT_CURRENCY','CREDIT_TYPE','CREDIT_ACTIVE']:
bur[col]=bur[col].apply(lambda x: str(x ).replace(" ","_"))
dummy=pd.DataFrame()
for col in ['CREDIT_CURRENCY','CREDIT_TYPE','CREDIT_ACTIVE']:
dummy=pd.concat([dummy,pd.get_dummies(bur[col],prefix='DUM_'+col)],axis=1 )
fi = gbrt.feature_importances_
importances = pd.DataFrame(fi,index=X.columns)
 importances.sort_values(by=0,ascending=False )<find_best_model_class>
bur_f=pd.concat([bur.drop(['CREDIT_CURRENCY','CREDIT_TYPE','CREDIT_ACTIVE'],axis=1),dummy],axis=1 )
for i in [150] :
ada = AdaBoostClassifier(
 DecisionTreeClassifier(max_depth=3), n_estimators=200,
algorithm="SAMME.R", learning_rate=0.22)
 ada.fit(X_train, Y_train)
 ada_score = cross_val_score(ada,X_train,Y_train,cv=10 ).mean()
print("i : {} ,교차 검증 점수 : {:.3f}".format(i,ada_score))
print("i : {} ,훈련 세트 정확도: {:.3f}".format(i,ada.score(X_train, Y_train)))
 print("i : {} ,테스트 세트 정확도: {:.3f}".format(i,ada.score(X_test, Y_test)))
 print('================================================')
<find_best_model_class>
bur_f['CALC_PER_CREDIT_MAX_OVERDUE']=bur_f['AMT_CREDIT_MAX_OVERDUE']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_CREDIT_SUM_DEBT']=bur_f['AMT_CREDIT_SUM_DEBT']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_CREDIT_SUM_LIMIT']=bur_f['AMT_CREDIT_SUM_LIMIT']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_CREDIT_SUM_OVERDUE']=bur_f['AMT_CREDIT_SUM_OVERDUE']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_PER_ANNUITY']=bur_f['AMT_ANNUITY']/bur_f['AMT_CREDIT_SUM']
bur_f['CALC_CREDIT_LIMIT_CROSSED']=bur_f['AMT_CREDIT_SUM_LIMIT']-bur_f['AMT_CREDIT_SUM']
bur_f['CALC_CREDIT_PER_DAY']=bur_f['AMT_CREDIT_SUM']/bur_f['DAYS_CREDIT_ENDDATE'].abs()
bur_f['CALC_CREDIT_CLOSED']=(bur_f['DAYS_ENDDATE_FACT'] < 0 ).astype(int )
xgb = XGBClassifier(objective ='reg:linear', colsample_bytree = 0.6, learning_rate = 0.35,
max_depth = 7, n_estimators = 13)
 xgb.fit(X_train, Y_train)
 xgb_score = cross_val_score(xgb,X_train,Y_train,cv=10 ).mean()
print("i : {} ,교차 검증 점수 : {:.3f}".format(0,xgb_score))
print("i : {} ,훈련 세트 정확도: {:.3f}".format(0,xgb.score(X_train, Y_train)))
 print("i : {} ,테스트 세트 정확도: {:.3f}".format(0,xgb.score(X_test, Y_test)))
 print('================================================' )<split>
del bur,dummy
gc.collect()
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size = 0.2 , random_state=40)
 gbrt = GradientBoostingClassifier(random_state=101,max_depth=5,learning_rate=0.12)
 ada = AdaBoostClassifier(
 DecisionTreeClassifier(max_depth=3), n_estimators=200,
algorithm="SAMME.R", learning_rate=0.22)
 xgb = XGBClassifier(objective ='reg:linear', colsample_bytree = 0.6, learning_rate = 0.35,
max_depth = 7, n_estimators = 13)
 voting_clf = VotingClassifier(
 estimators=[('ada', ada),('GB', gbrt),('xgb',xgb)],
voting='hard')
 voting_clf.fit(X_train, Y_train)
 vo_score = cross_val_score(voting_clf,X_train,Y_train,cv=7 ).mean()
print(",교차 검증 점수 : {:.3f}".format(vo_score))
print(",훈련 세트 정확도: {:.3f}".format(voting_clf.score(X_train, Y_train)))
 print(",테스트 세트 정확도: {:.3f}".format(voting_clf.score(X_test, Y_test)))
 print('================================================')
<prepare_x_and_y>
bur_bal['MONTHS_BALANCE']=bur_bal.MONTHS_BALANCE.abs()
<save_to_csv>
%%time
bur_bal_f=bur_bal.groupby(['SK_ID_BUREAU','STATUS'] ).aggregate({'STATUS':['count'],'MONTHS_BALANCE':['max','min']})
 bur_bal_f.reset_index(inplace=True)
 bur_bal_f.columns=['SK_ID_BUREAU','STATUS','STATUS_count','MONTHS_BALANCE_max','MONTHS_BALANCE_min']
sub = pd.read_csv('.. /input/smhrd12/sample_submission.csv')
 sub['income'] = gbrt_pred
sub.to_csv('submission.csv',index=False )<import_modules>
dummy=pd.get_dummies(bur_bal_f['STATUS'],prefix='DUM_STATUS' )
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer, make_column_transformer
from sklearn.pipeline import Pipeline<load_from_csv>
bur_bal_ff=pd.concat([bur_bal_f.drop(['STATUS'],axis=1),dummy],axis=1 )
town_data = pd.read_csv('/kaggle/input/town-data/town_data.csv', header=[1])
 town_data = town_data.replace('*', np.nan)
 town_data = town_data.replace('X', np.nan )<load_from_csv>
%%time
dummy_col=[x for x in bur_bal_ff.columns if 'DUM_' in x]
for col in dummy_col:
bur_bal_ff[col]=bur_bal_ff.apply(lambda x: x.STATUS_count if x[col]==1 else 0,axis=1 )
train_data = pd.read_csv('/kaggle/input/machine-learning-lab-cas-data-science-fs-20/houses_train.csv', index_col=0 )<merge>
bur_bal_ff.drop('STATUS_count',axis=1,inplace=True )
train_data = train_data.join(town_data.set_index('Gemeindename'), on='municipality_name' )<data_type_conversions>
bur_bal_cols=[x for x in list(bur_bal_ff.columns)if x not in ['SK_ID_BUREAU']]
bur_bal_agg={}
bur_bal_name=['SK_ID_BUREAU']
for col in bur_bal_cols:
if 'DUM_' in col:
bur_bal_agg[col]=['sum']
bur_bal_name.append(col)
 elif '_max' in col:
bur_bal_agg[col]=['max']
bur_bal_name.append(col)
 elif '_min' in col:
bur_bal_agg[col]=['min']
bur_bal_name.append(col)
 else:
bur_bal_agg[col]=['sum','mean']
bur_bal_name.append(col+'_'+'sum')
 bur_bal_name.append(col+'_'+'mean' )
train_data = train_data.fillna(-1 )<prepare_x_and_y>
%%time
bur_bal_fg=bur_bal_ff.groupby('SK_ID_BUREAU' ).aggregate(bur_bal_agg)
 bur_bal_fg.reset_index(inplace=True)
 bur_bal_fg.columns=bur_bal_name
X_data = train_data.drop(columns='price')
 y_data = train_data['price']<split>
del bur_bal,bur_bal_f,bur_bal_ff
gc.collect()
X_train, X_dev, y_train, y_dev = train_test_split(X_data, y_data, stratify=X_data['object_type_name'], test_size=0.1 )<categorify>
bur_combi=bur_f.join(bur_bal_fg.set_index('SK_ID_BUREAU'),on='SK_ID_BUREAU',lsuffix='_BU', rsuffix='_BUB' )
pipeline = Pipeline([ 
('pre', make_column_transformer(( OneHotEncoder(handle_unknown='ignore'), ['zipcode', 'municipality_name', 'object_type_name']), remainder='passthrough')) , 
('clf', LGBMRegressor(
 num_leaves=50,
learning_rate=0.05,
n_estimators=200 
))
] )<import_modules>
del bur_bal_fg
gc.collect()
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV<define_search_space>
bur_combi_cols=[x for x in list(bur_combi.columns)if x not in ['SK_ID_CURR','SK_ID_BUREAU']]
bur_combi_agg={}
bur_combi_name=['SK_ID_CURR','SK_ID_BUREAU']
for col in bur_combi_cols:
if 'DUM_' in col:
bur_combi_agg[col]=['sum']
bur_combi_name.append(col+'_'+'sum')
 elif 'AMT_' in col:
bur_combi_agg[col]=['sum','mean','max','min','var','std']
bur_combi_name.append(col+'_'+'sum')
 bur_combi_name.append(col+'_'+'mean')
 bur_combi_name.append(col+'_'+'max')
 bur_combi_name.append(col+'_'+'min')
 bur_combi_name.append(col+'_'+'var')
 bur_combi_name.append(col+'_'+'std')
 elif 'CNT_' in col:
bur_combi_agg[col]=['sum','max','min','count']
bur_combi_name.append(col+'_'+'sum')
 bur_combi_name.append(col+'_'+'max')
 bur_combi_name.append(col+'_'+'min')
 bur_combi_name.append(col+'_'+'count')
 elif 'DAYS_' in col:
bur_combi_agg[col]=['sum','max','min']
bur_combi_name.append(col+'_'+'sum')
 bur_combi_name.append(col+'_'+'max')
 bur_combi_name.append(col+'_'+'min')
 elif 'CALC_' in col:
bur_combi_agg[col]=['mean']
bur_combi_name.append(col+'_'+'mean')
 else:
bur_combi_agg[col]=['sum']
bur_combi_name.append(col+'_'+'sum')
parameters = {
'clf__num_leaves': [10,20,30,40,50,60,70,80,90,100],
'clf__max_depth': [3,4,5,6,7,8,9,10],
'clf__n_estimators': [100,150,200,500]
}
clf = RandomizedSearchCV(pipeline, parameters, cv=5, n_iter=40, verbose=1, n_jobs=-1 )<train_model>
%%time
bur_combi_f=bur_combi.groupby(['SK_ID_CURR','SK_ID_BUREAU'] ).aggregate(bur_combi_agg)
 bur_combi_f.reset_index(inplace=True)
 bur_combi_f.columns=bur_combi_name
clf.fit(X_train, y_train )<find_best_params>
bur_combi_cols=list(bur_combi_f.columns)
 bur_combi_agg={}
bur_combi_name=['SK_ID_CURR']
for col in bur_combi_cols:
if 'SK_ID_CURR'==col:
bur_combi_agg[col]=['count']
bur_combi_name.append('SK_ID_BUREAU_count')
 elif '_sum'==col:
bur_combi_agg[col]=['sum']
bur_combi_name.append(col)
 elif '_mean' in col:
bur_combi_agg[col]=['mean']
bur_combi_name.append(col)
 elif '_max' in col:
bur_combi_agg[col]=['max']
bur_combi_name.append(col)
 elif '_min' in col:
bur_combi_agg[col]=['min']
bur_combi_name.append(col)
 elif '_count' in col:
bur_combi_agg[col]=['sum']
bur_combi_name.append(col)
 elif '_var' in col:
bur_combi_agg[col]=['mean']
bur_combi_name.append(col)
 elif '_std' in col:
bur_combi_agg[col]=['mean']
bur_combi_name.append(col)
 else:
bur_combi_agg[col]=['sum']
bur_combi_name.append(col )
est = clf.best_estimator_<predict_on_test>
%%time
bur_combi_fg=bur_combi_f.groupby(['SK_ID_CURR'] ).aggregate(bur_combi_agg)
 bur_combi_fg.reset_index(inplace=True)
 bur_combi_fg.columns=bur_combi_name
y_dev_pred = est.predict(X_dev )<compute_test_metric>
bur_combi_fg.drop(['DUM_CREDIT_TYPE_Car_loan_sum', 'DUM_CREDIT_CURRENCY_currency_2_sum', 'AMT_CREDIT_SUM_OVERDUE_var'],axis=1,inplace=True )
def mean_absolute_percentage_error(y_true, y_pred):
return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 100<compute_test_metric>
df_final=df_final.join(bur_combi_fg.set_index('SK_ID_CURR'),on='SK_ID_CURR',lsuffix='_AP', rsuffix='_BU' )
mean_absolute_percentage_error(y_dev, y_dev_pred )<train_model>
del bur_combi,bur_combi_f,bur_combi_fg
gc.collect()
est.fit(X_data, y_data )<load_from_csv>
%%time
prev_appl=pd.read_csv(path1+'previous_application.csv')
 print('previous_application set reading complete...' )
X_test = pd.read_csv('/kaggle/input/machine-learning-lab-cas-data-science-fs-20/houses_test.csv', index_col=0 )<merge>
prev_appl_encoding={'NAME_YIELD_GROUP':{'middle':3,'low_action':1,'high':4,'low_normal':1,'XNA':0},
'WEEKDAY_APPR_PROCESS_START':{'MONDAY':1,'TUESDAY':2,'WEDNESDAY':3,'THURSDAY':4,'FRIDAY':5,'SATURDAY':6,'SUNDAY':7},
'FLAG_LAST_APPL_PER_CONTRACT':{'Y':1,'N':0}}
prev_appl.replace(prev_appl_encoding,inplace=True )
X_test = X_test.join(town_data.set_index('Gemeindename'), on='municipality_name')
 X_test = X_test.fillna(-1 )<predict_on_test>
%%time
days_col=[x for x in prev_appl.columns if 'DAYS_' in x]
for col in days_col:
prev_appl.replace(365243,np.nan,inplace= True )
y_test_pred = est.predict(X_test )<create_dataframe>
prev_appl_fs[prev_appl_fs.Data_type=='object']
X_test_submission = pd.DataFrame(index=X_test.index )<feature_engineering>
%%time
prev_appl_cf=list(prev_appl_fs[prev_appl_fs.Data_type=='object'].index)
 for col in prev_appl_cf:
prev_appl[col]=prev_appl[col].apply(lambda x: str(x ).replace(" ","_"))
print('previous application categorical features',len(prev_appl_cf))
prev_appl_cf.remove('NAME_CONTRACT_STATUS')
 prev_appl_cat=pd.DataFrame()
for col in prev_appl_cf:
dummy=pd.get_dummies(prev_appl[col],prefix='DUM_'+col)
 prev_appl_cat=pd.concat([prev_appl_cat,dummy],axis=1)
 display(prev_appl_cat.head() )
X_test_submission['price'] = y_test_pred<save_to_csv>
prev_appl_cat.drop(['DUM_NAME_CASH_LOAN_PURPOSE_XAP', 'DUM_PRODUCT_COMBINATION_nan'],axis=1,inplace=True )
X_test_submission.to_csv('lightgbm_submission_all.csv', header=True, index_label='id' )<set_options>
print('Newly created Dummy features:',len(prev_appl_cat.columns))
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<define_variables>
prev_appl_f=pd.concat([prev_appl.drop(prev_appl_cf,axis=1),prev_appl_cat],axis=1)
SEED = 42
VALIDATION_PCT = 0.1
IMAGE_SIZE_64 = 64
IMAGE_SIZE_224 = 224
BATCH_SIZE_64 = 64
BATCH_SIZE_224 = 224
PATH = Path('.. /input')
 TRAIN_PATH = PATH/'train'
TEST_FOLDER_PATH = "test/test"
SAMPLE_SUBMISSION_PATH = PATH/"sample_submission.csv"<categorify>
del prev_appl,prev_appl_cat,prev_appl_fs
gc.collect()
def load_data(path, image_size, batch_size, validation_pct = VALIDATION_PCT):
data =(ImageList.from_folder(path)
.split_by_rand_pct(validation_pct, seed = SEED)
.label_from_folder() 
.transform(get_transforms() , size = image_size)
.databunch(bs = batch_size)
.normalize(imagenet_stats))
return data<load_pretrained>
%%time
prev_appl_f['CALC_PERC_ANNUITY']=prev_appl_f['AMT_ANNUITY']/prev_appl_f['AMT_CREDIT']
prev_appl_f['CALC_PERC_AMT_APPLICATION']=prev_appl_f['AMT_APPLICATION']/prev_appl_f['AMT_CREDIT']
prev_appl_f['CALC_PERC_DOWN_PAYMENT']=prev_appl_f['AMT_DOWN_PAYMENT']/prev_appl_f['AMT_CREDIT']
prev_appl_f['CALC_PERC_APPL_DOWN_PAYMENT']=prev_appl_f['AMT_DOWN_PAYMENT']/prev_appl_f['AMT_APPLICATION']
prev_appl_f['CALC_PERC_GOODS_PRICE']=prev_appl_f['AMT_GOODS_PRICE']/prev_appl_f['AMT_CREDIT']
prev_appl_f['CALC_FLAG_PRIVILEGED_CUSTOMER']=(prev_appl_f['RATE_INTEREST_PRIVILEGED']>0 ).astype(int)
 prev_appl_f['CALC_TERMINATED']=(prev_appl_f['DAYS_TERMINATION']<0 ).astype(int)
 prev_appl_f['CALC_AMT_ANNUITY_PER_PAY']=prev_appl_f['AMT_ANNUITY']/prev_appl_f['CNT_PAYMENT']
prev_appl_f['CALC_AMT_APPLICATION_PER_PAY']=prev_appl_f['AMT_APPLICATION']/prev_appl_f['CNT_PAYMENT']
prev_appl_f['CALC_AMT_DOWN_PAYMENT_PER_PAY']=prev_appl_f['AMT_DOWN_PAYMENT']/prev_appl_f['CNT_PAYMENT']
prev_appl_f['CALC_AMT_APPL_DOWN_PAYMENT_PER_PAY']=prev_appl_f['AMT_DOWN_PAYMENT']/prev_appl_f['CNT_PAYMENT']
prev_appl_f['CALC_AMT_GOODS_PRICE_PER_PAY']=prev_appl_f['AMT_GOODS_PRICE']/prev_appl_f['CNT_PAYMENT']
prev_appl_f['CALC_AMT_CREDIT_APP_DIFF']=prev_appl_f['AMT_CREDIT']-prev_appl_f['AMT_APPLICATION']
prev_appl_f['CALC_LAST_DUE_AFTER_APPL']=(prev_appl_f['DAYS_LAST_DUE_1ST_VERSION']>0 ).astype(int )
data_64 = load_data(TRAIN_PATH, IMAGE_SIZE_64, BATCH_SIZE_224, VALIDATION_PCT)
 data_64<choose_model_class>
prev_cols=[x for x in list(prev_appl_f.columns)if x not in ['SK_ID_CURR','SK_ID_PREV','NAME_CONTRACT_STATUS']]
prev_agg={}
prev_name=['SK_ID_CURR','SK_ID_PREV','NAME_CONTRACT_STATUS']
for col in prev_cols:
if 'DUM_' in col:
prev_agg[col]=['sum','mean','std']
prev_name.append(col+'_'+'sum')
 prev_name.append(col+'_'+'mean')
 prev_name.append(col+'_'+'std')
 elif 'AMT_' in col:
prev_agg[col]=['sum','mean','max','min','var','std']
prev_name.append(col+'_'+'sum')
 prev_name.append(col+'_'+'mean')
 prev_name.append(col+'_'+'max')
 prev_name.append(col+'_'+'min')
 prev_name.append(col+'_'+'var')
 prev_name.append(col+'_'+'std')
 elif 'CNT_' in col:
prev_agg[col]=['sum','max','min','size','count']
prev_name.append(col+'_'+'sum')
 prev_name.append(col+'_'+'max')
 prev_name.append(col+'_'+'min')
 prev_name.append(col+'_'+'size')
 prev_name.append(col+'_'+'count')
 elif 'DAYS_' in col:
prev_agg[col]=['sum','max','min']
prev_name.append(col+'_'+'sum')
 prev_name.append(col+'_'+'max')
 prev_name.append(col+'_'+'min')
 elif 'CALC_FLAG_' in col:
prev_agg[col]=['sum']
prev_name.append(col+'_'+'sum')
 elif 'CALC_' in col:
prev_agg[col]=['mean']
prev_name.append(col+'_'+'mean')
 else:
prev_agg[col]=['sum','mean']
prev_name.append(col+'_'+'sum')
 prev_name.append(col+'_'+'mean')
learn = cnn_learner(data_64,
models.resnet152,
metrics = [error_rate, accuracy],
model_dir = '/tmp/model/' )<train_model>
%%time
prev_appl_ff=prev_appl_f.groupby(['SK_ID_CURR','SK_ID_PREV','NAME_CONTRACT_STATUS'] ).aggregate(prev_agg)
 prev_appl_ff.reset_index(inplace=True)
 prev_appl_ff.columns=prev_name
learn.fit_one_cycle(1, 1e-2 )<load_from_csv>
del prev_appl_f
gc.collect()
test_df = pd.read_csv(SAMPLE_SUBMISSION_PATH)
 test_df.head() 
<define_variables>
dummy=pd.get_dummies(prev_appl_ff['NAME_CONTRACT_STATUS'],prefix='DUM_'+'NAME_CONTRACT_STATUS' )
test_images = ImageList.from_df(test_df, PATH/"", folder = TEST_FOLDER_PATH )<load_pretrained>
prev_appl_fg=pd.concat([prev_appl_ff.iloc[:,:1],dummy,prev_appl_ff.iloc[:,3:]],axis=1 )
data_256 = load_data(TRAIN_PATH, IMAGE_SIZE_224, BATCH_SIZE_64, VALIDATION_PCT)
 data_256.add_test(test_images )<find_best_params>
del prev_appl_ff,dummy
gc.collect()
learn.data = data_256
learn.lr_find()
learn.recorder.plot()<train_model>
prev_cols=list(prev_appl_fg.columns)
 prev_agg={}
prev_name=['SK_ID_CURR']
for col in prev_cols:
if 'SK_ID_CURR'==col:
prev_agg[col]=['count']
prev_name.append('SK_ID_PREV_count')
 elif '_sum' in col:
prev_agg[col]=['sum']
prev_name.append(col)
 elif '_mean' in col:
prev_agg[col]=['mean']
prev_name.append(col)
 elif '_max' in col:
prev_agg[col]=['max']
prev_name.append(col)
 elif '_min' in col:
prev_agg[col]=['min']
prev_name.append(col)
 elif '_var' in col:
prev_agg[col]=['mean']
prev_name.append(col)
 elif '_std' in col:
prev_agg[col]=['mean']
prev_name.append(col)
 elif '_size' in col:
prev_agg[col]=['mean']
prev_name.append(col)
 elif '_count' in col:
prev_agg[col]=['sum']
prev_name.append(col)
 else:
prev_agg[col]=['sum']
prev_name.append(col)
learn.unfreeze()
learn.fit_one_cycle(30, max_lr = slice(1.0e-4, 1.0e-3))<predict_on_test>
%%time
prev_appl_fgg=prev_appl_fg.groupby(['SK_ID_CURR'] ).aggregate(prev_agg)
 prev_appl_fgg.reset_index(inplace=True)
 prev_appl_fgg.columns=prev_name
test_probabalities, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_predictions = [data_256.classes[pred] for pred in np.argmax(test_probabalities.numpy() , axis=-1)]<save_to_csv>
del prev_appl_fg
gc.collect()
test_df.predicted_class = test_predictions
test_df.to_csv("submission.csv", index=False)
 test_df.head()<import_modules>
df_final=df_final.join(prev_appl_fgg.set_index('SK_ID_CURR'),on='SK_ID_CURR',lsuffix='_AP', rsuffix='_PV' )
import sklearn<load_from_csv>
del prev_appl_fgg
gc.collect()
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 sample = pd.read_csv(".. /input/sample_sumbission.csv" )<import_modules>
%%time
inst_pay=pd.read_csv(path1+'installments_payments.csv')
 print('installments_payments set reading complete...' )
import numpy as np
import pandas as pd<import_modules>
for col in inst_pay.columns:
if 'DAYS_' in col:
inst_pay[col]=inst_pay[col].abs()
from sklearn.ensemble import BaggingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import VotingRegressor
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import Lars
from sklearn.linear_model import LassoLars
from sklearn.linear_model import OrthogonalMatchingPursuit
from sklearn.linear_model import PassiveAggressiveRegressor
from sklearn.linear_model import BayesianRidge
from sklearn.linear_model import ARDRegression
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import SGDRegressor
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveRegressor
from sklearn.linear_model import RANSACRegressor
from sklearn.linear_model import TheilSenRegressor
from sklearn.linear_model import HuberRegressor
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsRegressor
from sklearn.neighbors import RadiusNeighborsRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.svm import SVR
from sklearn.svm import NuSVR
from sklearn.svm import LinearSVR
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor<load_from_csv>
%%time
inst_pay['CALC_DAYS_LATE_PAYMENT']=inst_pay['DAYS_ENTRY_PAYMENT']-inst_pay['DAYS_INSTALMENT']
inst_pay['CALC_PERC_LESS_PAYMENT']=inst_pay['AMT_PAYMENT']/inst_pay['AMT_INSTALMENT']
inst_pay['CALC_PERC_LESS_PAYMENT'].replace(np.inf,0,inplace=True)
 inst_pay['CALC_DIFF_INSTALMENT']=inst_pay['AMT_INSTALMENT']-inst_pay['AMT_PAYMENT']
inst_pay['CALC_PERC_DIFF_INSTALMENT']=np.abs(inst_pay['CALC_DIFF_INSTALMENT'])/inst_pay['AMT_INSTALMENT']
inst_pay['CALC_PERC_DIFF_INSTALMENT'].replace(np.inf,0,inplace=True)
 inst_pay['CALC_INSTAL_PAID_LATE'] =(inst_pay['CALC_DAYS_LATE_PAYMENT'] > 0 ).astype(int)
 inst_pay['CALC_OVERPAID']=(inst_pay['CALC_DIFF_INSTALMENT'] < 0 ).astype(int )
train_data = pd.read_csv('.. /input/abalone-train.csv', index_col='id' )<load_from_csv>
inst_pay_cols=[x for x in list(inst_pay.columns)if x not in ['SK_ID_CURR','SK_ID_PREV']]
inst_pay_agg={}
inst_pay_name=['SK_ID_CURR','SK_ID_PREV']
for col in inst_pay_cols:
if 'NUM_INSTALMENT_VERSION'==col:
inst_pay_agg[col]=[cnt_unique]
inst_pay_name.append(col+'_'+'unique')
 elif 'NUM_INSTALMENT_NUMBER'==col:
inst_pay_agg[col]=['max','count']
inst_pay_name.append(col+'_'+'max')
 inst_pay_name.append(col+'_'+'count')
 elif 'AMT_' in col:
inst_pay_agg[col]=['sum','mean','max','min','var','std']
inst_pay_name.append(col+'_'+'sum')
 inst_pay_name.append(col+'_'+'mean')
 inst_pay_name.append(col+'_'+'max')
 inst_pay_name.append(col+'_'+'min')
 inst_pay_name.append(col+'_'+'var')
 inst_pay_name.append(col+'_'+'std')
 elif 'CALC_DAYS_' in col:
inst_pay_agg[col]=['sum']
inst_pay_name.append(col+'_'+'sum')
 elif 'DAYS_' in col:
inst_pay_agg[col]=['sum','max','min']
inst_pay_name.append(col+'_'+'sum')
 inst_pay_name.append(col+'_'+'max')
 inst_pay_name.append(col+'_'+'min')
 else:
inst_pay_agg[col]=['mean']
inst_pay_name.append(col+'_'+'mean' )
test_data = pd.read_csv('.. /input/abalone-test.csv', index_col='id' )<categorify>
%%time
inst_pay_f=inst_pay.groupby(['SK_ID_CURR','SK_ID_PREV'] ).aggregate(inst_pay_agg)
 inst_pay_f.reset_index(inplace=True)
 inst_pay_f.columns=inst_pay_name
train_data = pd.get_dummies(train_data)
 test_data = pd.get_dummies(test_data )<drop_column>
inst_pay_f['CALC_NUM_INSTALMENT_MISSED']=inst_pay_f['NUM_INSTALMENT_NUMBER_max']-inst_pay_f['NUM_INSTALMENT_NUMBER_count']
data = train_data
outliers = np.concatenate((
data[(data['height'] < 0.01)|(data['height'] > 0.3)].index,
data[(data['viscera_weight'] < 0.0001)|(data['viscera_weight'] > 0.6)].index 
), axis=0)
 train_data.drop(outliers, inplace=True )<prepare_x_and_y>
inst_pay_cols=[x for x in list(inst_pay_f.columns)if x not in ['SK_ID_PREV']]
inst_pay_agg={}
inst_pay_name=['SK_ID_CURR']
for col in inst_pay_cols:
if 'SK_ID_CURR'==col:
inst_pay_agg[col]=['count']
inst_pay_name.append('SK_ID_PREV_count')
 elif '_unique' in col:
inst_pay_agg[col]=['sum']
inst_pay_name.append(col)
 elif '_mean' in col:
inst_pay_agg[col]=['mean']
inst_pay_name.append(col)
 elif '_max' in col:
inst_pay_agg[col]=['max']
inst_pay_name.append(col)
 elif '_min' in col:
inst_pay_agg[col]=['min']
inst_pay_name.append(col)
 elif '_count' in col:
inst_pay_agg[col]=['sum']
inst_pay_name.append(col)
 else:
inst_pay_agg[col]=['sum']
inst_pay_name.append(col )
X_train = train_data.drop(['rings'], axis=1)
 y_train = train_data['rings']
print('Forma dos dados de treino:', X_train.shape, y_train.shape )<choose_model_class>
%%time
inst_pay_f.drop(['SK_ID_PREV'],axis=1,inplace=True)
 inst_pay_fg=inst_pay_f.groupby(['SK_ID_CURR'] ).aggregate(inst_pay_agg)
 inst_pay_fg.reset_index(inplace=True)
 inst_pay_fg.columns=inst_pay_name
models = []
models.append(( 'LinReg', LinearRegression(n_jobs=-1, fit_intercept=True, normalize=True)))
 models.append(( 'LogReg', LogisticRegression(n_jobs=-1, random_state=42, multi_class='auto', C=1000, solver='newton-cg')))
 models.append(( 'OMP', OrthogonalMatchingPursuit(n_nonzero_coefs=7, fit_intercept=True, normalize=True)))
 models.append(( 'PAR', PassiveAggressiveRegressor(random_state=42, C=0.1, fit_intercept=True, max_iter=1000, tol=0.001)))
 models.append(( 'PP', Perceptron(random_state=42, penalty='l2', alpha=1e-3, fit_intercept=True, max_iter=1000, tol=1e-3)))
 models.append(( 'RANSAC', RANSACRegressor(random_state=42, min_samples=0.75)))
 models.append(( 'Ridge', Ridge(random_state=42, alpha=0.001, fit_intercept=True, normalize=True)))
 models.append(( 'SGD', SGDRegressor(random_state=42, alpha=1e-6, fit_intercept=True, penalty=None, tol=1e-3)))
 models.append(( 'TSR', TheilSenRegressor(random_state=42, n_jobs=-1, fit_intercept=True)))
 models.append(( 'DTR', DecisionTreeRegressor(random_state=42, max_depth=4, min_samples_split=0.25)))
 models.append(( 'GPR', GaussianProcessRegressor(random_state=42, alpha=0.01, normalize_y=False)))
 models.append(( 'KRR', KernelRidge(alpha=0.1)))
 models.append(( 'GNB', GaussianNB(var_smoothing=0.001)))
 models.append(( 'kNN', KNeighborsRegressor(n_jobs=-1, n_neighbors=11, weights='distance')))
 models.append(( 'SVM', SVR(gamma='auto', kernel='linear')))
 models.append(( 'MLP', MLPRegressor(random_state=42, max_iter=500,
activation='tanh', hidden_layer_sizes=(5, 2), solver='lbfgs')))
 models.append(( 'RFR', RandomForestRegressor(random_state=42, n_jobs=-1, n_estimators=100, max_depth=9)))
 models.append(( 'GBR', GradientBoostingRegressor(random_state=42, learning_rate=0.05, n_estimators=100,
subsample=0.8, max_depth=6, max_features=0.75)))
 models.append(( 'ETR', ExtraTreesRegressor(random_state=42, n_jobs=-1, n_estimators=200, max_features=0.75)))
 models.append(( 'BDTR', BaggingRegressor(random_state=42, n_jobs=-1, base_estimator=DecisionTreeRegressor() ,
max_features=0.75, n_estimators=75)))
 models.append(( 'ABDTR', AdaBoostRegressor(random_state=42, n_estimators=200, base_estimator=DecisionTreeRegressor())))
 models.append(( 'XGBR', XGBRegressor(random_state=42, n_jobs=-1, learning_rate=0.1,
n_estimators=50, max_depth=5, objective='reg:squarederror')))
 models.append(( 'VR', VotingRegressor(estimators=[ 
('MLP', MLPRegressor(random_state=42, max_iter=500,
activation='tanh', hidden_layer_sizes=(5, 2), solver='lbfgs')) , 
('GPR', GaussianProcessRegressor(random_state=42, alpha=0.01, normalize_y=False)) , 
('GBR', GradientBoostingRegressor(random_state=42, learning_rate=0.05, n_estimators=100,
subsample=0.8, max_depth=6, max_features=0.75))
], n_jobs=-1, weights=(2,1,1))))<save_to_csv>
inst_pay_fg['CALC_CNT_INSTALMENT_VERSION_CHG']=inst_pay_fg['NUM_INSTALMENT_VERSION_unique']-inst_pay_fg['SK_ID_PREV_count']
sufixo_arquivo = '05set'
for name, model in models:
print(model, '
')
 model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 submission = pd.DataFrame({
'id': X_test.index,
'rings': y_pred
})
 submission.set_index('id', inplace=True)
 filename = 'abalone-submission-p-%s-%s.csv' %(sufixo_arquivo, name.lower())
 submission.to_csv(filename )<import_modules>
inst_pay_fg.drop(['DAYS_ENTRY_PAYMENT_max', 'AMT_PAYMENT_var', 'DAYS_INSTALMENT_sum'],axis=1,inplace=True )
import os
from PIL import Image
import numpy as np
import os
from random import shuffle
import matplotlib.pyplot as plt
import pandas as pd<categorify>
df_final=df_final.join(inst_pay_fg.set_index('SK_ID_CURR'),on='SK_ID_CURR',lsuffix='_AP', rsuffix='_INP' )
def label_img(name):
word_label = name.split('-')[0]
if word_label == 'hot dog': return np.array([1, 0])
 elif word_label == 'not hot dog' : return np.array([0, 1] )<define_variables>
del inst_pay,inst_pay_f,inst_pay_fg
gc.collect()
IMG_SIZE = 100
label_texts = ['hot dog', 'not hot dog']
def load_training_data() :
train_data = []
for label_t in label_texts:
count = 0
print('reading images for ' + label_t)
 for image_file in os.listdir('.. /input/skill-factory-final-hackathon/train/train/' + label_t):
label = label_img(label_t)
 count += 1
if count > 5500:
break
if "DS_Store" not in image_file:
img = Image.open('.. /input/skill-factory-final-hackathon/train/train/' + label_t + '/' + image_file)
 img = img.resize(( IMG_SIZE, IMG_SIZE), Image.ANTIALIAS)
 train_data.append([np.array(img), label])
 print('reading additional images for hotdog')
 for image_file in os.listdir('.. /input/hotdog2/archive'):
label = label_img('hot dog')
 if "DS_Store" not in image_file:
img = Image.open('.. /input/hotdog2/archive/' + image_file)
 img = img.resize(( IMG_SIZE, IMG_SIZE), Image.ANTIALIAS)
 train_data.append([np.array(img), label])
 print('reading imagen for hotdog')
 for image_file in os.listdir('.. /input/imggen2/hdresized/hdresized'):
label = label_img('hot dog')
 if "DS_Store" not in image_file:
img = Image.open('.. /input/imggen2/hdresized/hdresized/' + image_file)
 img = img.resize(( IMG_SIZE, IMG_SIZE), Image.ANTIALIAS)
 train_data.append([np.array(img), label])
 shuffle(train_data)
 return train_data 
<load_pretrained>
%%time
pos_cash=pd.read_csv(path1+'POS_CASH_balance.csv')
 print('POS_CASH_balance set reading complete...' )
train_data = load_training_data()<define_variables>
pos_cash['MONTHS_BALANCE']=pos_cash['MONTHS_BALANCE'].abs()
trainImages = np.array([i[0] for i in train_data] ).reshape(-1, IMG_SIZE, IMG_SIZE, 3)
 trainLabels = np.array([i[1] for i in train_data] )<import_modules>
pos_cash['CALC_PERC_REMAINING_INSTAL']=pos_cash['CNT_INSTALMENT_FUTURE']/pos_cash['CNT_INSTALMENT']
pos_cash['CALC_CNT_REMAINING_INSTAL']=pos_cash['CNT_INSTALMENT']-pos_cash['CNT_INSTALMENT_FUTURE']
pos_cash['CALC_DAYS_WITHOUT_TOLERANCE']=pos_cash['SK_DPD']-pos_cash['SK_DPD_DEF']
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, Convolution2D
from keras.layers.normalization import BatchNormalization
import numpy as np<set_options>
pos_cash['NAME_CONTRACT_STATUS']=pos_cash['NAME_CONTRACT_STATUS'].apply(lambda x: str(x ).replace(" ","_"))
dummy=pd.get_dummies(pos_cash['NAME_CONTRACT_STATUS'],prefix='DUM_NAME_CONTRACT_STATUS' )
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))<choose_model_class>
pos_cash_f=pd.concat([pos_cash.drop(['NAME_CONTRACT_STATUS'],axis=1),dummy],axis=1 )
model = Sequential()
model = Sequential()
model.add(Conv2D(32, kernel_size =(3, 3), activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 3)))
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(96, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(96, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dense(2, activation = 'softmax'))<choose_model_class>
pos_cash_cols=[x for x in list(pos_cash_f.columns)if x not in ['SK_ID_CURR']]
pos_cash_agg={}
pos_cash_name=['SK_ID_CURR','SK_ID_PREV']
for col in pos_cash_cols:
if 'SK_ID_PREV'==col:
pos_cash_agg[col]=['count']
pos_cash_name.append(col+'_'+'count')
 elif 'MONTHS_BALANCE'==col:
pos_cash_agg[col]=['max','min','count']
pos_cash_name.append(col+'_'+'max')
 pos_cash_name.append(col+'_'+'min')
 pos_cash_name.append(col+'_'+'count')
 elif 'DUM_' in col:
pos_cash_agg[col]=['sum','mean','max','min']
pos_cash_name.append(col+'_'+'sum')
 pos_cash_name.append(col+'_'+'mean')
 pos_cash_name.append(col+'_'+'max')
 pos_cash_name.append(col+'_'+'min')
 elif 'CNT_' in col:
pos_cash_agg[col]=['max','min','sum','count']
pos_cash_name.append(col+'_'+'max')
 pos_cash_name.append(col+'_'+'min')
 pos_cash_name.append(col+'_'+'sum')
 pos_cash_name.append(col+'_'+'count')
 else:
pos_cash_agg[col]=['sum','mean']
pos_cash_name.append(col+'_'+'sum')
 pos_cash_name.append(col+'_'+'mean' )
model.compile(loss='binary_crossentropy', optimizer='adam', metrics = ['accuracy'] )<train_model>
%%time
pos_cash_ff=pos_cash_f.groupby(['SK_ID_CURR','SK_ID_PREV'] ).aggregate(pos_cash_agg)
 pos_cash_ff.reset_index(inplace=True)
 pos_cash_ff.columns=pos_cash_name
model.fit(trainImages, trainLabels, batch_size = 64, epochs = 50, verbose = 1 )<load_pretrained>
pos_cash_cols=[x for x in list(pos_cash_ff.columns)if x not in ['SK_ID_CURR','SK_ID_PREV']]
pos_cash_agg={}
pos_cash_name=['SK_ID_CURR']
for col in pos_cash_cols:
if '_sum'==col:
pos_cash_agg[col]=['sum']
pos_cash_name.append(col)
 elif '_mean' in col:
pos_cash_agg[col]=['mean']
pos_cash_name.append(col)
 elif '_max' in col:
pos_cash_agg[col]=['max']
pos_cash_name.append(col)
 elif '_min' in col:
pos_cash_agg[col]=['min']
pos_cash_name.append(col)
 elif '_count' in col:
pos_cash_agg[col]=['sum']
pos_cash_name.append(col)
 else:
pos_cash_agg[col]=['sum']
pos_cash_name.append(col )
def load_training_data() :
test_data = []
leaderboard_filenames = []
for image_file in os.listdir('.. /input/skill-factory-final-hackathon/test/test/'):
if "DS_Store" not in image_file:
img = Image.open('.. /input/skill-factory-final-hackathon/test/test/' + image_file)
 img = img.resize(( IMG_SIZE, IMG_SIZE), Image.ANTIALIAS)
 test_data.append([np.array(img)])
 leaderboard_filenames.append(image_file)
 return test_data, leaderboard_filenames
test_data, leaderboard_filenames = load_training_data()<define_variables>
%%time
pos_cash_fg=pos_cash_ff.groupby(['SK_ID_CURR'] ).aggregate(pos_cash_agg)
 pos_cash_fg.reset_index(inplace=True)
 pos_cash_fg.columns=pos_cash_name
testImages = np.array([i[0] for i in test_data] ).reshape(-1, IMG_SIZE, IMG_SIZE, 3 )<predict_on_test>
df_final=df_final.join(pos_cash_fg.set_index('SK_ID_CURR'),on='SK_ID_CURR',lsuffix='_AP', rsuffix='_PC' )
pred = model.predict(testImages )<define_variables>
del pos_cash_fg,pos_cash,pos_cash_f,pos_cash_ff
gc.collect()
fp = []
for i in pred:
fp.append(round(i[0],2))<create_dataframe>
%%time
cc_bal=pd.read_csv(path1+'credit_card_balance.csv')
 print('credit_card_balance set reading complete...' )
submission = pd.DataFrame(
 {
'image_id': leaderboard_filenames,
'image_hot_dog_probability': fp
}
)<save_to_csv>
cc_bal['MONTHS_BALANCE']=cc_bal['MONTHS_BALANCE'].abs()
submission.to_csv('submit.csv', index=False )<define_variables>
cc_bal['CALC_PERC_BALANCE']=cc_bal['AMT_BALANCE']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_DRAWINGS_ATM_CURRENT']=cc_bal['AMT_DRAWINGS_ATM_CURRENT']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_DRAWINGS_CURRENT']=cc_bal['AMT_DRAWINGS_CURRENT']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_DRAWINGS_OTHER_CURRENT']=cc_bal['AMT_DRAWINGS_OTHER_CURRENT']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_DRAWINGS_POS_CURRENT']=cc_bal['AMT_DRAWINGS_POS_CURRENT']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_INST_MIN_REGULARITY']=cc_bal['AMT_INST_MIN_REGULARITY']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_PAYMENT_CURRENT']=cc_bal['AMT_PAYMENT_CURRENT']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_PAYMENT_TOTAL_CURRENT']=cc_bal['AMT_PAYMENT_TOTAL_CURRENT']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_RECEIVABLE_PRINCIPAL']=cc_bal['AMT_RECEIVABLE_PRINCIPAL']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_PERC_RECIVABLE']=cc_bal['AMT_RECIVABLE']/cc_bal['AMT_CREDIT_LIMIT_ACTUAL']
cc_bal['CALC_DAYS_WITHOUT_TOLERANCE']=cc_bal['SK_DPD']-cc_bal['SK_DPD_DEF']
CNT_DRAWING_LIST=['CNT_DRAWINGS_ATM_CURRENT','CNT_DRAWINGS_CURRENT','CNT_DRAWINGS_OTHER_CURRENT','CNT_DRAWINGS_POS_CURRENT']
cc_bal['CALC_CNT_DRAWINGS_TOTAL']=cc_bal[CNT_DRAWING_LIST].sum(axis=1 )
paths= ".. /input/SMEMI309-final-evaluation-challenge-2020/"
at4_test_path= paths+"AT4_test.csv"
at4_train_path = paths+"AT4_train.csv"
at2_test_path= paths+"AT2_test.csv"
at2_train_path = paths+"AT2_train.csv"
bt_test_path=paths+"BT_test.csv"
bt_train_path=paths+"BT_train.csv"<set_options>
cc_bal['NAME_CONTRACT_STATUS']=cc_bal['NAME_CONTRACT_STATUS'].apply(lambda x: str(x ).replace(" ","_"))
dummy=pd.get_dummies(cc_bal['NAME_CONTRACT_STATUS'],prefix='DUM_NAME_CONTRACT_STATUS' )
pd.set_option('display.max_columns', 30 )<load_from_csv>
cc_bal_f=pd.concat([cc_bal.drop(['NAME_CONTRACT_STATUS'],axis=1),dummy],axis=1 )
at4_train =pd.read_csv(at4_train_path,sep=",")
 at2_train =pd.read_csv(at2_train_path,sep=",")
 bt_train=pd.read_csv(bt_train_path,sep=",")
 test_data =pd.read_csv(at4_test_path,sep="," )<statistical_test>
cc_bal_cols=[x for x in list(cc_bal_f.columns)if x not in ['SK_ID_CURR']]
cc_bal_agg={}
cc_bal_name=['SK_ID_CURR','SK_ID_PREV']
for col in cc_bal_cols:
if 'SK_ID_PREV'==col:
cc_bal_agg[col]=['count']
cc_bal_name.append(col+'_'+'count')
 elif 'MONTHS_BALANCE'==col:
cc_bal_agg[col]=['max','min','count']
cc_bal_name.append(col+'_'+'max')
 cc_bal_name.append(col+'_'+'min')
 cc_bal_name.append(col+'_'+'count')
 elif 'AMT_' in col:
cc_bal_agg[col]=['sum','mean','max','min','var','std']
cc_bal_name.append(col+'_'+'sum')
 cc_bal_name.append(col+'_'+'mean')
 cc_bal_name.append(col+'_'+'max')
 cc_bal_name.append(col+'_'+'min')
 cc_bal_name.append(col+'_'+'var')
 cc_bal_name.append(col+'_'+'std')
 elif 'CNT_' in col:
cc_bal_agg[col]=['max','min','sum','count']
cc_bal_name.append(col+'_'+'max')
 cc_bal_name.append(col+'_'+'min')
 cc_bal_name.append(col+'_'+'sum')
 cc_bal_name.append(col+'_'+'count')
 else:
cc_bal_agg[col]=['mean']
cc_bal_name.append(col+'_'+'mean' )
for i in range(1,31):
anova=calculate_anova(at2_train[at2_train['label']==3].iloc[:,i],
at4_train[at4_train['label']==3].iloc[:,i],
bt_train[bt_train['label']==3].iloc[:,i])
 print(i)
 print(anova )<statistical_test>
%%time
cc_bal_ff=cc_bal_f.groupby(['SK_ID_CURR','SK_ID_PREV'] ).aggregate(cc_bal_agg)
 cc_bal_ff.reset_index(inplace=True)
 cc_bal_ff.columns=cc_bal_name
for i in range(1,31):
wilcoxon=calculate_wilcoxon(at2_train[at2_train['label']==2].iloc[:,i],
at4_train[at4_train['label']==2].iloc[:,i])
 stat, p_value = wilcoxon
print(i)
 if(p_value>0.05):
print('P-value larger than Alpha')
 print(wilcoxon )<groupby>
cc_bal_cols=[x for x in list(cc_bal_ff.columns)if x not in ['SK_ID_CURR','SK_ID_PREV']]
cc_bal_agg={}
cc_bal_name=['SK_ID_CURR']
for col in cc_bal_cols:
if '_sum'==col:
cc_bal_agg[col]=['sum']
cc_bal_name.append(col)
 elif '_var' in col:
cc_bal_agg[col]=['mean']
cc_bal_name.append(col)
 elif '_std' in col:
cc_bal_agg[col]=['mean']
cc_bal_name.append(col)
 elif '_mean' in col:
cc_bal_agg[col]=['mean']
cc_bal_name.append(col)
 elif '_max' in col:
cc_bal_agg[col]=['max']
cc_bal_name.append(col)
 elif '_min' in col:
cc_bal_agg[col]=['min']
cc_bal_name.append(col)
 elif '_count' in col:
cc_bal_agg[col]=['sum']
cc_bal_name.append(col)
 else:
cc_bal_agg[col]=['sum']
cc_bal_name.append(col )
bt_train.groupby('label')['label'].count()<define_search_space>
%%time
cc_bal_fg=cc_bal_ff.groupby(['SK_ID_CURR'] ).aggregate(cc_bal_agg)
 cc_bal_fg.reset_index(inplace=True)
 cc_bal_fg.columns=cc_bal_name
for i in range(10,21):
for j in range(i,21):
if(i !=j):
stat, p_value=calculate_wilcoxon(bt_train.iloc[:,i],bt_train.iloc[:,j])
 if(p_value>0.05 and i!=j):
print('columns {} and {} are similar'.format(i,j))<define_search_space>
df_final=df_final.join(cc_bal_fg.set_index('SK_ID_CURR'),on='SK_ID_CURR',lsuffix='_AP', rsuffix='_CB' )
for i in range(20,31):
for j in range(i,31):
if(i !=j):
stat, p_value=calculate_wilcoxon(bt_train.iloc[:,i],bt_train.iloc[:,j])
 if(p_value>0.05 and i!=j):
print('columns {} and {} are similar'.format(i,j))<feature_engineering>
del cc_bal_fg,cc_bal,cc_bal_f,cc_bal_ff
gc.collect()
at4_train.loc[:,at4_train.columns!='label'].apply(minMax )<feature_engineering>
df_final.drop(['SK_ID_CURR'],axis=1,inplace=True )
at4_train.loc[:,at4_train.columns!='label'].apply(minMax )<feature_engineering>
df_final.drop(drop_col,axis=1,inplace=True )
bt_train.loc[:,bt_train.columns!='label'].apply(minMax )<split>
del corr
gc.collect()
X_At4_train, X_At4_val, y_At4_train, y_At4_val=train_test_split(at4_train.loc[: , at4_train.columns!='label'],at4_train['label'],random_state=0)
 X_At2_train, X_At2_val, y_At2_train, y_At2_val=train_test_split(at2_train.loc[: , at2_train.columns!='label'],at2_train['label'],random_state=0)
 X_bt_train, X_bt_val, y_bt_train, y_bt_val=train_test_split(bt_train.loc[: , bt_train.columns!='label'],bt_train['label'],random_state=0 )<choose_model_class>
%%time
df_final['the_mean'] = df_final.mean(axis=1)
 print('the_mean calculated...')
 df_final['the_sum'] =df_final.sum(axis=1)
 print('the_sum calculated...')
 df_final['the_std'] = df_final.std(axis=1)
 print('the_std calculated...' )
def create_simple_mlp() :
mlp = Sequential()
mlp.add(Dense(32, activation='relu'))
mlp.add(Dense(16, activation='relu'))
mlp.add(Dense(8, activation='relu'))
mlp.add(Dense(4, activation='softmax'))
return mlp<choose_model_class>
X=df_final.iloc[:len(train),:]
y=train['TARGET']
test=df_final.iloc[len(train):,:]
opt = keras.optimizers.Adam(learning_rate=0.001)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=50 )<train_model>
del df_final
gc.collect()
mlp_bt=create_simple_mlp()
mlp_bt.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_bt = mlp_bt.fit(X_bt_train,y_bt_train ,batch_size=32, validation_data=(X_bt_val, y_bt_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_bt)
<compute_test_metric>
%%time
model=CatBoostClassifier(iterations=1000,
learning_rate=0.05,
depth=7,
l2_leaf_reg=40,
bootstrap_type='Bernoulli',
subsample=0.7,
scale_pos_weight=5,
eval_metric='AUC',
metric_period=50,
od_type='Iter',
od_wait=45,
random_seed=17,
allow_writing_files=False)
 df_preds=pd.DataFrame()
df_preds_x=pd.DataFrame()
k=1
splits=10
avg_score=0
skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=200)
 print('
Starting KFold iterations...')
 for train_index,test_index in skf.split(X,y):
df_X=X.iloc[train_index,:]
df_y=y.iloc[train_index]
val_X=X.iloc[test_index,:]
val_y=y.iloc[test_index]
model.fit(df_X,df_y)
 col_name='cat_predsx_'+str(k)
 preds_x=pd.Series(model.predict_proba(val_X)[:,1])
 df_preds_x[col_name]=pd.Series(model.predict_proba(X)[:,1])
 acc=roc_auc_score(val_y,preds_x)
 print('Iteration:',k,' roc_auc_score:',acc)
 if k==1:
score=acc
model1=model
preds=pd.Series(model.predict_proba(test)[:,1])
 col_name='cat_preds_'+str(k)
 df_preds[col_name]=preds
else:
preds1=pd.Series(model.predict_proba(test)[:,1])
 preds=preds+preds1
col_name='cat_preds_'+str(k)
 df_preds[col_name]=preds1
if score<acc:
score=acc
model1=model
avg_score=avg_score+acc
k=k+1
print('
Best score:',score,' Avg Score:',avg_score/splits)
 preds=preds/splits
score_bt=mlp_bt.evaluate(X_bt_val,y_bt_val)
 score_bt<train_model>
sample=pd.read_csv(path1+'sample_submission.csv' )
mlp_at2=create_simple_mlp()
mlp_at2.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_at2 = mlp_at2.fit(X_At2_train,y_At2_train ,batch_size=16, validation_data=(X_At4_val, y_At4_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_at2)
<train_model>
sample['TARGET']=preds
mlp_at4=create_simple_mlp()
mlp_at4.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_at4 = mlp_at4.fit(X_At4_train,y_At4_train ,batch_size=16, validation_data=(X_At4_val, y_At4_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_at4)
<create_dataframe>
sample.to_csv('submission.csv',index=False )
shuffeled=X_At4_val.copy()<statistical_test>
df_preds_x.to_csv('cat_preds_x.csv',index=False)
 df_preds.to_csv('cat_preds.csv',index=False )
<randomize_order><EOS> 
%%time
sample_ens=sample.copy()
df_preds_x.columns=['cat_preds_1','cat_preds_2','cat_preds_3','cat_preds_4','cat_preds_5','cat_preds_6',
'cat_preds_7','cat_preds_8','cat_preds_9','cat_preds_10']
df_final_ens=pd.concat([df_preds_x,df_preds],ignore_index=True)
 df_final_ens['the_mean'] = df_final_ens.mean(axis=1)
 print('the_mean calculated...')
 df_final_ens['the_sum'] =df_final_ens.sum(axis=1)
 print('the_sum calculated...')
 df_final_ens['the_std'] = df_final_ens.std(axis=1)
 print('the_std calculated...')
 df_final_ens['the_kur'] = df_final_ens.kurtosis(axis=1)
 print('the_kur calculated...')
 X=df_final_ens.iloc[:len(train),:]
y=train['TARGET']
test=df_final_ens.iloc[len(train):,:]
model=LogisticRegression()
k=1
splits=10
avg_score=0
skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=200)
 print('
Starting KFold iterations...')
 for train_index,test_index in skf.split(X,y):
df_X=X.iloc[train_index,:]
df_y=y.iloc[train_index]
val_X=X.iloc[test_index,:]
val_y=y.iloc[test_index]
model.fit(df_X,df_y)
 preds_x=pd.Series(model.predict_proba(val_X)[:,1])
 acc=roc_auc_score(val_y,preds_x)
 print('Iteration:',k,' roc_auc_score:',acc)
 if k==1:
score=acc
model1=model
preds=pd.Series(model.predict_proba(test)[:,1])
 else:
preds=preds+pd.Series(model.predict_proba(test)[:,1])
 if score<acc:
score=acc
model1=model
avg_score=avg_score+acc
k=k+1
print('
Best score:',score,' Avg Score:',avg_score/splits)
 preds=preds/splits
sample_ens['TARGET']=preds
sample_ens.to_csv('submission_ens.csv',index=False )