Kaggle competition: Don't Overfit! II
corr_pred = 0.6 * df['T2'] + 0.05 * df['T3'] + 0.05 * df['T4'] + 0.1 * df['T5'] + 0.2 * df['T1']
corr_pred.head()<save_to_csv>
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv')
 sub=pd.read_csv('.. /input/sample_submission.csv')
SuB = pd.DataFrame()
SuB['SK_ID_CURR'] = df['SK_ID_CURR']
SuB['TARGET'] = corr_pred
SuB.to_csv('corr_blend.csv', index=False )<define_variables>
train_feat=train.drop(columns=['id','target'],axis=1)
 train_feat.head()
deb=time.time()
path=".. /input"
listfiles={"application_train":("SK_ID_CURR",["SK_ID_CURR"]),"application_test":("SK_ID_CURR",["SK_ID_CURR"]),"bureau":("SK_ID_CURR",["SK_ID_CURR","SK_ID_BUREAU"]),
"bureau_balance":("SK_ID_BUREAU",["SK_ID_BUREAU"]),"POS_CASH_balance":("SK_ID_PREV",["SK_ID_CURR","SK_ID_PREV"]),"previous_application":("SK_ID_CURR",["SK_ID_CURR","SK_ID_PREV"]),
"installments_payments":("SK_ID_PREV",["SK_ID_CURR","SK_ID_PREV"]),"credit_card_balance":("SK_ID_PREV",["SK_ID_CURR","SK_ID_PREV"])}
links=[('bureau_balance','bureau'),('installments_payments','previous_application'),('credit_card_balance','previous_application'),('POS_CASH_balance','previous_application'), 
('previous_application','full'),('bureau','full')]
ftime=[("Start",time.time() ,0)]
def timer(title,showtime=True):
global ftime
newtime=(title,time.time() ,time.time() -ftime[-1][1])
 ftime.append(newtime)
 if showtime==True:
print(newtime[0] + " in " + str(np.round(newtime[2],2)))
 else:
print(newtime[0])
 data={x:(pd.read_csv(path +"/"+ x + '.csv')).sort_values(listfiles[x][0],ascending=True)for x in listfiles}
listetodel=list(set(list(data["application_train"])[44:91])^set(["FONDKAPREMONT_MODE", "HOUSETYPE_MODE", "WALLSMATERIAL_MODE", "EMERGENCYSTATE_MODE"]))
print("liste ok")
 for i in listfiles:
data[i].index=list(range(data[i].shape[0]))
data["full"]=pd.concat([data["application_train"],data["application_test"]],axis=0,sort=True)
 data["full"].index=list(range(data["full"].shape[0]))
indtest=pd.DataFrame(data["full"]["SK_ID_CURR"].iloc[data["application_train"].shape[0]:])
 train_shape,test_shape=data["application_train"].shape[0],data["application_test"].shape[0]
del data["application_train"],data["application_test"]
dicovars={}
timer("Files read and sorted by key")
 def agregate(dataf,key,table):
global dicovars
continuous, discrete,dichot=set_groups(dataf)
 if len(dichot)> 0 : dichot = dichot + [key]
vals=sorted(list(set(dataf[key])))
 size=len(vals)
 nbint=10
pace=size//nbint+nbint
indmin=dataf[key].index[0]
dicodf={}
for k,i in enumerate(list(range(pace,size+pace,pace-1))):
valmax=vals[i-pace:i-1][-1]
indmax=dataf[key][dataf[key]==valmax].index.tolist() [-1]
dicosets={}
if len(continuous)> 0 : dicosets["continuous"]=dataf[continuous].loc[indmin:indmax,:].groupby(key ).agg(["sum","max","min"])
 if len(dichot)> 0 : dicosets["dichot"]=dataf[dichot].loc[indmin:indmax,:].groupby(key ).agg(["sum"])
 dicosets["count"]=pd.DataFrame(dataf.loc[indmin:indmax,:].groupby(key)[key].agg("count"))
ind=pd.MultiIndex(levels=[[table],["count"]],labels=[[0],[0]])
 dicosets["count"].columns=ind
for x in list(dicosets): dicosets[x].columns=[x[0]+"_"+x[1] for x in dicosets[x].columns]
dicodf[k]=pd.concat([dicosets[x] for x in list(dicosets)],axis=1)
 del dicosets
indmin=indmax+1
dataf_tsf=pd.concat([dicodf[k] for k in list(dicodf)],axis=0)
 timer("Agreggation done")
 return dataf_tsf
def binar(series,table):
global dicovars
setserie=set(series)
 if len(setserie)==2:
var=pd.DataFrame(pd.get_dummies(series ).iloc[:,0])
 else: var=pd.get_dummies(series)
 name=series.name
var.columns=[str(x)+ "_"+ name for x in var.columns]
dicovars[table+ "_"+str(name)]=var.columns.tolist()
return var
def set_groups(dataf):
num=dataf.columns[(dataf.dtypes!="object" ).tolist() ].tolist()
nom=dataf.columns[(dataf.dtypes=="object" ).tolist() ].tolist()
continuous,discrete,dichot = [],nom,[]
for i in num:
if len(set(dataf[i].loc[:1000])) >3: continuous.append(i)
 else:
setvar=set(dataf[i])
 if len(setvar)>3: continuous.append(i)
 elif(setvar==set([0,1])and dataf[i].count() ==dataf.shape[0]): dichot.append(i)
 else: discrete.append(i)
 timer("Groups set")
 return(continuous, discrete,dichot)
 def fill_values(dataf,continuous, discrete,dichot,table,fill_cont=True):
dicodf={}
if len(discrete)>0:
dataf[discrete]=dataf[discrete].fillna('missing')
 timer("Missing values filled for discrete vars")
 dicodf["discrete"]=pd.concat([binar(dataf[x],table)for x in discrete],axis=1)
 liste=list(set(dataf)^set(discrete))
if len(liste)>0:
dicodf["Others"]=dataf[liste]
fusion=pd.concat([dicodf[x] for x in list(dicodf)],axis=1)
 return fusion
def process(dataf,keys,keyforagg,name):
liste=list(set(dataf)^set(keys))
dataf=dataf[liste+[keyforagg]]
columnswithoutkeys=list(set(dataf)^set([keyforagg]))
continuous, discrete,dichot =set_groups(dataf[columnswithoutkeys])
 dataf=fill_values(dataf,continuous, discrete,dichot,name,fill_cont=False)
 liste=dataf.columns.tolist()
for i in range(len(liste)) :
if liste[i] not in keys: liste[i]= str(liste[i]+"_" +name)
 dataf.columns=liste
dataf=agregate(dataf,keyforagg,name)
 dataf[keyforagg]=dataf.index.tolist()
dataf.index=list(range(dataf.shape[0]))
return dataf
diconame={"bureau":"B","bureau_balance":"BB","POS_CASH_balance":"PCB","previous_application":"PA","installments_payments":"IP","credit_card_balance":"CCB"}
for i in links:
timer("Processing " + i[0],showtime=False)
 data[i[0]]=process(data[i[0]],listfiles[i[0]][1],listfiles[i[0]][0],diconame[i[0]])
 gc.collect()
timer("Merging " + i[0] + " with " +i[1],showtime=False)
 print(data[i[0]].shape)
 data[i[1]]=data[i[1]].merge(right=data[i[0]],how='left',on=listfiles[i[0]][0])
 del data[i[0]]
timer("Merging completed")
 print(data[i[1]].shape)
 gc.collect()
columnswithoutkeys=list(set(data["full"])^set(["SK_ID_CURR"]))
continuous, discrete,dichot =set_groups(data["full"][columnswithoutkeys])
 data["full"]=fill_values(data["full"],continuous, discrete,dichot,"BB",fill_cont=False)
 gc.collect()
gc.collect()
print('total preprocessing time : ' + str(time.time() -deb))
data["full"]=data["full"][data["full"].count() [data["full"].count() >0].index.tolist() ]
data["full"]["Annuity_income_ratio"]=data["full"]["AMT_ANNUITY"]/data["full"]["AMT_INCOME_TOTAL"]
data["full"]['Credit_annuity_ratio'] = data["full"]['AMT_CREDIT'] / data["full"]['AMT_ANNUITY']
data["full"]['NEW_SOURCES_PROD'] = data["full"]['EXT_SOURCE_1'] * data["full"]['EXT_SOURCE_2'] * data["full"]['EXT_SOURCE_3']
data["full"]['NEW_EXT_SOURCES_MEAN'] = data["full"][['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 data["full"]['NEW_CREDIT_TO_GOODS_RATIO'] = data["full"]['AMT_CREDIT'] / data["full"]['AMT_GOODS_PRICE']
data["full"]['NEW_EMPLOY_TO_BIRTH_RATIO'] = data["full"]['DAYS_EMPLOYED'] / data["full"]['DAYS_BIRTH']
data["full"]['NEW_PHONE_TO_EMPLOY_RATIO'] = data["full"]['DAYS_LAST_PHONE_CHANGE'] / data["full"]['DAYS_EMPLOYED']
data["full"]['NEW_SCORES_STD'] = data["full"][['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 liste=list(filter(lambda x :x !="TARGET",list(data["full"])))
 X_train, X_val, y_train, y_val = train_test_split(( data["full"].iloc[: train_shape,:])[liste], data["full"]["TARGET"].iloc[:train_shape], test_size=0.2, random_state=42)
 X_test=(data["full"].iloc[train_shape:,:])[liste]
del data["full"]
gc.collect()
listetodel=['2_STATUS_BB_sum_B_min',
'3_STATUS_BB_sum_B_max',
'3_STATUS_BB_sum_B_min',
'4_STATUS_BB_sum_B_max',
'4_STATUS_BB_sum_B_min',
'AMT_BALANCE_CCB_sum_PA_sum',
'AMT_CREDIT_SUM_OVERDUE_B_min',
'AMT_DRAWINGS_ATM_CURRENT_CCB_min_PA_max',
'AMT_DRAWINGS_ATM_CURRENT_CCB_min_PA_min',
'AMT_DRAWINGS_CURRENT_CCB_min_PA_max',
'AMT_DRAWINGS_OTHER_CURRENT_CCB_min_PA_max',
'AMT_DRAWINGS_OTHER_CURRENT_CCB_min_PA_min',
'AMT_DRAWINGS_OTHER_CURRENT_CCB_min_PA_sum',
'AMT_DRAWINGS_OTHER_CURRENT_CCB_sum_PA_max',
'AMT_DRAWINGS_POS_CURRENT_CCB_min_PA_max',
'AMT_DRAWINGS_POS_CURRENT_CCB_min_PA_sum',
'AMT_INST_MIN_REGULARITY_CCB_min_PA_max',
'AMT_INST_MIN_REGULARITY_CCB_sum_PA_sum',
'AMT_RECIVABLE_CCB_min_PA_min',
'AMT_REQ_CREDIT_BUREAU_HOUR',
'AMT_TOTAL_RECEIVABLE_CCB_max_PA_sum',
'AMT_TOTAL_RECEIVABLE_CCB_min_PA_max',
'AMT_TOTAL_RECEIVABLE_CCB_sum_PA_sum',
'APARTMENTS_AVG',
'APARTMENTS_MEDI',
'APARTMENTS_MODE',
'Additional Service_NAME_GOODS_CATEGORY_PA_sum',
'Advertising_ORGANIZATION_TYPE',
'Agriculture_ORGANIZATION_TYPE',
'Amortized debt_NAME_CONTRACT_STATUS_PCB_sum_PA_max',
'Amortized debt_NAME_CONTRACT_STATUS_PCB_sum_PA_min',
'Amortized debt_NAME_CONTRACT_STATUS_PCB_sum_PA_sum',
'Animals_NAME_GOODS_CATEGORY_PA_sum',
'Another type of loan_CREDIT_TYPE_B_sum',
'Approved_NAME_CONTRACT_STATUS_CCB_sum_PA_max',
'Approved_NAME_CONTRACT_STATUS_CCB_sum_PA_min',
'Approved_NAME_CONTRACT_STATUS_CCB_sum_PA_sum',
'Approved_NAME_CONTRACT_STATUS_PCB_sum_PA_max',
'Approved_NAME_CONTRACT_STATUS_PCB_sum_PA_min',
'Approved_NAME_CONTRACT_STATUS_PCB_sum_PA_sum',
'BASEMENTAREA_AVG',
'BASEMENTAREA_MEDI',
'BASEMENTAREA_MODE',
'Bad debt_CREDIT_ACTIVE_B_sum',
'Building a house or an annex_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Business development_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Buying a garage_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Buying a holiday home / land_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Buying a home_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Buying a new car_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Buying a used car_NAME_CASH_LOAN_PURPOSE_PA_sum',
'CLIENT_CODE_REJECT_REASON_PA_sum',
'CNT_CREDIT_PROLONG_B_min',
'CNT_CREDIT_PROLONG_B_sum',
'CNT_DRAWINGS_ATM_CURRENT_CCB_min_PA_max',
'CNT_DRAWINGS_ATM_CURRENT_CCB_min_PA_min',
'CNT_DRAWINGS_ATM_CURRENT_CCB_min_PA_sum',
'CNT_DRAWINGS_CURRENT_CCB_min_PA_max',
'CNT_DRAWINGS_CURRENT_CCB_min_PA_min',
'CNT_DRAWINGS_CURRENT_CCB_min_PA_sum',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_max_PA_max',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_max_PA_min',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_min_PA_max',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_min_PA_min',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_min_PA_sum',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_sum_PA_max',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_sum_PA_min',
'CNT_DRAWINGS_POS_CURRENT_CCB_min_PA_sum',
'CNT_INSTALMENT_MATURE_CUM_CCB_min_PA_max',
'CNT_INSTALMENT_MATURE_CUM_CCB_min_PA_min',
'CNT_INSTALMENT_MATURE_CUM_CCB_min_PA_sum',
'COMMONAREA_AVG',
'COMMONAREA_MEDI',
'COMMONAREA_MODE',
'CREDIT_DAY_OVERDUE_B_min',
'Canceled_NAME_CONTRACT_STATUS_PCB_sum_PA_max',
'Canceled_NAME_CONTRACT_STATUS_PCB_sum_PA_min',
'Canceled_NAME_CONTRACT_STATUS_PCB_sum_PA_sum',
'Car dealer_CHANNEL_TYPE_PA_sum',
'Car repairs_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Cars_NAME_PORTFOLIO_PA_sum',
'Cash loan(non-earmarked)_CREDIT_TYPE_B_sum',
'Cashless from the account of the employer_NAME_PAYMENT_TYPE_PA_sum',
'Cleaning_ORGANIZATION_TYPE',
'Cooking staff_OCCUPATION_TYPE',
'Culture_ORGANIZATION_TYPE',
'DAYS_FIRST_DRAWING_PA_max',
'Demand_NAME_CONTRACT_STATUS_CCB_sum_PA_max',
'Demand_NAME_CONTRACT_STATUS_CCB_sum_PA_min',
'Demand_NAME_CONTRACT_STATUS_CCB_sum_PA_sum',
'Demand_NAME_CONTRACT_STATUS_PCB_sum_PA_min',
'Demand_NAME_CONTRACT_STATUS_PCB_sum_PA_sum',
'Direct Sales_NAME_GOODS_CATEGORY_PA_sum',
'ELEVATORS_AVG',
'ELEVATORS_MEDI',
'ELEVATORS_MODE',
'ENTRANCES_AVG',
'ENTRANCES_MEDI',
'ENTRANCES_MODE',
'Education_NAME_GOODS_CATEGORY_PA_sum',
'Electricity_ORGANIZATION_TYPE',
'Emergency_ORGANIZATION_TYPE',
'Everyday expenses_NAME_CASH_LOAN_PURPOSE_PA_sum',
'FLAG_CONT_MOBILE',
'FLAG_DOCUMENT_15',
'FLAG_DOCUMENT_2',
'FLAG_DOCUMENT_5',
'FLAG_DOCUMENT_9',
'FLAG_EMP_PHONE',
'FLOORSMAX_AVG',
'FLOORSMAX_MEDI',
'FLOORSMAX_MODE',
'FLOORSMIN_AVG',
'FLOORSMIN_MEDI',
'FLOORSMIN_MODE',
'Fitness_NAME_GOODS_CATEGORY_PA_sum',
'Furniture_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Gardening_NAME_GOODS_CATEGORY_PA_sum',
'Gasification / water supply_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Group of people_NAME_TYPE_SUITE_PA_sum',
'Hobby_NAME_CASH_LOAN_PURPOSE_PA_sum',
'House Construction_NAME_GOODS_CATEGORY_PA_sum',
'Industry: type 10_ORGANIZATION_TYPE',
'Industry: type 11_ORGANIZATION_TYPE',
'Industry: type 13_ORGANIZATION_TYPE',
'Industry: type 2_ORGANIZATION_TYPE',
'Industry: type 6_ORGANIZATION_TYPE',
'Industry: type 7_ORGANIZATION_TYPE',
'Industry: type 8_ORGANIZATION_TYPE',
'Insurance_NAME_GOODS_CATEGORY_PA_sum',
'Insurance_ORGANIZATION_TYPE',
'Interbank credit_CREDIT_TYPE_B_sum',
'Journey_NAME_CASH_LOAN_PURPOSE_PA_sum',
'LANDAREA_AVG',
'LANDAREA_MEDI',
'LANDAREA_MODE',
'LIVE_CITY_NOT_WORK_CITY',
'LIVINGAPARTMENTS_AVG',
'LIVINGAPARTMENTS_MEDI',
'LIVINGAPARTMENTS_MODE',
'LIVINGAREA_AVG',
'LIVINGAREA_MEDI',
'LIVINGAREA_MODE',
'Legal Services_ORGANIZATION_TYPE',
'Loan for business development_CREDIT_TYPE_B_sum',
'Loan for purchase of shares(margin lending)_CREDIT_TYPE_B_sum',
'Loan for the purchase of equipment_CREDIT_TYPE_B_sum',
'Loan for working capital replenishment_CREDIT_TYPE_B_sum',
'MLM partners_NAME_SELLER_INDUSTRY_PA_sum',
'MONTHS_BALANCE_CCB_max_PA_min',
'MONTHS_BALANCE_CCB_sum_PA_sum',
'Maternity leave_NAME_INCOME_TYPE',
'Medical Supplies_NAME_GOODS_CATEGORY_PA_sum',
'Medicine_NAME_GOODS_CATEGORY_PA_sum',
'Medicine_ORGANIZATION_TYPE',
'Mobile operator loan_CREDIT_TYPE_B_sum',
'Mobile_ORGANIZATION_TYPE',
'Money for a third person_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Monolithic_WALLSMATERIAL_MODE',
'NONLIVINGAPARTMENTS_AVG',
'NONLIVINGAPARTMENTS_MEDI',
'NONLIVINGAPARTMENTS_MODE',
'NONLIVINGAREA_AVG',
'NONLIVINGAREA_MEDI',
'NONLIVINGAREA_MODE',
'NUM_INSTALMENT_NUMBER_IP_min_PA_max',
'NUM_INSTALMENT_VERSION_IP_min_PA_min',
'Office Appliances_NAME_GOODS_CATEGORY_PA_sum',
'Other_B_NAME_TYPE_SUITE',
'Other_NAME_GOODS_CATEGORY_PA_sum',
'POS others without interest_PRODUCT_COMBINATION_PA_sum',
'Payments on other loans_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Private service staff_OCCUPATION_TYPE',
'Purchase of electronic equipment_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Real estate loan_CREDIT_TYPE_B_sum',
'Refusal to name the goal_NAME_CASH_LOAN_PURPOSE_PA_sum',
'Refused_NAME_CONTRACT_STATUS_CCB_sum_PA_max',
'Refused_NAME_CONTRACT_STATUS_CCB_sum_PA_min',
'Refused_NAME_CONTRACT_STATUS_CCB_sum_PA_sum',
'Religion_ORGANIZATION_TYPE',
'Returned to the store_NAME_CONTRACT_STATUS_PCB_sum_PA_min',
'SK_DPD_CCB_max_PA_max',
'SK_DPD_CCB_sum_PA_sum',
'SK_DPD_DEF_CCB_max_PA_sum',
'SK_DPD_DEF_PCB_min_PA_max',
'SK_DPD_DEF_PCB_min_PA_sum',
'SK_DPD_PCB_min_PA_max',
'SK_DPD_PCB_min_PA_sum',
'SYSTEM_CODE_REJECT_REASON_PA_sum',
'Sent proposal_NAME_CONTRACT_STATUS_CCB_sum_PA_max',
'Sent proposal_NAME_CONTRACT_STATUS_CCB_sum_PA_min',
'Sent proposal_NAME_CONTRACT_STATUS_CCB_sum_PA_sum',
'Services_ORGANIZATION_TYPE',
'Signed_NAME_CONTRACT_STATUS_CCB_sum_PA_max',
'Signed_NAME_CONTRACT_STATUS_CCB_sum_PA_min',
'Signed_NAME_CONTRACT_STATUS_CCB_sum_PA_sum',
'TOTALAREA_MODE',
'Telecom_ORGANIZATION_TYPE',
'Tourism_NAME_GOODS_CATEGORY_PA_sum',
'Tourism_NAME_SELLER_INDUSTRY_PA_sum',
'Trade: type 1_ORGANIZATION_TYPE',
'Trade: type 4_ORGANIZATION_TYPE',
'Trade: type 5_ORGANIZATION_TYPE',
'Trade: type 6_ORGANIZATION_TYPE',
'Transport: type 1_ORGANIZATION_TYPE',
'Transport: type 2_ORGANIZATION_TYPE',
'Unemployed_NAME_INCOME_TYPE',
'University_ORGANIZATION_TYPE',
'Unknown type of loan_CREDIT_TYPE_B_sum',
'Vehicles_NAME_GOODS_CATEGORY_PA_sum',
'Weapon_NAME_GOODS_CATEGORY_PA_sum',
'XNA_NAME_CLIENT_TYPE_PA_sum',
'XNA_NAME_CONTRACT_STATUS_PCB_sum_PA_max',
'XNA_NAME_CONTRACT_STATUS_PCB_sum_PA_sum',
'XNA_NAME_CONTRACT_TYPE_PA_sum',
'YEARS_BEGINEXPLUATATION_AVG',
'YEARS_BEGINEXPLUATATION_MEDI',
'YEARS_BEGINEXPLUATATION_MODE',
'YEARS_BUILD_AVG',
'YEARS_BUILD_MEDI',
'YEARS_BUILD_MODE',
'currency 4_CREDIT_CURRENCY_B_sum',
'missing_NAME_TYPE_SUITE',
'missing_PRODUCT_COMBINATION_PA_sum',
'4_STATUS_BB_sum_B_sum',
'Industry: type 5_ORGANIZATION_TYPE',
'N_FLAG_LAST_APPL_PER_CONTRACT_PA_sum',
'AMT_RECEIVABLE_PRINCIPAL_CCB_sum_PA_sum',
'Security_ORGANIZATION_TYPE',
'CNT_DRAWINGS_OTHER_CURRENT_CCB_sum_PA_sum',
'Jewelry_NAME_SELLER_INDUSTRY_PA_sum',
'Completed_NAME_CONTRACT_STATUS_CCB_sum_PA_max']
listfin=list(( set(listetodel)^set(X_train)) &set(X_train))
X_train=X_train[listfin]
X_val=X_val[listfin]
X_test=X_test[listfin]
gc.collect()
timer("Variables selected")
 X_train=pd.concat([X_train,X_val],axis=0)
 y_train=pd.concat([y_train,y_val],axis=0)
 del X_val,y_val
gc.collect()
timer("Data ready for modelling")
 vect=np.copy(y_train)
 vect[vect==1]=4
vect[vect==0]=1
clf = LGBMClassifier(learning_rate =0.075, num_boost_round=1500, nthread=8, seed=27,colsample_bytree=1, max_depth=3,
min_child_weight=87.5467,min_split_gain=0.0950,num_leaves=22,reg_alpha=0.0019,reg_lambda=0.0406,subsample=0.8709)
 clf.fit(X_train, y_train, eval_metric= 'auc', verbose= 100)
 timer("Model created.")
<save_to_csv>
test_feat=test.drop(columns='id',axis=1)
 test_feat.head()
sub=pd.Series(clf.predict_proba(X_test)[:,1],name="TARGET")
 sub.loc[sub<0]=0
sub.loc[sub>1]=1
sub.index=indtest.index
submission=pd.concat([indtest,sub],axis=1)
 submission.to_csv('submission.csv', index=False )<categorify>
target=pd.DataFrame(pred3 ).rename(columns={0:'target'})
 sub_id=sub[['id']]
submission=pd.concat([sub_id,target],axis=1)
 submission
warnings.simplefilter(action='ignore', category=FutureWarning)
 @contextmanager
def timer(title):
t0 = time.time()
yield
print("{} - done in {:.0f}s".format(title, time.time() - t0))
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
def application_train_test(num_rows = None, nan_as_category = False):
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
df = df[df['CODE_GENDER'] != 'XNA']
docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]
live = [_f for _f in df.columns if('FLAG_' in _f)&('FLAG_DOC' not in _f)&('_FLAG_' not in _f)]
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 inc_by_org = df[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE' ).median() ['AMT_INCOME_TOTAL']
df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)
 df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)
 df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] /(1 + df['CNT_CHILDREN'])
 df['NEW_INC_BY_ORG'] = df['ORGANIZATION_TYPE'].map(inc_by_org)
 df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] /(1 + df['AMT_INCOME_TOTAL'])
 df['NEW_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']
df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 df['NEW_SCORES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)
 df['NEW_SCORES_STD'] = df['NEW_SCORES_STD'].fillna(df['NEW_SCORES_STD'].mean())
 df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['NEW_PHONE_TO_BIRTH_RATIO_EMPLOYER'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 dropcolum=['FLAG_DOCUMENT_2','FLAG_DOCUMENT_4',
'FLAG_DOCUMENT_5','FLAG_DOCUMENT_6','FLAG_DOCUMENT_7',
'FLAG_DOCUMENT_8','FLAG_DOCUMENT_9','FLAG_DOCUMENT_10',
'FLAG_DOCUMENT_11','FLAG_DOCUMENT_12','FLAG_DOCUMENT_13',
'FLAG_DOCUMENT_14','FLAG_DOCUMENT_15','FLAG_DOCUMENT_16',
'FLAG_DOCUMENT_17','FLAG_DOCUMENT_18','FLAG_DOCUMENT_19',
'FLAG_DOCUMENT_20','FLAG_DOCUMENT_21']
df= df.drop(dropcolum,axis=1)
 del test_df
gc.collect()
return df
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': [ 'mean', 'var'],
'DAYS_CREDIT_ENDDATE': [ 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'CREDIT_DAY_OVERDUE': ['mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': [ 'max', 'mean'],
'AMT_APPLICATION': [ 'max','mean'],
'AMT_CREDIT': [ 'max', 'mean'],
'APP_CREDIT_PERC': [ 'max', 'mean'],
'AMT_DOWN_PAYMENT': [ 'max', 'mean'],
'AMT_GOODS_PRICE': [ 'max', 'mean'],
'HOUR_APPR_PROCESS_START': [ 'max', 'mean'],
'RATE_DOWN_PAYMENT': [ 'max', 'mean'],
'DAYS_DECISION': [ 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APPROVED_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REFUSED_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum','min','std' ],
'DBD': ['max', 'mean', 'sum','min','std'],
'PAYMENT_PERC': [ 'max','mean', 'var','min','std'],
'PAYMENT_DIFF': [ 'max','mean', 'var','min','std'],
'AMT_INSTALMENT': ['max', 'mean', 'sum','min','std'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum','std'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum','std']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INSTAL_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg([ 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg
def kfold_lightgbm(df, num_folds, stratified = False, debug= False):
train_df = df[df['TARGET'].notnull() ]
test_df = df[df['TARGET'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=47)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=47)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
objective = 'binary',
n_estimators=1400,
max_bin = 255,
learning_rate=0.01,
num_leaves = 31,
min_data_in_leaf = 1500,
feature_fraction = 0.7,
bagging_freq = 1,
bagging_fraction = 0.7,
lambda_l1 = 1,
lambda_l2=1, 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 1000, early_stopping_rounds= 200)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
if not debug:
test_df['TARGET'] = sub_preds
test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)
 display_importances(feature_importance_df)
 return feature_importance_df
def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:40].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances01.png')
 def main(debug = False):
num_rows = 10000 if debug else None
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Run LightGBM with kfold"):
feat_importance = kfold_lightgbm(df, num_folds= 5, stratified= False, debug= debug)
 if __name__ == "__main__":
submission_file_name = "submission_kerne2.csv"
with timer("Full model run"):
main()<set_options>
submission.to_csv('sub.csv',index=False )