Home Credit Default Risk- End to End Machine learning project.
By: Praveen Kotha
Tags: Loans, Data Science, Plotly, Machine Learning, Homecredit

Exploratory Data Analysis:

At first, import the necessary packages.

import pandas as pd

import sklearn

import numpy as np

import matplotlib.pyplot as plt

import os

import warnings

import seaborn as sns from sklearn.preprocessing import OneHotEncoder

from sklearn.impute import SimpleImputer

from sklearn.pipeline import Pipeline

from sklearn.compose import ColumnTransformer

from sklearn.preprocessing import StandardScaler

from sklearn.svm import LinearSVC

from sklearn.metrics import roc_auc_score

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import roc_auc_score

from sklearn.calibration import CalibratedClassifierCV

from sklearn.metrics import confusion_matrix

from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import accuracy_score

from sklearn.linear_model import SGDClassifier import plotly.offline as py

import plotly.graph_objs as go from plotly.offline import init_notebook_mode, iplot

from sklearn.model_selection import train_test_split

init_notebook_mode(connected=True) import cufflinks as cf

cf.go_offline() import pickle

import gc

import lightgbm as lgb warnings.filterwarnings('ignore')

%matplotlib inline

Load the data from given csv file into a pandas dataframe.

print('Reading the data....', end='')

application = pd.read_csv('application_train.csv')

print('done!!!')

print('The shape of data:',application.shape)

print('First 5 rows of data:')

application.head()

We are using ‘application_train.csv’ file :

1. This dataset consists of 307511 rows and 122 columns.

2. Each row has unique id ‘SK_ID_CURR’ and the output label is in the ‘TARGET’ column.

3. TARGET indicating 0: the loan was repaid or 1: the loan was not repaid.

4. The description of each column can be found in the file ‘HomeCredit_columns_description.csv’

Let us check for missing values in each column.

count = application.isnull().sum().sort_values(ascending=False)

percentage = ((application.isnull().sum()/len(application)*100)).sort_values(ascending=False) missing_application = pd.concat([count, percentage], axis=1, keys=['Count','Percentage'])

print('Count and percentage of missing values for top 20 columns:')

missing_application.head(20)

Observations:

1. There are lot of missing values in each column.

2. We need to somehow handle these missing values, we will see how to handle later in the case study.

Let’s check for duplicate data:

columns_without_id = [col for col in application.columns if col!='SK_ID_CURR'] #Checking for duplicates in the data.

application[application.duplicated(subset = columns_without_id, keep=False)] print('The no of duplicates in the data:',application[application.duplicated(subset = columns_without_id, keep=False)]

.shape[0])

Let’s check the distribution of data points among output class.

cf.set_config_file(theme='polar')

contract_val = application['NAME_CONTRACT_TYPE'].value_counts()

contract_df = pd.DataFrame({'labels': contract_val.index,

'values': contract_val.values

})

contract_df.iplot(kind='pie',labels='labels',values='values', title='Types of Loan', hole = 0.6)

Observations:

1. The data is imbalanced (91.9%(Loan repayed-0) and 8.07%(Loan not repayed-1)) and we need to handle this problem.

Distribution of AMT_INCOME_TOTAL.

application[application['AMT_INCOME_TOTAL'] < 2000000]['AMT_INCOME_TOTAL'].iplot(kind='histogram', bins=100,

xTitle = 'Total Income', yTitle ='Count of applicants',

title='Distribution of AMT_INCOME_TOTAL')

(application[application['AMT_INCOME_TOTAL'] > 1000000]['TARGET'].value_counts())/len(application[application['AMT_INCOME_TOTAL'] > 1000000])*100

Observations:

1. The distribution is right skewed and there are extreme values, we can apply log distribution.

2. People with high income(>1000000) are likely to repay the loan.

Types of loan available.

cf.set_config_file(theme='polar')

contract_val = application['NAME_CONTRACT_TYPE'].value_counts()

contract_df = pd.DataFrame({'labels': contract_val.index,

'values': contract_val.values

})

contract_df.iplot(kind='pie',labels='labels',values='values', title='Types of Loan', hole = 0.6)

Observations:

1. Many people are willing to take cash loan than revolving loan (https://www.investopedia.com/terms/r/revolving-loan-facility.asp).

Distribution of AMT_CREDIT

application['AMT_CREDIT'].iplot(kind='histogram', bins=100,

xTitle = 'Credit Amount',yTitle ='Count of applicants',

title='Distribution of AMT_CREDIT')

np.log(application['AMT_CREDIT']).iplot(kind='histogram', bins=100,

xTitle = 'log(Credit Amount)',yTitle ='Count of applicants',

title='Distribution of log(AMT_CREDIT)')

Observations:

1. People who are taking credit for large amount are very likely to repay the loan.

2. Originally the distribution is right skewed, we used log transformation to make it normal distributed.

Distribution of Name of type of the Suite in terms of loan is repayed or not.

cf.set_config_file(theme='polar')

suite_val = (application['NAME_TYPE_SUITE'].value_counts()/len(application))*100

suite_val.iplot(kind='bar', xTitle = 'Name of type of the Suite',

yTitle='Count of applicants in %',

title='Who accompanied client when applying for the application in % ')

suite_val = application['NAME_TYPE_SUITE'].value_counts() suite_val_y0 = []

suite_val_y1 = []

for val in suite_val.index:

suite_val_y1.append(np.sum(application['TARGET'][application['NAME_TYPE_SUITE']==val] == 1))

suite_val_y0.append(np.sum(application['TARGET'][application['NAME_TYPE_SUITE']==val] == 0)) data = [go.Bar(x = suite_val.index, y = ((suite_val_y1 / suite_val.sum()) * 100), name='Yes' ),

go.Bar(x = suite_val.index, y = ((suite_val_y0 / suite_val.sum()) * 100), name='No' )] layout = go.Layout(

title = "Who accompanied client when applying for the application in terms of loan is repayed or not in %",

xaxis=dict(

title='Name of type of the Suite',

),

yaxis=dict(

title='Count of applicants in %',

)

) fig = go.Figure(data = data, layout=layout)

fig.layout.template = 'plotly_dark' py.iplot(fig)

Distribution of Income sources of Applicants in terms of loan is repayed or not.

income_val = application['NAME_INCOME_TYPE'].value_counts() income_val_y0 = []

income_val_y1 = []

for val in income_val.index:

income_val_y1.append(np.sum(application['TARGET'][application['NAME_INCOME_TYPE']==val] == 1))

income_val_y0.append(np.sum(application['TARGET'][application['NAME_INCOME_TYPE']==val] == 0)) data = [go.Bar(x = income_val.index, y = ((income_val_y1 / income_val.sum()) * 100), name='Yes' ),

go.Bar(x = income_val.index, y = ((income_val_y0 / income_val.sum()) * 100), name='No' )] layout = go.Layout(

title = "Income sources of Applicants in terms of loan is repayed or not in %",

xaxis=dict(

title='Income source',

),

yaxis=dict(

title='Count of applicants in %',

)

) fig = go.Figure(data = data, layout=layout)

fig.layout.template = 'plotly_dark' py.iplot(fig)

Observations:

1. All the Students and Businessman are repaying loan.(Open the chart and hover over the plot to observe).

Distribution of Education of Applicants in terms of loan is repayed or not.

education_val = application['NAME_EDUCATION_TYPE'].value_counts() education_val_y0 = []

education_val_y1 = []

for val in education_val.index:

education_val_y1.append(np.sum(application['TARGET'][application['NAME_EDUCATION_TYPE']==val] == 1))

education_val_y0.append(np.sum(application['TARGET'][application['NAME_EDUCATION_TYPE']==val] == 0)) data = [go.Bar(x = education_val.index, y = ((education_val_y1 / education_val.sum()) * 100), name='Yes' ),

go.Bar(x = education_val.index, y = ((education_val_y0 / education_val.sum()) * 100), name='No' )] layout = go.Layout(

title = "Education sources of Applicants in terms of loan is repayed or not in %",

xaxis=dict(

title='Education of Applicants',

),

yaxis=dict(

title='Count of applicants in %',

)

) fig = go.Figure(data = data, layout=layout)

fig.layout.template = 'plotly_dark' py.iplot(fig)

Observations:

1. People with Academic Degree are more likely to repay the loan(Out of 164, only 3 applicants are not able to repay)

Distribution of Family status of Applicants in terms of loan is repayed or not.

Observations:

1. Widows are more likely to repay the loan when compared to appliants with the other family statuses.

Distribution of Housing type of Applicants in terms of loan is repayed or not.

Distribution of Clients Age

cf.set_config_file(theme='pearl')

(application['DAYS_BIRTH']/(-365)).iplot(kind='histogram',

xTitle = 'Age', bins=50,

yTitle='Count of type of applicants in %',

title='Distribution of Clients Age')

Distribution of years before the application the person started current employment.

cf.set_config_file(theme='pearl')

(application['DAYS_EMPLOYED']).iplot(kind='histogram',

xTitle = 'Days',bins=50,

yTitle='Count of applicants in %',

title='Days before the application the person started current employment')

Observations:

1. The data looks strange(we have -1000.66 years(-365243 days) of employment which is impossible) looks like there is data entry error.

error = application[application['DAYS_EMPLOYED'] == 365243]

print('The no of errors are :', len(error))

(error['TARGET'].value_counts()/len(error))*100

# Create an error flag column

application['DAYS_EMPLOYED_ERROR'] = application["DAYS_EMPLOYED"] == 365243 # Replace the error values with nan

application['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)

Created a seperate column ‘DAYS_EMPLOYED_ERROR’, which flags the error.

cf.set_config_file(theme='pearl')

(application['DAYS_EMPLOYED']/(-365)).iplot(kind='histogram', xTitle = 'Years of Employment',bins=50,

yTitle='Count of applicants in %',

title='Years before the application the person started current employment')

application[application['DAYS_EMPLOYED']>(-365*2)]['TARGET'].value_counts()/sum(application['DAYS_EMPLOYED']>(-365*2))

Observations:

1. The applicants with less than 2 years of employment are less likely to repay the loan.

Data Preparation:

Feature Engineering of Application data:

# Flag to represent when Total income is greater than Credit

application['INCOME_GT_CREDIT_FLAG'] = application['AMT_INCOME_TOTAL'] > application['AMT_CREDIT'] # Column to represent Credit Income Percent

application['CREDIT_INCOME_PERCENT'] = application['AMT_CREDIT'] / application['AMT_INCOME_TOTAL'] # Column to represent Annuity Income percent

application['ANNUITY_INCOME_PERCENT'] = application['AMT_ANNUITY'] / application['AMT_INCOME_TOTAL'] # Column to represent Credit Term

application['CREDIT_TERM'] = application['AMT_CREDIT'] / application['AMT_ANNUITY'] # Column to represent Days Employed percent in his life

application['DAYS_EMPLOYED_PERCENT'] = application['DAYS_EMPLOYED'] / application['DAYS_BIRTH'] # Shape of Application data

print('The shape of application data:",application.shape)

Using Bureau Data:

print('Reading the data....', end='')

bureau = pd.read_csv('bureau.csv')

print('done!!!')

print('The shape of data:',bureau.shape)

print('First 5 rows of data:')

bureau.head()

Joining Bureau data to Application data:

# Combining numerical features

grp = bureau.drop(['SK_ID_BUREAU'], axis = 1).groupby(by=['SK_ID_CURR']).mean().reset_index()

grp.columns = ['BUREAU_'+column if column !='SK_ID_CURR' else column for column in grp.columns]

application_bureau = application.merge(grp, on='SK_ID_CURR', how='left')

application_bureau.update(application_bureau[grp.columns].fillna(0)) # Combining categorical features

bureau_categorical = pd.get_dummies(bureau.select_dtypes('object'))

bureau_categorical['SK_ID_CURR'] = bureau['SK_ID_CURR'] grp = bureau_categorical.groupby(by = ['SK_ID_CURR']).mean().reset_index()

grp.columns = ['BUREAU_'+column if column !='SK_ID_CURR' else column for column in grp.columns]

application_bureau = application_bureau.merge(grp, on='SK_ID_CURR', how='left')

application_bureau.update(application_bureau[grp.columns].fillna(0)) # Shape of application and bureau data combined

print('The shape application and bureau data combined:',application_bureau.shape)

Feature Engineering of Bureau Data:

# Number of past loans per customer

grp = bureau.groupby(by = ['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index().rename(columns = {'SK_ID_BUREAU': 'BUREAU_LOAN_COUNT'}) application_bureau = application_bureau.merge(grp, on='SK_ID_CURR', how='left')

application_bureau['BUREAU_LOAN_COUNT'] = application_bureau['BUREAU_LOAN_COUNT'].fillna(0) # Number of types of past loans per customer

grp = bureau[['SK_ID_CURR', 'CREDIT_TYPE']].groupby(by = ['SK_ID_CURR'])['CREDIT_TYPE'].nunique().reset_index().rename(columns={'CREDIT_TYPE': 'BUREAU_LOAN_TYPES'}) application_bureau = application_bureau.merge(grp, on='SK_ID_CURR', how='left')

application_bureau['BUREAU_LOAN_TYPES'] = application_bureau['BUREAU_LOAN_TYPES'].fillna(0) # Debt over credit ratio

bureau['AMT_CREDIT_SUM'] = bureau['AMT_CREDIT_SUM'].fillna(0)

bureau['AMT_CREDIT_SUM_DEBT'] = bureau['AMT_CREDIT_SUM_DEBT'].fillna(0) grp1 = bureau[['SK_ID_CURR','AMT_CREDIT_SUM']].groupby(by=['SK_ID_CURR'])['AMT_CREDIT_SUM'].sum().reset_index().rename(columns={'AMT_CREDIT_SUM': 'TOTAL_CREDIT_SUM'}) grp2 = bureau[['SK_ID_CURR','AMT_CREDIT_SUM_DEBT']].groupby(by=['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT'].sum().reset_index().rename(columns={'AMT_CREDIT_SUM_DEBT':'TOTAL_CREDIT_SUM_DEBT'}) grp1['DEBT_CREDIT_RATIO'] = grp2['TOTAL_CREDIT_SUM_DEBT']/grp1['TOTAL_CREDIT_SUM'] del grp1['TOTAL_CREDIT_SUM'] application_bureau = application_bureau.merge(grp1, on='SK_ID_CURR', how='left')

application_bureau['DEBT_CREDIT_RATIO'] = application_bureau['DEBT_CREDIT_RATIO'].fillna(0)

application_bureau['DEBT_CREDIT_RATIO'] = application_bureau.replace([np.inf, -np.inf], 0)

application_bureau['DEBT_CREDIT_RATIO'] = pd.to_numeric(application_bureau['DEBT_CREDIT_RATIO'], downcast='float') # Overdue over debt ratio

bureau['AMT_CREDIT_SUM_OVERDUE'] = bureau['AMT_CREDIT_SUM_OVERDUE'].fillna(0)

bureau['AMT_CREDIT_SUM_DEBT'] = bureau['AMT_CREDIT_SUM_DEBT'].fillna(0) grp1 = bureau[['SK_ID_CURR','AMT_CREDIT_SUM_OVERDUE']].groupby(by=['SK_ID_CURR'])['AMT_CREDIT_SUM_OVERDUE'].sum().reset_index().rename(columns={'AMT_CREDIT_SUM_OVERDUE': 'TOTAL_CUSTOMER_OVERDUE'}) grp2 = bureau[['SK_ID_CURR','AMT_CREDIT_SUM_DEBT']].groupby(by=['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT'].sum().reset_index().rename(columns={'AMT_CREDIT_SUM_DEBT':'TOTAL_CUSTOMER_DEBT'}) grp1['OVERDUE_DEBT_RATIO'] = grp1['TOTAL_CUSTOMER_OVERDUE']/grp2['TOTAL_CUSTOMER_DEBT'] del grp1['TOTAL_CUSTOMER_OVERDUE'] application_bureau = application_bureau.merge(grp1, on='SK_ID_CURR', how='left')

application_bureau['OVERDUE_DEBT_RATIO'] = application_bureau['OVERDUE_DEBT_RATIO'].fillna(0)

application_bureau['OVERDUE_DEBT_RATIO'] = application_bureau.replace([np.inf, -np.inf], 0) application_bureau['OVERDUE_DEBT_RATIO'] = pd.to_numeric(application_bureau['OVERDUE_DEBT_RATIO'], downcast='float')

Using Previous Application Data:

print('Reading the data....', end='')

previous_applicaton = pd.read_csv('previous_application.csv')

print('done!!!')

print('The shape of data:',previous_applicaton.shape)

print('First 5 rows of data:')

previous_applicaton.head()

Joining Previous Application data to Application Bureau data:

# Number of previous applications per customer

grp = previous_applicaton[['SK_ID_CURR','SK_ID_PREV']].groupby(by=['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index().rename(columns={'SK_ID_PREV':'PREV_APP_COUNT'})

application_bureau_prev = application_bureau.merge(grp, on =['SK_ID_CURR'], how = 'left')

application_bureau_prev['PREV_APP_COUNT'] = application_bureau_prev['PREV_APP_COUNT'].fillna(0) # Combining numerical features

grp = previous_applicaton.drop('SK_ID_PREV', axis =1).groupby(by=['SK_ID_CURR']).mean().reset_index()

prev_columns = ['PREV_'+column if column != 'SK_ID_CURR' else column for column in grp.columns ]

grp.columns = prev_columns

application_bureau_prev = application_bureau_prev.merge(grp, on =['SK_ID_CURR'], how = 'left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0)) # Combining categorical features

prev_categorical = pd.get_dummies(previous_applicaton.select_dtypes('object'))

prev_categorical['SK_ID_CURR'] = previous_applicaton['SK_ID_CURR']

prev_categorical.head() grp = prev_categorical.groupby('SK_ID_CURR').mean().reset_index()

grp.columns = ['PREV_'+column if column != 'SK_ID_CURR' else column for column in grp.columns] application_bureau_prev = application_bureau_prev.merge(grp, on=['SK_ID_CURR'], how='left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0))

Using POS_CASH_balance data:

print('Reading the data....', end='')

pos_cash = pd.read_csv('POS_CASH_balance.csv')

print('done!!!')

print('The shape of data:',pos_cash.shape)

print('First 5 rows of data:')

pos_cash.head()

Joining POS_CASH_balance data to application_bureau_prev_data:

# Combining numerical features

grp = pos_cash.drop('SK_ID_PREV', axis =1).groupby(by=['SK_ID_CURR']).mean().reset_index()

prev_columns = ['POS_'+column if column != 'SK_ID_CURR' else column for column in grp.columns ]

grp.columns = prev_columns

application_bureau_prev = application_bureau_prev.merge(grp, on =['SK_ID_CURR'], how = 'left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0)) # Combining categorical features

pos_cash_categorical = pd.get_dummies(pos_cash.select_dtypes('object'))

pos_cash_categorical['SK_ID_CURR'] = pos_cash['SK_ID_CURR'] grp = pos_cash_categorical.groupby('SK_ID_CURR').mean().reset_index()

grp.columns = ['POS_'+column if column != 'SK_ID_CURR' else column for column in grp.columns] application_bureau_prev = application_bureau_prev.merge(grp, on=['SK_ID_CURR'], how='left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0))

Using installments_payments data:

print('Reading the data....', end='')

insta_payments = pd.read_csv('installments_payments.csv')

print('done!!!')

print('The shape of data:',insta_payments.shape)

print('First 5 rows of data:')

insta_payments.head()

Joining Installments Payments data to application_bureau_prev_data:

# Combining numerical features and there are no categorical features in this dataset

grp = insta_payments.drop('SK_ID_PREV', axis =1).groupby(by=['SK_ID_CURR']).mean().reset_index()

prev_columns = ['INSTA_'+column if column != 'SK_ID_CURR' else column for column in grp.columns ]

grp.columns = prev_columns

application_bureau_prev = application_bureau_prev.merge(grp, on =['SK_ID_CURR'], how = 'left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0))

Using Credit card balance data:

print('Reading the data....', end='')

credit_card = pd.read_csv('credit_card_balance.csv')

print('done!!!')

print('The shape of data:',credit_card.shape)

print('First 5 rows of data:')

credit_card.head()

Joining Credit card balance data to application_bureau_prev data:

# Combining numerical features

grp = credit_card.drop('SK_ID_PREV', axis =1).groupby(by=['SK_ID_CURR']).mean().reset_index()

prev_columns = ['CREDIT_'+column if column != 'SK_ID_CURR' else column for column in grp.columns ]

grp.columns = prev_columns

application_bureau_prev = application_bureau_prev.merge(grp, on =['SK_ID_CURR'], how = 'left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0)) # Combining categorical features

credit_categorical = pd.get_dummies(credit_card.select_dtypes('object'))

credit_categorical['SK_ID_CURR'] = credit_card['SK_ID_CURR'] grp = credit_categorical.groupby('SK_ID_CURR').mean().reset_index()

grp.columns = ['CREDIT_'+column if column != 'SK_ID_CURR' else column for column in grp.columns] application_bureau_prev = application_bureau_prev.merge(grp, on=['SK_ID_CURR'], how='left')

application_bureau_prev.update(application_bureau_prev[grp.columns].fillna(0))

Shape of final prepared data: (307511, 377)

Dividing final data into train, valid and test datasets:

y = application_bureau_prev.pop('TARGET').values X_train, X_temp, y_train, y_temp = train_test_split(application_bureau_prev.drop(['SK_ID_CURR'],axis=1), y, stratify = y, test_size=0.3, random_state=42)

X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, stratify = y_temp, test_size=0.5, random_state=42) print('Shape of X_train:',X_train.shape)

print('Shape of X_val:',X_val.shape)

print('Shape of X_test:',X_test.shape)

Featurizing the data:

# Seperation of columns into numeric and categorical columns

types = np.array([dt for dt in X_train.dtypes]) all_columns = X_train.columns.values

is_num = types != 'object' num_cols = all_columns[is_num]

cat_cols = all_columns[~is_num] # Featurization of numeric data

imputer_num = SimpleImputer(strategy='median')

X_train_num = imputer_num.fit_transform(X_train[num_cols])

X_val_num = imputer_num.transform(X_val[num_cols])

X_test_num = imputer_num.transform(X_test[num_cols]) scaler_num = StandardScaler()

X_train_num1 = scaler_num.fit_transform(X_train_num)

X_val_num1 = scaler_num.transform(X_val_num)

X_test_num1 = scaler_num.transform(X_test_num) X_train_num_final = pd.DataFrame(X_train_num1, columns=num_cols)

X_val_num_final = pd.DataFrame(X_val_num1, columns=num_cols)

X_test_num_final = pd.DataFrame(X_test_num1, columns=num_cols) # Featurization of categorical data

imputer_cat = SimpleImputer(strategy='constant', fill_value='MISSING')

X_train_cat = imputer_cat.fit_transform(X_train[cat_cols])

X_val_cat = imputer_cat.transform(X_val[cat_cols])

X_test_cat = imputer_cat.transform(X_test[cat_cols]) X_train_cat1= pd.DataFrame(X_train_cat, columns=cat_cols)

X_val_cat1= pd.DataFrame(X_val_cat, columns=cat_cols)

X_test_cat1= pd.DataFrame(X_test_cat, columns=cat_cols) ohe = OneHotEncoder(sparse=False,handle_unknown='ignore')

X_train_cat2 = ohe.fit_transform(X_train_cat1)

X_val_cat2 = ohe.transform(X_val_cat1)

X_test_cat2 = ohe.transform(X_test_cat1) cat_cols_ohe = list(ohe.get_feature_names(input_features=cat_cols)) X_train_cat_final = pd.DataFrame(X_train_cat2, columns = cat_cols_ohe)

X_val_cat_final = pd.DataFrame(X_val_cat2, columns = cat_cols_ohe)

X_test_cat_final = pd.DataFrame(X_test_cat2, columns = cat_cols_ohe) # Final complete data

X_train_final = pd.concat([X_train_num_final,X_train_cat_final], axis = 1)

X_val_final = pd.concat([X_val_num_final,X_val_cat_final], axis = 1)

X_test_final = pd.concat([X_test_num_final,X_test_cat_final], axis = 1)

print(X_train_final.shape)

print(X_val_final.shape)

print(X_test_final.shape)

Saving the files for future use:

# Saving the Dataframes into CSV files for future use

X_train_final.to_csv('X_train_final.csv')

X_val_final.to_csv('X_val_final.csv')

X_test_final.to_csv('X_test_final.csv') # Saving the numpy arrays into text files for future use

np.savetxt('y.txt', y)

np.savetxt('y_train.txt', y_train)

np.savetxt('y_val.txt', y_val)

np.savetxt('y_test.txt', y_test)

Selection of features:

model_sk = lgb.LGBMClassifier(boosting_type='gbdt', max_depth=7, learning_rate=0.01, n_estimators= 2000,

class_weight='balanced', subsample=0.9, colsample_bytree= 0.8, n_jobs=-1) train_features, valid_features, train_y, valid_y = train_test_split(X_train_final, y_train, test_size = 0.15, random_state = 42) model_sk.fit(train_features, train_y, early_stopping_rounds=100, eval_set = [(valid_features, valid_y)], eval_metric = 'auc', verbose = 200)

Training LGBM

feature_imp = pd.DataFrame(sorted(zip(model_sk.feature_importances_, X_train_final.columns)), columns=['Value','Feature'])

features_df = feature_imp.sort_values(by="Value", ascending=False)

selected_features = list(features_df[features_df['Value']>=50]['Feature']) # Saving the selected features into pickle file

with open('select_features.txt','wb') as fp:

pickle.dump(selected_features, fp) print('The no. of features selected:',len(selected_features))