Kaggle competition: Homework for Students
cols =('BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))<feature_engineering>
df_train = pd.read_csv('/kaggle/input/homework-for-students3/train.csv', index_col=0, parse_dates=['issue_d'])
 df_test =pd.read_csv('/kaggle/input/homework-for-students3/test.csv', index_col=0, parse_dates=['issue_d'])
 year = df_train.issue_d.dt.year
df_train = df_train[year >= 2014]
columns = df_train.columns
y_train = df_train.loan_condition
X_train = df_train.drop(['loan_condition'], axis = 1)
 X_test = df_test
del df_train
del df_test
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<feature_engineering>
drop_col = ['issue_d','earliest_cr_line']
X_train = X_train.drop(columns=drop_col)
 X_test = X_test.drop(columns=drop_col )
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.5
for feature in skewed_features:
all_data[feature] = boxcox1p(all_data[feature], lam )<categorify>
def capping(series, min_threshold, max_threshold):
series_filtered = series.copy()
index_outlier_up = [series_filtered >= max_threshold]
index_outlier_low = [series_filtered <= min_threshold]
series_filtered.iloc[index_outlier_up] = max_threshold
series_filtered.iloc[index_outlier_low] = min_threshold
return series_filtered
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
X_all = pd.concat([X_train,X_test],axis =0)
 X_all['Asset'] = X_all['tot_cur_bal'] + X_all['annual_inc']*2
X_train = X_all.iloc[:X_train.shape[0],:]
X_test = X_all.iloc[X_train.shape[0]:,:]
df_train = all_data[:ntrain]
df_test = all_data[ntrain:]<compute_train_metric>
col = 'grade'
mapping_dict = {
"grade": {
"A": 0,"B": 1,"C": 2,"D": 3,"E": 4,"F": 5,"G": 6
}
}
X_train = X_train.replace(mapping_dict)
 X_test = X_test.replace(mapping_dict)
 mapping_col = ['grade']
X_train[mapping_col] = X_train[mapping_col].fillna(-1)
 X_test[mapping_col] = X_test[mapping_col].fillna(-1)
 X_train[mapping_col] = X_train[mapping_col].astype(int)
 X_test[mapping_col] = X_test[mapping_col].astype(int)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(df_train.values)
 rmse= np.sqrt(-cross_val_score(model, df_train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
col = 'sub_grade'
mapping_dict = {
"sub_grade": {
"A1": 0,"A2": 1,"A3": 2,"A4": 3,"A5": 4,
"B1": 5,"B2": 6,"B3": 7,"B4": 8,"B5": 9,
"C1": 10,"C2": 11,"C3": 12,"C4": 13,"C5": 14,
"D1": 15,"D2": 16,"D3": 17,"D4": 18,"D5": 19,
"E1": 20,"E2": 21,"E3": 22,"E4": 23,"E5": 24,
"F1": 25,"F2": 26,"F3": 27,"F4": 28,"F5": 29,
"G1": 30,"G2": 31,"G3": 32,"G4": 33,"G5": 34
}
}
X_train = X_train.replace(mapping_dict)
 X_test = X_test.replace(mapping_dict)
 mapping_col = ['sub_grade']
X_train[mapping_col] = X_train[mapping_col].fillna(-1)
 X_test[mapping_col] = X_test[mapping_col].fillna(-1)
 X_train[mapping_col] = X_train[mapping_col].astype(int)
 X_test[mapping_col] = X_test[mapping_col].astype(int)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1)) 
<compute_test_metric>
col = 'emp_length'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
col = 'home_ownership'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))<compute_test_metric>
col = 'purpose'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
col = 'title'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
col = 'zip_code'
target = 'loan_condition'
X_train[col] = X_train[col].str[0:2]
X_test[col] = X_test[col].str[0:2]
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
max_threshold = 0.3
min_threshold = 0.1
X_train[col] = capping(X_train[col], min_threshold , max_threshold)
 X_test[col]= capping(X_test[col], min_threshold , max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.005,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=5,
loss='huber', random_state =5)
 score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
col = 'addr_state'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
max_threshold = 0.25
min_threshold = 0.12
X_train[col] = capping(X_train[col], min_threshold , max_threshold)
 X_test[col]= capping(X_test[col], min_threshold , max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
col = 'emp_title'
col4 = 'emp_title4'
target = 'loan_condition'
X_train[col4] = X_train[col].str[0:4]
X_test[col4] = X_test[col].str[0:4]
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col4])[target].mean()
X_test[col4] = X_test[col4].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col4])[target].mean()
enc_train.iloc[val_ix] = X_val[col4].map(summary)
 X_train[col4] = enc_train
max_threshold = 0.5
min_threshold = 0
X_train[col4] = capping(X_train[col4],min_threshold, max_threshold)
 X_test[col4]= capping(X_test[col4],min_threshold, max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col4].hist(density=True, alpha=0.5, bins=20)
 X_test[col4].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col4)
 plt.ylabel('density')
 plt.show()
col = 'emp_title'
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
max_threshold = 0.5
min_threshold = 0
X_train[col] = capping(X_train[col],min_threshold, max_threshold)
 X_test[col]= capping(X_test[col],min_threshold, max_threshold)
 plt.figure(figsize=[7,7])
 X_train[col].hist(density=True, alpha=0.5, bins=20)
 X_test[col].hist(density=True, alpha=0.5, bins=20)
 plt.xlabel(col)
 plt.ylabel('density')
 plt.show()
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
drop_col = ['title','acc_now_delinq','grade','installment']
X_train = X_train.drop(columns=drop_col)
 X_test = X_test.drop(columns=drop_col )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
<compute_train_metric>
X_train1 = X_train
X_train2 = X_train
X_train3 = X_train
X_test1 = X_test
X_test2 = X_test
X_test3 = X_test
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
drop_col = ['emp_title']
X_train1 = X_train1.drop(columns=drop_col)
 X_test1 = X_test1.drop(columns=drop_col )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
drop_col = ['emp_title']
X_train2 = X_train2.drop(columns=drop_col)
 X_test2 = X_test2.drop(columns=drop_col )
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))<compute_test_metric>
ratio_PCA = 0.95
sequence_col = ['sub_grade'
,'loan_amnt'
,'annual_inc'
,'dti'
,'Asset']
X_train_sequence = X_train2[sequence_col]
X_test_sequence = X_test2[sequence_col]
scaler = StandardScaler()
scaler.fit(X_train_sequence)
 X_train_sequence[sequence_col] = scaler.transform(X_train_sequence[sequence_col])
 X_test_sequence[sequence_col] = scaler.transform(X_test_sequence[sequence_col])
 X_train_sequence.fillna(X_train_sequence.median() , inplace=True)
 X_test_sequence.fillna(X_test_sequence.median() , inplace=True)
 pca = PCA(n_components=X_train_sequence.shape[1])
 pca.fit(X_train_sequence)
 sorted_variance = sorted(pca.explained_variance_ratio_,reverse=True)
 cum_sum = np.cumsum(sorted_variance)
 ratio = cum_sum / np.sum(sorted_variance)
 col_num = len(ratio)- len(ratio[ratio>ratio_PCA])+1
if col_num >= X_train_sequence.shape[1]:
col_num = X_train_sequence.shape[1]-1
sorted_variance = sorted(pca.explained_variance_ratio_,reverse=True)
 np.set_printoptions(suppress=True)
 svd = TruncatedSVD(n_components=col_num, n_iter=7, random_state=42)
 svd.fit(X_train_sequence)
 X_train_sequence = svd.transform(X_train_sequence)
 X_test_sequence = svd.transform(X_test_sequence)
 column_name = []
for c in range(0,col_num):
column_name.append('PCA' + str(c))
X_train_sequence = pd.DataFrame(X_train_sequence,columns = column_name)
 X_test_sequence = pd.DataFrame(X_test_sequence,columns = column_name)
 X_train_sequence.index = X_train.index
X_test_sequence.index = X_test.index
drop_col = sequence_col
X_train2 = X_train2.drop(columns=drop_col)
 X_test2 = X_test2.drop(columns=drop_col)
 X_train2 = pd.concat([X_train2, X_train_sequence], axis=1)
 X_test2 = pd.concat([X_test2, X_test_sequence], axis=1 )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
drop_col = ['emp_title4']
X_train3 = X_train3.drop(columns=drop_col)
 X_test3 = X_test3.drop(columns=drop_col )
stacked_averaged_models.fit(df_train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(df_train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(df_test.values))
print(rmsle(y_train, stacked_train_pred))<predict_on_test>
clf = LGBMClassifier(boosting_type = 'gbdt',class_weight='balanced')
 clf.fit(X_train1, y_train, eval_metric='auc')
 y_pred1 = clf.predict_proba(X_test1)[:,1]
clf = LGBMClassifier(boosting_type = 'gbdt',class_weight='balanced')
 clf.fit(X_train2, y_train, eval_metric='auc')
 y_pred2 = clf.predict_proba(X_test2)[:,1]
clf = LGBMClassifier(boosting_type = 'gbdt',class_weight='balanced')
 clf.fit(X_train3, y_train, eval_metric='auc')
 y_pred3 = clf.predict_proba(X_test3)[:,1]
y_pred =(y_pred1+y_pred2+y_pred3)/3
model_xgb.fit(df_train, y_train)
 xgb_train_pred = model_xgb.predict(df_train)
 xgb_pred = np.expm1(model_xgb.predict(df_test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
submission = pd.read_csv('/kaggle/input/homework-for-students3/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )