Kaggle competition: Exam for Students20200129
<SOS> metric: rmsle Kaggle data source: exam-for-students20200129<save_to_csv>
plt.style.use('ggplot')
 %matplotlib inline
print('Saving model predictions')
 fname = 'submission_5models_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.mean(y_preds_list_for_ensemble, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))
weights = [3.9, 11.4, 11.0, 11.0, 5.6]
fname = 'submission_5models_weighted_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.average(y_preds_list_for_ensemble, weights=weights, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))
best_3preds_plusNN = [y_preds_list_for_ensemble[1], y_preds_list_for_ensemble[2],
y_preds_list_for_ensemble[3], y_preds_list_for_ensemble[4]]
fname = 'submission_best3models_plusNN_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.mean(best_3preds_plusNN, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))
best_2preds = [y_preds_list_for_ensemble[1], y_preds_list_for_ensemble[2]]
fname = 'submission_2LightGBMs_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.mean(best_2preds, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))<load_from_csv>
pd.set_option('display.max_columns', 500 )
df_train = pd.read_csv('.. /input/homework-for-students4plus/train.csv', index_col=0)
 df_test = pd.read_csv('.. /input/homework-for-students4plus/test.csv', index_col=0)
<data_type_conversions>
df_train = pd.read_csv('.. /input/exam-for-students20200129/train.csv', index_col=0)
 df_test = pd.read_csv('.. /input/exam-for-students20200129/test.csv', index_col=0 )
df_train['FLG'] = 'train'
print('df_train' + str(df_train.shape))
df_test['FLG'] = 'test'
print('df_test' + str(df_test.shape))
df_temp = pd.concat([df_train,df_test])
 df_temp['issue_d_year'] = df_temp.issue_d.str[-4:].astype(int)
 df_temp = df_temp[df_temp.issue_d_year > 2010]
print('df_temp' + str(df_temp.shape))<data_type_conversions>
df_train.isnull().sum()
df_temp['earliest_cr_line_year'] = df_temp.earliest_cr_line.str[-4:].astype(int)
 df_temp['year_difference'] = df_temp['issue_d_year'] - df_temp['earliest_cr_line_year']<categorify>
df_test.isnull().sum()
col = 'sub_grade'
df_temp[col] = df_temp[col].str.replace('A1','0.5' ).str.replace('A2','2' ).str.replace('A3','3' ).str.replace('A4','4' ).str.replace('A5','5')\ 
.str.replace('B1','6' ).str.replace('B2','7' ).str.replace('B3','8' ).str.replace('B4','9' ).str.replace('B5','10')\ 
.str.replace('C1','11' ).str.replace('C2','12' ).str.replace('C3','13' ).str.replace('C4','14' ).str.replace('C5','15')\ 
.str.replace('D1','16' ).str.replace('D2','17' ).str.replace('D3','18' ).str.replace('D4','19' ).str.replace('D5','20')\ 
.str.replace('E1','21' ).str.replace('E2','22' ).str.replace('E3','23' ).str.replace('E4','24' ).str.replace('E5','25')\ 
.str.replace('F1','26' ).str.replace('F2','27' ).str.replace('F3','28' ).str.replace('F4','29' ).str.replace('F5','30')\ 
.str.replace('G1','31' ).str.replace('G2','32' ).str.replace('G3','33' ).str.replace('G4','34' ).str.replace('G5','35')
 df_temp[col] = pd.to_numeric(df_temp[col], errors='coerce')
 print(df_temp[col].unique() )<count_missing_values>
for col_name in df_train.columns:
print(df_train[col_name].value_counts())
 print(df_train[col_name].value_counts() / len(df_train[col_name]))
print(df_temp.isnull().sum() )<define_variables>
for col_name in df_test.columns:
print(df_test[col_name].value_counts())
 print(df_test[col_name].value_counts() / len(df_test[col_name]))
col = ['emp_title', 'emp_length','annual_inc', 'title','dti', 'delinq_2yrs', 'earliest_cr_line',
'inq_last_6mths', 'mths_since_last_delinq', 'mths_since_last_record','open_acc',
'pub_rec', 'revol_util', 'total_acc','collections_12_mths_ex_med','mths_since_last_major_derog',
'acc_now_delinq','tot_coll_amt', 'tot_cur_bal']
col2 = ['emp_title_flg', 'emp_length_flg','annual_inc_flg', 'title_flg','dti_flg', 'delinq_2yrs_flg', 'earliest_cr_line_flg',
'inq_last_6mths_flg', 'mths_since_last_delinq_flg', 'mths_since_last_record_flg','open_acc_flg',
'pub_rec_flg', 'revol_util_flg', 'total_acc_flg','collections_12_mths_ex_med_flg','mths_since_last_major_derog_flg',
'acc_now_delinq_flg','tot_coll_amt_flg', 'tot_cur_bal_flg']
df_temp[col2] = df_temp[col].copy()
for i in col2:
exec("df_temp." + str(i)+ "[df_temp." + str(i)+ ".notnull() ] = 1")
 exec("df_temp." + str(i)+ ".fillna(0, inplace=True)")
 print(df_temp.info() )<data_type_conversions>
df_country_info = pd.read_csv('.. /input/exam-for-students20200129/country_info.csv' )
col = 'emp_length'
df_temp[col] = df_temp[col].replace('[^0-9]+', '', regex=True)
 df_temp[col].fillna('0', inplace=True)
 df_temp[col] = pd.to_numeric(df_temp[col], errors='coerce')
 print(df_temp.info())
 print(df_temp[col].unique() )<feature_engineering>
df_country_info['Country'].value_counts()
df_temp['emp_length_max_flg'] = 1
df_temp['emp_length_max_flg'] = df_temp['emp_length_max_flg'].where(df_temp['emp_length']==10,0)
 df_temp['loan_amnt_max_flg'] = 1
df_temp['loan_amnt_max_flg'] = df_temp['loan_amnt_max_flg'].where(df_temp['loan_amnt']==35000,0)
 df_temp['revol_bal_0_flg'] = 1
df_temp['revol_bal_0_flg'] = df_temp['loan_amnt_max_flg'].where(df_temp['loan_amnt']==0,0)
 df_temp['tot_coll_amt_0_flg'] = 1
df_temp['tot_coll_amt_0_flg'] = df_temp['loan_amnt_max_flg'].where(df_temp['loan_amnt']==0,0 )<feature_engineering>
df_country_info_edit = df_country_info[['Country', 'Region']].copy()
df_country_info_edit
df_temp['emp_title'] = df_temp['emp_title'].str.upper()
df_temp['emp_title_flg'] = 1
df_temp['emp_title_flg'] = df_temp['emp_title_flg'].where(df_temp['emp_title'].str.contains('MANAGER'),0)
 df_temp['emp_title_flg2'] = 1
df_temp['emp_title_flg2'] = df_temp['emp_title_flg2'].where(df_temp['emp_title'].str.contains('TEACHER'),0)
<feature_engineering>
df_train_add_countryinfo = pd.merge(df_train, df_country_info_edit, how = 'left', on = ['Country'] ).copy()
df_test_add_countryinfo = pd.merge(df_test, df_country_info_edit, how = 'left', on = ['Country'] ).copy()
df_temp['zip_code_1'] = df_temp.zip_code.str[:1]
num_cols = ['loan_amnt','annual_inc','revol_bal','tot_cur_bal']
p01 = df_temp[num_cols].quantile(0.01)
 p99 = df_temp[num_cols].quantile(0.99)
 df_temp[num_cols] = df_temp[num_cols].clip(p01,p99,axis=1)
 labels=[10,9,8,7,6,5,4,3,2,1]
df_temp['loan_amnt_cut'] = pd.cut(df_temp.loan_amnt, 10, labels=labels ).astype(int)
 labels=[1,2,3,4,5,6,7,8,9,10]
df_temp['annual_inc_cut'] = pd.cut(df_temp.annual_inc, 10, labels=labels ).astype(int)
 df_temp['loan_risk'] = df_temp['loan_amnt_cut'] * df_temp['annual_inc_cut']
df_temp = df_temp.drop(columns=['loan_amnt_cut'])
 df_temp = df_temp.drop(columns=['annual_inc_cut'])
 df_temp['annual_inc_installment_ratio'] = df_temp['annual_inc'] / 12 / df_temp['installment']
df_temp['subgrade_loanamnt_mean'] = df_temp.groupby('sub_grade')['loan_amnt'].transform(lambda x: x.mean())
 df_temp['subgrade_loanamnt_std'] = df_temp.groupby('sub_grade')['loan_amnt'].transform(lambda x: x.std())
 df_temp['subgrade_loanamnt_var'] = df_temp.groupby('sub_grade')['loan_amnt'].transform(lambda x: x.var())
 df_temp['subgrade_loanamnt_difference'] = df_temp['subgrade_loanamnt_mean'] - df_temp['loan_amnt']
df_temp['sub_grade_annual_inc_mean'] = df_temp.groupby('sub_grade')['annual_inc'].transform(lambda x: x.mean())
 df_temp['sub_grade_annual_inc_std'] = df_temp.groupby('sub_grade')['annual_inc'].transform(lambda x: x.std())
 df_temp['sub_grade_annual_inc_var'] = df_temp.groupby('sub_grade')['annual_inc'].transform(lambda x: x.var())
 df_temp['sub_grade_annual_inc_difference'] = df_temp['loan_amnt'] / df_temp['sub_grade_annual_inc_mean']
df_temp['sub_grade_annual_inc_difference2'] = df_temp['sub_grade_annual_inc_mean'] / df_temp['loan_amnt']
df_temp['loan_amnt_sub_grade_ratio'] = df_temp['loan_amnt'] / df_temp['sub_grade']
df_temp['annual_inc_loanamnt_ratio'] = df_temp['annual_inc'] / df_temp['loan_amnt']
df_temp['loan_amnt_sub_grade_ratio'] = df_temp['loan_amnt'] / df_temp[col]
df_temp['loan_amnt_open_acc_ratio'] = df_temp['loan_amnt'] / df_temp['open_acc']
df_temp['annual_inc_sub_grade_ratio'] = df_temp['annual_inc'] / df_temp[col]
df_temp['annual_inc_open_acc_ratio'] = df_temp['annual_inc'] / df_temp['open_acc']
df_temp['revol_bal_sub_grade_ratio'] = df_temp['revol_bal'] / df_temp[col]
df_temp['revol_bal_open_acc_ratio'] = df_temp['revol_bal'] / df_temp['open_acc']
df_temp['tot_cur_bal_sub_grade_ratio'] = df_temp['tot_cur_bal'] / df_temp[col]
df_temp['tot_cur_bal_open_acc_ratio'] = df_temp['tot_cur_bal'] / df_temp['open_acc']
print(df_temp.shape)
 print(df_temp.info() )<categorify>
y_train = df_train_add_countryinfo['ConvertedSalary'].copy()
X_train = df_train_add_countryinfo.drop(['ConvertedSalary', 'Country'], axis=1 ).copy()
X_test = df_test_add_countryinfo.drop(['Country'], axis=1 ).copy()
df_temp = pd.get_dummies(df_temp, columns=['purpose','grade','home_ownership','zip_code_1'])
 df_temp.shape<drop_column>
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique() )
X_train = df_temp[df_temp["FLG"] == "train"]
X_train = X_train.drop(columns=["FLG"])
 print('X_train' + str(X_train.shape))
X_test = df_temp[df_temp["FLG"] == "test"]
X_test = X_test.drop(columns=["FLG"])
 print('X_test' + str(X_test.shape))<feature_engineering>
encoder = OrdinalEncoder(cols=cats)
 encoder
X_train['loan_amnt'] = X_train['loan_amnt'].apply(np.log1p)
 X_test['loan_amnt'] = X_test['loan_amnt'].apply(np.log1p)
 X_train['emp_length'] = X_train['emp_length'].apply(np.log1p)
 X_test['emp_length'] = X_test['emp_length'].apply(np.log1p)
 X_train['annual_inc'] = X_train['annual_inc'].apply(np.log1p)
 X_test['annual_inc'] = X_test['annual_inc'].apply(np.log1p)
 X_train['open_acc'] = X_train['open_acc'].apply(np.log1p)
 X_test['open_acc'] = X_test['open_acc'].apply(np.log1p)
 X_train['revol_bal'] = X_train['revol_bal'].apply(np.log1p)
 X_test['revol_bal'] = X_test['revol_bal'].apply(np.log1p)
 X_train['total_acc'] = X_train['total_acc'].apply(np.log1p)
 X_test['total_acc'] = X_test['total_acc'].apply(np.log1p)
 X_train['tot_cur_bal'] = X_train['tot_cur_bal'].apply(np.log1p)
 X_test['tot_cur_bal'] = X_test['tot_cur_bal'].apply(np.log1p)
<data_type_conversions>
X_train[cats] = encoder.fit_transform(X_train[cats])
 X_test[cats] = encoder.transform(X_test[cats] )
nums = ['loan_amnt', 'installment', 'annual_inc', 'dti', 'delinq_2yrs', 'inq_last_6mths',
'mths_since_last_delinq', 'mths_since_last_record', 'open_acc', 'pub_rec', 'revol_bal',
'revol_util', 'total_acc', 'collections_12_mths_ex_med', 'mths_since_last_major_derog',
'acc_now_delinq', 'tot_coll_amt', 'tot_cur_bal']
scaler = StandardScaler()
X_train[nums] = X_train[nums].astype(float)
 X_test[nums] = X_test[nums].astype(float)
 scaler.fit(X_train[nums])
 X_train[nums] = scaler.transform(X_train[nums])
 X_test[nums] = scaler.transform(X_test[nums] )<categorify>
target = 'ConvertedSalary'
t_encoding_col = ['Region', 'Employment', 'LastNewJob', 'YearsCodingProf', 'SalaryType', 'Currency', 'Age', 'Student', 'CompanySize', 'MilitaryUS', 'CareerSatisfaction', 'NumberMonitors', 'OperatingSystem', 'EducationParents']
for i, t_col in enumerate(t_encoding_col):
X_temp = pd.concat([X_train, y_train], axis=1)
 summary = X_temp.groupby([t_col])[target].mean()
enc_test = X_test[t_col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([t_col])[target].mean()
enc_train.iloc[val_ix] = X_val[t_col].map(summary)
 X_train[t_col] = enc_train
X_test[t_col] = enc_test
cats = ['issue_d','sub_grade',
'title','zip_code','addr_state','earliest_cr_line','initial_list_status','application_type']
oe = OrdinalEncoder(cols=cats, return_df=False)
 X_train[cats] = oe.fit_transform(X_train[cats])
 X_test[cats] = oe.transform(X_test[cats] )<categorify>
y_train_log = np.log1p(y_train ).copy()
TXT_train = X_train.emp_title.copy()
TXT_test = X_test.emp_title.copy()
TXT_train.head()
X_train.drop(['emp_title'], axis=1, inplace=True)
 X_test.drop(['emp_title'], axis=1, inplace=True)
 X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_train.median() , inplace=True )<categorify>
scores = []
kf = KFold(n_splits=5, random_state=71, shuffle=True)
 for i,(train_ix, test_ix)in tqdm(enumerate(kf.split(X_train, y_train_log))):
X_train_, y_train_ = X_train.values[train_ix], y_train_log.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train_log.values[test_ix]
reg = LGBMRegressor()
reg.fit(X_train_, y_train_, eval_metric='rmse')
 y_pred = reg.predict(X_val)
 score = mean_absolute_error(y_val, y_pred)
 scores.append(score)
 print('CV Score of Fold_%d is %f' %(i, score))
print(np.mean(scores))
print(scores )
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 cats = ['issue_d','sub_grade',
'title','zip_code','addr_state','earliest_cr_line','initial_list_status','application_type']
for col in cats:
summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=71, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train<prepare_x_and_y>
np.expm1(y_pred )
X_train.fillna(X_train.mean() , axis=0, inplace=True)
 X_test.fillna(X_train.mean() , axis=0, inplace=True )<prepare_x_and_y>
reg.fit(X_train, y_train_log, eval_metric='rmse')
 y_pred = np.expm1(reg.predict(X_test))
y_pred = np.zeros(X_test.shape[0])
 for i in range(30):
clf = LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.71,
importance_type='split', learning_rate=0.05, max_depth=-1,
min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
n_estimators=1000, n_jobs=-1, num_leaves=31, objective=None,
random_state=i, reg_alpha=1.0, reg_lambda=1.0, silent=True,
subsample=0.9, subsample_for_bin=200000, subsample_freq=0)
 clf.fit(X_train, y_train)
 y_pred += clf.predict_proba(X_test)[:,1]
y_pred /= 30<train_model>
submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission.ConvertedSalary = y_pred
submission.to_csv('submission.csv' )
<predict_on_test><EOS> 
fti = reg.feature_importances_
print('Feature Importances:')
 for i, feat in enumerate(X_train):
print('\t{0:20s} : {1:>.6f}'.format(feat, fti[i]))