Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<save_to_csv>
import pandas as pd
import numpy as np
import lightgbm as lgbm
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
submit = test_df[['SK_ID_CURR']]
submit['TARGET'] = pred
submit.to_csv('NN.csv', index = False )<set_options>
train = pd.read_csv(".. /input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-jan-2021/test.csv" )
%matplotlib inline
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 100)
 pd.set_option('display.max_columns', 200 )<load_from_csv>
cont_features = [col for col in train.columns if col.startswith("cont")]
len(cont_features )
app_train = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 app_test = pd.read_csv('.. /input/home-credit-default-risk/application_test.csv' )<train_model>
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof = np.zeros(len(train))
score_list = []
fold = 1
test_preds = []
seed_list = [None,2,3]
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
X_train = X_train.abs()
y_pred_list = []
for seed in seed_list:
dtrain = lgbm.Dataset(X_train[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 print(seed)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":7,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 dtrain = lgbm.Dataset(X_train[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 350,
"lambda_l1":7,
"lambda_l2":1,
"learning_rate":0.003,
'min_child_samples': 35,
"bagging_fraction":0.8,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=200,
init_model = model 
)
 y_pred_list.append(model.predict(X_val[cont_features]))
print(np.sqrt(mean_squared_error(y_val, np.mean(y_pred_list,axis=0))))
test_preds.append(model.predict(test[cont_features]))
oof[test_index] = np.mean(y_pred_list,axis=0)
 score = np.sqrt(mean_squared_error(y_val, oof[test_index]))
score_list.append(score)
 print(f"RMSE Fold-{fold} : {score}")
 fold+=1
np.mean(score_list )
print('Training data shape: ', app_train.shape)
 app_train.head() 
<count_values>
print(score_list)
 np.mean(score_list )
app_train['TARGET'].value_counts()<count_values>
train["1_preds"] = oof
test["1_preds"] = np.mean(test_preds,axis=0 )
app_train.dtypes.value_counts() 
<define_variables>
threshold = 8
train["target_class"] = train["target"].apply(lambda x: 1 if x > threshold else 0)
 train.groupby("target_class")["target"].mean()
np.linspace(20,70,num=11 )<feature_engineering>
y = train["target_class"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof = np.zeros(len(train))
score_list = []
fold = 1
test_preds = []
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
dtrain_0 = lgbm.Dataset(X_train[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "binary",
"metric": "auc",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 200,
"lambda_l1":7,
"lambda_l2":0,
"learning_rate":0.01,
'min_child_samples': 0,
"bagging_fraction":0.75,
"bagging_freq":1,
}
model_0 = lgbm.train(params,
dtrain_0,
valid_sets=[dtrain_0, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 y_pred_0 = model_0.predict(X_val[cont_features])
 score = roc_auc_score(y_val,y_pred_0)
 print(f"AUC Fold-{fold} : {score}")
 oof[test_index] = y_pred_0
score_list.append(score)
 test_preds.append(model_0.predict(test[cont_features]))
fold+=1
np.mean(score_list )
age_data=app_train[['TARGET','DAYS_BIRTH']]
age_data['DAYS_BIRTH']=-age_data['DAYS_BIRTH']
age_data['YEARS_BIRTH']=age_data['DAYS_BIRTH']/365
age_data['YEARS_BINNED']=pd.cut(age_data['YEARS_BIRTH'],bins=np.linspace(20,70,num=11))
age_data.head(10 )<groupby>
train["class_preds"] = oof
test["class_preds"] = np.mean(test_preds,axis=0)
 roc_auc_score(train["target_class"],train["class_preds"] )
age_groups = age_data.groupby('YEARS_BINNED' ).mean()
age_groups<filter>
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_2 = np.zeros(len(train))
score_list = []
fold = 1
test_preds_2 = []
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
X_train_0 = X_train[X_train.target_class==0]
y_train_0 = y_train[X_train.target_class==0]
X_train_1 = X_train[X_train.target_class==1]
y_train_1 = y_train[X_train.target_class==1]
dtrain_0 = lgbm.Dataset(X_train_0[cont_features], y_train_0)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":4,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
model_0 = lgbm.train(params,
dtrain_0,
valid_sets=[dtrain_0, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 y_pred_0 = model_0.predict(X_val[cont_features])
 score = np.sqrt(mean_squared_error(y_val, y_pred_0))
print(f"RMSE Fold-{fold} : {score}")
 dtrain_1 = lgbm.Dataset(X_train_1[cont_features], y_train_1)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":4,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
model_1 = lgbm.train(params,
dtrain_1,
valid_sets=[dtrain_1, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 y_pred_1 = model_1.predict(X_val[cont_features])
 score = np.sqrt(mean_squared_error(y_val, y_pred_1))
print(f"RMSE Fold-{fold} : {score}")
 oof_2[test_index] = y_pred_0*(1-X_val["class_preds"])+ y_pred_1*(X_val["class_preds"])
 score = np.sqrt(mean_squared_error(y_val,oof_2[test_index]))
print(f"RMSE Fold-Final-{fold} : {score}")
 score_list.append(score)
 test_preds_2.append(model_0.predict(test[cont_features])*(1-test["class_preds"])+ model_1.predict(test[cont_features])*(test["class_preds"]))
fold+=1
np.mean(score_list )
anom = app_train[app_train['DAYS_EMPLOYED'] == 365243]
non_anom = app_train[app_train['DAYS_EMPLOYED'] != 365243]
print('이상값이 아닌 data의 target 평균: %0.2f%%' %(100 * non_anom['TARGET'].mean()))
print('이상값인 data의 target 평균: %0.2f%%' %(100 * anom['TARGET'].mean())) 
<feature_engineering>
print(np.mean(score_list))
score_list
app_test['DAYS_EMPLOYED_ANOM']=app_test['DAYS_EMPLOYED']==365243
app_test['DAYS_EMPLOYED'].replace({365243:np.nan}, inplace=True)
 print('%d 개의 data 중에 testing data에서 %d 개의 이상값이 있다.'%(len(app_test),app_test['DAYS_EMPLOYED_ANOM'].sum()))<feature_engineering>
train["2_preds"] = oof_2
test["2_preds"] = np.mean(test_preds_2,axis=0)
 np.sqrt(mean_squared_error(train["target"],train["2_preds"]))
app_train['DAYS_BIRTH'] = app_train['DAYS_BIRTH'] / -365
app_test['DAYS_BIRTH'] = app_test['DAYS_BIRTH'] / -365
ext=app_train[['TARGET','EXT_SOURCE_1','EXT_SOURCE_2','EXT_SOURCE_3','DAYS_BIRTH']]
extcorr = ext.corr()
extcorr<count_values>
from tensorflow.keras.layers import Input,Dense,Dropout
from tensorflow.keras import Model
from tensorflow.keras.regularizers import l2
import tensorflow as tf
app_train.dtypes.value_counts() 
<data_type_conversions>
def get_DAE() :
inputs = Input(( 14,))
x = Dense(500, activation='relu' )(inputs)
 x = Dense(500, activation='relu', name="feature" )(x)
 x = Dense(500, activation='relu' )(x)
 outputs = Dense(14, activation='relu' )(x)
 model = Model(inputs=inputs, outputs=outputs)
 model.compile(optimizer='adam', loss='mse')
 return model
object_columns = app_train.dtypes[app_train.dtypes == 'object'].index.tolist()
object_columns<count_values>
alldata = pd.concat([train[cont_features],test[cont_features]],axis=0)
 print(alldata.shape)
 autoencoder = get_DAE()
autoencoder.fit(alldata[cont_features], alldata[cont_features],
epochs=15,
batch_size=256,
shuffle=True
)
cond_1 =(app_train['TARGET'] == 1)
 cond_0 =(app_train['TARGET'] == 0)
 for a in obj:
print(a)
 print('
연체인 경우
',app_train[cond_1][a].value_counts() /app_train[cond_1].shape[0])
 print('
연체가 아닌 경우
',app_train[cond_0][a].value_counts() /app_train[cond_0].shape[0])
 print('----------------------------' )<categorify>
test_denoised = test.copy()
test_denoised[cont_features] = autoencoder.predict(test_denoised[cont_features])
 train_denoised = train.copy()
train_denoised[cont_features] = autoencoder.predict(train_denoised[cont_features] )
def missing_values_table(df):
miss = df.isnull().sum()
miss_percent = 100 * miss / len(df)
 mis_table = pd.concat([miss, miss_percent], axis=1)
 mis_val_table = mis_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table = mis_val_table[
mis_val_table.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("선택된 데이터프레임은 " + str(df.shape[1])+ "개의 컬럼이 있다.
 "
"그중에서 " + str(mis_val_table.shape[0])+
" 개는 결측값이 있는 컬럼이다.")
 return mis_val_table<count_missing_values>
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_3 = np.zeros(len(train))
score_list = []
fold = 1
test_preds_3 = []
seed_list = [None,2,3]
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
X_train_denoised, X_val_denoised = train_denoised.iloc[train_index], train_denoised.iloc[test_index]
print(X_train_denoised.shape)
 X_train_denoised = pd.concat([X_train_denoised,X_train],axis=0)
 y_train = pd.concat([y_train,y_train],axis=0)
 print(X_train_denoised.shape)
 y_pred_list = []
for seed in seed_list:
dtrain = lgbm.Dataset(X_train_denoised[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":7,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 dtrain = lgbm.Dataset(X_train_denoised[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 350,
"lambda_l1":7,
"lambda_l2":1,
"learning_rate":0.003,
'min_child_samples': 35,
"bagging_fraction":0.8,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=200,
init_model = model 
)
 y_pred_list.append(model.predict(X_val[cont_features]))
print(np.sqrt(mean_squared_error(y_val, np.mean(y_pred_list,axis=0))))
test_preds_3.append(model.predict(test[cont_features]))
oof_3[test_index] = np.mean(y_pred_list,axis=0)
 score = np.sqrt(mean_squared_error(y_val, oof_3[test_index]))
score_list.append(score)
 print(f"RMSE Fold-{fold} : {score}")
 fold+=1
np.mean(score_list )
missing_values = missing_values_table(app_train)
 missing_values.head(20 )<concatenate>
print(np.mean(score_list))
score_list
apps = pd.concat([app_train,app_test])
 print(apps.shape )<count_values>
train["3_preds"] = oof_3
test["3_preds"] = np.mean(test_preds_3,axis=0 )
apps['TARGET'].value_counts(dropna=False )<feature_engineering>
train[["id","1_preds","2_preds","3_preds"]].to_csv("train_preds.csv",index=False)
 test[["id","1_preds","2_preds","3_preds"]].to_csv("test_preds.csv",index=False )
object_col = apps.dtypes[apps.dtypes == 'object'].index.tolist()
for column in object_col:
apps[column] = pd.factorize(apps[column])[0]<feature_engineering>
train["4_preds"] = pd.read_csv(".. /input/nn-with-categorical-embedding/nn_preds_train.csv")["4_preds"]
test["4_preds"] = pd.read_csv(".. /input/nn-with-categorical-embedding/nn_preds_test.csv")["4_preds"]
apps['CREDIT_INCOME_PERCENT'] = apps['AMT_CREDIT'] / apps['AMT_INCOME_TOTAL']
apps['ANNUITY_INCOME_PERCENT'] = apps['AMT_ANNUITY'] / apps['AMT_INCOME_TOTAL']
apps['CREDIT_TERM'] = apps['AMT_ANNUITY'] / apps['AMT_CREDIT']
apps['GOODS_CREDIT_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_CREDIT']
apps['CREDIT_GOODS_DIFF'] = apps['AMT_CREDIT'] - apps['AMT_GOODS_PRICE']
apps['GOODS_INCOME_RATIO'] = apps['AMT_GOODS_PRICE']/apps['AMT_INCOME_TOTAL']<feature_engineering>
print("1st model score: ",np.sqrt(mean_squared_error(train["target"], train["1_preds"])))
 print("2nd model score: ",np.sqrt(mean_squared_error(train["target"], train["2_preds"])))
 print("3rd model score: ",np.sqrt(mean_squared_error(train["target"], train["3_preds"])))
 print("4th model score: ",np.sqrt(mean_squared_error(train["target"], train["4_preds"])))
 print("Blending model score: ",np.sqrt(mean_squared_error(train["target"], train["4_preds"]*0.1+train["3_preds"]*0.3+train["2_preds"]*0.3+train["1_preds"]*0.3)) )
apps['APPS_EXT_SOURCE_MEAN'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)
 apps['APPS_EXT_SOURCE_STD'] = apps[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1 )<feature_engineering>
input_df = train[["1_preds","2_preds","3_preds","4_preds"]].copy()
sub_input_df = test[["1_preds","2_preds","3_preds","4_preds"]].copy()
apps['APPS_EXT_SOURCE_STD'] = apps['APPS_EXT_SOURCE_STD'].fillna(apps['APPS_EXT_SOURCE_STD'].mean() )<feature_engineering>
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_stack = np.zeros(len(train))
score_list= []
fold = 1
test_preds_stack = []
for train_index, test_index in kf.split(input_df):
X_train, X_val = input_df.iloc[train_index], input_df.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
reg = LinearRegression(fit_intercept=True ).fit(X_train, y_train)
 y_stack = reg.predict(X_val)
 oof_stack[test_index] = y_stack*1
score = np.sqrt(mean_squared_error(y_val, oof_stack[test_index]))
score_list.append(score)
 test_preds_stack.append(reg.predict(sub_input_df.values ).ravel())
 np.sqrt(mean_squared_error(y, oof_stack))
apps['EMPLOYED_BIRTH_RATIO'] = apps['DAYS_EMPLOYED']/apps['DAYS_BIRTH']
apps['INCOME_EMPLOYED_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_EMPLOYED']
apps['INCOME_BIRTH_RATIO'] = apps['AMT_INCOME_TOTAL']/apps['DAYS_BIRTH']
apps['CAR_BIRTH_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_BIRTH']
apps['CAR_EMPLOYED_RATIO'] = apps['OWN_CAR_AGE'] / apps['DAYS_EMPLOYED']<load_from_csv>
print(np.mean(score_list))
score_list
ccb = pd.read_csv('.. /input/home-credit-default-risk/credit_card_balance.csv' )<merge>
def final_features(df):
combinations_list = [("1_preds","2_preds"), 
("1_preds","3_preds"), 
("1_preds","4_preds"), 
("2_preds","3_preds"), 
("2_preds","4_preds"), 
("3_preds","4_preds")]
for i in range(len(combinations_list)) :
col1 = combinations_list[i][0]
col2 = combinations_list[i][1]
df.loc[:,f"{col1}_{col2}_multiply"] = df[col1] * df[col2]
df.loc[:,f"{col1}_{col2}_divide"] = df[col1] / df[col2]
combinations_list = [("1_preds","2_preds","3_preds"), 
("1_preds","2_preds","4_preds"), 
("1_preds","3_preds","4_preds"), 
("2_preds","3_preds","4_preds")]
for i in range(len(combinations_list)) :
col1 = combinations_list[i][0]
col2 = combinations_list[i][1]
col3 = combinations_list[i][2]
df.loc[:,f"{col1}_{col2}_{col3}_multiply"] = df[col1] * df[col2]* df[col3]
return df
input_df = train[["1_preds","2_preds","3_preds","4_preds"]].copy()
sub_input_df = test[["1_preds","2_preds","3_preds","4_preds"]].copy()
input_df = final_features(input_df)
 sub_input_df = final_features(sub_input_df)
 input_df.head()
app_ccb = ccb.merge(app_train, left_on='SK_ID_CURR', right_on='SK_ID_CURR', how='outer')
 app_ccb.shape<count_missing_values>
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_stack = np.zeros(len(train))
score_list= []
fold = 1
test_preds_stack = []
for train_index, test_index in kf.split(input_df):
X_train, X_val = input_df.iloc[train_index], input_df.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
reg = LinearRegression(fit_intercept=True ).fit(X_train, y_train)
 y_stack = reg.predict(X_val)
 oof_stack[test_index] = y_stack*1
score = np.sqrt(mean_squared_error(y_val, oof_stack[test_index]))
score_list.append(score)
 test_preds_stack.append(reg.predict(sub_input_df.values ).ravel())
 np.sqrt(mean_squared_error(y, oof_stack))
missing_values = missing_values_table(ccb)
 missing_values.head(20 )<groupby>
print(np.mean(score_list))
score_list
app_ccb.groupby('SK_ID_CURR' ).count()<groupby>
train["stacked_preds"] = oof_stack
test["target"] = np.mean(test_preds_stack,axis=0 )
<merge><EOS> 
train[["id","stacked_preds"]].to_csv("tarin_stacked_oof.csv",index=False)
 test[["id","target"]].to_csv("submission.csv",index=False )