Kaggle competition: Home Credit Default Risk
def to_int(obj):
return int(re.sub("[^\d]", '', obj))
df['Source IP'] = df['Source IP'].apply(to_int)
 df['Destination IP'] = df['Destination IP'].apply(to_int)
 df['Timestamp'] = df['Timestamp'].apply(to_int)
 df = df.drop(['Usage'], axis=1)
 df = df.drop(['Flow ID'], axis=1)
 df = df.replace([np.nan, 'Infinity'], 2**31-1 )<prepare_x_and_y>
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings("ignore")
 np.random.seed(5)
X = df.iloc[:, :-1]
y = df.iloc[:, -1]<normalization>
train_df = pd.read_csv('.. /input/preprocessed-home-credit/local_data_final.csv')
 test_df = pd.read_csv('.. /input/preprocessed-home-credit/kaggle_test_final.csv')
enc = LabelEncoder()
y = enc.fit_transform(y)
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y)
 ohe = OneHotEncoder()
y_train_ohe = ohe.fit_transform(y_train.reshape(-1, 1))
y_test_ohe = ohe.fit_transform(y_test.reshape(-1, 1))
y_ohe = ohe.fit_transform(y.reshape(-1, 1))
scaler = StandardScaler()
X_scale = scaler.fit_transform(X)
 X_train_scale = scaler.fit_transform(X_train)
 X_test_scale = scaler.transform(X_test )<choose_model_class>
def normalize(t, test_df):
train_dff = t.drop(columns = ['SK_ID_CURR',"TARGET"],axis = 1)
 X_train = train_dff.values
print(X_train.shape)
 Y_train = t['TARGET']
test_dff = test_df.drop(columns = ['SK_ID_CURR',"TARGET"],axis = 1)
 X_test = test_dff.values
print(X_test.shape)
 scaler = preprocessing.StandardScaler().fit(X_train)
 print(scaler)
 xtr = pd.DataFrame(( scaler.transform(X_train)).tolist())
 xte = pd.DataFrame(( scaler.transform(X_test)).tolist())
 return xtr, xte, Y_train
model = Sequential()
model.add(Dense(units=64, activation='relu', input_dim=83))
model.add(Dense(units=16, activation='relu'))
model.add(Dense(units=4, activation='softmax'))<choose_model_class>
X_train, X_test, Y_train = normalize(t, test_df )
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
dff = train_df.drop('SK_ID_CURR', axis = 1)
 dfff = dff.drop('TARGET', axis = 1)
 target = dff['TARGET']
X_train.columns = dfff.columns
X_test.columns = dfff.columns
train_dff = X_train
train_dff['TARGET'] = Y_train.tolist()
history = model.fit(X_train_scale,
y_train_ohe,
epochs=80,
batch_size=512,
validation_data=(X_test_scale, y_test_ohe))<predict_on_test>
def learning_rate_010_decay_power_099(current_iter):
base_learning_rate = 0.1
lr = base_learning_rate * np.power (.99, current_iter)
 return lr if lr > 1e-3 else 1e-3
def learning_rate_010_decay_power_0995(current_iter):
base_learning_rate = 0.1
lr = base_learning_rate * np.power (.995, current_iter)
 return lr if lr > 1e-3 else 1e-3
def learning_rate_005_decay_power_099(current_iter):
base_learning_rate = 0.05
lr = base_learning_rate * np.power (.99, current_iter)
 return lr if lr > 1e-3 else 1e-3
fit_params={"early_stopping_rounds":30,
"eval_metric" : 'auc',
'eval_names': ['valid'],
'verbose': 100,
'categorical_feature': 'auto'}
param_test ={'num_leaves': sp_randint(6, 50),
'min_child_samples': sp_randint(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': sp_uniform(loc=0.2, scale=0.8),
'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}
n_HP_points_to_test = 100
clf = lgb.LGBMClassifier(max_depth=-1, random_state=314, silent=True, metric='None', n_jobs=4, n_estimators=5000)
 gs = RandomizedSearchCV(
 estimator=clf, param_distributions=param_test,
n_iter=n_HP_points_to_test,
scoring='roc_auc',
cv=3,
refit=True,
random_state=314,
verbose=True)
 opt_parameters = {'colsample_bytree': 0.48263575356020577, 'min_child_samples': 311,
'min_child_weight': 1, 'num_leaves': 7, 'reg_alpha': 7, 'reg_lambda': 0.1,
'subsample': 0.3542761367404292}
enc.inverse_transform(model.predict_classes(X_scale))<create_dataframe>
def kfold_lightgbm(train_df,test_df,Y_train,num_folds,stratified = False, debug= False):
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(boosting_type='gbdt', class_weight=None,
colsample_bytree=0.48263575356020577, importance_type='split',
learning_rate=0.1, max_depth=-1, metric='None',
min_child_samples=311, min_child_weight=1, min_split_gain=0.0,
n_estimators=10000, n_jobs=4, num_leaves=7, objective=None,
random_state=None, reg_alpha=7, reg_lambda=0.1, silent=True,
subsample=0.3542761367404292, subsample_for_bin=200000,
subsample_freq=0,scale_pos_weight = 2)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 30,callbacks=[lgb.reset_parameter(learning_rate=learning_rate_010_decay_power_0995)])
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
folds_idx = [(trn_idx, val_idx)for trn_idx, val_idx in folds.split(train_df[feats], train_df['TARGET'])]
display_roc_curve(y_=Y_train,oof_preds_=oof_preds,sub_preds_ = sub_preds, folds_idx_=folds_idx)
 display_precision_recall(y_=Y_train, oof_preds_=oof_preds, folds_idx_=folds_idx)
 return feature_importance_df,sub_preds
def display_roc_curve(y_, oof_preds_,sub_preds_,folds_idx_):
plt.figure(figsize=(6,6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = roc_auc_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(fpr, tpr, lw=1, alpha=0.3, label='ROC fold %d(AUC = %0.4f)' %(n_fold + 1, score))
plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', label='Chance', alpha=.8)
 fpr, tpr, thresholds = roc_curve(y_, oof_preds_)
 score = roc_auc_score(y_, oof_preds_)
 plt.plot(fpr, tpr, color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2, alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('LightGBM ROC Curve')
 plt.legend(loc="lower right")
 plt.tight_layout()
plt.savefig('roc_curve.png')
 def display_precision_recall(y_, oof_preds_, folds_idx_):
plt.figure(figsize=(6,6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = average_precision_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(fpr, tpr, lw=1, alpha=0.3, label='AP fold %d(AUC = %0.4f)' %(n_fold + 1, score))
precision, recall, thresholds = precision_recall_curve(y_, oof_preds_)
 score = average_precision_score(y_, oof_preds_)
 plt.plot(precision, recall, color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2, alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.title('LightGBM Recall / Precision')
 plt.legend(loc="best")
 plt.tight_layout()
def compiler2(train_df,test_df,Y_train,debug = False):
num_rows = 10000 if debug else None
feat_importance,sub_preds = kfold_lightgbm(train_df,test_df,
Y_train, num_folds= 5,
stratified= True, debug= debug)
 return feat_importance,sub_preds
dftrain = pd.DataFrame(enc.inverse_transform(model.predict_classes(X)) , columns=['Label'], index=X.index ).reset_index()<filter>
feat_importance,sub_preds = compiler2(train_dff,test_dff,Y_train,debug = False)
 result = pd.DataFrame()
result['SK_ID_CURR'] = test_df['SK_ID_CURR'].values.tolist()
result['TARGET'] = sub_preds
dftrain[dftrain['Label'] != 'BENIGN']<load_from_csv>
result.to_csv('submission_kernel_lightGBM.csv', index= False)
<normalization><EOS> 
if lgb.compat.MATPLOTLIB_INSTALLED:
else:
raise ImportError('You need to install matplotlib for plot_example.py.')
 X_train_, X_test_, Y_train_, Y_test_ = train_test_split(X_train, Y_train, test_size=0.1,random_state=42)
 lgb_train = lgb.Dataset(X_train_, Y_train_)
 lgb_test = lgb.Dataset(X_test_, Y_test_, reference=lgb_train)
 params = {'colsample_bytree': 0.48263575356020577, 'min_child_samples': 311,
'min_child_weight': 1, 'num_leaves': 7, 'reg_alpha': 7, 'reg_lambda': 0.1,
'subsample': 0.3542761367404292, 'metric': 'roc_auc'}
print('Starting training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=100,
valid_sets=[lgb_train, lgb_test],
feature_name=[X_train.columns.tolist() [i] for i in range(len(X_train.columns.tolist())) ],
verbose_eval=10)