Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<train_model>
%matplotlib inline
sm = SMOTE(random_state=42)
 X_resamp_tr, y_resamp_tr = sm.fit_resample(X, Y)
 print('Resampled dataset shape %s' % Counter(y_resamp_tr))
X_resamp_tr = pd.DataFrame(X_resamp_tr)
 y_resamp_tr = pd.DataFrame({"target": y_resamp_tr} )<init_hyperparams>
def read_data(file_path):
print('Loading datasets...')
 train = pd.read_csv(file_path + 'train.csv', sep=',')
 test = pd.read_csv(file_path + 'test.csv', sep=',')
 print('Datasets loaded')
 return train, test
params = {'num_leaves': 9,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 16,
'learning_rate': 0.0123,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': 11,
'reg_alpha': 1.728910519108444,
'reg_lambda': 4.9847051755586085,
'random_state': 42,
'metric': 'auc',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01077313523861969,
'min_child_weight': 19.428902804238373,
'num_threads': 4}<split>
PATH = '.. /input/cat-in-the-dat-ii/'
train, test = read_data(PATH )
%%time
fold_n=3
folds = StratifiedKFold(n_splits=fold_n, shuffle=True, random_state=30)
 y_pred_lgb = np.zeros(len(test_X))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X_resamp_tr,y_resamp_tr)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X_resamp_tr.iloc[train_index], X_resamp_tr.iloc[valid_index]
y_train, y_valid = y_resamp_tr.iloc[train_index], y_resamp_tr.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 y_pred_lgb += lgb_model.predict(test_X, num_iteration=lgb_model.best_iteration)/3<save_to_csv>
def zoom_dataset(data):
Count_missing_val = data.isnull().sum()
Percent_missing =(data.isnull().sum() /data.isnull().count() *100)
 Percent_no_missing = 100 - Percent_missing
Count_unique = data.nunique()
Percent_unique_val = Count_unique / len(data)*100
Type = data.dtypes
data=[[i, Counter(data[i][data[i].notna() ] ).most_common(3)] for i in list(data)]
top = pd.DataFrame(data, columns=['Name', 'Top_3'] ).set_index(['Name'])
 tt = pd.concat([Percent_no_missing, Count_missing_val, Percent_missing, Count_unique, Percent_unique_val, Type], axis=1,
keys=['Percent_no_missing','Count_missing_val', 'Percent_missing', ' Count_unique', 'Percent_unique_val', 'Type'])
 tt = pd.concat([tt, top[['Top_3']]], axis=1, sort=False ).reset_index()
return(( tt))
submission_lgb_smote_2 = pd.DataFrame({
"ID_code": sctp_test["ID_code"],
"target": y_pred_lgb
})
 submission_lgb_smote_2.to_csv('submission_lgb_smote_2.csv', index=False )<load_from_csv>
def encoding(train, test, smooth):
print('Target encoding...')
 train.sort_index(inplace=True)
 target = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 cat_feat_to_encode = train.columns.tolist()
smoothing=smooth
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(train, target):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, target)
 train = oof.sort_index()
test = ce_target_encoder.transform(test)
 features = list(train)
 print('Target encoding done!')
 return train, test, test_id, features, target
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )<data_type_conversions>
def timer(start_time=None):
if not start_time:
start_time = datetime.now()
return start_time
elif start_time:
thour, temp_sec = divmod(( datetime.now() - start_time ).total_seconds() , 3600)
 tmin, tsec = divmod(temp_sec, 60)
 print('Time taken : %i hours %i minutes and %s seconds.' %(thour, tmin, round(tsec, 2)) )
train_x=train.drop(['ID_code','target'],axis=1)
 corr_matrix = train_x.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.995)]<create_dataframe>
def run_model(splits, features, target, train):
model = lgb.LGBMClassifier(**{
'learning_rate': 0.05,
'feature_fraction': 0.1,
'min_data_in_leaf' : 12,
'max_depth': 3,
'reg_alpha': 1,
'reg_lambda': 1,
'objective': 'binary',
'metric': 'auc',
'n_jobs': -1,
'n_estimators' : 5000,
'feature_fraction_seed': 42,
'bagging_seed': 42,
'boosting_type': 'gbdt',
'verbose': 1,
'is_unbalance': True,
'boost_from_average': False})
 plt.rcParams['figure.figsize']=(6,4)
 start_time = timer(None)
 print('LGBM modeling...')
 cms= []
tprs = []
aucs = []
y_real = []
y_proba = []
recalls = []
roc_aucs = []
mean_tpr = []
mean_fpr = []
f1_scores = []
accuracies = []
precisions = []
oof = np.zeros(len(train))
mean_fpr = np.linspace(0,1,100)
 feature_importance_df = pd.DataFrame()
i = 1
folds = StratifiedKFold(n_splits=splits, shuffle=True, random_state=42)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print('Fold:', fold_)
 model = model.fit(train.iloc[trn_idx][features], target.iloc[trn_idx],
eval_set = [(train.iloc[trn_idx][features], target.iloc[trn_idx]), 
(train.iloc[val_idx][features], target.iloc[val_idx])],
verbose = 1000,
eval_metric = 'auc',
early_stopping_rounds = 1000)
 oof[val_idx] = model.predict_proba(train.iloc[val_idx][features])[:,1]
f = plt.figure(1)
 fpr, tpr, t = roc_curve(target[val_idx], oof[val_idx])
 tprs.append(interp(mean_fpr, fpr, tpr))
roc_auc = auc(fpr, tpr)
 aucs.append(roc_auc)
 plt.plot(fpr, tpr, lw=2, alpha=0.5, label='ROC fold %d(AUC = %0.4f)' %(i,roc_auc))
g = plt.figure(2)
 precision, recall, _ = precision_recall_curve(target[val_idx], oof[val_idx])
 y_real.append(target[val_idx])
 y_proba.append(oof[val_idx])
 plt.plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' %(i))
i= i+1
roc_aucs.append(roc_auc_score(target[val_idx], oof[val_idx]))
accuracies.append(accuracy_score(target[val_idx], oof[val_idx].round()))
recalls.append(recall_score(target[val_idx], oof[val_idx].round()))
precisions.append(precision_score(target[val_idx], oof[val_idx].round()))
f1_scores.append(f1_score(target[val_idx], oof[val_idx].round()))
cms.append(confusion_matrix(target[val_idx], oof[val_idx].round()))
print(
 '
CV roc score : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)) ,
'
CV accuracy score : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)) ,
'
CV recall score : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)) ,
'
CV precision score : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)) ,
'
CV f1 score : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores)))
 print('Modeling done!
')
 timer(start_time)
 return model, cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr, train
def graph_metrics(cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr):
f = plt.figure(1)
 plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')
 mean_tpr = np.mean(tprs, axis=0)
 mean_auc = auc(mean_fpr, mean_tpr)
 plt.plot(mean_fpr, mean_tpr, color='blue',
label=r'Mean ROC(AUC = %0.4f)' %(( mean_auc)) ,lw=2, alpha=1)
 plt.grid(b=True, which='major', color='grey', linewidth=0.4)
 plt.xlabel('False Positive Rate', fontsize=12)
 plt.ylabel('True Positive Rate', fontsize=12)
 plt.title('LGBM - ROC by folds', fontsize=18)
 plt.legend(loc="lower right")
 g = plt.figure(2)
 plt.plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')
 y_real = np.concatenate(y_real)
 y_proba = np.concatenate(y_proba)
 precision, recall, _ = precision_recall_curve(y_real, y_proba)
 plt.plot(recall, precision, color='blue',
label=r'Mean P|R')
 plt.grid(b=True, which='major', color='grey', linewidth=0.4)
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.title('LGBM P|R curve by folds', fontsize=18)
 plt.legend(loc="lower left")
 plt.rcParams["axes.grid"] = False
cm = np.average(cms, axis=0)
 class_names = [0,1]
plt.figure()
plot_confusion_matrix(cm,
classes=class_names,
title='LGBM Confusion matrix [averaged/folds]')
 plt.show()
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=2019)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
target=train.target
train_id=train.ID_code
test_id=test.ID_code
test_df=test.drop('ID_code',axis=1)
 test_df=test_df.drop(to_drop,axis=1 )<init_hyperparams>
train, test, test_id, features, target = encoding(train, test, 0.3)
 model, cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr, df_ml = run_model(5, features, target, train)
 graph_metrics(cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr )
param = {'num_leaves': 21,
'min_data_in_leaf': 35,
'objective':'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.6,
"bagging_fraction": 0.8,
"bagging_seed": 42,
"metric": 'auc',
"verbosity": -1,
"nthread": 4,
"random_state": 2019}<train_model>
pd.DataFrame({'id': test_id, 'target': model.predict_proba(test)[:,1]} ).to_csv('submission.csv', index=False )