Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<define_variables>
%matplotlib inline
allcolumns = df_train.columns
binaries = [c for c in df_train.columns if 'bin' in c]
nominals = [c for c in df_train.columns if 'nom' in c]
ordinals = [c for c in df_train.columns if 'ord' in c]<feature_engineering>
def read_data(file_path):
print('Loading datasets...')
 train = pd.read_csv(file_path + 'train.csv', sep=',')
 test = pd.read_csv(file_path + 'test.csv', sep=',')
 print('Datasets loaded')
 return train, test
df_train['nans'] = df_train.isnull().sum(axis=1)
 df_test['nans'] = df_test.isnull().sum(axis=1 )<prepare_x_and_y>
PATH = '.. /input/cat-in-the-dat-ii/'
train, test = read_data(PATH )
df_train = df_train.set_index('id')
 df_test = df_test.set_index('id')
 y_train = df_train.target
del df_train['target']<categorify>
def zoom_dataset(data):
Count_missing_val = data.isnull().sum()
Percent_missing =(data.isnull().sum() /data.isnull().count() *100)
 Percent_no_missing = 100 - Percent_missing
Count_unique = data.nunique()
Percent_unique_val = Count_unique / len(data)*100
Type = data.dtypes
data=[[i, Counter(data[i][data[i].notna() ] ).most_common(3)] for i in list(data)]
top = pd.DataFrame(data, columns=['Name', 'Top_3'] ).set_index(['Name'])
 tt = pd.concat([Percent_no_missing, Count_missing_val, Percent_missing, Count_unique, Percent_unique_val, Type], axis=1,
keys=['Percent_no_missing','Count_missing_val', 'Percent_missing', ' Count_unique', 'Percent_unique_val', 'Type'])
 tt = pd.concat([tt, top[['Top_3']]], axis=1, sort=False ).reset_index()
return(( tt))
ord_0_mapping = {1 : 0, 2 : 1, 3 : 2}
ord_1_mapping = {'Novice' : 0, 'Contributor' : 1, 'Expert' : 2, 'Master': 3, 'Grandmaster': 4}
ord_2_mapping = { 'Freezing': 0, 'Cold': 1, 'Warm' : 2, 'Hot': 3, 'Boiling Hot' : 4, 'Lava Hot' : 5}
df_train['real_ord_0'] = df_train.loc[df_train.ord_0.notnull() , 'ord_0'].map(ord_0_mapping)
 df_train['real_ord_1'] = df_train.loc[df_train.ord_1.notnull() , 'ord_1'].map(ord_1_mapping)
 df_train['real_ord_2'] = df_train.loc[df_train.ord_2.notnull() , 'ord_2'].map(ord_2_mapping)
 df_test['real_ord_0'] = df_test.loc[df_test.ord_0.notnull() , 'ord_0'].map(ord_0_mapping)
 df_test['real_ord_1'] = df_test.loc[df_test.ord_1.notnull() , 'ord_1'].map(ord_1_mapping)
 df_test['real_ord_2'] = df_test.loc[df_test.ord_2.notnull() , 'ord_2'].map(ord_2_mapping)
 otherordinals = ['ord_3','ord_4','ord_5']
for c in otherordinals:
print(c)
 df_train['real_'+c] = df_train[[c]].apply(lambda a: OrdinalConverter(a[c])if not pd.isnull(a[c])else np.nan,axis=1)
 df_test['real_'+c] = df_test[[c]].apply(lambda a: OrdinalConverter(a[c])if not pd.isnull(a[c])else np.nan,axis=1)
<drop_column>
def encoding(train, test, smooth):
print('Target encoding...')
 train.sort_index(inplace=True)
 target = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 cat_feat_to_encode = train.columns.tolist()
smoothing=smooth
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(train, target):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, target)
 train = oof.sort_index()
test = ce_target_encoder.transform(test)
 features = list(train)
 print('Target encoding done!')
 return train, test, test_id, features, target
df_train.drop(ordinals,inplace=True,axis=1)
 df_test.drop(ordinals,inplace=True,axis=1 )<concatenate>
def timer(start_time=None):
if not start_time:
start_time = datetime.now()
return start_time
elif start_time:
thour, temp_sec = divmod(( datetime.now() - start_time ).total_seconds() , 3600)
 tmin, tsec = divmod(temp_sec, 60)
 print('Time taken : %i hours %i minutes and %s seconds.' %(thour, tmin, round(tsec, 2)) )
df = pd.concat([df_train,df_test] )<categorify>
def run_model(splits, features, target, train):
model = lgb.LGBMClassifier(**{
'learning_rate': 0.05,
'feature_fraction': 0.1,
'min_data_in_leaf' : 12,
'max_depth': 3,
'reg_alpha': 1,
'reg_lambda': 1,
'objective': 'binary',
'metric': 'auc',
'n_jobs': -1,
'n_estimators' : 5000,
'feature_fraction_seed': 42,
'bagging_seed': 42,
'boosting_type': 'gbdt',
'verbose': 1,
'is_unbalance': True,
'boost_from_average': False})
 plt.rcParams['figure.figsize']=(6,4)
 start_time = timer(None)
 print('LGBM modeling...')
 cms= []
tprs = []
aucs = []
y_real = []
y_proba = []
recalls = []
roc_aucs = []
mean_tpr = []
mean_fpr = []
f1_scores = []
accuracies = []
precisions = []
oof = np.zeros(len(train))
mean_fpr = np.linspace(0,1,100)
 feature_importance_df = pd.DataFrame()
i = 1
folds = StratifiedKFold(n_splits=splits, shuffle=True, random_state=42)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print('Fold:', fold_)
 model = model.fit(train.iloc[trn_idx][features], target.iloc[trn_idx],
eval_set = [(train.iloc[trn_idx][features], target.iloc[trn_idx]), 
(train.iloc[val_idx][features], target.iloc[val_idx])],
verbose = 1000,
eval_metric = 'auc',
early_stopping_rounds = 1000)
 oof[val_idx] = model.predict_proba(train.iloc[val_idx][features])[:,1]
f = plt.figure(1)
 fpr, tpr, t = roc_curve(target[val_idx], oof[val_idx])
 tprs.append(interp(mean_fpr, fpr, tpr))
roc_auc = auc(fpr, tpr)
 aucs.append(roc_auc)
 plt.plot(fpr, tpr, lw=2, alpha=0.5, label='ROC fold %d(AUC = %0.4f)' %(i,roc_auc))
g = plt.figure(2)
 precision, recall, _ = precision_recall_curve(target[val_idx], oof[val_idx])
 y_real.append(target[val_idx])
 y_proba.append(oof[val_idx])
 plt.plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' %(i))
i= i+1
roc_aucs.append(roc_auc_score(target[val_idx], oof[val_idx]))
accuracies.append(accuracy_score(target[val_idx], oof[val_idx].round()))
recalls.append(recall_score(target[val_idx], oof[val_idx].round()))
precisions.append(precision_score(target[val_idx], oof[val_idx].round()))
f1_scores.append(f1_score(target[val_idx], oof[val_idx].round()))
cms.append(confusion_matrix(target[val_idx], oof[val_idx].round()))
print(
 '
CV roc score : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)) ,
'
CV accuracy score : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)) ,
'
CV recall score : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)) ,
'
CV precision score : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)) ,
'
CV f1 score : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores)))
 print('Modeling done!
')
 timer(start_time)
 return model, cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr, train
def graph_metrics(cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr):
f = plt.figure(1)
 plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')
 mean_tpr = np.mean(tprs, axis=0)
 mean_auc = auc(mean_fpr, mean_tpr)
 plt.plot(mean_fpr, mean_tpr, color='blue',
label=r'Mean ROC(AUC = %0.4f)' %(( mean_auc)) ,lw=2, alpha=1)
 plt.grid(b=True, which='major', color='grey', linewidth=0.4)
 plt.xlabel('False Positive Rate', fontsize=12)
 plt.ylabel('True Positive Rate', fontsize=12)
 plt.title('LGBM - ROC by folds', fontsize=18)
 plt.legend(loc="lower right")
 g = plt.figure(2)
 plt.plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')
 y_real = np.concatenate(y_real)
 y_proba = np.concatenate(y_proba)
 precision, recall, _ = precision_recall_curve(y_real, y_proba)
 plt.plot(recall, precision, color='blue',
label=r'Mean P|R')
 plt.grid(b=True, which='major', color='grey', linewidth=0.4)
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.title('LGBM P|R curve by folds', fontsize=18)
 plt.legend(loc="lower left")
 plt.rcParams["axes.grid"] = False
cm = np.average(cms, axis=0)
 class_names = [0,1]
plt.figure()
plot_confusion_matrix(cm,
classes=class_names,
title='LGBM Confusion matrix [averaged/folds]')
 plt.show()
df = pd.get_dummies(df, dummy_na=False, columns=binaries)
<categorify>
train, test, test_id, features, target = encoding(train, test, 0.3)
 model, cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr, df_ml = run_model(5, features, target, train)
 graph_metrics(cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr )
<split><EOS> 
pd.DataFrame({'id': test_id, 'target': model.predict_proba(test)[:,1]} ).to_csv('submission.csv', index=False )