Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<predict_on_test>
!pip install --upgrade scikit-learn
py.init_notebook_mode(connected=True)
preds = learn.get_preds(ds_type=DatasetType.Test)[0][:,1].numpy()<save_to_csv>
path='.. /input/cat-in-the-dat-ii/'
train=pd.read_csv(path+'train.csv')
 test=pd.read_csv(path+'test.csv')
 submission=pd.read_csv(path+'sample_submission.csv')
submission = pd.DataFrame({'id':sample_submission['id'],'target':preds})
 submission.to_csv('my_submission.csv', header=True, index=False )<import_modules>
def resumetable(df):
print(f"Dataset Shape: {df.shape}")
 summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Total'] = df.count().values
summary['Missing Percentage']=(summary['Missing']/summary['Total'])*100
summary['Uniques'] = df.nunique().values
return summary
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.preprocessing import OrdinalEncoder,OneHotEncoder
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold,StratifiedKFold
from sklearn.metrics import roc_auc_score
import lightgbm as lgb
from sklearn.preprocessing import StandardScaler
import string
import category_encoders as ce
import time<load_from_csv>
bin_features=[i for i in train.columns if i.split('_')[0]=='bin']
ord_features=[i for i in train.columns if i.split('_')[0]=='ord']
nom_features=[i for i in train.columns if i.split('_')[0]=='nom']
cyc_features=[i for i in train.columns if i in ['day','month']]
rawtrain=pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 rawtest=pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 sub=pd.read_csv('.. /input/cat-in-the-dat-ii/sample_submission.csv')
 target=rawtrain['target']
train=rawtrain.drop(['id','target'],axis=1)
 test=rawtest.drop('id',axis=1)
<feature_engineering>
train_copy=train.dropna()
cate_ord=['ord_1','ord_2']
for c in cate_ord:
print(rawtrain[c].unique())
 levelmap={c:i for i,c in enumerate(['Novice','Contributor', 'Expert', 'Master','Grandmaster'])}
train['ord_1']=train['ord_1'].replace(levelmap)
 test['ord_1']=test['ord_1'].replace(levelmap)
 tempratmap={c:i for i,c in enumerate(['Freezing','Cold', 'Warm','Hot' , 'Boiling Hot' ,'Lava Hot' ])}
train['ord_2']=train['ord_2'].replace(tempratmap)
 test['ord_2']=test['ord_2'].replace(tempratmap)
 lowermap={c:i for i,c in enumerate(string.ascii_lowercase)}
train['ord_3']=train['ord_3'].replace(lowermap)
 test['ord_3']=test['ord_3'].replace(lowermap)
 upperletter=rawtrain['ord_4'].unique().tolist()
upperletter.remove(np.nan)
 upperletter.sort()
uppermap={c:i for i,c in enumerate(string.ascii_uppercase)}
train['ord_4']=train['ord_4'].replace(uppermap)
 test['ord_4']=test['ord_4'].replace(uppermap)
 alletter=string.ascii_letters
allmap={c:i for i,c in enumerate(alletter)}
def getP(x,p):
if pd.isnull(x):
return x
else:
if p==0:
return x[0]
else:
return x[1]
train['ord_5_0']=rawtrain['ord_5'].apply(lambda x: getP(x,0)).replace(allmap)
 train['ord_5_1']=rawtrain['ord_5'].apply(lambda x: getP(x,1)).replace(allmap)
 test['ord_5_0']=rawtest['ord_5'].apply(lambda x: getP(x,0)).replace(allmap)
 test['ord_5_1']=rawtest['ord_5'].apply(lambda x: getP(x,1)).replace(allmap)
 train=train.drop('ord_5',axis=1)
 test=test.drop('ord_5',axis=1)
<categorify>
def cramers_v(x, y):
confusion_matrix = pd.crosstab(x,y)
 chi2 = ss.chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum().sum()
phi2 = chi2/n
r,k = confusion_matrix.shape
phi2corr = max(0, phi2-(( k-1)*(r-1)) /(n-1))
rcorr = r-(( r-1)**2)/(n-1)
 kcorr = k-(( k-1)**2)/(n-1)
 return np.sqrt(phi2corr/min(( kcorr-1),(rcorr-1)))
 all_feat=bin_features+nom_features+ord_features+cyc_features
normcol59=['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
train_cluster=train.drop(normcol59,axis=1)
 test_cluster=test.drop(normcol59,axis=1)
 for c in train_cluster.columns:
test_cluster[c].fillna(train_cluster[c].mode() [0], inplace = True)
 train_cluster[c].fillna(train_cluster[c].mode() [0], inplace = True)
 bincol_labeled=['bin_3', 'bin_4']
binOE=OrdinalEncoder()
train_cluster[bincol_labeled]=binOE.fit_transform(train_cluster[bincol_labeled])
 test_cluster[bincol_labeled]=binOE.transform(test_cluster[bincol_labeled])
 normcol_labeled=['nom_0','nom_1','nom_2', 'nom_3', 'nom_4']
binOE=OrdinalEncoder()
train_cluster[normcol_labeled]=binOE.fit_transform(train_cluster[normcol_labeled])
 test_cluster[normcol_labeled]=binOE.transform(test_cluster[normcol_labeled])
 km = KModes(n_clusters=2, init = "Cao", n_init = 1, verbose=1,random_state=1990)
 train['cluster'] = km.fit_predict(train_cluster)
 test['cluster'] = km.predict(test_cluster )<categorify>
def encoding(train, test, smooth):
print('Target encoding...')
 train.sort_index(inplace=True)
 target = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 cat_feat_to_encode = bin_features+nom_features+ord_features+cyc_features
smoothing=smooth
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(train, target):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, target)
 train = oof.sort_index()
test = ce_target_encoder.transform(test)
 features = list(train)
 print('Target encoding done!')
 return train, test, test_id, features, target
bincol=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']
normcol=['nom_0','nom_1','nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
def TargetEncode(trainc,testc,targetc, smooth):
print('Target encoding...')
 smoothing=smooth
oof = np.zeros(len(trainc))
for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(trainc, targetc):
train_x=trainc.iloc[tr_idx].reset_index(drop=True)
 valid_x=trainc.iloc[oof_idx].reset_index(drop=True)
 target_train=targetc.iloc[tr_idx].reset_index(drop=True)
 prior = target_train.mean()
tmp = target_train.groupby(train_x ).agg(['sum', 'count'])
 tmp['mean'] = tmp['sum'] / tmp['count']
smoothing = 1 /(1 + np.exp(-(tmp["count"] - 1)/ smoothing))
cust_smoothing = prior *(1 - smoothing)+ tmp['mean'] * smoothing
tmp['smoothing'] = cust_smoothing
tmp = tmp['smoothing'].to_dict()
oof[oof_idx]=valid_x.map(tmp ).values
prior = targetc.mean()
tmp = targetc.groupby(trainc ).agg(['sum', 'count'])
 tmp['mean'] = tmp['sum'] / tmp['count']
smoothing = 1 /(1 + np.exp(-(tmp["count"] - 1)/ smoothing))
cust_smoothing = prior *(1 - smoothing)+ tmp['mean'] * smoothing
tmp['smoothing'] = cust_smoothing
tmp = tmp['smoothing'].to_dict()
testc=testc.map(tmp)
 return oof, testc
for n in normcol+bincol:
train[n],test[n]=TargetEncode(train[n],test[n],target,0.3)
<data_type_conversions>
train_encode, test_encode, test_id, features, target = encoding(train, test, 0.3 )
floatlist=['ord_1','ord_2','ord_3','ord_4','ord_5_0','ord_5_1']
train[floatlist]=train[floatlist].astype(float)
 test[floatlist]=test[floatlist].astype(float)
 usedfeatures=test.columns.tolist()
cat_cols=['cluster']
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=1990)
 params = {
'learning_rate': 0.05,
'feature_fraction': 0.15,
'min_data_in_leaf' : 80,
'max_depth': 6,
'objective': 'binary',
'num_leaves':25,
'metric': 'auc',
'n_jobs': -1,
'feature_fraction_seed': 42,
'bagging_seed': 42,
'boosting_type': 'gbdt',
'verbose': 1,
'is_unbalance': True,
'boost_from_average': False}
t1=time.clock()
traintion = np.zeros(len(train))
validation = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train,target)) :
print("fold n°{}".format(fold_))
train_x=train.iloc[trn_idx][usedfeatures].reset_index(drop=True)
 valid_x=train.iloc[val_idx][usedfeatures].reset_index(drop=True)
 target_train=target.iloc[trn_idx].reset_index(drop=True)
 target_valid=target.iloc[val_idx].reset_index(drop=True)
 trn_data = lgb.Dataset(train_x,
label=target_train,
categorical_feature=cat_cols 
)
 val_data = lgb.Dataset(valid_x,
label=target_valid,
categorical_feature=cat_cols 
)
 num_round = 1000000
clf = lgb.train(params,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=250,
early_stopping_rounds = 250)
 traintion[trn_idx] += clf.predict(train_x, num_iteration=clf.best_iteration)/(folds.n_splits-1)
 validation[val_idx] = clf.predict(valid_x, num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = usedfeatures
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[usedfeatures], num_iteration=clf.best_iteration)/ folds.n_splits
t2=time.clock() -t1
print("Time used: {:<8.5f}".format(t2))
print("Train AUC score: {:<8.5f}".format(roc_auc_score(target,traintion)))
 print("Valid AUC score: {:<8.5f}".format(roc_auc_score(target,validation)))
 sub['target']=predictions
sub.to_csv('submission.csv',index=False)
<set_options>
train_encode=pd.concat([train_encode,target],axis=1,ignore_index=True)
 train_encode.columns=list(train.columns)+['target']
warnings.filterwarnings("ignore" )<load_from_csv>
X, y = train_encode[all_feat], train_encode['target']
path_in = '.. /input/cat-in-the-dat-ii/'
X_train= pd.read_csv(path_in+'train.csv')
 X_test= pd.read_csv(path_in+'test.csv')
 samp_subm = pd.read_csv(path_in+'sample_submission.csv' )<concatenate>
def make_classifier() :
clf = CatBoostClassifier(
 loss_function='CrossEntropy',
eval_metric="AUC",
task_type="CPU",
learning_rate=0.05,
n_estimators =100,
early_stopping_rounds=10,
random_seed=2019,
silent=True 
)
 return clf
train_test =pd.concat([X_train, X_test],sort=False ).reset_index()
y_train = X_train['target']
train_test = train_test.drop(['index','id','target'], axis = 1 )<define_variables>
scoring = "roc_auc"
HistGBM_param = {
'l2_regularization': 0.0,
'loss': 'auto',
'max_bins': 255,
'max_depth': 15,
'max_leaf_nodes': 31,
'min_samples_leaf': 20,
'n_iter_no_change': 50,
'scoring': scoring,
'tol': 1e-07,
'validation_fraction': 0.15,
'verbose': 0,
'warm_start': False
}
folds = StratifiedKFold(n_splits=3, shuffle=True, random_state=1)
 fold_preds = np.zeros([test_encode.shape[0],3])
 oof_preds = np.zeros([X.shape[0],3])
 results = {}
estimators = [ 
('histgbm', HistGradientBoostingClassifier(**HistGBM_param)) , 
('catboost', make_classifier())
 ]
f, ax = plt.subplots(1,3,figsize = [14,5])
 for i,(trn_idx, val_idx)in enumerate(folds.split(X,y)) :
print(f"Fold {i} stacking.... ")
 clf = StackingClassifier(
 estimators=estimators,
final_estimator=LogisticRegression() , 
)
 clf.fit(X.loc[trn_idx,:], y.loc[trn_idx])
 tmp_pred = clf.predict_proba(X.loc[val_idx,:])[:,1]
oof_preds[val_idx,0] = tmp_pred
fold_preds[:,0] += clf.predict_proba(test_encode)[:,1] / folds.n_splits
estimator_performance = {}
estimator_performance['stack_score'] = metrics.roc_auc_score(y.loc[val_idx], tmp_pred)
 for ii, est in enumerate(estimators):
model = clf.named_estimators_[est[0]]
pred = model.predict_proba(X.loc[val_idx,:])[:,1]
oof_preds[val_idx, ii+1] = pred
fold_preds[:,ii+1] += model.predict_proba(test_encode)[:,1] / folds.n_splits
estimator_performance[est[0]+"_score"] = metrics.roc_auc_score(y.loc[val_idx], pred)
 stack_coefficients = {x+"_coefficient":y for(x,y)in zip([x[0] for x in estimators], clf.final_estimator_.coef_[0])}
stack_coefficients['intercept'] = clf.final_estimator_.intercept_[0]
results["Fold {}".format(str(i+1)) ] = [
estimator_performance,
stack_coefficients
]
plot_roc_curve(clf, X.loc[val_idx,:], y.loc[val_idx], ax=ax[i])
 ax[i].plot([0.0, 1.0])
 ax[i].set_title("Fold {} - ROC AUC".format(str(i)))
 plt.tight_layout(pad=2)
 plt.show()
f, ax = plt.subplots(1,2,figsize = [11,5])
 sns.heatmap(pd.DataFrame(oof_preds, columns = ['stack','histgbm','catboost'] ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="magma",ax=ax[0])
 ax[0].set_title("OOF PRED - Correlation Plot")
 sns.heatmap(pd.DataFrame(fold_preds, columns = ['stack','histgbm','catboost'] ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="inferno",ax=ax[1])
 ax[1].set_title("TEST PRED - Correlation Plot")
 plt.tight_layout(pad=3)
 plt.show()
<categorify><EOS> 
submission['target'] =fold_preds[:,0]
submission.to_csv('submission.csv', index=None)
 submission.head()