Kaggle competition: Categorical Feature Encoding Challenge II
cols=[]
for i in range(10):
cols.append("nom_{}".format(i))<correct_missing_values>
!pip install catboost -U
data=data.fillna(-1 )<drop_column>
import numpy as np
import pandas as pd
from time import time
import pprint
import joblib
from catboost import CatBoostClassifier, Pool
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, average_precision_score
from sklearn.metrics import make_scorer
data=data.drop('id', axis=1 )<drop_column>
X = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/train.csv")
 Xt = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/test.csv" )
data=data.drop('index', axis=1 )<define_variables>
y = X.target.values
id_train = X.id
id_test = Xt.id
X.drop(['id', 'target'], axis=1, inplace=True)
 Xt.drop(['id'], axis=1, inplace=True)
 binary_vars = [c for c in X.columns if 'bin_' in c]
nominal_vars = [c for c in X.columns if 'nom_' in c]
high_cardinality = [c for c in nominal_vars if len(X[c].unique())> 16]
low_cardinality = [c for c in nominal_vars if len(X[c].unique())<= 16]
ordinal_vars = [c for c in X.columns if 'ord_' in c]
time_vars = ['day', 'month']
features=[i for i in data.columns]<categorify>
X['ord_5_1'] = X['ord_5'].apply(lambda x: x[0] if type(x)== str else np.nan)
 X['ord_5_2'] = X['ord_5'].apply(lambda x: x[1] if type(x)== str else np.nan)
 Xt['ord_5_1'] = Xt['ord_5'].apply(lambda x: x[0] if type(x)== str else np.nan)
 Xt['ord_5_2'] = Xt['ord_5'].apply(lambda x: x[1] if type(x)== str else np.nan)
 ordinal_vars += ['ord_5_1', 'ord_5_2']
le=LabelEncoder()
for i in features:
data[i]=le.fit_transform(data[i].astype(str))<split>
ordinals = {
'ord_1' : {
'Novice' : 0,
'Contributor' : 1,
'Expert' : 2,
'Master' : 3,
'Grandmaster' : 4
},
'ord_2' : {
'Freezing' : 0,
'Cold' : 1,
'Warm' : 2,
'Hot' : 3,
'Boiling Hot' : 4,
'Lava Hot' : 5
}
}
def return_order(X, Xt, var_name):
mode = X[var_name].mode() [0]
el = sorted(set(X[var_name].fillna(mode ).unique())|set(Xt[var_name].fillna(mode ).unique()))
return {v:e for e, v in enumerate(el)}
for mapped_var in ordinal_vars:
if mapped_var not in ordinals:
mapped_values = return_order(X, Xt, mapped_var)
 X[mapped_var + '_num'] = X[mapped_var].replace(mapped_values)
 Xt[mapped_var + '_num'] = Xt[mapped_var].replace(mapped_values)
 else:
X[mapped_var + '_num'] = X[mapped_var].replace(ordinals[mapped_var])
 Xt[mapped_var + '_num'] = Xt[mapped_var].replace(ordinals[mapped_var] )
train_data=data[:600000]
test_data=data[600000:]<split>
label_encoders = [LabelEncoder() for _ in range(X.shape[1])]
for col, column in enumerate(X.columns):
unique_values = pd.Series(X[column].append(Xt[column] ).unique())
 unique_values = unique_values[unique_values.notnull() ]
label_encoders[col].fit(unique_values)
 X.loc[X[column].notnull() , column] = label_encoders[col].transform(X.loc[X[column].notnull() , column])
 Xt.loc[Xt[column].notnull() , column] = label_encoders[col].transform(Xt.loc[Xt[column].notnull() , column] )
X_train, X_test, y_train, y_test=train_test_split(train_data, target, test_size=.20 )<choose_model_class>
X = X.fillna(-1)
 Xt = Xt.fillna(-1 )
classifier=RandomForestClassifier(n_estimators=1200, max_depth=100, n_jobs=-1, verbose=2, max_features='sqrt' )<train_model>
def frequency_encoding(column, df, df_test=None):
frequencies = df[column].value_counts().reset_index()
df_values = df[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).iloc[:,-1].values
if df_test is not None:
df_test_values = df_test[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).fillna(1 ).iloc[:,-1].values
else:
df_test_values = None
return df_values, df_test_values
for column in X.columns:
train_values, test_values = frequency_encoding(column, X, Xt)
 X[column+'_counts'] = train_values
Xt[column+'_counts'] = test_values
classifier.fit(X_train, y_train, )<predict_on_test>
cat_feat_to_encode = binary_vars + ordinal_vars + nominal_vars + time_vars
smoothing = 0.3
enc_x = np.zeros(X[cat_feat_to_encode].shape)
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(X, y):
encoder = cat_encs.TargetEncoder(cols=cat_feat_to_encode, smoothing=smoothing)
 encoder.fit(X[cat_feat_to_encode].iloc[tr_idx], y[tr_idx])
 enc_x[oof_idx, :] = encoder.transform(X[cat_feat_to_encode].iloc[oof_idx], y[oof_idx])
 encoder.fit(X[cat_feat_to_encode], y)
 enc_xt = encoder.transform(Xt[cat_feat_to_encode] ).values
for idx, new_var in enumerate(cat_feat_to_encode):
new_var = new_var + '_enc'
X[new_var] = enc_x[:,idx]
Xt[new_var] = enc_xt[:, idx]
pred=classifier.predict(X_test )<import_modules>
from sklearn.metrics import f1_score<compute_test_metric>
X = X.astype(np.float32)
 Xt = Xt.astype(np.float32)
 cat_features = nominal_vars + ordinal_vars
X[cat_features] = X[cat_features].astype(np.int64)
 Xt[cat_features] = Xt[cat_features].astype(np.int64 )
fs=f1_score(y_test, pred )<import_modules>
_ = gc.collect()
import lightgbm as lgb<init_hyperparams>
best_params = {'bagging_temperature': 0.8,
'depth': 5,
'iterations': 1000,
'l2_leaf_reg': 30,
'learning_rate': 0.05,
'random_strength': 0.8}
param = {
'num_leaves': 31,
'objective': 'binary'
'metric': ''}
param['metric'] = ['auc', 'binary_logloss']<import_modules>
SEED = 42
FOLDS = 20
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED )
import category_encoders as ce
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Sequential
from tensorflow.keras.layers import MaxPool1D, Flatten, Dense, Dropout, BatchNormalization
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras import callbacks
from sklearn.metrics import accuracy_score, roc_auc_score, auc
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn import metrics, preprocessing
from tensorflow.keras import backend as K<load_from_csv>
roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
cv_test_preds = np.zeros(len(Xt))
best_iteration = list()
for train_idx, test_idx in skf.split(X, y):
X_train, y_train = X.iloc[train_idx, :], y[train_idx]
X_test, y_test = X.iloc[test_idx, :], y[test_idx]
train = Pool(data=X_train,
label=y_train,
feature_names=list(X_train.columns),
cat_features=cat_features)
 val = Pool(data=X_test,
label=y_test,
feature_names=list(X_test.columns),
cat_features=cat_features)
 catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=4,
verbose = False)
 catb.fit(train,
verbose_eval=100,
early_stopping_rounds=50,
eval_set=val,
use_best_model=True,
plot=False)
 best_iteration.append(catb.best_iteration_)
 preds = catb.predict_proba(X_test)
 oof[test_idx] = preds[:,1]
Xt_pool = Pool(data=Xt[list(X_train.columns)],
feature_names=list(X_train.columns),
cat_features=cat_features)
 cv_test_preds += catb.predict_proba(Xt_pool)[:,1] / FOLDS
roc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))
average_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))
sample_submission = pd.read_csv(".. /input/cat-in-the-dat-ii/sample_submission.csv")
 test = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv" )<categorify>
oof = pd.DataFrame({'id':id_train, 'catboost_oof': oof})
 oof.to_csv("oof.csv", index=False)
 cv_submission = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/sample_submission.csv")
 cv_submission.target = cv_test_preds
cv_submission.to_csv("./catboost_cv_submission.csv", index=False )
def encoding(train, test, smooth):
print('Target encoding...')
 train.sort_index(inplace=True)
 target = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 cat_feat_to_encode = train.columns.tolist()
smoothing=smooth
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True ).split(train, target):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, target)
 train = oof.sort_index()
test = ce_target_encoder.transform(test)
 features = list(train)
 print('Target encoding done!')
 return train, test, test_id, features, target<categorify>
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof.catboost_oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof.catboost_oof))
train, test, test_id, features, target = encoding(train, test, 0.3 )<normalization>
Xt_dnn = pd.read_csv("/kaggle/input/categorical-feature-encoding-with-tensorflow/dnn_cv_submission.csv")
 Xt_tab = pd.read_csv("/kaggle/input/tensorflow-tabnet/submission.csv")
 cv_submission.target = cv_submission.target * 0.5 + Xt_dnn.target * 0.35 + Xt_tab.target * 0.15
cv_submission.to_csv("./catboost_dnn_cv_submission.csv", index=False )