Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<set_options>
pip install --upgrade scikit-learn
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df<categorify>
print(sklearn.__version__ )
train_test=Convert_to_numeric(train_test)
 train_test = reduce_mem_usage(train_test,use_float16=True)
 X_train=train_test[:len(X_train)]
X_test=train_test[-len(X_test):] 
<categorify>
notebookstart = time.time()
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes'.format(name, round(( time.time() - t0)/60,2)) )
def preproc(X_train, X_val, X_test):
input_list_train = []
input_list_val = []
input_list_test = []
for c in X_train.select_dtypes(include=categoricals):
raw_vals = np.unique(X_train[c])
 val_map = {}
for i in range(len(raw_vals)) :
val_map[raw_vals[i]] = i
input_list_train.append(X_train[c].map(val_map ).values)
 input_list_val.append(X_val[c].map(val_map ).fillna(0 ).values)
 input_list_test.append(X_test[c].map(val_map ).fillna(0 ).values)
 other_cols = X_train.select_dtypes(include=numerics ).columns.tolist()
input_list_train.append(X_train[other_cols].values)
 input_list_val.append(X_val[other_cols].values)
 input_list_test.append(X_test[other_cols].values)
 return input_list_train, input_list_val, input_list_test<define_variables>
seed = 50
debug = False
if debug:
nrow = 5000
else:
nrow = None
def make_model() :
input_models=[]
output_embeddings=[]
numerics = ['float16', 'float32', 'float64']
categoricals=['int8','int16', 'int32', 'int64']
for categorical_var in X_train.select_dtypes(include=categoricals):
cat_emb_name= categorical_var.replace(" ", "")+'_Embedding'
no_of_unique_cat = X_train[categorical_var].nunique()
embedding_size = int(min(np.ceil(( no_of_unique_cat)/2), 50))
input_model = Input(shape=(1,))
output_model = Embedding(no_of_unique_cat, embedding_size, name=cat_emb_name )(input_model)
 output_model = SpatialDropout1D(0.3 )(output_model)
 output_model = Reshape(target_shape=(embedding_size,))(output_model)
 input_models.append(input_model)
 output_embeddings.append(output_model)
 shape_numeric=len(X_train.select_dtypes(include=numerics ).columns.tolist())
 input_numeric = Input(shape=(shape_numeric,))
embedding_numeric = BatchNormalization()(input_numeric)
 input_models.append(input_numeric)
 output_embeddings.append(embedding_numeric)
 output = Concatenate()(output_embeddings)
 output = BatchNormalization()(output)
 output = Dense(317,activation='relu',kernel_initializer="uniform" )(output)
 output= Dropout(0.3 )(output)
 output = BatchNormalization()(output)
 output = Dense(150,activation='relu',kernel_initializer="uniform" )(output)
 output= Dropout(0.2 )(output)
 output = BatchNormalization()(output)
 output = Dense(30,activation='relu' )(output)
 output= Dropout(0.1 )(output)
 output = BatchNormalization()(output)
 output = Dense(1, activation='sigmoid' )(output)
 model = Model(inputs=input_models, outputs=output)
 return model<choose_model_class>
with timer("Load"):
PATH = "/kaggle/input/cat-in-the-dat-ii/"
train = pd.read_csv(PATH + "train.csv", index_col = 'id', nrows = nrow)
 test = pd.read_csv(PATH + "test.csv", index_col = 'id')
 submission_df = pd.read_csv(PATH + "sample_submission.csv")
 [print(x.shape)for x in [train, test, submission_df]]
traindex = train.index
testdex = test.index
y = train.target.copy()
print("Target Distribution:
",y.value_counts(normalize = True ).to_dict())
 df = pd.concat([train.drop('target',axis = 1), test], axis = 0)
 del train, test, submission_df
cls =CatBoostClassifier(eval_metric='AUC',
loss_function='CrossEntropy',
learning_rate=0.05,
l2_leaf_reg=3)
 keras_oof = np.zeros(len(X_train))
cls_oof = np.zeros(len(X_train))
keras_preds = np.zeros(len(X_test))
cls_preds = np.zeros(len(X_test))
NFOLDS = 10
folds = StratifiedKFold(n_splits=NFOLDS, shuffle=False)
 training_start_time = time()
for fold,(train_idx, valid_idx)in enumerate(folds.split(X_train, y_train)) :
train_x, train_y = X_train.iloc[train_idx], y_train.iloc[train_idx]
valid_x, valid_y = X_train.iloc[valid_idx], y_train.iloc[valid_idx]
X_train_list,X_val_list,X_test_list = preproc(train_x,valid_x, X_test)
 start_time = time()
print(f'Training on fold {fold+1}')
 K.clear_session()
model = make_model()
adam = Adam(learning_rate=1e-4)
 model.compile(loss='binary_crossentropy', optimizer=adam ,metrics=[tf.keras.metrics.AUC() ])
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=6,
verbose=1, mode='min', baseline=None, restore_best_weights=True)
 rl = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,min_delta=0.001,
patience=2, min_lr=1e-6, mode='min', verbose=0)
 history = model.fit(X_train_list,train_y, validation_data=(X_val_list,valid_y), epochs = 20,
batch_size = 1024, callbacks=[es,rl],verbose= 0)
 keras_fold_val= model.predict(X_val_list ).ravel()
keras_oof[valid_idx] += keras_fold_val/folds.n_splits
keras_fold_auc=roc_auc_score(valid_y, keras_fold_val)
 keras_preds += model.predict(X_test_list ).ravel() /folds.n_splits
layer_name = 'batch_normalization_2'
intermediate_layer_model = Model(inputs=model.input,outputs=model.get_layer(layer_name ).output)
 X_train_k = pd.DataFrame(intermediate_layer_model.predict(X_train_list))
X_val_k = pd.DataFrame(intermediate_layer_model.predict(X_val_list))
X_test_k = pd.DataFrame(intermediate_layer_model.predict(X_test_list))
cls.fit(X_train_k, train_y, eval_set=(X_val_k, valid_y),early_stopping_rounds=50,verbose=100,plot=False)
 cls_fold_val=cls.predict_proba(X_val_k)[:,1]
cls_oof[valid_idx] += cls_fold_val/folds.n_splits
cls_fold_AUC=roc_auc_score(valid_y, cls_fold_val)
 cls_preds +=cls.predict_proba(X_test_k)[:,1]/folds.n_splits
print("
")
 print('-' * 30)
 print('Fold {} - Keras_OOF = {}'.format(fold + 1,keras_fold_auc))
print('Fold {} - CatBoost_OOF = {}'.format(fold + 1,cls_fold_AUC))
print('-' * 30)
 print("
")
 keras_val_auc=roc_auc_score(y_train, keras_oof)
 cls_val_auc=roc_auc_score(y_train, cls_oof)
 mix_val_auc=roc_auc_score(y_train,(keras_oof+cls_oof)/2)
 print('-' * 30)
 print('
print('
print('
print('-' * 30)
 print("
")
<save_to_csv>
with timer("FE 1"):
drop_cols=["bin_0"]
df["ord_5a"]=df["ord_5"].str[0]
df["ord_5b"]=df["ord_5"].str[1]
drop_cols.append("ord_5")
 xor_cols = []
nan_cols = []
for col in df.columns:
tmp_null = df.loc[:,col].isnull().sum()
if tmp_null > 0:
print("{} has {} missing values.. Filling".format(col, tmp_null))
nan_cols.append(col)
 if df.loc[:,col].dtype == "O":
df.loc[:,col].fillna("NAN", inplace=True)
 else:
df.loc[:,col].fillna(-1, inplace=True)
 train_vals = set(df.loc[traindex, col].unique())
 test_vals = set(df.loc[testdex, col].unique())
 xor_cat_vals=train_vals ^ test_vals
if xor_cat_vals:
df.loc[df[col].isin(xor_cat_vals), col]="xor"
print("{} has {} xor factors, {} rows".format(col, len(xor_cat_vals),df.loc[df[col] == 'xor',col].shape[0]))
xor_cols.append(col)
 ordinal_cols=['ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5a', 'day', 'month']
X_oh=df[df.columns.difference(ordinal_cols)]
oh1=pd.get_dummies(X_oh, columns=X_oh.columns, drop_first=True, sparse=True)
 ohc1=oh1.sparse.to_coo()
predictions=(keras_preds+cls_preds)/2
num = samp_subm.id
output = pd.DataFrame({'id': num,
'target': predictions})
 output.to_csv('keras-Embedding-catboost.csv', index=False)
 output.head()<import_modules>
class ThermometerEncoder(TransformerMixin):

 def __init__(self, sort_key=None):
self.sort_key = sort_key
self.value_map_ = None
def fit(self, X, y=None):
self.value_map_ = {val: i for i, val in enumerate(sorted(X.unique() , key=self.sort_key)) }
return self
def transform(self, X, y=None):
values = X.map(self.value_map_)
 possible_values = sorted(self.value_map_.values())
 idx1 = []
idx2 = []
all_indices = np.arange(len(X))
for idx, val in enumerate(possible_values[:-1]):
new_idxs = all_indices[values > val]
idx1.extend(new_idxs)
 idx2.extend(repeat(idx, len(new_idxs)))
 result = scipy.sparse.coo_matrix(( [1] * len(idx1),(idx1, idx2)) , shape=(len(X), len(possible_values)) , dtype="int8")
 return result
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from tensorflow.keras import layers
from tensorflow.keras import Sequential
from tensorflow.keras import Model
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
import os
import gc
import joblib
from sklearn.model_selection import StratifiedKFold
from sklearn import metrics
from tensorflow.keras import optimizers
from tensorflow.keras.models import Model, load_model
from tensorflow.keras import callbacks
from tensorflow.keras import backend as K
from tensorflow.keras import utils
import tensorflow as tf<load_from_csv>
other_classes = ["NAN", 'xor']
with timer("Thermometer Encoder"):
thermos=[]
for col in ordinal_cols:
if col=="ord_1":
sort_key=(other_classes + ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'] ).index
elif col=="ord_2":
sort_key=(other_classes + ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot'] ).index
elif col in ["ord_3", "ord_4", "ord_5a"]:
sort_key=str
elif col in ["day", "month"]:
sort_key=int
else:
raise ValueError(col)
 enc=ThermometerEncoder(sort_key=sort_key)
 thermos.append(enc.fit_transform(df[col]))
train=pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test=pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )<train_model>
LogisticRegression().get_params()
print("training data size: ", train.shape)
 print("training data size: ", test.shape )<choose_model_class>
scoring = "roc_auc"
model_names = ['stack',
'logistic1',
'logistic2'
]
n_models = len(model_names)
 Logistic_params_1 = {
"C": 0.123,
"penalty":"l2",
"solver": "lbfgs",
"max_iter": 5000
}
Logistic_params_2 = {
"C": 50,
'penalty':"l2",
"solver": "lbfgs",
"max_iter": 5000
}
folds = KFold(n_splits=3, shuffle=True, random_state=1)
 fold_preds = np.zeros([test.shape[0],n_models])
 oof_preds = np.zeros([X.shape[0],n_models])
 results = {}
with timer("Fit Model"):
estimators = [ 
('logistic_regression_2', LogisticRegression(**Logistic_params_1)) , 
('logistic_regression_1', LogisticRegression(**Logistic_params_2))
]
f, ax = plt.subplots(1,3,figsize = [14,5])
 for i,(trn_idx, val_idx)in enumerate(folds.split(X)) :
clf = StackingClassifier(
 estimators=estimators,
final_estimator=LogisticRegression() , 
)
 with timer("Fold {}".format(str(i))):
clf.fit(X[trn_idx], y.loc[trn_idx])
 tmp_pred = clf.predict_proba(X[val_idx])[:,1]
oof_preds[val_idx,0] = tmp_pred
fold_preds[:,0] += clf.predict_proba(test)[:,1] / folds.n_splits
estimator_performance = {}
estimator_performance['stack_score'] = metrics.roc_auc_score(y.loc[val_idx], tmp_pred)
 for ii, est in enumerate(estimators):
model = clf.named_estimators_[est[0]]
plot_roc_curve(model, X[val_idx], y.loc[val_idx], ax=ax[i])
 pred = model.predict_proba(X[val_idx])[:,1]
oof_preds[val_idx, ii+1] = pred
fold_preds[:,ii+1] += model.predict_proba(test)[:,1] / folds.n_splits
estimator_performance[est[0]+"_score"] = metrics.roc_auc_score(y.loc[val_idx], pred)
 stack_coefficients = {x+"_coefficient":y for(x,y)in zip([x[0] for x in estimators], clf.final_estimator_.coef_[0])}
stack_coefficients['intercept'] = clf.final_estimator_.intercept_[0]
results["Fold {}".format(str(i+1)) ] = [
estimator_performance,
{est[0]+"_iterations":clf.named_estimators_[est[0]].n_iter_ for est in estimators},
stack_coefficients
]
plot_roc_curve(clf, X[val_idx], y.loc[val_idx], ax=ax[i])
 ax[i].plot([0.0, 1.0])
 ax[i].set_title("Fold {} - ROC AUC".format(str(i + 1)))
 plt.tight_layout(pad=2)
 plt.show()
f, ax = plt.subplots(1,2,figsize = [11,5])
 sns.heatmap(pd.DataFrame(oof_preds, columns = model_names ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="magma",ax=ax[0])
 ax[0].set_title("OOF PRED - Correlation Plot")
 sns.heatmap(pd.DataFrame(fold_preds, columns = model_names ).corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Correlation Coefficient'},cmap="inferno",ax=ax[1])
 ax[1].set_title("TEST PRED - Correlation Plot")
 plt.tight_layout(pad=3)
 plt.show()
def create_model(data, features):
inputs=[]
output=[]
for c in features:
num_unique_values = int(data[c].nunique())
 embed_dim = int(min(np.ceil(( num_unique_values)/2), 50))
inp = layers.Input(shape=(1,))
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c )(inp)
 out = layers.SpatialDropout1D(0.3 )(out)
 out = layers.Reshape(target_shape=(embed_dim,))(out)
 inputs.append(inp)
 output.append(out)
 x=layers.Concatenate()(output)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300, activation="relu" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 y = layers.Dense(2, activation="softmax" )(x)
 model = Model(inputs=inputs, outputs=y)
 return model<compute_test_metric>
results_pd = pd.DataFrame(results ).T.reset_index()
results_pd.columns = ['Fold','Score','EarlyStopping', 'Coefficients']
results_pd = pd.concat(
 [pd.io.json.json_normalize(results_pd['Score'] ).reset_index(drop=True),
pd.io.json.json_normalize(results_pd['EarlyStopping'] ).reset_index(drop=True),
pd.io.json.json_normalize(results_pd['Coefficients'] ).reset_index(drop=True),
results_pd.reset_index(drop=True)
 ], axis = 1)
 display(results_pd )
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<categorify>
with timer("Submission"):
pd.DataFrame({'id': testdex, 'target': fold_preds[:,0]} ).to_csv('logistic_stacked_oof_submission.csv', index=False)
 pd.DataFrame({'id': testdex, 'target': fold_preds[:,1]} ).to_csv(estimators[0][0] + '_oof_submission.csv', index=False)
 pd.DataFrame({'id': testdex, 'target': fold_preds[:,2]} ).to_csv(estimators[1][0] + '_oof_submission.csv', index=False )
test["target"] = -1
data = pd.concat([train, test] ).reset_index(drop=True)
 features = [x for x in train.columns if x not in ["id", "target"]]
for feat in features:
lbl_enc = LabelEncoder()
data[feat] = lbl_enc.fit_transform(data[feat].fillna("-1" ).astype(str ).values)
 train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True)
 test_data = [test.loc[:, features].values[:, k] for k in range(test.loc[:, features].values.shape[1])]
oof_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test)))
 skf = StratifiedKFold(n_splits=50)
 for train_index, test_index in skf.split(train, train.target.values):
X_train, X_test = train.iloc[train_index, :], train.iloc[test_index, :]
X_train = X_train.reset_index(drop=True)
 X_test = X_test.reset_index(drop=True)
 y_train, y_test = X_train.target.values, X_test.target.values
model = create_model(data, features)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])
 X_train = [X_train.loc[:, features].values[:, k] for k in range(X_train.loc[:, features].values.shape[1])]
X_test = [X_test.loc[:, features].values[:, k] for k in range(X_test.loc[:, features].values.shape[1])]
es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,
verbose=1, mode='max', baseline=None, restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5,
patience=3, min_lr=1e-6, mode='max', verbose=1)
 model.fit(X_train,
utils.to_categorical(y_train),
validation_data=(X_test, utils.to_categorical(y_test)) ,
verbose=1,
batch_size=1024,
callbacks=[es, rlr],
epochs=100 
)
 valid_fold_preds = model.predict(X_test)[:, 1]
test_fold_preds = model.predict(test_data)[:, 1]
oof_preds[test_index] = valid_fold_preds.ravel()
test_preds += test_fold_preds.ravel()
print(metrics.roc_auc_score(y_test, valid_fold_preds))
K.clear_session()<save_model>
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))