Kaggle competition: Categorical Feature Encoding Challenge II
random_seed = 121
X_train, X_val, Y_train, Y_val = train_test_split(x_train, y_train, test_size = 0.15, random_state=random_seed)
<choose_model_class>
import numpy as np
import pandas as pd
from time import time
import pprint
import joblib
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, average_precision_score
from sklearn.metrics import make_scorer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import RobustScaler, StandardScaler
import matplotlib.pyplot as plt
import seaborn as sns
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(GlobalAveragePooling2D())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.2))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
import tensorflow as tf
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam, Nadam
from keras.layers import Input, Embedding, Reshape, GlobalAveragePooling1D
from keras.layers import Flatten, concatenate, Concatenate, Lambda, Dropout, SpatialDropout1D
from keras.layers import Reshape, MaxPooling1D,BatchNormalization, AveragePooling1D, Conv1D
from keras.layers import Activation, LeakyReLU
from keras.optimizers import SGD, Adam, Nadam
from keras.models import Model, load_model
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.regularizers import l2, l1_l2
from keras.losses import binary_crossentropy
from keras.utils import get_custom_objects
from keras.layers import Activation, LeakyReLU
from keras.models import load_model
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
X = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/train.csv")
 Xt = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/test.csv" )
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
y = X.target.values
id_train = X.id
id_test = Xt.id
X.drop(['id', 'target'], axis=1, inplace=True)
 Xt.drop(['id'], axis=1, inplace=True )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 50
batch_size = 86<define_variables>
binary_vars = [c for c in X.columns if 'bin_' in c]
nominal_vars = [c for c in X.columns if 'nom_' in c]
high_cardinality = [c for c in nominal_vars if len(X[c].unique())> 16]
low_cardinality = [c for c in nominal_vars if len(X[c].unique())<= 16]
ordinal_vars = [c for c in X.columns if 'ord_' in c]
time_vars = ['day', 'month']
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
X['ord_5_1'] = X['ord_5'].apply(lambda x: x[0] if type(x)== str else np.nan)
 X['ord_5_2'] = X['ord_5'].apply(lambda x: x[1] if type(x)== str else np.nan)
 Xt['ord_5_1'] = Xt['ord_5'].apply(lambda x: x[0] if type(x)== str else np.nan)
 Xt['ord_5_2'] = Xt['ord_5'].apply(lambda x: x[1] if type(x)== str else np.nan)
 ordinal_vars += ['ord_5_1', 'ord_5_2']
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
ordinals = {
'ord_1' : {
'Novice' : 0,
'Contributor' : 1,
'Expert' : 2,
'Master' : 3,
'Grandmaster' : 4
},
'ord_2' : {
'Freezing' : 0,
'Cold' : 1,
'Warm' : 2,
'Hot' : 3,
'Boiling Hot' : 4,
'Lava Hot' : 5
}
}
def return_order(X, Xt, var_name):
mode = X[var_name].mode() [0]
el = sorted(set(X[var_name].fillna(mode ).unique())|set(Xt[var_name].fillna(mode ).unique()))
return {v:e for e, v in enumerate(el)}
for mapped_var in ordinal_vars:
if mapped_var not in ordinals:
mapped_values = return_order(X, Xt, mapped_var)
 X[mapped_var].replace(mapped_values, inplace=True)
 Xt[mapped_var].replace(mapped_values, inplace=True)
 else:
X[mapped_var].replace(ordinals[mapped_var], inplace=True)
 Xt[mapped_var].replace(ordinals[mapped_var], inplace=True )
results = model.predict(test_df)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
X_cat, Xt_cat = list() , list()
categorical_counts = dict()
for hc in binary_vars+nominal_vars+ordinal_vars+time_vars:
levels = set(X[hc].astype(str ).fillna("NAN" ).unique())|set(Xt[hc].astype(str ).fillna("NAN" ).unique())
 levels = np.array(list(levels))
categorical_counts[hc] = len(levels)
 le = LabelEncoder()
le.fit(np.ravel(levels.reshape(-1,1)))
 X_cat.append(le.transform(X[hc].astype(str ).fillna("NAN" ).values))
Xt_cat.append(le.transform(Xt[hc].astype(str ).fillna("NAN" ).values))
print("Countings for high cardinality variables:")
 print(categorical_counts )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<set_options>
def frequency_encoding(column, df, df_test=None):
frequencies = df[column].value_counts().reset_index()
df_values = df[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).iloc[:,-1].values
if df_test is not None:
df_test_values = df_test[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).fillna(1 ).iloc[:,-1].values
else:
df_test_values = None
return df_values, df_test_values
freq_encoded = list()
for column in X.columns:
train_values, test_values = frequency_encoding(column, X, Xt)
 X[column+'_counts'] = train_values
Xt[column+'_counts'] = test_values
freq_encoded.append(column+'_counts' )
sns.set() 
<load_from_csv>
cat_feat_to_encode = binary_vars + ordinal_vars + nominal_vars + time_vars
smoothing = 0.3
enc_x = np.zeros(X[cat_feat_to_encode].shape)
 for tr_idx, oof_idx in StratifiedKFold(n_splits=3, random_state=42, shuffle=True ).split(X, y):
encoder = cat_encs.TargetEncoder(cols=cat_feat_to_encode, smoothing=smoothing)
 encoder.fit(X[cat_feat_to_encode].iloc[tr_idx], y[tr_idx])
 enc_x[oof_idx, :] = encoder.transform(X[cat_feat_to_encode].iloc[oof_idx], y[oof_idx])
 encoder.fit(X[cat_feat_to_encode], y)
 enc_xt = encoder.transform(Xt[cat_feat_to_encode] ).values
target_encoded = list()
for idx, new_var in enumerate(cat_feat_to_encode):
new_var = new_var + '_enc'
X[new_var] = enc_x[:,idx]
Xt[new_var] = enc_xt[:, idx]
target_encoded.append(new_var )
digits_PATH = '/kaggle/input/digit-recognizer/'
digits_train = pd.read_csv(digits_PATH+'train.csv' )<prepare_x_and_y>
ssc = StandardScaler()
selection = freq_encoded + target_encoded + ordinal_vars + time_vars
X_ohe = ssc.fit_transform(X[selection].fillna(X[selection].median()))
Xt_ohe = ssc.transform(Xt[selection].fillna(X[selection].median()))
X, y = digits_train.iloc[:,1:].values/255, digits_train.iloc[:,0].values
X = X.reshape(-1,28, 28, 1 )<split>
def gelu(x):
return 0.5 * x *(1 + tf.tanh(tf.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3))))
get_custom_objects().update({'gelu': Activation(gelu)})
 get_custom_objects().update({'leaky-relu': Activation(LeakyReLU(alpha=0.2)) } )
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9, random_state=0 )<train_on_grid>
def tabular_dnn(numeric_variables, categorical_variables, categorical_counts,
feature_selection_dropout=0.2, categorical_dropout=0.1,
first_dense = 256, second_dense = 256, dense_dropout = 0.2,
activation_type=gelu):
numerical_inputs = Input(shape=(numeric_variables,))
numerical_normalization = BatchNormalization()(numerical_inputs)
 numerical_feature_selection = Dropout(feature_selection_dropout )(numerical_normalization)
 categorical_inputs = []
categorical_embeddings = []
for category in categorical_variables:
categorical_inputs.append(Input(shape=[1], name=category))
category_counts = categorical_counts[category]
categorical_embeddings.append(
 Embedding(category_counts+1,
min(int(category_counts/1.5 + 1), 3),
name = category + "_embed" )(categorical_inputs[-1]))
categorical_logits = Concatenate(name = "categorical_conc" )([Flatten()(SpatialDropout1D(categorical_dropout )(cat_emb))
for cat_emb in categorical_embeddings])
 x = concatenate([numerical_feature_selection, categorical_logits])
 x = BatchNormalization()(x)
 x = Dense(first_dense, activation=activation_type )(x)
 x = BatchNormalization()(x)
 x = Dropout(dense_dropout )(x)
 x = Dense(second_dense, activation=activation_type )(x)
 x = Dropout(dense_dropout )(x)
 x = BatchNormalization()(x)
 output = Dense(1, activation="sigmoid" )(x)
 model = Model([numerical_inputs] + categorical_inputs, output)
 return model
class MultiCNN() :
def __init__(self, model_generator, num_models=1):
self.models = []
self.create_models(model_generator, num_models)
 def create_models(self, model_generator, num_models=1):
for i in range(0,num_models):
m = keras.models.Sequential(model_generator())
 m.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 self.models.append(m)
 def fit(self, train_inputs, train_targets, early_stop=True, verbose=2, batch_size=200, epochs=100):
callback = []
if early_stop:
callback.append(keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-6, patience=5))
for m in self.models:
X_train2, X_val2, y_train2, y_val2 = train_test_split(train_inputs, train_targets, test_size = 0.2)
 m.fit(X_train2, y_train2,
batch_size= batch_size,
epochs = epochs,
callbacks=callback,
validation_data =(X_val2, y_val2),
verbose=verbose 
)
 def fit_generator(self, generator, train_inputs, train_targets, early_stop=True, verbose=2, batch_size=32, epochs=100):
callback = []
if early_stop:
callback.append(keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-6, patience=5))
for m in self.models:
X_train2, X_val2, y_train2, y_val2 = train_test_split(train_inputs, train_targets, test_size = 0.2)
 m.fit_generator(generator.flow(X_train2,y_train2, batch_size=batch_size),
epochs = epochs,
callbacks=callback,
validation_data =(X_val2, y_val2),
verbose=verbose 
)
 def predict(self, test_inputs):
prediction = [model.predict(test_inputs)for model in self.models]
return sum(prediction)/len(prediction)
 def acuracia(self, test_input, test_target):
prediction = self.predict(test_input)
 prediction = np.argmax(prediction, axis=1)
 return accuracy_score(prediction, test_target)
 def confusion_matrix(self, test_input, test_target):
y_pred = self.predict(test_input)
 y_pred = np.argmax(y_pred, axis=1)
 confusao = confusion_matrix(y_pred, test_target)
 fig, ax = plt.subplots(1, figsize=(10,10))
sns.heatmap(pd.DataFrame(confusao), ax=ax, cbar=False, annot=True)
 ax.set_title('Matriz de confusão', size=20)
 ax.set_yticklabels(ax.get_xticklabels() , rotation=0, size=15)
 ax.set_xticklabels(ax.get_yticklabels() , rotation=0, size=15)
 plt.show()
def acuracia_individual(self, test_input, test_target):
return [accuracy_score(np.argmax(m.predict(X_test), axis=1), y_test)for m in self.models]<choose_model_class>
def auroc(y_true, y_pred):
try:
return tf.py_function(roc_auc_score,(y_true, y_pred), tf.double)
 except:
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double)
 get_custom_objects().update({'auroc': auroc})
 def mAP(y_true, y_pred):
try:
return tf.py_function(average_precision_score,(y_true, y_pred), tf.double)
 except:
return tf.py_func(average_precision_score,(y_true, y_pred), tf.double)
 get_custom_objects().update({'mAP': mAP})
 def compile_model(model, loss, metrics, optimizer):
model.compile(loss=loss, metrics=metrics, optimizer=optimizer)
 return model
def plot_keras_history(history, measures):

 rows = len(measures)// 2 + len(measures)% 2
fig, panels = plt.subplots(rows, 2, figsize=(15, 5))
plt.subplots_adjust(top = 0.99, bottom=0.01, hspace=0.4, wspace=0.2)
 try:
panels = [item for sublist in panels for item in sublist]
except:
pass
for k, measure in enumerate(measures):
panel = panels[k]
panel.set_title(measure + ' history')
 panel.plot(history.epoch, history.history[measure], label="Train "+measure)
 panel.plot(history.epoch, history.history["val_"+measure], label="Validation "+measure)
 panel.set(xlabel='epochs', ylabel=measure)
 panel.legend()
plt.show(fig )
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_default = MultiCNN(make_CNN, num_models=15)
 multiCNN_default.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_default.acuracia(X_test, y_test )<choose_model_class>
SEED = 42
FOLDS = 20
MAX_EPOCHS = 100
BATCH_SIZE = 1024 * 4
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_plus_layer = MultiCNN(make_CNN, num_models=15)
 multiCNN_plus_layer.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_plus_layer.acuracia(X_test, y_test )<choose_model_class>
measure_to_monitor = 'val_auroc'
modality = 'max'
early_stopping = EarlyStopping(monitor=measure_to_monitor,
mode=modality,
patience=5,
verbose=0)
 model_checkpoint = ModelCheckpoint('best.model',
monitor=measure_to_monitor,
mode=modality,
save_best_only=True,
verbose=0)
 model_reduce_lr = ReduceLROnPlateau(monitor=measure_to_monitor,
mode=modality,
factor=0.25,
patience=2,
min_lr=1e-6,
verbose=1 )
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.AveragePooling2D(pool_size =(2,2), strides = 2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.AveragePooling2D(pool_size =(2,2), strides = 2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_pooling = MultiCNN(make_CNN, num_models=15)
 multiCNN_pooling.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_pooling.acuracia(X_test, y_test )<choose_model_class>
model_params = {
"numeric_variables" : X_ohe.shape[1],
"categorical_variables" : categorical_counts.keys() ,
"categorical_counts" : categorical_counts,
"feature_selection_dropout" : 0.0,
"categorical_dropout" : 0.3,
"first_dense" : 512,
"second_dense" : 512,
"dense_dropout" : 0.3,
"activation_type" : 'relu'
}
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_batch_normalization = MultiCNN(make_CNN, num_models=15)
 multiCNN_batch_normalization.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_batch_normalization.acuracia(X_test, y_test )<choose_model_class>
skf = StratifiedKFold(n_splits=FOLDS,
shuffle=True,
random_state=SEED)
 roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
best_iteration = list()
cv_test_preds = np.zeros(len(Xt))
for fold,(train_idx, test_idx)in enumerate(skf.split(X, y)) :
model = compile_model(tabular_dnn(**model_params),
binary_crossentropy,
[auroc, mAP],
Adam(learning_rate=0.0001))
X_cv_ohe = X_ohe[train_idx].astype(np.float32)
 X_cv_cat = [X_cat[i][train_idx] for i in range(len(X_cat)) ]
y_cv = y[train_idx]
X_oof_ohe = X_ohe[test_idx].astype(np.float32)
 X_oof_cat = [X_cat[i][test_idx] for i in range(len(X_cat)) ]
y_oof = y[test_idx]
train_batch = batch_generator(X_cv_ohe,
X_cv_cat,
y_cv,
batch_size=BATCH_SIZE,
random_state=SEED)
 val_batch = batch_generator(X_oof_ohe,
X_oof_cat,
y_oof,
batch_size=BATCH_SIZE,
random_state=SEED)
 train_steps = len(y_cv)// BATCH_SIZE
validation_steps = len(y_oof)// BATCH_SIZE
history = model.fit_generator(train_batch,
validation_data=val_batch,
epochs=MAX_EPOCHS,
steps_per_epoch=train_steps,
validation_steps=validation_steps,
callbacks=[model_checkpoint, early_stopping, model_reduce_lr],
verbose=1)
 print("
FOLD %i" % fold)
 plot_keras_history(history, measures=['auroc', 'loss'])
 best_iteration.append(np.argmax(history.history['val_auroc'])+ 1)
 model = load_model('best.model')
 preds = model.predict([X_oof_ohe] + X_oof_cat,
verbose=1,
batch_size=1024 ).flatten()
oof[test_idx] = preds
roc_auc.append(roc_auc_score(y_true=y_oof, y_score=preds))
average_precision.append(average_precision_score(y_true=y_oof, y_score=preds))
cv_test_preds += model.predict([Xt_ohe] + Xt_cat,
verbose=1,
batch_size=1024 ).flatten() / FOLDS
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=64, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_dropout = MultiCNN(make_CNN, num_models=15)
 multiCNN_dropout.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_dropout.acuracia(X_test, y_test )<choose_model_class>
oof = pd.DataFrame({'id':id_train, 'dnn_oof': oof})
 oof.to_csv("oof.csv", index=False)
 submission = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/sample_submission.csv")
 submission.target = cv_test_preds
submission.to_csv("./dnn_cv_submission.csv", index=False )
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_200 = MultiCNN(make_CNN, num_models=15)
 multiCNN_200.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_200.acuracia(X_test, y_test )<choose_model_class>
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof.dnn_oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof.dnn_oof))
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_bigger_kernel = MultiCNN(make_CNN, num_models=15)
 multiCNN_bigger_kernel.fit(X_train, y_train, verbose = 0, early_stop=True)
 multiCNN_bigger_kernel.acuracia(X_test, y_test )<choose_model_class>
train_batch = batch_generator(X_ohe, X_cat, y,
batch_size=BATCH_SIZE,
random_state=SEED)
 train_steps = len(y)// BATCH_SIZE
model = compile_model(tabular_dnn(**model_params),
binary_crossentropy,
[auroc, mAP],
Adam(learning_rate=0.0001))
history = model.fit_generator(train_batch,
epochs=int(np.median(best_iteration)) ,
steps_per_epoch=train_steps,
verbose=1 )
def make_CNN() :
return [
keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation="relu", input_shape=(28, 28, 1)) ,
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation="relu"),
keras.layers.MaxPooling2D(pool_size =(2,2), strides = 2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Flatten() ,
keras.layers.Dense(units=200, activation="relu"),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=10, activation="softmax")
 ]
multiCNN_min_epochs = MultiCNN(make_CNN, num_models=15)
 multiCNN_min_epochs.fit(X_train, y_train, verbose = 0, early_stop=True, epochs=10)
 multiCNN_min_epochs.acuracia(X_test, y_test )<define_search_model>
test_preds = model.predict([Xt_ohe] + Xt_cat,
verbose=1,
batch_size=1024 ).flatten()
submission = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/sample_submission.csv")
 submission.target = test_preds
submission.to_csv("./submission.csv", index=False )
datagen = keras.preprocessing.image.ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
zca_epsilon=1e-06,
rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
brightness_range=None,
zoom_range=0.1,
fill_mode="nearest",
cval=0.0,
horizontal_flip=False,
vertical_flip=False,
rescale=None,
preprocessing_function=None,
data_format=None,
validation_split=0.0,
dtype=None, 
)
 datagen.fit(X_train )<choose_model_class>
submission.target =(submission.target + pd.read_csv("./dnn_cv_submission.csv" ).target)/ 2
submission.to_csv("./blend_submission.csv", index=False )