Kaggle competition: Amazon.com - Employee Access Challenge
<SOS> metric: AUC Kaggle data source: amazon.com-employee-access-challenge<concatenate>
!pip install catboost -U
v_model = pd.concat([passengerID, y_pred_v], axis=1 )<save_to_csv>
!git clone https://github.com/lmassaron/deep_learning_for_tabular_data.git
v_model.to_csv('voting.csv', index= False )<load_from_csv>
warnings.filterwarnings("ignore")
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
X, Xt = amazon()
y = X["ACTION"].apply(lambda x: 1 if x == 1 else 0 ).values
X.drop(["ACTION"], axis=1, inplace=True )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
label_encoders = [LabelEncoder() for _ in range(X.shape[1])]
for col, column in enumerate(X.columns):
label_encoders[col].fit(X[column].append(Xt[column]))
X[column] = label_encoders[col].transform(X[column])
 Xt[column] = label_encoders[col].transform(Xt[column] )
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1 )<feature_engineering>
def frequency_encoding(column, df, df_test=None):
frequencies = df[column].value_counts().reset_index()
df_values = df[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).iloc[:,-1].values
if df_test is not None:
df_test_values = df_test[[column]].merge(frequencies, how='left',
left_on=column, right_on='index' ).fillna(1 ).iloc[:,-1].values
else:
df_test_values = None
return df_values, df_test_values
for column in X.columns:
train_values, test_values = frequency_encoding(column, X, Xt)
 X[column+'_counts'] = train_values
Xt[column+'_counts'] = test_values
X_train = X_train / 255.0
test = test / 255<split>
categorical_variables = [col for col in X.columns if '_counts' not in col]
numeric_variables = [col for col in X.columns if '_counts' in col]
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=2 )<choose_model_class>
X[categorical_variables].nunique()
model = keras.Sequential([
keras.layers.Flatten(input_shape=(28, 28, 1)) ,
keras.layers.Dense(128, activation=tf.nn.relu),
keras.layers.Dense(10, activation=tf.nn.softmax)
 ] )<choose_model_class>
best_params = {'bagging_temperature': 0.6,
'border_count': 200,
'depth': 8,
'iterations': 350,
'l2_leaf_reg': 30,
'learning_rate': 0.30,
'random_strength': 0.01,
'scale_pos_weight': 0.48}
catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=2,
verbose = False )
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] )<train_model>
SEED = 42
FOLDS = 5
skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED )
history = model.fit(X_train, Y_train, epochs=5 )<prepare_output>
roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
best_iteration = list()
for train_idx, test_idx in skf.split(X, y):
X_train, y_train = X.iloc[train_idx, :], y[train_idx]
X_test, y_test = X.iloc[test_idx, :], y[test_idx]
train = Pool(data=X_train,
label=y_train,
feature_names=list(X_train.columns),
cat_features=categorical_variables)
 test = Pool(data=X_test,
label=y_test,
feature_names=list(X_test.columns),
cat_features=categorical_variables)
 catb.fit(train,
verbose_eval=100,
early_stopping_rounds=50,
eval_set=test,
use_best_model=True,
plot=False)
 best_iteration.append(catb.best_iteration_)
 preds = catb.predict_proba(X_test)
 oof[test_idx] = preds[:,1]
roc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))
average_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))
hist = pd.DataFrame(history.history)
 hist['epoch'] = history.epoch
hist.tail()<compute_test_metric>
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof))
test_loss, test_acc = model.evaluate(X_val, Y_val)
 print('Test accuracy:', test_acc )<choose_model_class>
best_params = {'bagging_temperature': 0.6,
'border_count': 200,
'depth': 8,
'iterations': int(np.median(best_iteration)* 1.3),
'l2_leaf_reg': 30,
'learning_rate': 0.30,
'random_strength': 0.01,
'scale_pos_weight': 0.48}
catb = CatBoostClassifier(**best_params,
loss_function='Logloss',
eval_metric = 'AUC',
nan_mode='Min',
thread_count=2,
verbose = False)
 train = Pool(data=X,
label=y,
feature_names=list(X_train.columns),
cat_features=categorical_variables)
 catb.fit(train,
verbose_eval=100,
plot=False)
 submission = pd.DataFrame(Xt.id)
 Xt_pool = Pool(data=Xt[list(X_train.columns)],
feature_names=list(X_train.columns),
cat_features=categorical_variables)
 submission['Action'] = catb.predict_proba(Xt_pool)[:,1]
submission.to_csv("catboost_submission.csv", index=False)
 cat_boost_submission = submission.copy()
model = keras.Sequential([
tf.keras.layers.Conv2D(32,(3,3), padding='same', activation=tf.nn.relu,
input_shape=(28, 28, 1)) ,
tf.keras.layers.Conv2D(32,(3,3), padding='same', activation=tf.nn.relu),
tf.keras.layers.MaxPooling2D(( 2, 2), strides=2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(64,(3,3), padding='same', activation=tf.nn.relu),
tf.keras.layers.Conv2D(64,(3,3), padding='same', activation=tf.nn.relu),
tf.keras.layers.MaxPooling2D(( 2, 2), strides=2),
tf.keras.layers.Dropout(0.25),
keras.layers.Flatten() ,
keras.layers.Dense(256, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.50),
keras.layers.Dense(10, activation=tf.nn.softmax)
 ] )<choose_model_class>
import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam, Nadam
from tensorflow.keras.layers import Input, Embedding, Reshape, GlobalAveragePooling1D
from tensorflow.keras.layers import Flatten, concatenate, Concatenate, Lambda, Dropout, SpatialDropout1D
from tensorflow.keras.layers import Reshape, MaxPooling1D,BatchNormalization, AveragePooling1D, Conv1D
from tensorflow.keras.layers import Activation, LeakyReLU
from tensorflow.keras.optimizers import SGD, Adam, Nadam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from tensorflow.keras.regularizers import l2, l1_l2
from keras.losses import binary_crossentropy
from sklearn.metrics import roc_auc_score
from sklearn.metrics import average_precision_score
import matplotlib.pyplot as plt
model.compile(optimizer = 'adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] )<train_model>
get_custom_objects().update({'gelu': Activation(gelu)})
 get_custom_objects().update({'mish': Mish(mish)})
 get_custom_objects().update({'leaky-relu': Activation(LeakyReLU(alpha=0.2)) } )
history = model.fit(X_train, Y_train, epochs=30 )<compute_test_metric>
def tabular_dnn(numeric_variables, categorical_variables, categorical_counts,
feature_selection_dropout=0.2, categorical_dropout=0.1,
first_dense = 256, second_dense = 256, dense_dropout = 0.2,
activation_type=gelu):
numerical_inputs = Input(shape=(len(numeric_variables),))
numerical_normalization = BatchNormalization()(numerical_inputs)
 numerical_feature_selection = Dropout(feature_selection_dropout )(numerical_normalization)
 categorical_inputs = []
categorical_embeddings = []
for category in categorical_variables:
categorical_inputs.append(Input(shape=[1], name=category))
category_counts = categorical_counts[category]
categorical_embeddings.append(
 Embedding(category_counts+1,
int(np.log1p(category_counts)+1),
name = category + "_embed" )(categorical_inputs[-1]))
categorical_logits = Concatenate(name = "categorical_conc" )([Flatten()(SpatialDropout1D(categorical_dropout )(cat_emb))
for cat_emb in categorical_embeddings])
 x = concatenate([numerical_feature_selection, categorical_logits])
 x = Dense(first_dense, activation=activation_type )(x)
 x = Dropout(dense_dropout )(x)
 x = Dense(second_dense, activation=activation_type )(x)
 x = Dropout(dense_dropout )(x)
 output = Dense(1, activation="sigmoid" )(x)
 model = Model([numerical_inputs] + categorical_inputs, output)
 return model
test_loss, test_acc = model.evaluate(X_val, Y_val)
 print('Test accuracy:', test_acc )<predict_on_test>
SEED = 42
FOLDS = 5
BATCH_SIZE = 512
predictions = model.predict(test )<prepare_output>
measure_to_monitor = 'val_auc'
modality = 'max'
early_stopping = EarlyStopping(monitor=measure_to_monitor,
mode=modality,
patience=5,
verbose=0)
 model_checkpoint = ModelCheckpoint('best.model',
monitor=measure_to_monitor,
mode=modality,
save_best_only=True,
verbose=0)
 reduce_learning = ReduceLROnPlateau(monitor=measure_to_monitor, mode=modality,
factor=0.25, patience=2, min_lr=1e-6, verbose=0 )
np.argmax(predictions[0] )<prepare_output>
skf = StratifiedKFold(n_splits=FOLDS,
shuffle=True,
random_state=SEED)
 roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
best_iteration = list()
for fold,(train_idx, test_idx)in enumerate(skf.split(X, y)) :
tb = TabularTransformer(numeric = numeric_variables,
ordinal = [],
lowcat = [],
highcat = categorical_variables)
 tb.fit(X.iloc[train_idx])
 sizes = tb.shape(X.iloc[train_idx])
 categorical_levels = dict(zip(categorical_variables, sizes[1:]))
print(f"Input array sizes: {sizes}")
 print(f"Categorical levels: {categorical_levels}
")
 model = tabular_dnn(numeric_variables, categorical_variables,
categorical_levels,
feature_selection_dropout=0.1,
categorical_dropout=0.1,
first_dense = 256,
second_dense = 256,
dense_dropout = 0.1,
activation_type=gelu)
 model = compile_model(model, binary_crossentropy, [AUC(name='auc'), mAP], Adam(learning_rate=0.0001))
train_batch = DataGenerator(X.iloc[train_idx],
y[train_idx],
tabular_transformer=tb,
batch_size=BATCH_SIZE,
shuffle=True)
 history = model.fit_generator(train_batch,
validation_data=(tb.transform(X.iloc[test_idx]), y[test_idx]),
epochs=30,
callbacks=[model_checkpoint, early_stopping, reduce_learning],
class_weight=[1.0,(np.sum(y==0)/ np.sum(y==1)) ],
verbose=1)
 print("
FOLD %i" % fold)
 plot_keras_history(history, measures=['auc', 'loss'])
 best_iteration.append(np.argmax(history.history['val_auc'])+ 1)
 preds = model.predict(tb.transform(X.iloc[test_idx]),
verbose=1,
batch_size=1024 ).flatten()
oof[test_idx] = preds
roc_auc.append(roc_auc_score(y_true=y[test_idx], y_score=preds))
average_precision.append(average_precision_score(y_true=y[test_idx], y_score=preds))
predictions = np.argmax(predictions,axis = 1)
 predictions = pd.Series(predictions,name="Label" )<save_to_csv>
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof))
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),predictions],axis = 1)
 submission.to_csv("mnist_submission_v6.csv",index=False )<import_modules>
tb = TabularTransformer(numeric = numeric_variables,
ordinal = [],
lowcat = [],
highcat = categorical_variables)
 tb.fit(X)
 sizes = tb.shape(X)
 categorical_levels = dict(zip(categorical_variables, sizes[1:]))
print(f"Input array sizes: {sizes}")
 print(f"Categorical levels: {categorical_levels}
")
 model = tabular_dnn(numeric_variables, categorical_variables,
categorical_levels,
feature_selection_dropout=0.1,
categorical_dropout=0.1,
first_dense = 256,
second_dense = 256,
dense_dropout = 0.1,
activation_type=gelu)
 model = compile_model(model, binary_crossentropy, [AUC(name='auc'), mAP], Adam(learning_rate=0.0001))
train_batch = DataGenerator(X, y,
tabular_transformer=tb,
batch_size=BATCH_SIZE,
shuffle=True)
 history = model.fit_generator(train_batch,
epochs=int(np.median(best_iteration)) ,
class_weight=[1.0,(np.sum(y==0)/ np.sum(y==1)) ],
verbose=1 )
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import GridSearchCV<set_options>
preds = model.predict(tb.transform(Xt[X.columns]),
verbose=1,
batch_size=1024 ).flatten()
submission = pd.DataFrame(Xt.id)
 submission['Action'] = preds
submission.to_csv("tabular_dnn_submission.csv", index=False)
 tabular_dnn_submission = submission.copy()
<load_from_csv><EOS> 
dnn_rank = rankdata(tabular_dnn_submission.Action, method='dense')/ len(Xt)
 cat_rank = rankdata(cat_boost_submission.Action, method='dense')/ len(Xt)
 submission = pd.DataFrame(Xt.id)
 submission['Action'] = 0.5 * dnn_rank + 0.5 * cat_rank
submission.to_csv("blended_submission.csv", index=False )