Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<normalization>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
ssc = StandardScaler()
selection = freq_encoded + target_encoded + ordinal_vars + time_vars
X_ohe = ssc.fit_transform(X[selection].fillna(X[selection].median()))
Xt_ohe = ssc.transform(Xt[selection].fillna(X[selection].median()))<normalization>
df_train = pd.read_csv(".. /input/train.csv", sep=",")
 df_test = pd.read_csv(".. /input/test.csv", sep=",")
 df_train.head()
def gelu(x):
return 0.5 * x *(1 + tf.tanh(tf.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3))))
get_custom_objects().update({'gelu': Activation(gelu)})
 get_custom_objects().update({'leaky-relu': Activation(LeakyReLU(alpha=0.2)) } )<choose_model_class>
df_train[['Sex','Survived']].groupby(['Sex'],as_index=False ).mean().sort_values(by='Survived',ascending=False ).round(2 )
def tabular_dnn(numeric_variables, categorical_variables, categorical_counts,
feature_selection_dropout=0.2, categorical_dropout=0.1,
first_dense = 256, second_dense = 256, dense_dropout = 0.2,
activation_type=gelu):
numerical_inputs = Input(shape=(numeric_variables,))
numerical_normalization = BatchNormalization()(numerical_inputs)
 numerical_feature_selection = Dropout(feature_selection_dropout )(numerical_normalization)
 categorical_inputs = []
categorical_embeddings = []
for category in categorical_variables:
categorical_inputs.append(Input(shape=[1], name=category))
category_counts = categorical_counts[category]
categorical_embeddings.append(
 Embedding(category_counts+1,
min(int(category_counts/1.5 + 1), 3),
name = category + "_embed" )(categorical_inputs[-1]))
categorical_logits = Concatenate(name = "categorical_conc" )([Flatten()(SpatialDropout1D(categorical_dropout )(cat_emb))
for cat_emb in categorical_embeddings])
 x = concatenate([numerical_feature_selection, categorical_logits])
 x = BatchNormalization()(x)
 x = Dense(first_dense, activation=activation_type )(x)
 x = BatchNormalization()(x)
 x = Dropout(dense_dropout )(x)
 x = Dense(second_dense, activation=activation_type )(x)
 x = Dropout(dense_dropout )(x)
 x = BatchNormalization()(x)
 output = Dense(1, activation="sigmoid" )(x)
 model = Model([numerical_inputs] + categorical_inputs, output)
 return model<compute_test_metric>
df_train = df_train.drop(["PassengerId", "Name"], axis=1)
 df_test = df_test.drop(["Name"], axis=1 )
def auroc(y_true, y_pred):
try:
return tf.py_function(roc_auc_score,(y_true, y_pred), tf.double)
 except:
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double)
 get_custom_objects().update({'auroc': auroc})
 def mAP(y_true, y_pred):
try:
return tf.py_function(average_precision_score,(y_true, y_pred), tf.double)
 except:
return tf.py_func(average_precision_score,(y_true, y_pred), tf.double)
 get_custom_objects().update({'mAP': mAP})
 def compile_model(model, loss, metrics, optimizer):
model.compile(loss=loss, metrics=metrics, optimizer=optimizer)
 return model
def plot_keras_history(history, measures):

 rows = len(measures)// 2 + len(measures)% 2
fig, panels = plt.subplots(rows, 2, figsize=(15, 5))
plt.subplots_adjust(top = 0.99, bottom=0.01, hspace=0.4, wspace=0.2)
 try:
panels = [item for sublist in panels for item in sublist]
except:
pass
for k, measure in enumerate(measures):
panel = panels[k]
panel.set_title(measure + ' history')
 panel.plot(history.epoch, history.history[measure], label="Train "+measure)
 panel.plot(history.epoch, history.history["val_"+measure], label="Validation "+measure)
 panel.set(xlabel='epochs', ylabel=measure)
 panel.legend()
plt.show(fig )<define_variables>
df_train['Cabin'].isna().sum()
SEED = 42
FOLDS = 20
MAX_EPOCHS = 100
BATCH_SIZE = 1024 * 4<choose_model_class>
df_train = df_train.drop(["Cabin"], axis=1)
 df_test = df_test.drop(["Cabin"], axis=1 )
measure_to_monitor = 'val_auroc'
modality = 'max'
early_stopping = EarlyStopping(monitor=measure_to_monitor,
mode=modality,
patience=5,
verbose=0)
 model_checkpoint = ModelCheckpoint('best.model',
monitor=measure_to_monitor,
mode=modality,
save_best_only=True,
verbose=0)
 model_reduce_lr = ReduceLROnPlateau(monitor=measure_to_monitor,
mode=modality,
factor=0.25,
patience=2,
min_lr=1e-6,
verbose=1 )<init_hyperparams>
df_train['Ticket'] = df_train['Ticket'].astype("category" ).cat.codes
df_train['Sex']= df_train['Sex'].astype("category" ).cat.codes
df_train['Embarked'] = df_train['Embarked'].astype("category" ).cat.codes
df_train['Age'] = round(df_train['Age'])
 is_alone_train=[]
for x in df_train['SibSp']:
if(x==0):is_alone_train.append(0)
 else: is_alone_train.append(1)
 df_train['is_alone'] = is_alone_train
age_group_train=[]
for x in df_train['Age']:
if x <=16: age_group_train.append(1)
 elif x >=17 and x <= 32:age_group_train.append(2)
 elif x >= 33 and x <= 42:age_group_train.append(3)
 else:age_group_train.append(4)
 df_train['age_group'] = age_group_train
df_test['Ticket'] = df_test['Ticket'].astype("category" ).cat.codes
df_test['Sex']= df_test['Sex'].astype("category" ).cat.codes
df_test['Embarked'] = df_test['Embarked'].astype("category" ).cat.codes
df_test['Age'] = round(df_test['Age'])
 is_alone_test=[]
for x in df_test['SibSp']:
if(x==0): is_alone_test.append(0)
 else: is_alone_test.append(1)
 df_test['is_alone'] = is_alone_test
age_group_test=[]
for x in df_test['Age']:
if x <=16: age_group_test.append(1)
 elif x >=17 and x <= 32: age_group_test.append(2)
 elif x >= 33 and x <= 42: age_group_test.append(3)
 else: age_group_test.append(4)
 df_test['age_group'] = age_group_test
model_params = {
"numeric_variables" : X_ohe.shape[1],
"categorical_variables" : categorical_counts.keys() ,
"categorical_counts" : categorical_counts,
"feature_selection_dropout" : 0.0,
"categorical_dropout" : 0.3,
"first_dense" : 512,
"second_dense" : 512,
"dense_dropout" : 0.3,
"activation_type" : 'relu'
}<choose_model_class>
df_train['Age'].fillna(0,inplace=True)
 df_train['Embarked'].fillna(df_train['Embarked'].mode() [0],inplace=True)
 df_train['Fare'].fillna(0, inplace=True)
 df_test['Age'].fillna(0,inplace=True)
 df_test['Embarked'].fillna(df_test['Embarked'].mode() [0],inplace=True)
 df_test['Fare'].fillna(0, inplace=True )
skf = StratifiedKFold(n_splits=FOLDS,
shuffle=True,
random_state=SEED)
 roc_auc = list()
average_precision = list()
oof = np.zeros(len(X))
best_iteration = list()
cv_test_preds = np.zeros(len(Xt))
for fold,(train_idx, test_idx)in enumerate(skf.split(X, y)) :
model = compile_model(tabular_dnn(**model_params),
binary_crossentropy,
[auroc, mAP],
Adam(learning_rate=0.0001))
X_cv_ohe = X_ohe[train_idx].astype(np.float32)
 X_cv_cat = [X_cat[i][train_idx] for i in range(len(X_cat)) ]
y_cv = y[train_idx]
X_oof_ohe = X_ohe[test_idx].astype(np.float32)
 X_oof_cat = [X_cat[i][test_idx] for i in range(len(X_cat)) ]
y_oof = y[test_idx]
train_batch = batch_generator(X_cv_ohe,
X_cv_cat,
y_cv,
batch_size=BATCH_SIZE,
random_state=SEED)
 val_batch = batch_generator(X_oof_ohe,
X_oof_cat,
y_oof,
batch_size=BATCH_SIZE,
random_state=SEED)
 train_steps = len(y_cv)// BATCH_SIZE
validation_steps = len(y_oof)// BATCH_SIZE
history = model.fit_generator(train_batch,
validation_data=val_batch,
epochs=MAX_EPOCHS,
steps_per_epoch=train_steps,
validation_steps=validation_steps,
callbacks=[model_checkpoint, early_stopping, model_reduce_lr],
verbose=1)
 print("
FOLD %i" % fold)
 plot_keras_history(history, measures=['auroc', 'loss'])
 best_iteration.append(np.argmax(history.history['val_auroc'])+ 1)
 model = load_model('best.model')
 preds = model.predict([X_oof_ohe] + X_oof_cat,
verbose=1,
batch_size=1024 ).flatten()
oof[test_idx] = preds
roc_auc.append(roc_auc_score(y_true=y_oof, y_score=preds))
average_precision.append(average_precision_score(y_true=y_oof, y_score=preds))
cv_test_preds += model.predict([Xt_ohe] + Xt_cat,
verbose=1,
batch_size=1024 ).flatten() / FOLDS<save_to_csv>
y = df_train['Survived']
export = pd.DataFrame()
export['PassengerId'] = df_test['PassengerId']
df_test = df_test.drop(['PassengerId'], axis=1)
 df_train = df_train.drop('Survived', axis=1)
 X = df_train
oof = pd.DataFrame({'id':id_train, 'dnn_oof': oof})
 oof.to_csv("oof.csv", index=False)
 submission = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/sample_submission.csv")
 submission.target = cv_test_preds
submission.to_csv("./dnn_cv_submission.csv", index=False )<compute_test_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.18, random_state=0)
 clf = RandomForestClassifier(n_estimators=50,
max_depth=6,
n_jobs = -1,
random_state=0)
 clf.fit(X_train, y_train)
 print("Teste: {}%".format(clf.score(X_test, y_test ).round(2)))
 print("Train: {}%".format(clf.score(X_train, y_train ).round(2)))
 print('-----------------------------')
 print('The mae of prediction is:', metrics.mean_absolute_error(y_train, clf.predict(X_train)).round(2))
print('Mean Squared Error:', metrics.mean_squared_error(y_train, clf.predict(X_train)).round(2))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_train, clf.predict(X_train)) ).round(2))
print("Average cv roc auc score %0.3f ± %0.3f" %(np.mean(roc_auc), np.std(roc_auc)))
 print("Average cv roc average precision %0.3f ± %0.3f" %(np.mean(average_precision), np.std(average_precision)))
 print("Roc auc score OOF %0.3f" % roc_auc_score(y_true=y, y_score=oof.dnn_oof))
print("Average precision OOF %0.3f" % average_precision_score(y_true=y, y_score=oof.dnn_oof))<train_model>
X_test = df_test
y_pred_test = clf.predict(X_test)
 export['Survived'] = y_pred_test
export_csv = export.to_csv(r'titanic.csv', index = None, header=True )
<save_to_csv><EOS> 
sc = StandardScaler()
X_scale = sc.fit_transform(X)
 cv = KFold(n_splits = 6, shuffle = True)
 result = cross_validate(clf,X_scale,y,cv=cv, return_train_score=False)
 print("Cross validate median {}%".format(np.median(result['test_score'] ).round(2)*100))
print("Cross validate average {}%".format(np.average(result['test_score'] ).round(2)*100))