Kaggle competition: Titanic - Machine Learning from Disaster
N_SPLITS = 5
BATCH_SIZE = 256
EPOCHS = 100
EARLY_STOPPING_PATIENCE = 15
OPTIMIZER = tf.keras.optimizers.Nadam()
LOSS='binary_crossentropy'
METRICS=[tf.keras.metrics.AUC() ]<choose_model_class>
data_test = pd.read_csv('.. /input/titanic/test.csv',index_col='PassengerId')
 data_train = pd.read_csv('.. /input/titanic/train.csv',index_col='PassengerId')
 data_train
def get_cnn_model_1() :
model = tf.keras.models.Sequential([
tf.keras.layers.Reshape(( len(features)* 1, 1), input_shape=(len(features)* 1,)) ,
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(1024, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(1, activation='sigmoid'),
])
 return model
def get_cnn_model() :
model = tf.keras.models.Sequential([
tf.keras.layers.Reshape(( len(features)* 1, 1), input_shape=(len(features)* 1,)) ,
tf.keras.layers.Conv1D(32, 1, activation='elu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv1D(64, 1, activation='elu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(1024, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(512, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.005)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(1, activation='sigmoid'),
])
 return model
def get_nn_model_2() :
model = tf.keras.models.Sequential([
tf.keras.layers.Reshape(( len(features)* 1, 1), input_shape=(len(features)* 1,)) ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(16, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(1, activation='sigmoid'),
])
 return model<prepare_x_and_y>
data_train.isnull().sum()
kfold = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=42)
 models = []
histories = []
for fold_num,(train_index, val_index)in tqdm(enumerate(kfold.split(train_df[features].values, train_df['target'].values)) , total=N_SPLITS):
print(f'Fold {fold_num+1}/{N_SPLITS}:')
 X_train = train_df.loc[train_index, features].values
y_train = train_df.loc[train_index, 'target'].values.reshape(-1, 1)
 X_val = train_df.loc[val_index, features].values
y_val = train_df.loc[val_index, 'target'].values.reshape(-1, 1)
 model = get_cnn_model_1()
model.compile(optimizer=OPTIMIZER, loss=LOSS, metrics=METRICS)
 early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='max', patience=EARLY_STOPPING_PATIENCE, restore_best_weights=True)
 history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=[early_stopping_callback])
 histories.append(history)
 val_preds = model.predict(X_val)
 val_auc = roc_auc_score(y_val, val_preds)
 print(f'Fold validation AUC: {val_auc}')
 print()
models.append(model )<predict_on_test>
for i in data_train.columns:
print(i ,': ',len(data_train[i].unique()))
train_preds = np.zeros(train_df.shape)
 test_preds = np.zeros(test_df.shape)
 for model in models:
pred_train = model.predict(train_df[features].values)
 pred_test = model.predict(test_df[features].values)
 train_preds += pred_train
test_preds += pred_test
train_preds /= len(models)
 test_preds /= len(models )<split>
columnsForDrop = ['Name', 'Cabin','Ticket','SibSp','Parch']
data_train.drop(columns=columnsForDrop, inplace=True)
 data_test.drop(columns=columnsForDrop, inplace=True)
 data_train
train_preds = train_preds[:, 0]
test_preds = test_preds[:, 0]<compute_test_metric>
print(data_train.Sex.value_counts())
 print('----------------------------------------------')
 print(data_train.Embarked.value_counts() )
train_auc = roc_auc_score(train_df['target'], train_preds)
 print(f'Train AUC: {train_auc}' )<load_from_csv>
y = data_train.Survived
X = data_train.drop(columns=['Survived'] )
test_df = pd.read_csv('test_small_with_targets.csv' )<compute_test_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y)
if test_df['target'][0] != -1:
test_auc = roc_auc_score(test_df['target'], test_preds)
 print(f'Test AUC: {test_auc}' )<save_to_csv>
my_imputer = SimpleImputer()
imputed_X_train = pd.DataFrame(my_imputer.fit_transform(X_train))
imputed_X_test = pd.DataFrame(my_imputer.transform(X_test))
imputed_X_train.columns = X_train.columns
imputed_X_test.columns = X_test.columns
sub = pd.DataFrame({'ID_code': test_df['ID_code'], 'target': test_preds})
 sub.to_csv('submission.csv', index=False )<import_modules>
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score, classification_report, f1_score
from sklearn.neighbors import KNeighborsClassifier
FileLink('submission.csv' )<load_from_csv>

 parameters = {'max_depth': list(range(6, 30, 10)) ,
'max_leaf_nodes': list(range(50, 500, 100)) ,
'n_estimators': list(range(50, 1001, 150)) }
gsearch = GridSearchCV(estimator=RandomForestClassifier() ,
param_grid = parameters,
scoring='f1',
n_jobs=4,cv=5,verbose=7)
 gsearch.fit(imputed_X_train, y_train )
train = pd.read_csv(r".. /input/train.csv")
 test = pd.read_csv(r".. /input/test.csv" )<prepare_x_and_y>
print(gsearch.best_params_.get('max_leaf_nodes'))
print(gsearch.best_params_.get('max_depth'))
cols = train.columns.values.tolist() [2: ]
predictors = train[cols]
target = train[['target']]
pre_test = test[cols]<split>
data_test.Age.fillna(X.Age.mean() , inplace=True)
 data_test.Fare.fillna(X.Fare.mean() , inplace=True)
 data_test.isna().sum()
%%time
sfl = StratifiedKFold(n_splits = 3, shuffle=True)
 pred_test_y = np.zeros(( test.shape[0]))
seed = 2019
N = 0
for train_indices, test_indices in sfl.split(predictors, target):
params = {
'num_leaves': 15,
'max_bin': 119,
'min_data_in_leaf': 11,
'learning_rate': 0.02,
'min_sum_hessian_in_leaf': 0.00245,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 4.972,
'lambda_l2': 2.276,
'min_gain_to_split': 0.65,
'max_depth': 14,
'save_binary': True,
'seed': seed,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
}
X_train, X_test = predictors.iloc[train_indices], predictors.iloc[test_indices]
y_train, y_test = target.iloc[train_indices], target.iloc[test_indices]
lgtrain = lgb.Dataset(X_train, label=y_train)
 lgval = lgb.Dataset(X_test, label=y_test)
 evals_result = {}
model2 = lgb.train(params, lgtrain, 10000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=20,
evals_result=evals_result)
 pred_val = model2.predict(X_test, num_iteration=model2.best_iteration)
 pred_test_y += model2.predict(pre_test, num_iteration=model2.best_iteration)
 print("Validation score is :", roc_auc_score(y_test, pred_val))
print(N, "Iteration completed")
 seed+= 2000
N+=1
pred_test = pred_test_y/N<define_variables>
preds = final_model.predict(data_test)
 print(preds.shape)
 print(data_test.shape )
<split><EOS> 
test_out = pd.DataFrame({
'PassengerId': data_test.index,
'Survived': preds
})
 test_out.to_csv('submission.csv', index=False)
 print('Done' )