Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
%matplotlib inline
sns.set_style('whitegrid')
 ExtraTreesClassifier, GradientBoostingClassifier, VotingClassifier)
 LogisticRegression, PassiveAggressiveClassifier, RidgeClassifier)
 data_raw = pd.read_csv('/kaggle/input/titanic/train.csv')
 data_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 train = data_raw.copy(deep=True)
 data_all = [train, data_test]
def get_data_batch(batch_size, randomize=False):
if randomize:
curr_idx = np.random.permutation(train.shape[0])
 else:
curr_idx = range(train.shape[0])
 for batch_n in range(int(np.ceil(train.shape[0] / batch_size))):
batch_data = train.iloc[curr_idx[batch_n*batch_size:(batch_n+1)*batch_size]]
group_sizes = batch_data['groupSize'].values*std_vals['groupSize'] + mean_vals['groupSize']
yield batch_data.drop(columns=['winPlacePerc', 'matchId', 'groupId'] ).values, batch_data['winPlacePerc'].values, group_sizes<categorify>
for dataset in data_all:
dataset.drop(['Ticket', 'Cabin'], axis=1, inplace=True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace=True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace=True)
 for c in set(dataset['Pclass']):
for s in set(dataset['Sex']):
age_median = dataset[(dataset['Pclass'] == c)&(dataset['Sex'] == s)]['Age'].median()
dataset.loc[(dataset['Age'].isnull())&(dataset['Pclass'] == c)&(dataset['Sex'] == s), 'Age'] = age_median
train.drop(['PassengerId'], axis=1, inplace=True)
 def get_titles(series):
return series.str.extract('([a-zA-Z]+)\.', expand=False)
 for dataset in data_all:
dataset['FamilySize'] = dataset['Parch'] + dataset['SibSp'] + 1
dataset['IsAlone'] =(dataset['FamilySize'] == 1 ).astype(int)
 dataset['Title'] = get_titles(dataset['Name'])
 title_counts = dataset['Title'].value_counts()
dataset['Title'] = dataset['Title'].map(lambda t: t if title_counts[t] >= 10 else 'Rare')
 dataset.drop('Name', axis=1, inplace=True)
 dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'], 5)
 categorical_cols = ['Sex', 'Embarked', 'Title', 'AgeBin', 'FareBin']
numerical_cols = ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'FamilySize', 'IsAlone']
le = LabelEncoder()
ohe = OneHotEncoder(sparse=False)
 for dataset in data_all:
for col in categorical_cols:
dataset[col + '_Code'] = le.fit_transform(dataset[col])
dropout_rate = [0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5]
def leaky_relu(z, name=None):
return tf.maximum(0.01*z, z, name=name)
 tf.reset_default_graph()
dynamic_dropout = tf.placeholder_with_default(0.5, shape=(None), name='dynamic_dropout')
 X = tf.placeholder(dtype=tf.float32, shape=[None, train.shape[1]-3], name='X')
 y = tf.placeholder(dtype=tf.float32, shape=(None), name='y')
 group_sizes = tf.placeholder(dtype=tf.float32, shape=(None), name='group_sizes')
 training = tf.placeholder_with_default(False, shape=(None), name='training')
 with tf.name_scope('layers')as scope:
hidden1 = tf.layers.dense(X, 100, name='hidden1',
kernel_initializer=tf.variance_scaling_initializer())
 hidden1 = tf.layers.batch_normalization(hidden1, training=training, momentum=0.9, name='hidden1_bn')
 hidden1 = tf.nn.elu(hidden1, name='hidden1_activation')
 hidden2 = tf.layers.dense(hidden1, 60, name='hidden2',
kernel_initializer=tf.variance_scaling_initializer())
 hidden2 = tf.layers.batch_normalization(hidden2, training=training, momentum=0.9, name='hidden2_bn')
 hidden2 = tf.nn.elu(hidden2, name='hidden2_activation')
 hidden3 = tf.layers.dense(hidden2, 60, name='hidden3',
kernel_initializer=tf.variance_scaling_initializer())
 hidden3 = tf.layers.batch_normalization(hidden3, training=training, momentum=0.9, name='hidden3_bn')
 hidden3 = tf.nn.elu(hidden3, name='hidden3_activation')
 hidden4 = tf.layers.dense(hidden3, 60, name='hidden4',
kernel_initializer=tf.variance_scaling_initializer())
 hidden4 = tf.layers.batch_normalization(hidden4, training=training, momentum=0.9, name='hidden4_bn')
 hidden4 = tf.nn.elu(hidden4, name='hidden4_activation')
 hidden5 = tf.layers.dense(hidden4, 60, name='hidden5',
kernel_initializer=tf.variance_scaling_initializer())
 hidden5 = tf.layers.batch_normalization(hidden5, training=training, momentum=0.9, name='hidden5_bn')
 hidden5 = tf.nn.elu(hidden5, name='hidden5_activation')
 hidden5 = tf.layers.dropout(hidden5, rate=dropout_rate[5], training=training, name='dropout5')
 output = tf.layers.dense(hidden5, 1, name='output',
kernel_initializer=tf.variance_scaling_initializer() )<compute_train_metric>
train = train.dropna(axis=0, subset=['Survived'])
 targ_col = 'Survived'
feature_cols = ['Pclass', 'FamilySize', 'IsAlone', 'Sex_Code', 'Embarked_Code',
'Title_Code', 'AgeBin_Code', 'FareBin_Code']
with tf.name_scope('loss')as scope:
diff_vector = tf.abs(tf.reshape(output, shape=[-1])- y)
 sum_group_sizes = tf.reduce_sum(group_sizes)
 loss_mse = tf.reduce_sum(tf.square(diff_vector)*group_sizes)/sum_group_sizes
loss_mae = tf.reduce_sum(diff_vector*group_sizes)/sum_group_sizes
clipped_diff_vector = tf.clip_by_value(diff_vector, 0.0, 1.0)
 lin_err = 2*(diff_vector - clipped_diff_vector)
 loss_hybrid = tf.reduce_sum(( tf.square(clipped_diff_vector)+ lin_err)*group_sizes)/sum_group_sizes
with tf.name_scope('training')as scope:
lr_high = 0.0008
lr_low = 0.00001
learning_rate = tf.Variable(0.0001, trainable=False, name='learning_rate')
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.6)
 training_op = optimizer.minimize(loss_hybrid)
 val_batch_size = 10000
def get_mse_mae(sess, batch_size=10000):
y_diffs = np.empty(shape=0)
 curr_group_sizes = np.empty(shape=0)
 for X_batch, y_batch, batch_gr_sizes in get_data_batch(batch_size=batch_size):
curr_output_vals = sess.run(output, feed_dict = {X:X_batch} ).reshape(-1)
 y_diffs = np.r_[y_diffs, np.abs(y_batch - curr_output_vals)]
curr_group_sizes = np.r_[curr_group_sizes, batch_gr_sizes]
loss_mae_val = np.sum(y_diffs*curr_group_sizes)/np.sum(curr_group_sizes)
 loss_mse_val = np.sum(( y_diffs**2)*curr_group_sizes)/np.sum(curr_group_sizes)
 return loss_mse_val, loss_mae_val
extra_training_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
 init = tf.global_variables_initializer()
saver = tf.train.Saver()<init_hyperparams>
classifiers = [
KNeighborsClassifier(**{'n_jobs': -1, 'n_neighbors': 7}),
SVC(**{'C': 0.390625, 'kernel': 'poly', 'probability': True, 'random_state': 0}),
LinearSVC(**{'C': 25.0, 'dual': False, 'loss': 'squared_hinge', 'max_iter': 25000, 'penalty': 'l1', 'random_state': 0}),
DecisionTreeClassifier(**{'criterion': 'entropy', 'max_depth': 10, 'min_samples_split': 8, 'random_state': 0}),
RandomForestClassifier(**{'criterion': 'entropy', 'max_depth': 12, 'min_samples_split': 8, 'n_estimators': 300, 'n_jobs': -1, 'random_state': 0}),
AdaBoostClassifier(**{'algorithm': 'SAMME', 'learning_rate': 0.3, 'n_estimators': 300, 'random_state': 0}),
BaggingClassifier(**{'max_samples': 0.1, 'n_estimators': 50, 'n_jobs': -1, 'random_state': 0}),
ExtraTreesClassifier(**{'criterion': 'gini', 'max_depth': 12, 'min_samples_split': 12, 'n_estimators': 200, 'n_jobs': -1, 'random_state': 0}),
GradientBoostingClassifier(**{'learning_rate': 0.3, 'max_depth': 2, 'min_samples_split': 2, 'n_estimators': 10, 'random_state': 0}),
BernoulliNB(**{'alpha': 0.1}),
GaussianNB() ,
LinearDiscriminantAnalysis(**{'shrinkage': None, 'solver': 'svd'}),
QuadraticDiscriminantAnalysis() ,
Perceptron(**{'early_stopping': True, 'n_jobs': -1, 'penalty': 'l2', 'random_state': 0}),
LogisticRegression(**{'C': 0.09765625, 'max_iter': 20, 'n_jobs': -1, 'penalty': 'l2', 'random_state': 0, 'solver': 'lbfgs'}),
PassiveAggressiveClassifier(**{'C': 0.01220703125, 'early_stopping': True, 'loss': 'hinge', 'max_iter': 5, 'n_jobs': -1, 'random_state': 0}),
RidgeClassifier(**{'alpha': 0.5, 'normalize': True, 'random_state': 0}),
GaussianProcessClassifier(**{'max_iter_predict': 10, 'n_jobs': -1, 'random_state': 0}),
XGBClassifier(**{'booster': 'gbtree', 'learning_rate': 0.1, 'max_depth': 10, 'n_jobs': -1, 'random_state': 0, 'reg_alpha': 0.16, 'reg_lambda': 2.56}),
LGBMClassifier(**{'boosting_type': 'goss', 'learning_rate': 0.1, 'n_estimators': 1000, 'n_jobs': -1, 'random_state': 0, 'reg_alpha': 0.16, 'reg_lambda': 0}),
]
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0 )
gc.collect()
batch_size = 5000
n_epochs = 1000
max_epochs_wo_improvement = 50
max_epochs_wo_lr_change = 10
max_time = 2
loss_data = {'mse_train':[], 'mae_train':[], 'lr':[]}
start_time = datetime.datetime.now()
def get_dropout(initial_dropout, epoch):
return np.maximum(0, initial_dropout*(1 - epoch/200))
with tf.Session() as sess:
init.run()
min_loss_mae = np.inf
epochs_wo_improvement = 0
epochs_since_lr_change = 0
batch_lr = lr_high
for epoch in range(n_epochs):
for X_batch, y_batch, batch_gr_sizes in get_data_batch(batch_size=batch_size, randomize=True):
feed_dict = {X: X_batch, y: y_batch, training:True, learning_rate:batch_lr, group_sizes:batch_gr_sizes}
sess.run([training_op, extra_training_ops], feed_dict=feed_dict)
 loss_mse_train, loss_mae_train = get_mse_mae(sess)
 loss_data['mse_train'].append(loss_mse_train)
 loss_data['mae_train'].append(loss_mae_train)
 loss_data['lr'].append(batch_lr)
 print('epoch', epoch, 'time passed:',(datetime.datetime.now() - start_time), 'learning rate: {:<5.5f}'.format(batch_lr))
print('MSE:', loss_mse_train, 'MAE:', loss_mae_train)
 if loss_mae_train <= min_loss_mae:
epochs_wo_improvement = 0
epochs_since_lr_change = 0
min_loss_mae = loss_mae_train
save_path = saver.save(sess, '.. /DNN_data/dnn_state.ckpt')
 print('- best so far!')
 else:
epochs_wo_improvement += 1
epochs_since_lr_change += 1
if(epochs_wo_improvement > max_epochs_wo_improvement)or(( datetime.datetime.now() - start_time ).seconds > max_time*60*60):
print('early breaking!')
 break
if epochs_since_lr_change >= max_epochs_wo_lr_change:
batch_lr = batch_lr -(batch_lr - lr_low)*0.3
epochs_since_lr_change = 0
saver.restore(sess, save_path)
 loss_mse_val, loss_mae_val = get_mse_mae(sess)
 print('Final MSE:', loss_mse_val, 'final MAE:', loss_mae_val )<drop_column>
clf_scores = pd.DataFrame(columns=['Classifier', 'Test Score', 'Test Score 3*STD'])
 clf_preds = pd.DataFrame(train[targ_col])
 for i in range(len(classifiers)) :
clf = classifiers[i]
clf_name = clf.__class__.__name__
clf.fit(train[feature_cols], train[targ_col])
 cv_results = cross_val_score(clf, train[feature_cols], train[targ_col], cv=skf)
 clf_scores.loc[i] = [clf_name, cv_results.mean() , cv_results.std() * 3]
clf_preds[clf_name] = clf.predict(train[feature_cols])
 clf_scores
group_cols = [col for col in train.columns if re.search(r'group', col)is not None]
group_cols = [col for col in group_cols if re.search(r'_rank', col)is None]
group_cols.remove('groupId')
 other_cols = [col for col in train.columns if re.search(r'match|group', col)is None]
other_cols.remove('num_opponents')
 other_cols.remove('winPlacePerc' )<data_type_conversions>
def print_scores_info(model_name, scores):
mean = scores.mean() * 100
std_3 = scores.std() * 100 * 3
print(model_name, 'score mean: ', mean)
 print(model_name, 'score 3 std range: ', mean - std_3, '—', mean + std_3 )
<define_search_space><EOS> 
vote_classifiers = [ 
('knn', KNeighborsClassifier(**{'n_jobs': -1, 'n_neighbors': 7})) , 
('bag', BaggingClassifier(**{'max_samples': 0.1, 'n_estimators': 50, 'n_jobs': -1, 'random_state': 0})) , 
('gbc', GradientBoostingClassifier(**{'learning_rate': 0.3, 'max_depth': 2, 'min_samples_split': 2, 'n_estimators': 10, 'random_state': 0})) , 
('bnb', BernoulliNB(**{'alpha': 0.1})) , 
('gnb', GaussianNB()), 
('lda', LinearDiscriminantAnalysis(**{'shrinkage': None, 'solver': 'svd'})) , 
('qda', QuadraticDiscriminantAnalysis()), 
('log', LogisticRegression(**{'C': 0.09765625, 'max_iter': 20, 'n_jobs': -1, 'penalty': 'l2', 'random_state': 0, 'solver': 'lbfgs'})) , 
('gpc', GaussianProcessClassifier(**{'max_iter_predict': 10, 'n_jobs': -1, 'random_state': 0})) , 
('xgb', XGBClassifier(**{'booster': 'gbtree', 'learning_rate': 0.1, 'max_depth': 10, 'n_jobs': -1, 'random_state': 0, 'reg_alpha': 0.16, 'reg_lambda': 2.56})) ,
]
vote_hard_clf = VotingClassifier(vote_classifiers, voting='hard')
 print_scores_info('Vote Hard', cross_val_score(vote_hard_clf, train[feature_cols], train[targ_col], cv=skf))
vote_hard_clf.fit(train[feature_cols], train[targ_col])
 preds_test = vote_hard_clf.predict(data_test[feature_cols])
 output = pd.DataFrame({'PassengerId': data_test.PassengerId, 'Survived': preds_test})
 output.to_csv('submission.csv', index=False )