Kaggle competition: Titanic - Machine Learning from Disaster
df_corr=X_train.corr().abs().unstack().sort_values().reset_index()
df_corr=df_corr[df_corr['level_0']!= df_corr['level_1']]
df_corr.head()<feature_engineering>
%matplotlib inline
%config InlineBackend.figure_format='retina'
sns.set_style("darkgrid")
X_train['sum'] = X_train.sum(axis=1)
 X_train['min'] = X_train.min(axis=1)
 X_train['max'] = X_train.max(axis=1)
 X_train['mean'] = X_train.mean(axis=1)
 X_train['median'] = X_train.median(axis=1)
 X_train['std'] = X_train.std(axis=1)
 X_train['skew'] = X_train.skew(axis=1)
 X_train.head()<feature_engineering>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 all_data = pd.concat([train,test], axis = 0)
 print('train dataset has {} rows, {} columns'.format(train.shape[0],train.shape[1]))
print('test dataset has {} rows, {} columns'.format(test.shape[0],test.shape[1]))
train.sample(20 )
X_test['sum'] = X_test.sum(axis=1)
 X_test['min'] = X_test.min(axis=1)
 X_test['max'] = X_test.max(axis=1)
 X_test['mean'] = X_test.mean(axis=1)
 X_test['median'] = X_test.median(axis=1)
 X_test['std'] = X_test.std(axis=1)
 X_test['skew'] = X_test.skew(axis=1)
 X_test.head()<define_variables>
num_col = [col for col in train.columns if train[col].dtypes != object]
unique = [len(train[col].unique())for col in num_col]
num_unique = dict(zip(num_col,unique))
print('There are {} Numerical columns
'.format(len(num_col)))
 print('Numerical unique values: {}'.format(num_unique))
skewed_columns=['var_0', 'var_1', 'var_2', 'var_5', 'var_13', 'var_18','var_19','var_20','var_21', 'var_22',
'var_24','var_26', 'var_35', 'var_36','var_40','var_44','var_45','var_47','var_48','var_49',
'var_51','var_52','var_54','var_56','var_61', 'var_66','var_67','var_70','var_74','var_75',
'var_76','var_80','var_81','var_82','var_83','var_86','var_87','var_90', 'var_92','var_94',
'var_97', 'var_99',
'var_100', 'var_101', 'var_102', 'var_107', 'var_109','var_110', 'var_115', 'var_117',
'var_118', 'var_122','var_123', 'var_127', 'var_128', 'var_136', 'var_137', 'var_139',
'var_147', 'var_149',
'var_150','var_151','var_154','var_155','var_157','var_160','var_163','var_164','var_165',
'var_167','var_170','var_172','var_173','var_174','var_177','var_178','var_179','var_180',
'var_184','var_187','var_188','var_190','var_191','var_196','var_198','var_199']<feature_engineering>
cat_col = [col for col in train.columns if train[col].dtypes == object]
unique = [len(train[col].unique())for col in cat_col]
cat_unique = dict(zip(cat_col,unique))
print('There are {} Categorical columns
'.format(len(cat_col)))
 print('Categorical unique values: {}'.format(cat_unique))
def bin_feature(col_name,X):
interval=pd.qcut(X[col_name],5 ).value_counts().index.sort_values()
X.loc[(X[col_name] <= interval[0].right),col_name] = 0
X.loc[(( X[col_name] > interval[1].left)&(X[col_name] <= interval[1].right)) ,col_name] = 1
X.loc[(( X[col_name] > interval[2].left)&(X[col_name] <= interval[2].right)) ,col_name] = 2
X.loc[(( X[col_name] > interval[3].left)&(X[col_name] <= interval[3].right)) ,col_name] = 3
X.loc[(X[col_name] > interval[4].left),col_name] = 4
return X<feature_engineering>
all_data['Family_Group'] = all_data.SibSp + all_data.Parch
for col in skewed_columns:
binned_col = col + '_bin'
X_train[binned_col] = X_train[col]
X_train[binned_col] = bin_feature(binned_col, X_train )<feature_engineering>
all_data['Ticket_short'] = all_data.Ticket.apply(lambda x: x[:1] )
for col in skewed_columns:
binned_col = col + '_bin'
X_test[binned_col] = X_test[col]
X_test[binned_col] = bin_feature(binned_col, X_test )<split>
all_data['Ticket_short'].replace(['7', 'W', '4', 'F', 'L', '9', '6', '5', '8'], 'O', inplace=True )
X_dev, X_val, Y_dev, Y_val=train_test_split(X_train, Y_train, test_size=0.4, random_state=0)
 X_dev.shape, X_val.shape, Y_dev.shape, Y_val.shape<import_modules>
all_data['Embarked'] = all_data['Embarked'].transform(lambda x: x.fillna(x.mode() [0]))
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import xgboost as xgb
from sklearn.ensemble import GradientBoostingClassifier<train_model>
all_data['Age'] = all_data.groupby(['Title'])['Age'].transform(lambda x: x.fillna(x.median()))
def xgb_train(X_train, Y_train, X_test):
xgb_param = {
'objective' : "binary:logistic",
'eval_metric' : "auc",
'max_depth' : 6,
'eta' : 0.1,
'gamma' : 5,
'subsample' : 0.7,
'colsample_bytree' : 0.7,
'min_child_weight' : 50,
'colsample_bylevel' : 0.7,
'lambda' : 1,
'alpha' : 0,
'booster' : "gbtree",
'silent' : 1,
"random_state": 4950
}
n_folds = 10
oof_train = np.zeros(X_train.shape[0])
 oof_test = np.zeros(X_test.shape[0])
 oof_test_kf = np.empty(( n_folds, X_test.shape[0]))
oof_train.shape, oof_test.shape, oof_test_kf.shape
kf = KFold(n_splits=n_folds, shuffle=True, random_state=2019)
 for fold_,(dev_idx, val_idx)in enumerate(kf.split(X_train)) :
print('Fold: ' + str(fold_))
X_dev, Y_dev = X_train.iloc[dev_idx, :], Y_train[dev_idx]
X_val, Y_val = X_train.iloc[val_idx, :], Y_train[val_idx]
xgb_dev = xgb.DMatrix(data=X_dev, label=Y_dev)
 xgb_val = xgb.DMatrix(data=X_val, label=Y_val)
 xgb_test = xgb.DMatrix(X_test)
 watchlist = [(xgb_dev, 'train'),(xgb_val, 'valid')]
xgb_model = xgb.train(xgb_param, xgb_dev, num_boost_round=1000, evals=watchlist,maximize=True,verbose_eval=50,early_stopping_rounds = 50)
 Y_pred = xgb_model.predict(xgb_val, ntree_limit=xgb_model.best_ntree_limit+50)
 Y_test = xgb_model.predict(xgb_test, ntree_limit=xgb_model.best_ntree_limit+50)
 oof_train[val_idx] = Y_pred
oof_test_kf[fold_, :] = Y_test
val_acc = np.round(roc_auc_score(Y_val, Y_pred), 4)
 val_acc = np.round(roc_auc_score(Y_train, oof_train), 4)
 print('Overall Training Accuracy: {0}'.format(val_acc))
oof_test = oof_test_kf.mean(axis=0)
 return oof_train, oof_test<train_model>
all_data.drop(columns = ['Name', 'Ticket', 'PassengerId', 'Fare'], inplace =True)
 all_data.head()
oof_xgb_train, oof_xgb_test = xgb_train(X_train, Y_train, X_test )<load_from_csv>
num_col = [col for col in all_data.columns if all_data[col].dtypes != 'object']
num_col.remove('Survived')
 cat_col = [col for col in all_data.columns if all_data[col].dtypes == 'object']
train_final = all_data[:train.shape[0]]
test_final = all_data[train.shape[0]:]
X = train_final.drop(columns = 'Survived')
 y = train_final.Survived
df_sample_submission= pd.read_csv('.. /input/sample_submission.csv')
 df_sample_submission.head()<save_to_csv>
skf = StratifiedKFold(n_splits=5, random_state=2020, shuffle=True)
 numerical_transformer = make_pipeline(StandardScaler())
 categorical_transformer = make_pipeline(SimpleImputer(strategy='most_frequent'),
OneHotEncoder(handle_unknown='ignore',sparse=False))
preprocess = ColumnTransformer(transformers=[ 
('num', numerical_transformer, num_col), 
('cat', categorical_transformer, cat_col)] )
output=pd.DataFrame({
"ID_code" : df_sample_submission['ID_code'],
"target" : oof_xgb_test
})
 output.to_csv('SCP_xgb_test.csv', index=False)
 output.head()<save_to_csv>
preprocess.fit_transform(X,y)
 enc_cat_col = preprocess.named_transformers_['cat']['onehotencoder'].get_feature_names()
labels = np.concatenate([num_col, enc_cat_col])
 X_transformed = pd.DataFrame(preprocess.fit_transform(X), columns=labels)
 X_transformed
id_train=df_train['ID_code']
df_out = pd.DataFrame({'id_train':id_train.values, 'target':oof_xgb_train})
 df_out.to_csv('SCP_xgb_train.csv', index=False )<compute_test_metric>
random_state = 2020
clf_list = [('LogisticRegression', LogisticRegression(random_state = random_state)) , 
('KNN_Classifier', KNeighborsClassifier()), 
('SVC', SVC(random_state = random_state, probability = True)) , 
('RandomForestClassifier', RandomForestClassifier(random_state = random_state)) , 
('XGBClassifier', XGBClassifier(random_state = random_state)) , 
('AdaBoostClassifier', AdaBoostClassifier(base_estimator = DecisionTreeClassifier(random_state = random_state), random_state = random_state)) ]
roc_auc_score(Y_train, oof_xgb_train )<load_from_csv>
estimators = [('lr', clf_list[0][1]), 
('knn', clf_list[1][1]),('svc', clf_list[2][1]), 
('rf', clf_list[3][1]),('xgb', clf_list[4][1]), 
('adb', clf_list[5][1])]
base_voting_hard = VotingClassifier(estimators = estimators , voting = 'hard')
 base_voting_soft = VotingClassifier(estimators = estimators , voting = 'soft')
 cv_hard = cross_val_score(base_voting_hard, X_transformed, y, cv = skf)
 cv_soft = cross_val_score(base_voting_soft, X_transformed, y, cv = skf)
 print('Baseline Models - Ensemble
--------------------------')
 print('Hard Voting: {}%'.format(np.round(cv_hard.mean() *100, 1)))
 print('Soft Voting: {}%'.format(np.round(cv_soft.mean() *100, 1)) )
train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 train_knowledge = pd.read_csv('.. /input/santander-2019-distillation/lgbm_train.csv' )<prepare_x_and_y>
final_pipe = make_pipeline(preprocess, LogisticRegression())
 param_grid = {'logisticregression__max_iter' : [100],
'logisticregression__penalty' : ['l1', 'l2'],
'logisticregression__C' : np.logspace(-2, 2, 20),
'logisticregression__solver' : ['lbfgs', 'liblinear']}
grid_lr = GridSearchCV(final_pipe, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_lr = grid_lr.fit(X, y)
 lr_param = best_grid_lr.best_params_
print(f'best_params_: {lr_param}')
 print(f'score: {best_grid_lr.best_score_:.5f}' )
y = train['target']
y_knowledge = train_knowledge['target']
id_code_train = train['ID_code']
id_code_test = test['ID_code']
features = [c for c in train.columns if c not in ['ID_code', 'target']]<feature_engineering>
lr = LogisticRegression()
param_grid = {'max_iter' : [100],
'penalty' : ['l1', 'l2'],
'C' : np.logspace(-2, 2, 20),
'solver' : ['lbfgs', 'liblinear']}
grid_lr = GridSearchCV(lr, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_lr = grid_lr.fit(X_transformed, y)
 lr_param = best_grid_lr.best_params_
print(f'best_params_: {lr_param}')
 print(f'
score: {best_grid_lr.best_score_:.5f}' )
for feature in features:
train['sq_'+feature] =(train[feature])**2
train['c_'+feature] =(train[feature])**3
train['r2_'+feature] = np.round(train[feature], 2)
<feature_engineering>
knn = KNeighborsClassifier()
final_pipe = make_pipeline(preprocess, knn)
 param_grid = {'kneighborsclassifier__n_neighbors' : np.arange(3, 30, 2),
'kneighborsclassifier__weights': ['uniform', 'distance'],
'kneighborsclassifier__algorithm': ['auto'],
'kneighborsclassifier__p': [1, 2]}
grid_knn = GridSearchCV(final_pipe, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_knn = grid_knn.fit(X, y)
 knn_param = best_grid_knn.best_params_
print(f'best_params_: {knn_param}')
 print(f'score: {best_grid_knn.best_score_:.5f}' )
for feature in features:
test['sq_'+feature] =(test[feature])**2
test['c_'+feature] =(test[feature])**3
test['r2_'+feature] = np.round(test[feature], 2 )<normalization>
knn = KNeighborsClassifier()
param_grid = {'n_neighbors' : np.arange(3, 30, 2),
'weights': ['uniform', 'distance'],
'algorithm': ['auto'],
'p': [1, 2]}
grid_knn = GridSearchCV(knn, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_knn = grid_knn.fit(X_transformed, y)
 knn_param = best_grid_knn.best_params_
print(f'best_params_: {knn_param}')
 print(f'
score: {best_grid_knn.best_score_:.5f}' )
class GaussRankScaler() :
def __init__(self):
self.epsilon = 1e-9
self.lower = -1 + self.epsilon
self.upper = 1 - self.epsilon
self.range = self.upper - self.lower
def fit_transform(self, X):
i = np.argsort(X, axis = 0)
 j = np.argsort(i, axis = 0)
 assert(j.min() == 0 ).all()
assert(j.max() == len(j)- 1 ).all()
j_range = len(j)- 1
self.divider = j_range / self.range
transformed = j / self.divider
transformed = transformed - self.upper
transformed = scipy.special.erfinv(transformed)
 return transformed<normalization>
svc = SVC(probability = True)
 param_grid = tuned_parameters = [{'kernel': ['rbf'],
'gamma': [0.01, 0.1, 0.5, 1, 2, 5],
'C': [.1, 1, 2, 5]},
{'kernel': ['linear'],
'C': [.1, 1, 2, 10]},
{'kernel': ['poly'],
'degree' : [2, 3, 4, 5],
'C': [.1, 1, 10]}]
grid_svc = GridSearchCV(svc, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_svc = grid_svc.fit(X_transformed, y)
 svc_param = best_grid_svc.best_params_
print(f'best_params_: {svc_param}')
 print(f'
score: {best_grid_svc.best_score_:.5f}' )
SPLIT = len(train)
 train = train.append(test)
 del test; gc.collect()
scaler = GaussRankScaler()
sc = StandardScaler()
for feat in tqdm(features):
train[feat] = sc.fit_transform(train[feat].values.reshape(-1, 1))
train[feat+'_r'] = rankdata(train[feat] ).astype('float32')
 train[feat+'_n'] = norm.cdf(train[feat] ).astype('float32')
 feats = [c for c in train.columns if c not in(['ID_code', 'target'] + features)]
for feat in tqdm(feats):
train[feat] = sc.fit_transform(train[feat].values.reshape(-1, 1))
train = train.drop(['target', 'ID_code'], axis=1)
 test = train[SPLIT:].values
train = train[:SPLIT].values
print('Done!!')
 print(train.shape)
<split>
rf = RandomForestClassifier(random_state = 2020)
 param_grid = {'n_estimators': [50, 150, 300, 450],
'criterion': ['entropy'],
'bootstrap': [True],
'max_depth': [3, 5, 10],
'max_features': ['auto','sqrt'],
'min_samples_leaf': [2, 3],
'min_samples_split': [2, 3]}
grid_rf = GridSearchCV(rf, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_rf = grid_rf.fit(X_transformed, y)
 rf_param = best_grid_rf.best_params_
print(f'best_params_: {rf_param}')
 print(f'
score: {best_grid_rf.best_score_:.5f}' )
x_train, x_valid, y_train, y_valid, y_knowledge_train, y_knowledge_valid = train_test_split(train, y, y_knowledge, stratify=y, test_size=0.2, random_state=8 )<choose_model_class>
xgb = XGBClassifier(random_state = 2020)
 param_grid = {'n_estimators': [15, 25, 50, 100],
'colsample_bytree': [0.65, 0.75, 0.80],
'max_depth': [None],
'reg_alpha': [1],
'reg_lambda': [1, 2, 5],
'subsample': [0.50, 0.75, 1.00],
'learning_rate': [0.01, 0.1, 0.5],
'gamma': [0.5, 1, 2, 5],
'min_child_weight': [0.01],
'sampling_method': ['uniform']}
grid_xgb = GridSearchCV(xgb, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_xgb = grid_xgb.fit(X_transformed, y)
 xgb_param = best_grid_xgb.best_params_
print(f'best_params_: {xgb_param}')
 print(f'
score: {best_grid_xgb.best_score_:.5f}' )
function = 'relu'
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 x = Dense(16, activation=function )(input_tensor)
 x = Flatten()(x)
 out_put = Dense(n_out, activation='sigmoid' )(x)
 model = Model(input_tensor, out_put)
 return model<compute_test_metric>
adblogis = AdaBoostClassifier(base_estimator = DecisionTreeClassifier(random_state = random_state), random_state=random_state)
 param_grid = {'algorithm': ['SAMME', 'SAMME.R'],
'base_estimator__criterion' : ['gini', 'entropy'],
'base_estimator__splitter' : ['best', 'random'],
'n_estimators': [2, 5, 10, 50],
'learning_rate': [0.01, 0.1, 0.2, 0.3, 1, 2]}
grid_adblogis = GridSearchCV(adblogis, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_adblogis = grid_adblogis.fit(X_transformed, y)
 adblogis_param = best_grid_adblogis.best_params_
print(f'best_params_: {adblogis_param}')
 print(f'
score: {best_grid_adblogis.best_score_:.5f}' )
def auc(y_true, y_pred):
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double )<compute_train_metric>
clf_list = [("LogisticRegression", best_grid_lr.best_estimator_), 
("KNN_Classifier", best_grid_knn.best_estimator_), 
('SVC', best_grid_svc.best_estimator_), 
('RandomForestClassifier', best_grid_rf.best_estimator_), 
("XGBClassifier", best_grid_xgb.best_estimator_), 
("AdaBoostClassifier", best_grid_adblogis.best_estimator_)]
print('='*30 +'AFTER TUNED MODEL' + '='*30)
 mean_score = []
sd_score = []
model_name = []
for name, clf in clf_list:
final_pipe = make_pipeline(preprocess, clf)
 score = cross_val_score(final_pipe, X, y, scoring = 'accuracy', cv = skf, n_jobs = -1)
 print(f'{name:<25} {score.mean() :.5f} ±{score.std() :.3f}')
 mean_score.append(score.mean())
 sd_score.append(score.std())
 model_name.append(name)
 base = pd.DataFrame({'model':model_name, 'mean_score':mean_score, 'sd_score':sd_score})
 sns.barplot(x="mean_score", y="model", data=base, orient = 'h',**{'xerr': sd_score})
 plt.title('Baseline: Cross Validation Accuracy Scores')
 plt.axvline(x = np.mean(mean_score), color = 'firebrick', linestyle = '--');
print('
Total Mean Score: {:.5f}'.format(np.mean(mean_score)) )
gamma = 2.0
alpha=.25
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
pt_1 = y_pred * y_true
pt_1 = K.clip(pt_1, epsilon, 1-epsilon)
 CE_1 = -K.log(pt_1)
 FL_1 = alpha* K.pow(1-pt_1, gamma)* CE_1
pt_0 =(1-y_pred)*(1-y_true)
 pt_0 = K.clip(pt_0, epsilon, 1-epsilon)
 CE_0 = -K.log(pt_0)
 FL_0 =(1-alpha)* K.pow(1-pt_0, gamma)* CE_0
loss = K.sum(FL_1, axis=1)+ K.sum(FL_0, axis=1)
 return loss<categorify>
test_final_transformed = pd.DataFrame(preprocess.fit_transform(test_final), columns = preprocess.transformers_[0][2] + list(preprocess.transformers_[1][1][1].get_feature_names()))
test_final_transformed.sample(7 )
def mixup_data(x, y, alpha=1.0):
if alpha > 0:
lam = np.random.beta(alpha, alpha)
 else:
lam = 1
sample_size = x.shape[0]
index_array = np.arange(sample_size)
 np.random.shuffle(index_array)
 mixed_x = lam * x +(1 - lam)* x[index_array]
mixed_y =(lam * y)+(( 1 - lam)* y[index_array])
 return mixed_x, mixed_y
def make_batches(size, batch_size):
nb_batch = int(np.ceil(size/float(batch_size)))
 return [(i*batch_size, min(size,(i+1)*batch_size)) for i in range(0, nb_batch)]
def batch_generator(X,y,batch_size=128,shuffle=True,mixup=False):
y = np.array(y)
 sample_size = X.shape[0]
index_array = np.arange(sample_size)
 while True:
if shuffle:
np.random.shuffle(index_array)
 batches = make_batches(sample_size, batch_size)
 for batch_index,(batch_start, batch_end)in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
X_batch = X[batch_ids]
y_batch = y[batch_ids]
if mixup:
X_batch,y_batch = mixup_data(X_batch,y_batch,alpha=1.0)
 yield X_batch,y_batch<choose_model_class>
best_lr = best_grid_lr.best_estimator_
best_knn = best_grid_knn.best_estimator_
best_svc = best_grid_svc.best_estimator_
best_rf = best_grid_rf.best_estimator_
best_xgb = best_grid_xgb.best_estimator_
best_ada = best_grid_adblogis.best_estimator_
estimators = [('lr', best_lr),('knn', best_knn),('svc', best_svc), 
('rf', best_rf),('xgb', best_xgb),('dt', best_ada)]
tuned_voting_hard = VotingClassifier(estimators = estimators, voting = 'hard', n_jobs = -1)
 tuned_voting_soft = VotingClassifier(estimators = estimators, voting = 'soft', n_jobs = -1)
 tuned_voting_hard.fit(X_transformed, y)
 tuned_voting_soft.fit(X_transformed, y)
 cv_hard = cross_val_score(tuned_voting_hard, X_transformed, y, cv = skf)
 cv_soft = cross_val_score(tuned_voting_soft, X_transformed, y, cv = skf)
 print('Tuned Models - Ensemble
-----------------------')
 print('Hard Voting: {}%'.format(np.round(cv_hard.mean() *100, 2)))
 print('Soft Voting: {}%'.format(np.round(cv_soft.mean() *100, 2)))
 y_pred_tuned_hd = tuned_voting_hard.predict(test_final_transformed ).astype(int)
 y_pred_tuned_sf = tuned_voting_soft.predict(test_final_transformed ).astype(int )
model = create_model(( NUM_FEATURES,1), 1)
 model.compile(loss='binary_crossentropy', optimizer='adam')
 model.summary()
checkpoint = ModelCheckpoint('feed_forward_model.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='min', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 callbacks_list = [checkpoint, reduceLROnPlat, early]
tr_gen = batch_generator(x_train,y_train,batch_size=BATCH_SIZE,shuffle=True,mixup=True)
 history = model.fit_generator(
 tr_gen,
steps_per_epoch=np.ceil(float(len(x_train)) / float(BATCH_SIZE)) ,
epochs=10,
verbose=1,
callbacks=callbacks_list,
validation_data=(x_valid, y_valid))<predict_on_test>
base_voting_hard.fit(X_transformed, y)
 base_voting_soft.fit(X_transformed, y)
 y_pred_base_hard = base_voting_hard.predict(test_final_transformed)
 y_pred_base_soft = base_voting_hard.predict(test_final_transformed )
model.load_weights('feed_forward_model.h5')
 prediction = model.predict(x_valid, batch_size=512, verbose=1)
 roc_auc_score(y_valid, prediction )<concatenate>
test_df = pd.DataFrame(pd.read_csv('.. /input/titanic/test.csv')['PassengerId'])
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_base_hard.astype(int)} ).to_csv('01-Baseline_Hard_voting.csv', index = False)
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_base_soft.astype(int)} ).to_csv('02-Baseline_Soft_voting.csv', index = False)
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_tuned_hd.astype(int)} ).to_csv('03-Tuned_Hard_Voting.csv', index = False)
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_tuned_sf.astype(int)} ).to_csv('04-Tuned_Soft_Voting.csv', index = False )