Kaggle competition: Titanic - Machine Learning from Disaster
space_RF ={
'n_estimators' : hp.choice('n_estimators', np.arange(10, 200, dtype=int)) ,
'max_depth' : hp.choice("max_depth", np.arange(3, 15, dtype=int)) ,
'min_samples_split' : hp.choice("min_samples_split", np.arange(20, 100, dtype=int)) ,
'min_samples_leaf' : hp.choice("min_samples_leaf", np.arange(10, 100, dtype=int)) ,
'criterion' : hp.choice('criterion', ["gini", "entropy"]),
'class_weight' : hp.choice('class_weight', ['balanced_subsample', None]),
'n_jobs' : -1,
'oob_score' : True,
'random_state' : hp.randint('random_state',2000000)
 } 
<compute_train_metric>
def read_and_concat_dataset(training_path, test_path):
train = pd.read_csv(training_path)
 train['train'] = 1
test = pd.read_csv(test_path)
 test['train'] = 0
data = train.append(test, ignore_index=True)
 return train, test, data
train, test, data = read_and_concat_dataset('/kaggle/input/titanic/train.csv', '/kaggle/input/titanic/test.csv')
 data = data.set_index('PassengerId')
 data.info()
def objective_RF(space):
global obj_call_count, cur_best_score, X_train, y_train, test, X_test, y_test, test_id
obj_call_count += 1
print('
LightGBM objective call
sorted_params = sorted(space.items() , key=lambda z: z[0])
 print('Params:', str.join(' ', ['{}={}'.format(k, v)for k, v in sorted_params if not k.startswith('column:')]))
params = sample(space)
 mdl = RandomForestClassifier(**params)
 cv_score = cross_val_score(mdl, X_train, y_train ).mean()
print('CV finished ;
cv_score={:7.5f}'.format(cv_score))
_model = mdl.fit(X_train, y_train)
 predictions = _model.predict_proba(X_test)[:,1]
score = roc_auc_score(y_test, predictions)
 print('valid score={}'.format(score))
do_submit = score > 0.64
if score > cur_best_score:
cur_best_score = score
print('NEW BEST SCORE={}'.format(cur_best_score))
do_submit = True
if do_submit:
submit_guid = uuid.uuid4()
print('Compute submissions guid={}'.format(submit_guid))
y_submission = _model.predict_proba(test_ext)[:,1]
submission_filename = 'rf_score_{:13.11f}_submission_guid_{}.csv'.format(score,submit_guid)
 pd.DataFrame(
 {'ids':test_id, 'prob':y_submission}
).to_csv(submission_filename, index=False)
 loss = 1 - score
return {'loss': loss, 'status': STATUS_OK}<choose_model_class>
def counting_values(data, variable1, variable2):
return data[[variable1, variable2]][data[variable2].isnull() ==False].groupby([variable1], as_index=False ).mean().sort_values(by=variable2, ascending=False )
trials = Trials()
best = fmin(fn=objective_RF,
space=space_RF,
algo=HYPEROPT_ALGO,
max_evals=N_HYPEROPT_PROBES,
trials=trials,
verbose=1)
 print('-'*50)
 print('The best params for RF:')
 print(best)
 print('
' )<define_search_space>
counting_values(data, 'Sex','Survived' )
space_XGB ={
'max_depth' : hp.choice("max_depth", np.arange(5, 15,dtype=int)) ,
'learning_rate' : hp.loguniform('learning_rate', -4.9, -3.0),
'n_estimators' : hp.choice('n_estimators', np.arange(10, 100,dtype=int)) ,
'objective' : 'binary:logistic',
'booster' : 'gbtree',
'reg_alpha' : hp.uniform('reg_alpha', 1e-5, 1e-1),
'reg_lambda' : hp.uniform('reg_lambda', 1e-5, 1e-1),
'colsample_bytree' : hp.uniform('colsample_bytree', 0.5, 0.8),
'min_child_weight ': hp.uniform('min_child_weight', 0.5, 0.8),
'random_state' : hp.randint('random_state',2000000)
 }<compute_train_metric>
data['Women'] = np.where(data.Sex=='female',1,0)
 comparing(data, 'Women','Survived' )
def objective_XGB(space):
global obj_call_count, cur_best_score, X_train, y_train, test, X_test, y_test, test_id
obj_call_count += 1
print('
LightGBM objective call
sorted_params = sorted(space.items() , key=lambda z: z[0])
 print('Params:', str.join(' ', ['{}={}'.format(k, v)for k, v in sorted_params if not k.startswith('column:')]))
params = sample(space)
 mdl = XGBClassifier(**params)
 cv_score = cross_val_score(mdl, X_train, y_train ).mean()
print('CV finished ;
cv_score={:7.5f}'.format(cv_score))
_model = mdl.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_test, y_test)], eval_metric='auc',verbose=True,early_stopping_rounds =30)
 params.update({'n_estimators': _model.best_iteration})
 predictions = _model.predict_proba(X_test)[:,1]
score = roc_auc_score(y_test, predictions)
 print('valid score={}'.format(score))
do_submit = score > 0.64
if score > cur_best_score:
cur_best_score = score
print('NEW BEST SCORE={}'.format(cur_best_score))
do_submit = True
if do_submit:
submit_guid = uuid.uuid4()
print('Compute submissions guid={}'.format(submit_guid))
y_submission = _model.predict_proba(test_ext)[:,1]
submission_filename = 'xgb_score_{:13.11f}_submission_guid_{}.csv'.format(score,submit_guid)
 pd.DataFrame(
 {'ids':test_id, 'prob':y_submission}
).to_csv(submission_filename, index=False)
 loss = 1 - score
return {'loss': loss, 'status': STATUS_OK}<choose_model_class>
data.isnull().sum()
trials = Trials()
best = fmin(fn=objective_XGB,
space=space_XGB,
algo=HYPEROPT_ALGO,
max_evals=N_HYPEROPT_PROBES,
trials=trials,
verbose=1)
 print('-'*50)
 print('The best params for XGB:')
 print(best)
 print('
' )<create_dataframe>
data.isnull().sum()
train_stack = train_backup.copy()
test_stack = test_backup.copy()<filter>
data.groupby('Pclass' ).Fare.mean()
data.Fare = data.Fare.fillna(0 )
meta_data = get_meta(train_stack)
 col_ordinal = meta_data[(meta_data.level == 'ordinal')&(meta_data.keep)&(meta_data.role != 'target')].index
col_nominal = meta_data[(meta_data.level == 'nominal')&(meta_data.keep)&(meta_data.role != 'target')].index
col_interval = meta_data[(meta_data.level == 'interval')&(meta_data.keep)&(meta_data.role != 'target')].index
col_binary = meta_data[(meta_data.level == 'binary')&(meta_data.keep)&(meta_data.role != 'target')].index 
<correct_missing_values>
print(data.Embarked.value_counts())
 data.Embarked = data.Embarked.fillna('S' )
train_stack = train_stack.replace(-1, np.NaN)
 d_median = train_stack.median(axis=0)
 d_mean = train_stack.mean(axis=0)
 train_stack = train_stack.fillna(-1 )<import_modules>
data.Cabin = data.Cabin.fillna('Unknown_Cabin')
 data['Cabin'] = data['Cabin'].str[0]
from sklearn import preprocessing<categorify>
print(data.Cabin.value_counts() )
def transform(df, ohe, d_median, d_mean):
dcol = [c for c in d_median.index if c in d_mean.index and c !='target']
for c in dcol:
if 'bin_' not in c:
df[c+'_median_range'] =(df[c].values > d_median[c])
 df[c+'_mean_range'] =(df[c].values > d_mean[c])
 for c in one_hot:
if len(one_hot[c])>2 and len(one_hot[c])< 7:
for val in one_hot[c]:
df[c+'_oh_' + str(val)] =(df[c].values == val)
 return df<categorify>
data.groupby('Pclass' ).Cabin.value_counts()
train_stack = transform(train_stack, one_hot,d_median, d_mean )<categorify>
data['Cabin'] = np.where(( data.Pclass==1)&(data.Cabin=='U'),'C',
np.where(( data.Pclass==2)&(data.Cabin=='U'),'D',
np.where(( data.Pclass==3)&(data.Cabin=='U'),'G',
np.where(data.Cabin=='T','C',data.Cabin))))
test_stack = transform(test_stack, one_hot, d_median, d_mean )<feature_engineering>
data['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(data['Title'], data['Sex'])
 data = data.drop('Name',axis=1 )
train_stack = create_extra_features(train_stack)
 test_stack = create_extra_features(test_stack)
 train_stack['bin_sum'] = train_stack[col_binary].sum(axis=1)
 test_stack['bin_sum'] = test_stack[col_binary].sum(axis=1 )<define_variables>
data['Title'] = np.where(( data.Title=='Capt')|(data.Title=='Countess')|(data.Title=='Don')|(data.Title=='Dona')
 |(data.Title=='Jonkheer')|(data.Title=='Lady')|(data.Title=='Sir')|(data.Title=='Major')|(data.Title=='Rev')|(data.Title=='Col'),'Other',data.Title)
 data['Title'] = data['Title'].replace('Ms','Miss')
 data['Title'] = data['Title'].replace('Mlle','Miss')
 data['Title'] = data['Title'].replace('Mme','Mrs' )
col = [c for c in train_stack.columns if c not in ['id','target']]
col = [c for c in col if not c.startswith('ps_calc_')]<count_duplicates>
data.groupby('Title' ).Age.mean()
dups = train_stack[train_stack.duplicated(subset=col, keep=False)]<filter>
data['Age'] = np.where(( data.Age.isnull())&(data.Title=='Master'),5,
np.where(( data.Age.isnull())&(data.Title=='Miss'),22,
np.where(( data.Age.isnull())&(data.Title=='Mr'),32,
np.where(( data.Age.isnull())&(data.Title=='Mrs'),37,
np.where(( data.Age.isnull())&(data.Title=='Other'),45,
np.where(( data.Age.isnull())&(data.Title=='Dr'),44,data.Age))))))
train_stack = train_stack[~(train_stack.index.isin(dups.index)) ]<define_variables>
data['FamilySize'] = data.SibSp + data.Parch + 1
data['Mother'] = np.where(( data.Title=='Mrs')&(data.Parch >0),1,0)
 data['Free'] = np.where(data['Fare']==0, 1,0)
 data = data.drop(['SibSp','Parch','Sex'],axis=1 )
target_stack = train_stack['target']<drop_column>
TypeOfTicket = []
for i in range(len(data.Ticket)) :
ticket = data.Ticket.iloc[i]
for c in string.punctuation:
ticket = ticket.replace(c,"")
 splited_ticket = ticket.split(" ")
 if len(splited_ticket)== 1:
TypeOfTicket.append('NO')
 else:
TypeOfTicket.append(splited_ticket[0])
 data['TypeOfTicket'] = TypeOfTicket
data.TypeOfTicket.value_counts()
data['TypeOfTicket'] = np.where(( data.TypeOfTicket!='NO')&(data.TypeOfTicket!='PC')&(data.TypeOfTicket!='CA')& 
(data.TypeOfTicket!='A5')&(data.TypeOfTicket!='SOTONOQ'),'other',data.TypeOfTicket)
 data = data.drop('Ticket',axis=1 )
train_stack = train_stack[col]<drop_column>
counting_values(data, 'Title','Survived' )
test_stack = test_stack[col]<filter>
TypeOfTicket vs Survived
meta_data = get_meta(train_stack)
 col_ordinal = meta_data[(meta_data.level == 'ordinal')&(meta_data.keep)&(meta_data.role != 'target')].index
col_nominal = meta_data[(meta_data.level == 'nominal')&(meta_data.keep)&(meta_data.role != 'target')].index
col_interval = meta_data[(meta_data.level == 'interval')&(meta_data.keep)&(meta_data.role != 'target')].index
col_binary = meta_data[(meta_data.level == 'binary')&(meta_data.keep)&(meta_data.role != 'target')].index 
<prepare_x_and_y>
counting_values(data, 'TypeOfTicket','Survived' )
X_stack = pd.concat([train_stack[col_interval],train_stack[col_ordinal], pd.get_dummies(train_stack[col_binary])], axis=1)
 test_stack_val = pd.concat([test_stack[col_interval],test_stack[col_ordinal], pd.get_dummies(test_stack[col_binary])], axis=1)
 y_stack = target_stack<drop_column>
counting_values(data, 'Cabin','Survived' )
X_stack = X_stack.drop(columns=['gender_-1','facebook_profile_-1'], axis = 1)
<compute_test_metric>
data = pd.get_dummies(data)
 data.head(10 )
cross_val_model(X_stack, y_stack, rf_model )<compute_test_metric>
trainX, testX, trainY, testY = train_test_split(data[data.Survived.isnull() ==False].drop('Survived',axis=1),data.Survived[data.Survived.isnull() ==False],test_size=0.30, random_state=2019 )
cross_val_model(X_stack, y_stack, XGB_model )<train_model>
Results = pd.DataFrame({'Model': [],'Accuracy Score': [], 'Recall':[], 'F1score':[]})
class Ensemble(object):
def __init__(self, n_splits, stacker, base_models):
self.n_splits = n_splits
self.stacker = stacker
self.base_models = base_models
def fit_predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=2017 ).split(X, y))
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_holdout = X[test_idx]
print("Fit %s fold %d" %(str(clf ).split('(')[0], j+1))
clf.fit(X_train, y_train)
 y_pred = clf.predict_proba(X_holdout)[:,1]
S_train[test_idx, i] = y_pred
S_test_i[:, j] = clf.predict_proba(T)[:,1]
S_test[:, i] = S_test_i.mean(axis=1)
 results = cross_val_score(self.stacker, S_train, y, cv=3, scoring='roc_auc')
 print("Stacker score: %.5f" %(results.mean()))
self.stacker.fit(S_train, y)
 res = self.stacker.predict_proba(S_test)[:,1]
return res<init_hyperparams>
model = DecisionTreeClassifier(max_depth=4)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['DecisionTreeClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
rf_params = {}
rf_params['n_estimators'] = 80
rf_params['max_depth'] = 12
rf_params['min_samples_split'] = 50
rf_params['min_samples_leaf'] = 23 
<init_hyperparams>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
xgb_params = {}
xgb_params['learning_rate'] =0.03660642032718193
xgb_params['n_estimators'] = 70
xgb_params['max_depth'] = 7
xgb_params['reg_alpha'] = 0.1
xgb_params['reg_lambda'] = 0.1
xgb_params['colsample_bytree'] = 0.6162725690461764
xgb_params['min_child_weight'] = 0.751826989118936 
<choose_model_class>
model = RandomForestClassifier(n_estimators=2500, max_depth=4)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['RandomForestClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
rf_model = RandomForestClassifier(**rf_params, random_state=584867 )<choose_model_class>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
xgb_model = XGBClassifier(**xgb_params, random_state=2943 )<choose_model_class>
model = XGBClassifier(learning_rate=0.001,n_estimators=2500,
max_depth=4, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['XGBClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
log_model = LogisticRegression(random_state=29 )<choose_model_class>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
stack = Ensemble(n_splits=3,
stacker = log_model,
base_models =(rf_model, xgb_model))<predict_on_test>
trainX = data[data.Survived.isnull() ==False].drop(['Survived','train'],axis=1)
 trainY = data.Survived[data.Survived.isnull() ==False]
testX = data[data.Survived.isnull() ==True].drop(['Survived','train'],axis=1)
 model = XGBClassifier(learning_rate=0.001,n_estimators=2500,
max_depth=4, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model.fit(trainX, trainY)
 test = data[data.train==0]
test['Survived'] = model.predict(testX ).astype(int)
 test = test.reset_index()
test[['PassengerId','Survived']].to_csv("submissionXGB.csv",index=False)
 print("done1" )