Kaggle competition: Titanic - Machine Learning from Disaster
reg_alpha = [1e-5, 1e-2, 0.1, 1, 100]
hyperparams = {'reg_alpha': reg_alpha}
gd = GridSearchCV(estimator=XGBClassifier(learning_rate=0.2, n_estimator=50, max_depth=8, min_child_weight=2, gamma=0, subsample=0.7, colsample_bytree=0.65),
param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 train_Id=train['PassengerId']
train=train.drop(['PassengerId'],axis=1)
 test_Id=test['PassengerId']
test=test.drop(['PassengerId'],axis=1)
 y=train['Survived']
n_restarts_optimizer = [0, 1, 2, 3]
max_iter_predict = [1, 2, 5, 10, 20, 35, 50, 100]
warm_start = [True, False]
hyperparams = {'n_restarts_optimizer': n_restarts_optimizer, 'max_iter_predict': max_iter_predict, 'warm_start': warm_start}
gd = GridSearchCV(estimator=GaussianProcessClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
alldata=pd.concat([train,test])
n_estimators = [10, 100, 200, 500]
learning_rate = [0.001, 0.01, 0.1, 0.5, 1, 1.5, 2]
hyperparams = {'n_estimators': n_estimators, 'learning_rate': learning_rate}
gd = GridSearchCV(estimator=AdaBoostClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
alldata.isna().sum()
n_neighbors = [1, 2, 3, 4, 5]
algorithm = ['auto']
weights = ['uniform', 'distance']
leaf_size = [1, 2, 3, 4, 5, 10]
hyperparams = {'algorithm':algorithm, 'weights': weights, 'leaf_size': leaf_size, 'n_neighbors': n_neighbors}
gd=GridSearchCV(estimator=KNeighborsClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
alldata['Fare']=alldata['Fare'].fillna(alldata['Fare'].mode().values[0])
 alldata['Age']=alldata['Age'].fillna(alldata['Age'].median())
 alldata['Embarked']=alldata['Embarked'].fillna(alldata['Embarked'].mode().values[0])
n_estimators = [10, 50, 100, 200]
max_depth = [3, None]
max_features = [0.1, 0.2, 0.5, 0.8]
min_samples_split = [2, 6]
min_samples_leaf = [2, 6]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd = GridSearchCV(estimator=RandomForestClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
alldata['Surename']=alldata.Name.apply(lambda x: x.split(',')[0])
 alldata['Title']=alldata.Name.apply(lambda x: x.split(',')[1].split('.')[0])
 alldata['SurrnameFreq']=alldata.Surename.apply(lambda x: alldata.groupby('Surename' ).count().Age[x])
 alldata['Deck']=alldata.Cabin.apply(lambda x: str(x)[0])
 alldata['Family_Size']=alldata['SibSp']+alldata['Parch']
alldata['Age*Class']=alldata['Age']*alldata['Pclass']
alldata['Fare_Per_Person']=alldata['Fare']/(alldata['Family_Size']+1)
 alldata['FareBin'] = pd.qcut(alldata['Fare'], 4)
 alldata['AgeBin'] = pd.cut(alldata['Age'].astype(int), 5)
 alldata=alldata.drop(['Cabin','Name','Ticket','Surename'],axis=1 )
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [0.1, 0.2, 0.5, 0.8]
min_samples_split = [2, 10]
min_samples_leaf = [2, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features, 'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd = GridSearchCV(estimator=ExtraTreesClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
alldata.isna().sum()
n_estimators = [10, 50, 75, 100, 200]
max_samples = [0.1, 0.2, 0.5, 0.8, 1.0]
max_features = [0.1, 0.2, 0.5, 0.8, 1.0]
hyperparams = {'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features}
gd = GridSearchCV(estimator=BaggingClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<choose_model_class>
alldata=pd.get_dummies(alldata)
 alldata.info()
ran = RandomForestClassifier(max_depth=None, max_features=0.1, min_samples_leaf=2, min_samples_split=6, n_estimators=100, random_state=1)
 knn = KNeighborsClassifier(leaf_size=1, n_neighbors=5, weights='distance')
 log = LogisticRegression(C=1.0, penalty='l2')
 xgb = XGBClassifier(learning_rate=0.2, n_estimators=50, max_depth=8, min_child_weight=2, gamma=0, subsample=0.7, colsample_bytree=0.65, reg_alpha=1)
 gbc = GradientBoostingClassifier(learning_rate=0.2, max_depth=3, n_estimators=1000)
 svc = SVC(probability=True, gamma=0.001, C=20)
 ext = ExtraTreesClassifier(max_depth=None, max_features=0.1, min_samples_leaf=2, min_samples_split=2, n_estimators=50, random_state=1)
 ada = AdaBoostClassifier(learning_rate=0.5, n_estimators=200, random_state=1)
 gpc = GaussianProcessClassifier(max_iter_predict=1, n_restarts_optimizer=0, warm_start=True)
 bag = BaggingClassifier(max_features=0.5, max_samples=1.0, n_estimators=75, random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
model_names = ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier']
scores3 = {}
for ind, mod in enumerate(models):
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores3[model_names[ind]] = acc<choose_model_class>
num_feat=alldata.dtypes[alldata.dtypes!="object"].index
scX = StandardScaler()
alldata[num_feat] = scX.fit_transform(alldata[num_feat].values)
grid_hard = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbor', knn), 
('Bagging Classifier', bag)], voting='hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train, y_train, cv=10)
 grid_hard.fit(X_train, y_train)
 print("Hard voting on test set score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))<choose_model_class>
from sklearn.model_selection import train_test_split,KFold,cross_val_score
grid_soft = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbor', knn), 
('Bagging Classifier', bag)], voting='soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train, y_train, cv=10)
 grid_soft.fit(X_train, y_train)
 print("Soft voting on test set score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))<save_to_csv>
alldata=alldata.drop(['Survived'],axis=1)
 train=alldata[:len(train)]
test=alldata[len(train):]
predictions = grid_hard.predict(X_test)
 submission = pd.concat([pd.DataFrame(passId), pd.DataFrame(predictions)], axis='columns')
 submission.columns = ["PassengerId", "Survived"]
submission.to_csv('titanic_submission1.csv', header=True, index=False )<save_to_csv>
kfolds = KFold(n_splits=5, shuffle=True,random_state=1)
 def acc_cv(model):
acc= cross_val_score(model, train.values, y.values, scoring="accuracy", cv = kfolds.get_n_splits(train.values))
return acc
predictions = grid_soft.predict(X_test)
 submission = pd.concat([pd.DataFrame(passId), pd.DataFrame(predictions)], axis='columns')
 submission.columns = ["PassengerId", "Survived"]
submission.to_csv('titanic_submission2.csv', header=True, index=False )<set_options>
X_train, X_test, y_train, y_test = train_test_split(train.values, y.values, test_size=0.4,random_state=100)
%matplotlib inline 
<load_from_csv>
class StackNet(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models,meta_final_model, meta_models1=None, meta_models2=None,add_prev_out=True, n_folds=10):
self.base_models = base_models
self.meta_models1 = meta_models1
self.meta_models2 = meta_models2
self.meta_final_model=meta_final_model
self.n_folds = n_folds
self.add_prev_out=add_prev_out
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
if self.meta_models1!=None:
self.meta_models1_ = [list() for x in self.meta_models1]
if self.meta_models2!=None:
self.meta_models2_ = [list() for x in self.meta_models2]
self.meta_final_model_=clone(self.meta_final_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions=np.hstack(( out_of_fold_predictions,X))
if self.meta_models1!=None:
out_of_fold_predictions2 = np.zeros(( X.shape[0],len(self.meta_models1)))
 if self.meta_models1!=None:
for i, model in enumerate(self.meta_models1):
for train_index, holdout_index in kfold.split(out_of_fold_predictions, y):
instance = clone(model)
 self.meta_models1_[i].append(instance)
 instance.fit(out_of_fold_predictions[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions[holdout_index])
 out_of_fold_predictions2[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions2=np.hstack(( out_of_fold_predictions2,X))
else:
out_of_fold_predictions2=out_of_fold_predictions
if self.meta_models2!=None:
out_of_fold_predictions3 = np.zeros(( X.shape[0],len(self.meta_models2)))
 if self.meta_models2!=None:
for i, model in enumerate(self.meta_models2):
for train_index, holdout_index in kfold.split(out_of_fold_predictions2, y):
instance = clone(model)
 self.meta_models2_[i].append(instance)
 instance.fit(out_of_fold_predictions2[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions2[holdout_index])
 out_of_fold_predictions3[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions3=np.hstack(( out_of_fold_predictions3,X))
else:
out_of_fold_predictions3=out_of_fold_predictions2
self.meta_final_model_.fit(out_of_fold_predictions3, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_])
 if self.add_prev_out:
meta_features=np.hstack(( meta_features,X))
if self.meta_models1!=None:
meta_features2 = np.column_stack([
np.column_stack([model.predict(meta_features)for model in meta_models1_] ).mean(axis=1)
 for meta_models1_ in self.meta_models1_ ])
 if self.add_prev_out:
meta_features2=np.hstack(( meta_features2,X))
else:
meta_features2=meta_features
if self.meta_models2!=None:
meta_features3 = np.column_stack([
np.column_stack([model.predict(meta_features2)for model in meta_models2_] ).mean(axis=1)
 for meta_models2_ in self.meta_models2_ ])
 if self.add_prev_out:
meta_features3=np.hstack(( meta_features3,X))
else:
meta_features3=meta_features2
return self.meta_final_model_.predict(meta_features3 )
train_df = pd.read_csv(".. /input/titanic/train.csv")
 test_df = pd.read_csv(".. /input/titanic/test.csv")
 combine = [train_df, test_df]<count_missing_values>
lr=LogisticRegression(random_state=1)
 xgbm=xgb.XGBClassifier(objective='binary:hinge',random_state=1)
 lgbmm=lgb.LGBMClassifier(objective='huber',random_state=1)
 gbc=GradientBoostingClassifier(random_state=1)
 adc=AdaBoostClassifier(random_state=1)
 rf=RandomForestClassifier(random_state=1,n_jobs=-1,n_estimators=100)
 bc=BaggingClassifier(xgb.XGBClassifier(objective='binary:hinge',random_state=1),n_estimators =3,random_state=1)
 bc.fit(X_train,y_train)
 print(bc.__class__.__name__,accuracy_score(bc.predict(X_test),y_test))
sn=StackNet(( lr,gbc,adc,rf,bc,lgbmm,xgbm),xgbm)
 sn.fit(X_train,y_train)
 print(sn.__class__.__name__,accuracy_score(sn.predict(X_test),y_test))
train_df.isnull().sum()<count_missing_values>
sn.fit(train.values,y.values)
 submission = pd.read_csv('.. /input/gender_submission.csv')
 submission['Survived'] = sn.predict(test.values)
 submission.to_csv('submission.csv', index=False )