Kaggle competition: Titanic - Machine Learning from Disaster
history = model.fit(train_df.values, y_value.values, nb_epoch=30, batch_size = 64, verbose=1)
 predict = model.predict(test_df.values )<save_to_csv>
train_df = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_df.head()
submission_nn = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_nn['Cover_Type'] = predict
submission_nn.to_csv('nn_last.csv', index=False )<merge>
%matplotlib inline
def calculate_correlation(base_df, target_df):
source = base_df.copy()
source = source.merge(target_df,on='ID')
 corr_df = source.corr()
corr = corr_df.ix['Cover_Type_x']['Cover_Type_y']
del corr_df, source
return corr<merge>
test_df = pd.read_csv('.. /input/titanic/test.csv' )
source = submission.copy()
source = source.merge(submission_nn,on='ID')
 source<compute_test_metric>
combine = [train_df, test_df]
calculate_correlation(submission, submission_nn )<train_model>
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape
class SklearnWrapper(object):
def __init__(self, clf, params=None, **kwargs):
seed = kwargs.get('seed', 0)
 params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train, x_cross=None, y_cross=None):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict_proba(x)[:,1]<choose_model_class>
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()
class LgbmWrapper(object):
def __init__(self, params=None, **kwargs):
seed = kwargs.get('seed', 0)
 num_rounds = kwargs.get('num_rounds', 1000)
 early_stopping = kwargs.get('ealry_stopping', 100)
 eval_function = kwargs.get('eval_function', None)
 verbose_eval = kwargs.get('verbose_eval', 100)
 self.param = params
self.param['seed'] = seed
self.num_rounds = num_rounds
self.early_stopping = early_stopping
self.eval_function = eval_function
self.verbose_eval = verbose_eval
def train(self, x_train, y_train, x_cross=None, y_cross=None):
need_cross_validation = True
if x_cross is None:
need_cross_validation = False
if isinstance(y_train, pd.DataFrame)is True:
y_train = y_train[y_train.columns[0]]
if need_cross_validation is True:
y_cross = y_cross[y_cross.columns[0]]
if need_cross_validation is True:
dtrain = lgbm.Dataset(x_train, label=y_train, silent=True)
 dvalid = lgbm.Dataset(x_cross, label=y_cross, silent=True)
 self.clf = lgbm.train(self.param, train_set=dtrain, num_boost_round=self.num_rounds, valid_sets=dvalid,
feval=self.eval_function, early_stopping_rounds=self.early_stopping,
verbose_eval=self.verbose_eval)
 else:
dtrain = lgbm.Dataset(x_train, label=y_train, silent= True)
 self.clf = lgbm.train(self.param, dtrain, self.num_rounds)
 def predict(self, x):
return self.clf.predict(x, num_iteration=self.clf.best_iteration)
 def get_params(self):
return self.param<prepare_x_and_y>
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
def get_oof(clf, x_train, y_train, x_test, eval_func, **kwargs):
nfolds = kwargs.get('NFOLDS', 5)
 kfold_shuffle = kwargs.get('kfold_shuffle', True)
 kfold_random_state = kwargs.get('kfold_random_sate', 0)
 ntrain = x_train.shape[0]
ntest = x_test.shape[0]
kf = StratifiedKFold(n_splits= nfolds, shuffle=kfold_shuffle, random_state=kfold_random_state)
 oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( nfolds, ntest))
cv_sum = 0
try:
if clf.clf is not None:
print(clf.clf)
 except:
print(clf)
 print(clf.get_params())
 for i,(train_index, cross_index)in enumerate(kf.split(x_train, y_train)) :
x_tr, x_cr = None, None
y_tr, y_cr = None, None
if isinstance(x_tr, pd.DataFrame):
x_tr, x_cr = x_train.iloc[train_index], x_train.iloc[cross_index]
y_tr, y_cr = y_train.iloc[train_index], y_train.iloc[cross_index]
else:
x_tr, x_cr = x_train[train_index], x_train[cross_index]
y_tr, y_cr = y_train[train_index], y_train[cross_index]
clf.train(x_tr, y_tr, x_cr, y_cr)
 oof_train[cross_index] = clf.predict(x_cr)
 cv_score = eval_func(y_cr, oof_train[cross_index])
 print('Fold %d / ' %(i+1), 'CV-Score: %.6f' % cv_score)
 cv_sum = cv_sum + cv_score
score = cv_sum / nfolds
print("Average CV-Score: ", score)
 clf.train(x_train, y_train)
 oof_test = clf.predict(x_test)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )<init_hyperparams>
train_df = train_df.drop(['Parch', 'SibSp'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp'], axis=1)
 combine = [train_df, test_df]
train_df.head()
lgbm_param1 = {
'boosting_type': 'dart',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 31,
"max_depth": 7,
"colsample_bytree": 0.8,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8,
'drop_rate':0.1,
'skip_drop':0.5,
'max_drop':50,
'top_rate':0.1,
'other_rate':0.1
}
lgbm_param2 = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 10,
"max_depth": 4,
"colsample_bytree": 0.5,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8
}
lgbm_param3 = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 24,
"max_depth": 6,
"colsample_bytree": 0.5,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8
}
rf_params = {
'criterion':'gini', 'max_leaf_nodes':24, 'n_estimators':200, 'min_impurity_split':0.0000001,
'max_features':0.4, 'max_depth':6, 'min_samples_leaf':20, 'min_samples_split':2,
'min_weight_fraction_leaf':0.0, 'bootstrap':True,
'random_state':1, 'verbose':False
}
et_parmas = {
'criterion':'gini', 'max_leaf_nodes':31, 'n_estimators':200, 'min_impurity_split':0.0000001,
'max_features':0.6, 'max_depth':10, 'min_samples_leaf':20, 'min_samples_split':2,
'min_weight_fraction_leaf':0.0, 'bootstrap':True,
'random_state':1, 'verbose':False
}<merge>
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port
org_train_df, org_test_df = tree_data_preprocessing(train, test)
 train_df = org_train_df.copy()
test_df = org_test_df.copy()<prepare_x_and_y>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port )
y_value = train_df["Cover_Type"]
del train_df["Cover_Type"], train_df["ID"]
del test_df["Cover_Type"], test_df["ID"]<choose_model_class>
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()
et_model = SklearnWrapper(clf = ExtraTreesClassifier, params=et_parmas)
 rf_model = SklearnWrapper(clf = RandomForestClassifier, params=rf_params )<categorify>
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()
et_train, et_test = get_oof(et_model, train_df.values, y_value, test_df.values, log_loss, NFOLDS=3)
 rf_train, rf_test = get_oof(rf_model, train_df.values, y_value, test_df.values, log_loss, NFOLDS=3 )<concatenate>
guess_ages = np.zeros(( 2,3))
guess_ages
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
x_train_second_layer = np.concatenate(( rf_train, et_train), axis=1)
 x_test_second_layer = np.concatenate(( rf_test, et_test), axis=1 )<create_dataframe>
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape
x_train = pd.DataFrame(x_train_second_layer)
 x_test = pd.DataFrame(x_test_second_layer )<split>
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
lgbm_meta_params = {
'boosting':'gbdt', 'num_leaves':28, 'learning_rate':0.03, 'min_sum_hessian_in_leaf':0.1,
'max_depth':7, 'feature_fraction':0.6, 'min_data_in_leaf':30, 'poission_max_delta_step':0.7,
'bagging_fraction':0.8, 'min_gain_to_split':0,
'objective':'binary', 'seed':1,'metric': 'binary_logloss'
}
NFOLD = 3
folds = StratifiedKFold(n_splits= NFOLD, shuffle=True, random_state=2018)
 total_score = 0
best_iteration = 0
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(x_train, y_value)) :
train_x, train_y = train_df.iloc[train_idx], y_value.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], y_value.iloc[valid_idx]
evals_result_dict = {}
dtrain = lgbm.Dataset(train_x, label=train_y)
 dvalid = lgbm.Dataset(valid_x, label=valid_y)
 clf = lgbm.train(lgbm_meta_params, train_set=dtrain, num_boost_round=5000, valid_sets=[dtrain, dvalid],
early_stopping_rounds=200, evals_result=evals_result_dict, verbose_eval=500)
 predict = clf.predict(valid_x)
 cv_score = log_loss(valid_y, predict)
 total_score += cv_score
best_iteration = max(best_iteration, clf.best_iteration)
 print('Fold {} LogLoss : {}'.format(n_fold + 1, cv_score))
lgbm.plot_metric(evals_result_dict)
 plt.show()
print("Best Iteration", best_iteration)
 print("Total LogLoss", total_score / NFOLD)
 print("Baseline model Score Diff", total_score / NFOLD - baseline_tree_score )<train_model>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest
dtrain = lgbm.Dataset(x_train, label=y_value)
 clf = lgbm.train(lgbm_meta_params, train_set=dtrain, num_boost_round=5000)
 predict_stacking = clf.predict(x_test )<save_to_csv>
scaler = StandardScaler()
scaler.fit(X_train)
 X_train = scaler.transform(X_train)
 X_test = scaler.transform(X_test )
submission_stacking = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_stacking['Cover_Type'] = predict_stacking
submission_stacking.to_csv('submission_stacking.csv', index=False )<save_to_csv>
from sklearn.neural_network import MLPClassifier
submission_et = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_et['Cover_Type'] = et_test
submission_et.to_csv('submission_et.csv', index=False )<save_to_csv>
clf = MLPClassifier(solver='lbfgs', alpha=1e-5,hidden_layer_sizes=(5, 2), random_state=1)
 clf.fit(X_train, Y_train)
 MLPClassifier(alpha=1e-05, hidden_layer_sizes=(5, 2), random_state=1, solver='lbfgs')
 Y_pred = clf.predict(X_test)
 Y_pred
submission_rf = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission_rf['Cover_Type'] = rf_test
submission_rf.to_csv('submission_rf.csv', index=False )<load_from_csv>
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
} )
<load_from_csv><EOS> 
submission.to_csv('my_submission1.csv', index=False )