Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<remove_duplicates>
%matplotlib inline
print(os.path.dirname(os.getcwd())+':', os.listdir(os.path.dirname(os.getcwd())));
print(os.getcwd() +':', os.listdir(os.getcwd())) ;
train_data=train_data.drop_duplicates()<groupby>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_df.info()
print('_'*40)
 test_df.info()
train_data[['Sex','Survived']].groupby(['Sex'], as_index=False ).mean()<count_missing_values>
print(train_df.isnull().sum())
 print('')
 print(test_df.isnull().sum() )
print("Train_data:",train_data.isnull().values.any())
 print("Test_data:",test_data.isnull().values.any() )<count_missing_values>
train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data.isnull().sum()<groupby>
train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data[['Pclass','Age']].groupby(['Pclass'], as_index=False ).mean()<feature_engineering>
train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_data.loc[(train_data['Pclass'] == 1), 'Age'] = 38
train_data.loc[(train_data['Pclass'] == 2), 'Age'] = 30
train_data.loc[(train_data['Pclass'] == 3), 'Age'] = 25
train_data['Embarked'].fillna(train_data['Embarked'].mode() [0], inplace=True )<count_missing_values>
train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_data.isnull().sum()<groupby>
data_df = pd.concat(( train_df, test_df)).reset_index(drop=True)
 print('data_df.shape = ', data_df.shape )
test_data[['Pclass','Age']].groupby(['Pclass'], as_index=False ).mean()<feature_engineering>
data_df['Sex'] = data_df['Sex'].map({'female': 1, 'male': 0} ).astype(int )
test_data.loc[(test_data['Pclass'] == 1), 'Age'] = 38
test_data.loc[(test_data['Pclass'] == 2), 'Age'] = 30
test_data.loc[(test_data['Pclass'] == 3), 'Age'] = 25
test_data['Fare'].fillna(test_data['Fare'].mean() , inplace=True )<count_missing_values>
print(data_df.Embarked.isnull().values.sum() ,'missing values')
 freq_port = data_df.Embarked.dropna().mode() [0]
print(freq_port,'= most frequent');
data_df['Embarked'] = data_df['Embarked'].fillna(freq_port)
 data_df[['Embarked', 'Survived']].groupby(['Embarked'],
as_index=False ).mean().sort_values(by='Survived', ascending=False )
print("Train_data:",train_data.isnull().values.any())
 print("Test_data:",test_data.isnull().values.any() )<categorify>
print(data_df.Age.isnull().values.sum() , 'missing values')
 guess_ages = np.zeros(( 2,3)) ;
for i in range(0, 2):
for j in range(0, 3):
guess_df = data_df[(data_df['Sex'] == i)& 
(data_df['Pclass'] == j+1)]['Age'].dropna()
age_mean = guess_df.mean()
age_std = guess_df.std()
age_guess = np.random.normal(age_mean, age_std)
 guess_ages[i,j] = int(( age_guess/0.5 + 0.5)* 0.5)
 for i in range(0, 2):
for j in range(0, 3):
data_df.loc[(data_df.Age.isnull())&(data_df.Sex == i)& 
(data_df.Pclass == j+1),'Age'] = guess_ages[i,j]
data_df['Age'] = data_df['Age'].astype(int)
 print(data_df.Age.isnull().values.sum() , 'missing values')
train_data['Sex']=train_data['Sex'].map({'male': 1,'female':2})
 test_data['Sex']=test_data['Sex'].map({'male': 1,'female':2})
 train_data['Embarked']=train_data['Embarked'].map({'S':1, 'C':2, 'Q':3})
 test_data['Embarked']=test_data['Embarked'].map({'S':1, 'C':2, 'Q':3} )<prepare_x_and_y>
data_df.loc[data_df['Age'] <= 16, 'AgeBin'] = 0
data_df.loc[(data_df['Age'] > 16)&(data_df['Age'] <= 32), 'AgeBin'] = 1
data_df.loc[(data_df['Age'] > 32)&(data_df['Age'] <= 48), 'AgeBin'] = 2
data_df.loc[(data_df['Age'] > 48)&(data_df['Age'] <= 64), 'AgeBin'] = 3
data_df.loc[data_df['Age'] > 64, 'AgeBin']
data_df = data_df.drop(['AgeBand'], axis=1)
 print(data_df.AgeBin.isnull().values.sum() , 'missing values')
X_train= train_data.iloc[:, 1:]
y_train= train_data['Survived'].values.reshape(-1,1)
 X_test= test_data<predict_on_test>
print(data_df.Fare.isnull().values.sum() , 'missing values')
 data_df['Fare'].fillna(data_df['Fare'].dropna().median() , inplace=True)
 print(data_df.Fare.isnull().values.sum() , 'missing values')
 data_df['FareBand'] = pd.qcut(data_df['Fare'], 5)
 data_df[['FareBand', 'Survived']].groupby(['FareBand'],
as_index=False ).mean().sort_values(by='FareBand', ascending=True )
lg= LogisticRegression()
lg.fit(X_train, y_train.ravel())
 Y_pred= lg.predict(X_test)
 print(lg.score(X_train, y_train))<choose_model_class>
data_df.loc[ data_df['Fare'] <= 7.91, 'FareBin'] = 0
data_df.loc[(data_df['Fare'] > 7.91)&(data_df['Fare'] <= 14.454), 'FareBin'] = 1
data_df.loc[(data_df['Fare'] > 14.454)&(data_df['Fare'] <= 31), 'FareBin'] = 2
data_df.loc[ data_df['Fare'] > 31, 'FareBin'] = 3
data_df['FareBin'] = data_df['FareBin'].astype(int)
 data_df = data_df.drop(['FareBand'], axis=1)
rf = RandomForestClassifier()
rf.fit(X_train, y_train)
 Y_pred = rf.predict(X_test)
 print(rf.score(X_train, y_train))<train_model>
data_df['Title'] = data_df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Survived'], data_df['Title'] )
knn = KNeighborsClassifier()
knn.fit(X_train, y_train)
 Y_pred = knn.predict(X_test)
 print(knn.score(X_train, y_train))<choose_model_class>
data_df['Title'] = data_df['Title'].replace(['Lady', 'Countess','Capt', 'Col',
'Don', 'Dr', 'Major', 'Rev', 'Sir',
'Jonkheer', 'Dona'], 'Rare')
 data_df['Title'] = data_df['Title'].replace('Mlle', 'Miss')
 data_df['Title'] = data_df['Title'].replace('Ms', 'Miss')
 data_df['Title'] = data_df['Title'].replace('Mme', 'Mrs')
 print('missing titles = ', data_df.Title.isnull().sum())
dtree = DecisionTreeClassifier()
dtree.fit(X_train, y_train)
 Y_pred = dtree.predict(X_test)
 print(dtree.score(X_train, y_train)) 
<load_from_csv>
data_df['FamilySize'] = data_df['SibSp'] + data_df['Parch'] + 1
data_df[['FamilySize', 'Survived']].groupby(['FamilySize'],
as_index=False ).mean().sort_values(by='Survived', ascending=False )
passenger_id = pd.read_csv(".. /input/titanic/test.csv")[["PassengerId"]].values<create_dataframe>
data_df['IsAlone'] = 0
data_df.loc[data_df['FamilySize'] == 1, 'IsAlone'] = 1
data_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()
submission = {'PassengerId': passenger_id.ravel() , 'Survived': Y_pred}
submission_pred = pd.DataFrame(submission ).set_index(['PassengerId'] )<save_to_csv>
data_df['AgeBin*Pclass'] = data_df.AgeBin * data_df.Pclass
data_df[['AgeBin*Pclass', 'Survived']].groupby(['AgeBin*Pclass'], as_index=False ).mean()
submission_pred.to_csv('titanic_prediction.csv' )<load_from_csv>
data_df['NameLength'] = data_df['Name'].apply(len)
X_full = pd.read_csv('.. /input/titanic/train.csv', index_col='PassengerId')
 X_test_full = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId')
 print(X_full.columns)
 y = X_full.Survived
features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch','Fare', 'Cabin', 'Embarked']
X = X_full[features].copy()
X_test = X_test_full[features].copy()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2,
random_state=0 )<categorify>
data_df['HasCabin'] = data_df["Cabin"].apply(lambda x: 0 if type(x)== float else 1)
 data_df[['HasCabin', 'Survived']].groupby(['HasCabin'], as_index=False ).mean()
numerical_transformer = SimpleImputer(strategy='constant')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ])
 X_train = preprocessor.fit_transform(X_train)
 X_valid = preprocessor.transform(X_valid,)
 X_test = preprocessor.transform(X_test,)
<train_model>
if True:
drop_ft = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'AgeBin', 'AgeBin*Pclass',
'FareBin']
data_df = data_df.drop(drop_ft, axis=1)
 print("data_df = ", data_df.shape )
dtrain = xgb.DMatrix(X_train, label=y_train)
 dvalid = xgb.DMatrix(X_valid, label=y_valid)
 dtest = xgb.DMatrix(X_test)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
param = {'min_child_weight': 100, 'eta': 0.04, 'colsample_bytree': 0.8, 'max_depth': 100,
'subsample': 0.75, 'lambda': 2, 'nthread': -1, 'booster' : 'gbtree', 'silent': 1, 'gamma' : 0,
'eval_metric': 'rmse', 'objective': 'reg:linear'}
model = xgb.train(param, dtrain, 500, watchlist, early_stopping_rounds=250,
maximize=False, verbose_eval=15)
 preds = model.predict(dtest)
<save_to_csv>
cols = data_df.select_dtypes(exclude = [np.number] ).columns.values
print('numerical columns:', data_df.select_dtypes(include =
[np.number] ).columns.values.shape[0])
 print('categorial columns:', cols.shape[0])
 if True:
data_df = pd.get_dummies(data_df ).copy()
else:
for c in cols:
lbl = sklearn.preprocessing.LabelEncoder()
lbl.fit(list(data_df[c].values))
data_df[c] = lbl.transform(list(data_df[c].values))
print('data_df.shape = ', data_df.shape )
test_predict = np.where(preds > 0.5, 1, 0)
 test_predict = pd.DataFrame({"PassengerId":X_test_full.index,"Survived":test_predict})
 test_predict.to_csv("submission.csv",index=False)
<set_options>
print(data_df.isnull().sum())
 data_df.head()
%matplotlib inline 
<load_from_csv>
def normalize_data(data):
rs = sklearn.preprocessing.RobustScaler()
rs.fit(data)
 data = rs.transform(data)
 return data
def get_accuracy(y_target, y_pred):
y_target_class = get_classes(y_target ).reshape(-1,)
 y_pred = get_classes(y_pred ).reshape(-1,)
 return np.mean(y_target_class == y_pred)
 def get_classes(y_proba):
return np.greater(y_proba, 0.5 ).astype(np.int)
 x_train_valid = data_df.drop(['Survived'],axis=1)[:train_df.shape[0]].copy().values
y_train_valid = data_df['Survived'][:train_df.shape[0]].copy().values.reshape(-1,)
 x_test = data_df.drop(['Survived'],axis=1)[train_df.shape[0]:].copy().values
features_df = pd.DataFrame(data_df.drop(["Survived"], axis=1 ).columns)
 features_df.columns = ['Features']
print(features_df)
 print('')
 x_train_valid = normalize_data(x_train_valid)
 x_test = normalize_data(x_test)
 print('x_train_valid.shape = ', x_train_valid.shape)
 print('y_train_valid.shape = ', y_train_valid.shape)
 print('x_test.shape = ', x_test.shape )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<define_variables>
perm_array_train = np.array([])
 index_in_epoch = 0
def next_batch(batch_size, x_train, y_train):
global index_in_epoch, perm_array_train
start = index_in_epoch
index_in_epoch += batch_size
if not len(perm_array_train)== len(x_train):
perm_array_train = np.arange(len(x_train))
if index_in_epoch > x_train.shape[0]:
np.random.shuffle(perm_array_train)
 start = 0
index_in_epoch = batch_size
end = index_in_epoch
x_tr = x_train[perm_array_train[start:end]]
y_tr = y_train[perm_array_train[start:end]].reshape(-1,1)
 return x_tr, y_tr
def create_nn_graph(num_input_features = 10, num_output_features = 1):
tf.reset_default_graph()
x_size = num_input_features
y_size = num_output_features
n_n_fc1 = 1024;
n_n_fc2 = 1024;
x_data = tf.placeholder('float', shape=[None, x_size])
 y_data = tf.placeholder('float', shape=[None, y_size])
 W_fc1 = tf.Variable(tf.truncated_normal(shape = [x_size, n_n_fc1], stddev = 0.1))
b_fc1 = tf.Variable(tf.constant(0.1, shape = [n_n_fc1]))
h_fc1 = tf.nn.relu(tf.matmul(x_data, W_fc1)+ b_fc1)
 tf_keep_prob = tf.placeholder('float')
 h_fc1_drop = tf.nn.dropout(h_fc1, tf_keep_prob)
 W_fc2 = tf.Variable(tf.truncated_normal(shape = [n_n_fc1, n_n_fc2], stddev = 0.1))
b_fc2 = tf.Variable(tf.constant(0.1, shape = [n_n_fc2]))
h_fc2 = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2)+ b_fc2)
 h_fc2_drop = tf.nn.dropout(h_fc2, tf_keep_prob)
 W_fc3 = tf.Variable(tf.truncated_normal(shape = [n_n_fc2, y_size], stddev = 0.1))
b_fc3 = tf.Variable(tf.constant(0.1, shape = [y_size]))
z_pred = tf.matmul(h_fc2_drop, W_fc3)+ b_fc3
cross_entropy = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
 labels=y_data, logits=z_pred)) ;
tf_learn_rate = tf.placeholder(dtype='float', name="tf_learn_rate")
 train_step = tf.train.AdamOptimizer(tf_learn_rate ).minimize(cross_entropy)
 y_pred_proba = tf.cast(tf.nn.sigmoid(z_pred),dtype = tf.float32);
y_pred_class = tf.cast(tf.greater(y_pred_proba, 0.5),'float')
 accuracy = tf.reduce_mean(tf.cast(tf.equal(y_pred_class, y_data), 'float'))
tf_tensors = {'train_step': train_step, 'cross_entropy': cross_entropy,
'y_pred_proba': y_pred_proba,'accuracy': accuracy,
'tf_keep_prob': tf_keep_prob, 'tf_learn_rate': tf_learn_rate,
'x_data': x_data, 'y_data': y_data}
return tf_tensors
def train_nn_graph(tf_tensors, x_train, y_train, x_valid, y_valid, verbose = False):
train_step = tf_tensors['train_step']
cross_entropy = tf_tensors['cross_entropy']
y_pred_proba = tf_tensors['y_pred_proba']
accuracy = tf_tensors['accuracy']
tf_keep_prob = tf_tensors['tf_keep_prob']
tf_learn_rate = tf_tensors['tf_learn_rate']
x_data = tf_tensors['x_data']
y_data = tf_tensors['y_data']
keep_prob = 0.5;
learn_rate_range = [0.01,0.005,0.0025,0.001,0.001,0.001,0.00075,0.0005,0.00025,0.0001,
0.0001,0.0001,0.0001];
learn_rate_step = 10
batch_size = 10
n_epoch = 10
cv_num = 10
n_step = -1;
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
 for i in range(int(n_epoch*x_train.shape[0]/batch_size)) :
if i%int(learn_rate_step*x_train.shape[0]/batch_size)== 0:
n_step += 1;
learn_rate = learn_rate_range[n_step];
if verbose:
print('set learnrate = ', learn_rate)
 x_batch, y_batch = next_batch(batch_size, x_train, y_train)
 sess.run(train_step, feed_dict={x_data: x_batch, y_data: y_batch,
tf_keep_prob: keep_prob, tf_learn_rate: learn_rate})
 if verbose and i%int(1.*x_train.shape[0]/batch_size)== 0:
train_loss = sess.run(cross_entropy,feed_dict={x_data: x_train,
y_data: y_train,
tf_keep_prob: 1.0})
 train_acc = accuracy.eval(feed_dict={x_data: x_train,
y_data: y_train,
tf_keep_prob: 1.0})
 valid_loss = sess.run(cross_entropy,feed_dict={x_data: x_valid,
y_data: y_valid,
tf_keep_prob: 1.0})
 valid_acc = accuracy.eval(feed_dict={x_data: x_valid,
y_data: y_valid,
tf_keep_prob: 1.0})
 print('%.2f epoch: train/val loss = %.4f/%.4f, train/val acc = %.4f/%.4f'%(
(i+1)*batch_size/x_train.shape[0], train_loss, valid_loss, train_acc,
valid_acc))
y_train_pred_proba = y_pred_proba.eval(feed_dict={x_data: x_train,
tf_keep_prob: 1.0} ).flatten()
y_valid_pred_proba = y_pred_proba.eval(feed_dict={x_data: x_valid,
tf_keep_prob: 1.0} ).flatten()
y_test_pred_proba = y_pred_proba.eval(feed_dict={x_data: x_test,
tf_keep_prob: 1.0} ).flatten()
sess.close() ;
return(y_train_pred_proba, y_valid_pred_proba, y_test_pred_proba )
cols_target = ['obscene','insult','toxic','severe_toxic','identity_hate','threat']<filter>
if False:
y_train_pred_proba = {}
y_valid_pred_proba = {}
train_acc = {}
valid_acc = {}
tf_tensors = create_nn_graph(x_train_valid.shape[1], 1)
 cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True)
 for train_index, valid_index in kfold.split(x_train_valid):
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index] 
(y_train_pred_proba['nn'],
y_valid_pred_proba['nn'],
y_test_pred['nn'])= train_nn_graph(tf_tensors, x_train,
y_train.reshape(-1,1), x_valid,
y_valid.reshape(-1,1), False)
 train_acc['nn'] = get_accuracy(y_train_pred_proba['nn'], y_train)
 valid_acc['nn'] = get_accuracy(y_valid_pred_proba['nn'], y_valid)
 print('nn: train/valid accuracy = %.4f/%.4f'%(train_acc['nn'], valid_acc['nn']))
unlabelled_in_all = train_df[(train_df['toxic']!=1)&(train_df['severe_toxic']!=1)&(train_df['obscene']!=1)& 
(train_df['threat']!=1)&(train_df['insult']!=1)&(train_df['identity_hate']!=1)]
print('Percentage of unlabelled comments is ', len(unlabelled_in_all)/len(train_df)*100 )<filter>
logreg = sklearn.linear_model.LogisticRegression()
extra_trees = sklearn.ensemble.ExtraTreesClassifier(max_depth = 4,n_estimators=10)
 gradient_boost = sklearn.ensemble.GradientBoostingClassifier(max_depth = 4,n_estimators=10)
 random_forest = sklearn.ensemble.RandomForestClassifier(max_depth = 4,n_estimators=10)
 decision_tree = sklearn.tree.DecisionTreeClassifier(max_depth = 4)
 gaussianNB = sklearn.naive_bayes.GaussianNB()
ada_boost = sklearn.ensemble.AdaBoostClassifier(n_estimators=10)
 base_models = {'logreg': logreg, 'extra_trees': extra_trees, 'ada_boost': ada_boost,
'gradient_boost': gradient_boost, 'random_forest': random_forest,
'decision_tree': decision_tree, 'gaussianNB': gaussianNB}
take_models = ['logreg', 'extra_trees', 'gradient_boost', 'gaussianNB',
'random_forest', 'decision_tree', 'nn', 'ada_boost']
train_acc = {}
valid_acc = {}
y_test_pred_proba = {}
y_train_pred_proba = {}
y_valid_pred_proba = {}
for mn in take_models:
train_acc[mn] = 0
valid_acc[mn] = 0
y_test_pred_proba[mn] = 0
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True)
 print('Training {} models with {}-fold cross validation'.format(len(take_models), cv_num))
print('')
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
print('{}.training of models in progress'.format(i+1))
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
for mn in take_models:
if mn == 'nn':
tf_tensors = create_nn_graph(x_train_valid.shape[1], 1)
 params = train_nn_graph(tf_tensors, x_train, y_train.reshape(-1,1),
x_valid, y_valid.reshape(-1,1), False)
 train_acc['nn'] += get_accuracy(params[0], y_train)
 valid_acc['nn'] += get_accuracy(params[1], y_valid)
 y_test_pred_proba['nn'] += params[2]
else:
model = sklearn.base.clone(base_models[mn])
 model.fit(x_train, y_train)
 train_acc[mn] += get_accuracy(model.predict_proba(x_train)[:,1], y_train)
 valid_acc[mn] += get_accuracy(model.predict_proba(x_valid)[:,1], y_valid)
 y_test_pred_proba[mn] += model.predict_proba(x_test)[:,1]
print('')
 for mn in take_models:
train_acc[mn] /= cv_num
valid_acc[mn] /= cv_num
y_test_pred_proba[mn] /= cv_num
print(mn,'train/valid accuracy = %.3f/%.3f'%(train_acc[mn], valid_acc[mn]))
take_model_avg = ['logreg', 'random_forest', 'decision_tree']
train_acc['majority_vote'] = 0
valid_acc['majority_vote'] = 0
y_test_pred_proba['majority_vote'] = 0
for mn in take_model_avg:
train_acc['majority_vote'] += train_acc[mn] / len(take_model_avg)
 valid_acc['majority_vote'] += valid_acc[mn] / len(take_model_avg)
 y_test_pred_proba['majority_vote'] += y_test_pred_proba[mn] / len(take_model_avg)
 print('')
 print('Average models:', take_model_avg)
 print('Soft majority vote: train/valid accuracy = %.3f/%.3f'%(train_acc['majority_vote'],
valid_acc['majority_vote']))
no_comment = train_df[train_df['comment_text'].isnull() ]
len(no_comment )<filter>
y_test_pred_df = pd.DataFrame({})
 for key in y_test_pred_proba.keys() :
y_test_pred_df[key] = get_classes(y_test_pred_proba[key])
 corrmat = y_test_pred_df.corr()
corrmat
plt.subplots(figsize=(10,5))
plt.title('Correlation of Test Predictions')
 sns.heatmap(corrmat, vmax=1, square=True )
no_comment = test_df[test_df['comment_text'].isnull() ]
no_comment<feature_engineering>
take_models = ['logreg', 'random_forest', 'gradient_boost', 'ada_boost', 'nn']
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True)
 print('Ensemble learning with', len(take_models),'models')
 print('Pasting with', cv_num, 'out-of-fold predictions')
 print('')
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
for mn in take_models:
if mn == 'nn':
tf_tensors = create_nn_graph(x_train_valid.shape[1],1)
 params = train_nn_graph(tf_tensors, x_train, y_train.reshape(-1,1),
x_valid, y_valid.reshape(-1,1), False)
 y_train_pred_proba[mn,i] = params[0].reshape(-1,)
 y_valid_pred_proba[mn,i] = params[1].reshape(-1,)
 y_test_pred_proba[mn,i] = params[2].reshape(-1,)
 else:
model = sklearn.base.clone(base_models[mn])
 model.fit(x_train, y_train)
 y_train_pred_proba[mn,i] = model.predict_proba(x_train)[:,1]
y_valid_pred_proba[mn,i] = model.predict_proba(x_valid)[:,1]
y_test_pred_proba[mn,i] = model.predict_proba(x_test)[:,1]
print(i, mn,': train/valid accuracy = %.3f/%.3f'%(
 get_accuracy(y_train_pred_proba[mn,i], y_train),
get_accuracy(y_valid_pred_proba[mn,i], y_valid)))
 if i==0:
y_train_meta = y_valid
else:
y_train_meta = np.concatenate([y_train_meta, y_valid])
 if False:
for i in range(cv_num):
tmp = 0
for mn in take_models:
tmp += y_valid_pred_proba[mn,i] / len(take_models)
 x_test_meta += y_test_pred_proba[mn,i] /(len(take_models)*cv_num)
 x_train_meta = np.concatenate([x_train_meta, tmp])
 if True:
for i in range(cv_num):
for j,mn in enumerate(take_models):
if j==0:
tmp1 = y_valid_pred_proba[mn,i].reshape(-1,1)
 tmp2 = y_test_pred_proba[mn,i].reshape(-1,1)
 else:
tmp1 = np.concatenate([tmp1, y_valid_pred_proba[mn,i].reshape(-1,1)], axis=1)
 tmp2 = np.concatenate([tmp2, y_test_pred_proba[mn,i].reshape(-1,1)], axis=1)
 if i==0:
x_train_meta = tmp1
x_test_meta = tmp2 / cv_num
else:
x_train_meta = np.concatenate([x_train_meta, tmp1], axis=0)
 x_test_meta += tmp2 / cv_num
print('')
 print('Ensemble method: soft majority vote of all models')
 y_valid_pred_proba['majority_vote'] = np.mean(x_train_meta, axis=1)
 y_test_pred_proba['majority_vote'] = x_test_meta
print('Soft majority vote: valid accuracy = %.3f'%get_accuracy(y_train_meta,
y_valid_pred_proba['majority_vote']))
train_df['char_length'] = train_df['comment_text'].apply(lambda x: len(str(x)) )<create_dataframe>
take_meta_model = 'logreg'
print('')
 print('Ensemble method: stacking of base models using the meta-model', take_meta_model)
 print('x_train_meta.shape =', x_train_meta.shape)
 print('y_train_meta.shape =', y_train_meta.shape)
 print('x_test_meta.shape =', x_test_meta.shape)
 model = sklearn.base.clone(base_models[take_meta_model])
 model.fit(x_train_meta, y_train_meta)
 y_train_pred_proba['meta_model'] = model.predict_proba(x_train_meta)[:,1]
y_test_pred_proba['meta_model'] = model.predict_proba(x_test_meta)[:,1]
print('Meta model: train accuracy = %.3f'%get_accuracy(y_train_meta,
y_train_pred_proba['meta_model']))
data = train_df[cols_target]<feature_engineering>
y_test_submit = get_classes(y_test_pred_proba['meta_model'])
 y_test_submit.mean()
test_df['char_length'] = test_df['comment_text'].apply(lambda x: len(str(x)) )<categorify>
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": y_test_submit
})
 submission.to_csv('submission.csv', index=False)
 submission.head()