Kaggle competition: Titanic - Machine Learning from Disaster
def build_network() :
network = Sequential([
layers.Dense(64, activation='relu', kernel_regularizer=regularizers.l2(_lambda)) ,
layers.Dropout(rate=0.5),
layers.Dense(32, activation='relu', kernel_regularizer=regularizers.l2(_lambda)) ,
layers.Dropout(rate=0.5),
layers.Dense(1, activation='sigmoid')
 ])
 network.build(input_shape=(batchsz, 8))
network.compile(optimizer=optimizers.Adam(lr=0.001),loss=losses.BinaryCrossentropy(from_logits=False), metrics='accuracy')
 network.summary()
return network<split>
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import tree
import graphviz
train = all_data[0:891]
test = all_data[891:len(all_data)]
x_train, y_train = TrainPreprocess(train)
 x_test = TestPreprocess(test)
 db_train = tf.data.Dataset.from_tensor_slices(( x_train, y_train))
db_train = db_train.shuffle(1000 ).batch(batchsz)
 network = build_network()
network.fit(db_train, epochs=300)
 y_test = network.predict(x_test)
 print(y_test)
 with open('survival_rate.csv','w+',newline='')as f:
csv_file = csv.writer(f)
 csv_file.writerows(y_test)
 my_prediction = []
for i in range(len(y_test)) :
my_prediction.append(1 if(y_test[i] > 0.5)else 0)
 my_submission = pd.DataFrame({'PassengerId': PassengerId, 'Survived': my_prediction})
 my_submission.to_csv('my_submission.csv', index=False, sep=',' )<set_options>
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
import seaborn as sns
from sklearn.metrics import confusion_matrix, roc_curve, auc
%matplotlib inline 
<load_from_csv>
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
from xgboost import XGBClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import AdaBoostClassifier
from keras.models import Sequential,Model
from keras.layers import Dense, Activation, Embedding, concatenate, Flatten, Input
from keras.optimizers import Adam
from sklearn.preprocessing import LabelEncoder
from sklearn.svm import SVC
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<drop_column>
Titanic_data = pd.read_csv(".. /input/train.csv", header=0)
 Titanic_data_test_pID = pd.read_csv(".. /input/test.csv", header=0)
 Titanic_data_test = pd.read_csv(".. /input/test.csv", header=0 )
train.set_index('PassengerId', inplace=True)
 test.set_index('PassengerId', inplace=True )<drop_column>
for i in range(len(Titanic_data)) :
a = str(Titanic_data['Cabin'][i])
 if a.count(' ')>=1:
Titanic_data['Fare'][i] =(Titanic_data['Fare'][i]/(a.count(' ')+1))
train.drop(['Ticket','Cabin'], axis=1, inplace=True)
 test.drop(['Ticket','Cabin'], axis=1, inplace=True )<rename_columns>
for i in range(len(Titanic_data_test)) :
a = str(Titanic_data_test['Cabin'][i])
 if a.count(' ')>=1:
Titanic_data_test['Fare'][i] =(Titanic_data_test['Fare'][i]/(a.count(' ')+1))
train['Embarked'].fillna('S', inplace=True )<filter>
Titanic_data = Titanic_data.drop(['PassengerId' , 'Name' , 'Ticket','Cabin'], axis=1)
 Titanic_data_test = Titanic_data_test.drop(['PassengerId' , 'Name' , 'Ticket','Cabin'], axis=1 )
test[test['Fare'].isnull() ]<correct_missing_values>
num_attr = ['Age','Fare']
num_attr
test['Fare'].fillna(7.9, inplace=True )<categorify>
cat_attr = ['Sex','Embarked','Survived','Pclass','SibSp','Parch']
cat_attr
train['Sex'] = preprocessing.LabelEncoder().fit_transform(train['Sex'].values)
 test['Sex'] = preprocessing.LabelEncoder().fit_transform(test['Sex'].values)
 train['Embarked'] = preprocessing.LabelEncoder().fit_transform(train['Embarked'].values)
 test['Embarked'] = preprocessing.LabelEncoder().fit_transform(test['Embarked'].values )<data_type_conversions>
Titanic_data['Age'] = round(( Titanic_data['Age']),0)
 Titanic_data['Fare'] = round(( Titanic_data['Fare']),0)
 Titanic_data_test['Age'] = round(( Titanic_data_test['Age']),0)
 Titanic_data_test['Fare'] = round(( Titanic_data_test['Fare']),0 )
for dataset in train, test:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int )<data_type_conversions>
Titanic_data['Age'].fillna(( Titanic_data['Age'].median()), inplace=True)
 Titanic_data_test['Age'].fillna(( Titanic_data_test['Age'].median()), inplace=True)
 Titanic_data_test['Fare'].fillna(( Titanic_data_test['Fare'].median()), inplace=True )
train["Age"] = train[['Pclass', 'Sex', 'Age']].groupby(['Pclass', 'Sex'] ).transform(lambda x: x.fillna(x.median()))
test["Age"] = test[['Pclass', 'Sex', 'Age']].groupby(['Pclass', 'Sex'] ).transform(lambda x: x.fillna(x.median()))<feature_engineering>
for i in cat_attr:
Titanic_data[i] = Titanic_data[i].astype('category')
for dataset in train, test:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
dataset['Age'] = dataset['Age'].astype(int )<drop_column>
for i in cat_attr:
if i not in ['Survived']:
Titanic_data_test[i] = Titanic_data_test[i].astype('category' )
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True )<prepare_x_and_y>
for i in num_attr:
Titanic_data[i] = Titanic_data[i].astype('int64' )
y_train = train["Survived"]
train.drop('Survived', axis=1, inplace=True)
 train.shape, y_train.shape, test.shape<train_model>
for i in num_attr:
Titanic_data_test[i] = Titanic_data_test[i].astype('int64' )
random_forest = RandomForestClassifier()
random_forest.fit(train, y_train)
 scores = cross_validate(random_forest, train, y_train, scoring='roc_auc', cv=5, return_train_score=True)
 scores<define_search_space>
Titanic_data_test.isna().sum()
n_est = [int(x)for x in np.linspace(start = 200, stop = 2000, num = 10)]
max_feat = ['auto', 'sqrt']
max_depth = [int(x)for x in np.linspace(5, 110, num = 11)]
max_depth.append(None)
 min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
random_grid = {'n_estimators': n_est,
'max_features': max_feat,
'max_depth': max_depth,
'criterion' :['gini', 'entropy'],
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
random_grid<train_model>
Titanic_data.isna().sum()
model = RandomForestClassifier(bootstrap=False, criterion='entropy', max_depth=5,
min_samples_leaf=2, min_samples_split=10,
n_estimators=1800)
 model.fit(train, y_train)
 y_pred = model.predict(test )<save_to_csv>
y= Titanic_data["Survived"]
X= Titanic_data.drop('Survived',axis=1 )
submission = pd.DataFrame({
"PassengerId": test.index,
"Survived": y_pred
})
 submission.to_csv('./submission.csv', index=False )<load_from_csv>
X_train,X_test,y_train,y_test = train_test_split(X,y, test_size = 0.3 , random_state = 2, stratify=y)
 X_train.shape
train= pd.read_csv('.. /input/titanic/train.csv')
 test= pd.read_csv('.. /input/titanic/test.csv')
<count_missing_values>
cat_attr_x = ['Sex','Embarked','Pclass','SibSp','Parch']
cat_attr_x
X_train_imputed_dummies = pd.get_dummies(columns=cat_attr_x, data=X_train,prefix=cat_attr_x, prefix_sep="_", drop_first=True)
 X_test_imputed_dummies = pd.get_dummies(columns=cat_attr_x, data=X_test,prefix=cat_attr_x, prefix_sep="_", drop_first=True)
 Titanic_data_test_dummies = pd.get_dummies(columns=cat_attr_x, data=Titanic_data_test,prefix=cat_attr_x, prefix_sep="_", drop_first=True)
 X_train_imputed_dummies.columns
train.isnull().sum()<count_missing_values>
Titanic_data_test_dummies=Titanic_data_test_dummies.drop(['Parch_9'], axis=1 )
train.isnull().sum()<count_unique_values>
scaler = StandardScaler()
scaler.fit(X_train[num_attr])
 X_train_imputed_dummies[num_attr]=scaler.transform(X_train_imputed_dummies[num_attr])
 X_test_imputed_dummies[num_attr]=scaler.transform(X_test_imputed_dummies[num_attr])
 Titanic_data_test_dummies[num_attr]=scaler.transform(Titanic_data_test_dummies[num_attr] )
train['Embarked'].nunique()<data_type_conversions>
clf = tree.DecisionTreeClassifier(max_depth=7)
 clf = clf.fit(X_train_imputed_dummies,y_train )
def preprocess_data(data):
data= data.drop(['Name', 'Ticket'], axis=1)
 data['Sex'].replace({'male':'0', 'female':'1'}, inplace= True)
 data['Embarked'] = data['Embarked'].fillna(0)
 data['Embarked'].replace({'S':'0', 'C':'1', 'Q':'2'}, inplace= True)
 data['Sex']= data['Sex'].astype(int)
 data['Embarked']= data['Embarked'].astype(int)
 data['Age']= data['Age'].fillna(method= 'ffill')
 data["Deck"] = data["Cabin"].str.slice(0,1)
 data["Room"] = data["Cabin"].str.slice(1,5 ).str.extract("([0-9]+)", expand=False ).astype("float")
 data['Deck']= data['Deck'].fillna(0)
 data['Room']= data['Room'].fillna(0)
 data['Fare'] = data['Fare'].fillna(0)
 data['Deck'].replace({'A':'0', 'B':'1', 'C':'2', 'D':'3', 'F':'4', 'G':'5', 'T':'6','E':'7'}, inplace= True)
 data['Deck']= data['Deck'].astype(int)
 return data<categorify>
dot_data = tree.export_graphviz(clf, out_file=None,
feature_names=X_train_imputed_dummies.columns,
class_names='target',
filled=True, rounded=True, special_characters=True)
 graph = graphviz.Source(dot_data)
 graph
final_train= preprocess_data(train)
 final_test= preprocess_data(test )<drop_column>
train_pred = clf.predict(X_train_imputed_dummies)
 print(accuracy_score(y_train,train_pred))
test_pred = clf.predict(X_test_imputed_dummies)
 print(accuracy_score(y_test,test_pred))
final_train = final_train.drop(['Cabin'], axis=1)
 final_train<count_missing_values>
Titanic_data_test_pred = clf.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred })
 submission_svc.to_csv("submission_5_DT.csv", index=False )
final_test= final_test.drop(['Cabin'], axis=1)
 final_test.isnull().sum()<prepare_x_and_y>
BaggingClassifier = BaggingClassifier(max_features=8,max_samples=0.3)
 BaggingClassifier.fit(X_train_imputed_dummies, y_train)
 y_pred_train = BaggingClassifier.predict(X_train_imputed_dummies)
 y_pred = BaggingClassifier.predict(X_test_imputed_dummies)
 bagging_accy = round(accuracy_score(y_pred, y_test), 20)
 print(bagging_accy)
 bagging_accy_train = round(accuracy_score(y_pred_train, y_train), 20)
 print(bagging_accy_train )
y = final_train['Survived']
X = final_train.drop(['Survived'], axis=1 )<split>
Titanic_data_test_pred_Bg = BaggingClassifier.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred })
 submission_svc.to_csv("submission_1_Bagging.csv", index=False )
x_train, x_test, y_train, y_test = train_test_split(X, y, random_state= 42, test_size=0.20)
 lin= LinearRegression()<predict_on_test>
np.random.seed(0)
 lr = LogisticRegression(solver='lbfgs')
 gnb = GaussianNB()
svc = LinearSVC(C=1.0)
 rfc = RandomForestClassifier(n_estimators=100)
 plt.figure(figsize=(10, 10))
ax1 = plt.subplot2grid(( 3, 1),(0, 0), rowspan=2)
 ax2 = plt.subplot2grid(( 3, 1),(2, 0))
ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated")
 for clf, name in [(lr, 'Logistic'), 
(gnb, 'Naive Bayes'), 
(svc, 'Support Vector Classification'), 
(rfc, 'Random Forest')]:
clf.fit(X_train_imputed_dummies, y_train)
 if hasattr(clf, "predict_proba"):
prob_pos = clf.predict_proba(X_test_imputed_dummies)[:, 1]
else:
prob_pos = clf.decision_function(X_test_imputed_dummies)
 prob_pos = \ 
(prob_pos - prob_pos.min())/(prob_pos.max() - prob_pos.min())
 fraction_of_positives, mean_predicted_value = \
calibration_curve(y_test, prob_pos, n_bins=10)
 ax1.plot(mean_predicted_value, fraction_of_positives, "s-",
label="%s" %(name,))
ax2.hist(prob_pos, range=(0, 1), bins=10, label=name,
histtype="step", lw=2)
 ax1.set_ylabel("Fraction of positives")
 ax1.set_ylim([-0.05, 1.05])
 ax1.legend(loc="lower right")
 ax1.set_title('Calibration plots(reliability curve)')
 ax2.set_xlabel("Mean predicted value")
 ax2.set_ylabel("Count")
 ax2.legend(loc="upper center", ncol=2)
 plt.tight_layout()
plt.show()
model= RandomForestClassifier(random_state=1, n_estimators=200, max_depth=10, criterion='entropy', verbose=1)
 model.fit(x_train, y_train)
 prediction= model.predict(x_test)
 final_predict = model.predict(final_test)
<train_model>
classifier = RandomForestClassifier(n_estimators=10000,max_depth=7)
 clf = Pipeline(memory = './',steps=[('classifier', classifier)])
 clf.fit(X=X_train_imputed_dummies, y=y_train)
 y_pred = clf.predict(X_train_imputed_dummies)
 print(accuracy_score(y_train,y_pred))
y_pred = clf.predict(X_test_imputed_dummies)
 print("Test Accuracy = ",accuracy_score(y_test,y_pred))
X, y = make_classification(n_samples=100, n_features=7,
n_informative=2, n_redundant=0,
random_state=1, shuffle=True)
 clf = BaggingClassifier(base_estimator=RandomForestClassifier() ,
n_estimators=10, random_state=1 ).fit(x_train, y_train)
 clf.predict(x_test)
 final_predict2= clf.predict(final_test )<save_to_csv>
Titanic_data_test_pred = clf.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred })
 submission_svc.to_csv("submission_3_RF.csv", index=False )
output = pd.DataFrame({'PassengerId': final_test.PassengerId, 'Survived': final_predict})
 output.to_csv('my_submission_rf.csv', index=False)
 print(output )<save_to_csv>
XGB_model = XGBClassifier(n_estimators=10000, gamma=2,learning_rate=0.05,max_depth=8,subsample=0.8,reg_lambda= 2)
 %time XGB_model.fit(X_train_imputed_dummies, y_train)
 y_pred = XGB_model.predict(X_test_imputed_dummies)
 print(accuracy_score(y_test,y_pred))
y_pred_train = XGB_model.predict(X_train_imputed_dummies)
 print(accuracy_score(y_train,y_pred_train))
output = pd.DataFrame({'PassengerId': final_test.PassengerId, 'Survived': final_predict2})
 output.to_csv('my_submission_clf.csv', index=False)
 print(output )<import_modules>
Titanic_data_test_pred_xg = XGB_model.predict(Titanic_data_test_dummies)
 submission_svc = pd.DataFrame({ 'PassengerId': Titanic_data_test_pID['PassengerId'],
'Survived': Titanic_data_test_pred_xg })
 submission_svc.to_csv("submission_4_XG.csv", index=False )
import seaborn as sns
import matplotlib.pyplot as plt<import_modules>
encoding_dim = 9
actual_dim = X_train_imputed_dummies.shape[1]
input_img = Input(shape =(actual_dim,))
encoded = Dense(encoding_dim,activation='relu' )(input_img)
 decoded = Dense(actual_dim,activation='sigmoid' )(encoded)
 autoencoder = Model(input_img,decoded)
 print(autoencoder.summary() )
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier<import_modules>
autoencoder.compile(optimizer='adam', loss='binary_crossentropy')
 autoencoder.fit(X_train_imputed_dummies, X_train_imputed_dummies, epochs=100, batch_size=80)
 encoder = Model(input_img,encoded)
 print(encoder.summary() )
from sklearn.ensemble import ExtraTreesClassifier<load_from_csv>
X_train_nonLinear_features = encoder.predict(X_train_imputed_dummies)
 X_test_nonLinear_features = encoder.predict(X_test_imputed_dummies)
 titanic_nonLinear_features = encoder.predict(Titanic_data_test_dummies)
 X_train1=np.concatenate(( X_train_imputed_dummies, X_train_nonLinear_features), axis=1)
 X_test1=np.concatenate(( X_test_imputed_dummies, X_test_nonLinear_features), axis=1)
 Titatanic_data1=np.concatenate(( Titanic_data_test_dummies, titanic_nonLinear_features), axis=1 )
df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 combine = [df, test_df]<count_missing_values>
clf = SVC()
clf.fit(X_train1,y_train)
 print("Train Accuracy on Support Vector MAchine is -----", clf.score(X_train1,y_train))
print("Train Accuracy on Support Vector MAchine is -----",clf.score(X_test1,y_test))
df.isnull().sum()<feature_engineering>
model = Sequential()
model.add(Dense(32, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(32, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(16, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(16, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(4, input_dim=X_train1.shape[1], activation='relu'))
model.add(Dense(1, activation='sigmoid'))
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0 )<categorify>
model.fit(X_train1, y_train, epochs=80, verbose=1,batch_size=90, validation_data=(X_test1, y_test))
<set_options><EOS> 
model.evaluate(X_test1, y_test, )