Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
mean_bmi = train['bmi'].mean()
train['bmi'] = train['bmi'].fillna(mean_bmi)
 test['bmi'] = test['bmi'].fillna(mean_bmi )<groupby>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train = train_data.copy()
test = test_data.copy()
train.groupby('stroke' ).size().reset_index(name='counts' )<sort_values>
train.drop(['PassengerId'],axis=1,inplace=True)
 test.drop(['PassengerId'],axis=1,inplace=True)
 pred = train_data['Survived']
train[['gender', 'stroke']].groupby(['gender'], as_index=False ).mean().sort_values(by='stroke', ascending=False )<count_values>
train.isnull().sum()
train['gender'].value_counts()<categorify>
train.isnull().sum()
gender_target_dict =(train.groupby(['gender'])['stroke'].agg(['mean'])).to_dict() ['mean']
gender_target_dict['Other'] = global_mean_stroke
train['gender_target_enc'] = train['gender'].replace(gender_target_dict)
 test['gender_target_enc'] = test['gender'].replace(gender_target_dict )<sort_values>
test.isnull().sum()
work_type_data = train[['work_type', 'stroke']].groupby(['work_type'], as_index=False ).mean().sort_values(by='stroke', ascending=False)
 work_type_data<count_values>
test.isnull().sum()
train['work_type'].value_counts()<categorify>
sex1 = pd.get_dummies(train['Sex'])
 sex2 = pd.get_dummies(test['Sex'])
 train.drop(['Sex'],axis=1,inplace=True)
 test.drop(['Sex'],axis=1,inplace=True)
 train = pd.concat([train,sex1],axis=1)
 test = pd.concat([test,sex2],axis=1 )
n = train.groupby('work_type' ).size()
mean_by_work_type = train.groupby('work_type')['stroke'].mean()
m = 10
work_type_target_dict =(( mean_by_work_type*n + global_mean_stroke*m)/(n+m)).to_dict()
train['work_type_smoothed_target_enc'] = train['work_type'].replace(work_type_target_dict)
 test['work_type_smoothed_target_enc'] = test['work_type'].replace(work_type_target_dict)
<feature_engineering>
train.drop(['female'],axis=1,inplace=True)
 test.drop(['female'],axis=1,inplace=True )
for work_type in train['work_type'].unique() :
train['work_type_is_{}'.format(work_type)] =(train['work_type'] == work_type)*1
test['work_type_is_{}'.format(work_type)] =(test['work_type'] == work_type)*1<drop_column>
plt.figure(figsize=(8, 5))
sns.set_style('whitegrid')
 sns.countplot(x = 'Pclass',hue='Survived',data=train_data,palette='deep' )
train = train.drop('Residence_type', axis=1)
 test = test.drop('Residence_type', axis=1 )<categorify>
Fare_0 = []
Fare_1 = []
for i in range(0,891):
if train_data["Survived"][i] == 0:
Fare_0.append(train["Fare"][i])
 else:
Fare_1.append(train["Fare"][i] )
train['ever_married'] = train['ever_married'].replace({'Yes':1, 'No':0 })
 test['ever_married'] = test['ever_married'].replace({'Yes':1, 'No':0 } )<categorify>
train["Embarked"].fillna("S", inplace = True)
 test["Embarked"].fillna("S", inplace = True )
features = [ 'age', 'hypertension', 'heart_disease', 'ever_married',
'avg_glucose_level',
'gender_target_enc', 'work_type_smoothed_target_enc']
X_embedded = TSNE(n_components=2, random_state=21 ).fit_transform(train[features] )<find_best_model_class>
embark1 = pd.get_dummies(train['Embarked'])
 embark2 = pd.get_dummies(test['Embarked'])
 train.drop(['Embarked'],axis=1,inplace=True)
 test.drop(['Embarked'],axis=1,inplace=True)
 train = pd.concat([train,embark1],axis=1)
 test = pd.concat([test,embark2],axis=1 )
def your_cross_validation_for_roc_auc(clf, X, y ,cv=5):
X = np.array(X.copy())
 y = np.array(y.copy())
 kf = KFold(n_splits=cv)
 kf.get_n_splits(X)
 scores = []
for train_index, test_index in kf.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
clf.fit(X_train, y_train)
 prediction_on_this_fold = clf.predict_proba(X_test)[:,1]
score = roc_auc_score(y_score=prediction_on_this_fold, y_true=y_test)
 scores.append(score)
 return scores<choose_model_class>
def fam(x):
if(x['SibSp'] + x['Parch'])> 0:
return 1
else:
return 0
train['Family'] = train.apply(fam, axis = 1)
 test['Family'] = test.apply(fam, axis = 1 )
models = [
SGDClassifier(loss='log', penalty = 'elasticnet', max_iter=50),
XGBClassifier(n_estimators=45, max_depth=3),
RandomForestClassifier() ,
]
models_names = dict(zip(models, ['svc', 'xgb', 'gnb']))<define_variables>
train = train.drop(['SibSp','Parch'],axis=1)
 test = test.drop(['SibSp','Parch'],axis=1 )
all_features = [ 'age', 'hypertension', 'heart_disease', 'ever_married',
'avg_glucose_level', 'bmi',
'work_type_is_children',
'work_type_is_Private', 'work_type_is_Never_worked',
'work_type_is_Self-employed', 'work_type_is_Govt_job',
'gender_target_enc', 'work_type_smoothed_target_enc']<feature_engineering>
train["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in train['Cabin'] ])
 test["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in test['Cabin'] ] )
scores = {}
for model in models:
features_scores = {}
print(models_names[model])
 for f in all_features:
scores = your_cross_validation_for_roc_auc(model, train[[f]] , train['stroke'])
 print(f, np.mean(scores))
features_scores[f] = np.mean(scores)
<define_variables>
train["Cabin"] = train["Cabin"].map({"X":0, "A":1, "B" : 2 , "C":3, "D":4, "E":5, "F":6, "G":7,"T":0})
 train["Cabin"] = train["Cabin"].astype(int)
 test["Cabin"] = test["Cabin"].map({"X":0, "A":1, "B" : 2 , "C":3, "D":4, "E":5, "F":6, "G":7,"T":0})
 test["Cabin"] = test["Cabin"].astype(int )
features_target = ['age', 'hypertension', 'heart_disease', 'ever_married', 'avg_glucose_level', 'bmi',
'work_type_smoothed_target_enc', 'gender_target_enc']<define_variables>
train_title = [i.split(",")[1].split(".")[0].strip() for i in train["Name"]]
train["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title )
features_onehot = [ 'age', 'hypertension', 'heart_disease', 'ever_married',
'avg_glucose_level', 'bmi',
'work_type_is_children',
'work_type_is_Private', 'work_type_is_Never_worked',
'work_type_is_Self-employed', 'work_type_is_Govt_job',
'gender_target_enc']<choose_model_class>
train = train.drop(['Name'], axis = 1)
 test = test.drop(['Name'], axis = 1 )
models = [
SGDClassifier(loss='log', penalty = 'elasticnet', max_iter=100),
XGBClassifier(n_estimators=75, max_depth=3),
RandomForestClassifier(n_estimators=75, max_depth=3),
]
models_names = dict(zip(models, ['svc', 'xgb', 'rfc']))
for model in models:
print(models_names[model])
 scores_target = your_cross_validation_for_roc_auc(model, train[features_target] , train['stroke'])
 scores_onehot = your_cross_validation_for_roc_auc(model, train[features_onehot] , train['stroke'])
 print(np.mean(scores_target), np.mean(scores_onehot)) 
<save_to_csv>
train["Title"] = train["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train["Title"] = train["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train["Title"] = train["Title"].astype(int)
 test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )
best_model = XGBClassifier(n_estimators=75, max_depth=3)
 best_model.fit(train[features_target], train['stroke'])
 Y_pred = pd.DataFrame(best_model.predict_proba(test[features_target])) [1]
submission = pd.DataFrame({
"id": test["id"],
"stroke": Y_pred
})
 submission.to_csv(".. /working/submit.csv", index=False)
 submission.sample(10 )<import_modules>
Ticket1 = []
for i in list(train.Ticket):
if not i.isdigit() :
Ticket1.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket1.append("X")
 train["Ticket"] = Ticket1
Ticket2 = []
for j in list(test.Ticket):
if not j.isdigit() :
Ticket2.append(j.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket2.append("X")
 test["Ticket"] = Ticket2
import pandas as pd
import numpy as np
import sklearn
from sklearn import preprocessing
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn import svm
import statistics
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis 
<load_from_csv>
np.union1d(train["Ticket"], test["Ticket"] )
t_data = pd.read_csv(".. /input/train.csv")
 ts_data = pd.read_csv(".. /input/test.csv" )<drop_column>
train= pd.get_dummies(train, columns = ["Ticket"], prefix="T")
 test = pd.get_dummies(test, columns = ["Ticket"], prefix="T" )
ddata = t_data.drop(["PlayerID","Name"], axis=1)
 sdata = ts_data.drop(["PlayerID","Name"], axis=1 )<normalization>
train = train.drop(['T_SP','T_SOP','T_Fa','T_LINE','T_SWPP','T_SCOW','T_PPP','T_AS','T_CASOTON'],axis = 1)
 test = test.drop(['T_SCA3','T_STONOQ','T_AQ4','T_A','T_LP','T_AQ3'],axis = 1 )
ddata = ddata.interpolate()
ddata = ddata.replace([np.inf], np.float64.max)
 ddata = ddata.replace([-np.inf], np.float64.min)
 features = ddata.loc[:, ddata.columns.values[:len(ddata.columns.values)-1]].values
labels = ddata.loc[:, ['TARGET_5Yrs']].values
st_features = preprocessing.StandardScaler().fit_transform(features)
 sdata = sdata.interpolate()
sdata = sdata.replace([np.inf], np.float64.max)
 sdata = sdata.replace([-np.inf], np.float64.min)
 sfeatures = sdata.loc[:, sdata.columns.values].values
st_sfeatures = preprocessing.StandardScaler().fit_transform(sfeatures )<categorify>
train.drop(['Survived'],axis=1,inplace=True )
pca = PCA(n_components=10)
 pca.fit(features)
 principalComponents = pca.transform(features)
 test_principalComponenta = pca.transform(sfeatures)
 print(principalComponents.shape, "
", test_principalComponenta.shape )<predict_on_test>
print(train.isnull().sum())
 print("Number of columns are :",train.isnull().sum().count() )
testf = sfeatures 
<choose_model_class>
print(test.isnull().sum())
 print("Number of columns are :",test.isnull().sum().count() )
names = ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
"Naive Bayes", "QDA"]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0)) ,
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
MLPClassifier(alpha=1),
AdaBoostClassifier() ,
GaussianNB() ,
QuadraticDiscriminantAnalysis() ]
res = []
res2 = []
for name, clf in zip(names, classifiers):
clf.fit(inputf, labels)
 res.append(clf.predict(testf))
temp = clf.predict(inputf)
 res2.append(temp)
 print(name, " : ", sklearn.metrics.accuracy_score(labels, temp)) 
<define_variables>
sc = StandardScaler()
train2 = sc.fit_transform(train)
 test2 = sc.transform(test )
res2 = np.array(res2)
 res3 = []
for i in range(res2.shape[1]):
s = 0
for j in res2:
s = s + j[i]
if s >= 5:
res3.append(1)
 else:
res3.append(0 )<compute_test_metric>
KFold_Score = pd.DataFrame()
classifiers = ['Linear SVM', 'Radial SVM', 'LogisticRegression',
'RandomForestClassifier', 'AdaBoostClassifier',
'XGBoostClassifier', 'KNeighborsClassifier','GradientBoostingClassifier']
models = [svm.SVC(kernel='linear'),
svm.SVC(kernel='rbf'),
LogisticRegression(max_iter = 1000),
RandomForestClassifier(n_estimators=200, random_state=0),
AdaBoostClassifier(random_state = 0),
xgb.XGBClassifier(n_estimators=100),
KNeighborsClassifier() ,
GradientBoostingClassifier(random_state=0)
 ]
j = 0
for i in models:
model = i
cv = KFold(n_splits=5, random_state=0, shuffle=True)
 KFold_Score[classifiers[j]] =(cross_val_score(model, train, np.ravel(pred), scoring = 'accuracy', cv=cv))
j = j+1
sklearn.metrics.accuracy_score(labels, res3 )<define_variables>
mean = pd.DataFrame(KFold_Score.mean() , index= classifiers)
 KFold_Score = pd.concat([KFold_Score,mean.T])
 KFold_Score.index=['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5','Mean']
KFold_Score.T.sort_values(by=['Mean'], ascending = False )
res = np.array(res)
 res3 = []
for i in range(res.shape[1]):
s = 0
for j in res:
s = s + j[i]
if s >= 5:
res3.append(1)
 else:
res3.append(0 )<save_to_csv>
col_name1[0],col_name1[2] = col_name1[2],col_name1[0]
col_name2[0],col_name2[2] = col_name2[2],col_name2[0]
cols = { 'PlayerID': [i+901 for i in range(440)] , 'TARGET_5Yrs': res3 }
submission = pd.DataFrame(cols)
 submission.to_csv("submission.csv", index=False)
 print(submission )<import_modules>
train_new = train[col_name1]
test_new = test[col_name2]
print(os.listdir(".. /input")) 
<load_from_csv>
train_new = train_new.drop(['Cabin'],axis = 1)
 test_new = test_new.drop(['Cabin'],axis = 1 )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
<prepare_x_and_y>
sc = StandardScaler()
train3 = sc.fit_transform(train_new)
 test3 = sc.transform(test_new )
<data_type_conversions>
rfc = RandomForestClassifier(random_state=0 )
X = train.values[:,2:20]
y = train.values[:,21]
y = y.astype('int')
<normalization>
param_grid = {
'n_estimators': [ 200,300],
'max_features': ['auto', 'sqrt'],
'max_depth' : [6,7,8],
'criterion' :['gini', 'entropy']
}
imputer = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
 imputer = imputer.fit(X)
 X = imputer.transform(X)
<categorify>
CV_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5)
 CV_rfc.fit(train3,pred)
 CV_rfc.best_params_
Xt = test.values[:,2:20]
imputer = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
 imputer = imputer.fit(Xt)
 Xt = imputer.transform(Xt)
<normalization>
rfc1=RandomForestClassifier(random_state=0, n_estimators= 200, criterion = 'gini',max_features = 'auto',max_depth = 8)
 rfc1.fit(train3, pred )
sc = StandardScaler()
sct = StandardScaler()
X = sc.fit_transform(X)
 Xt = sc.transform(Xt)
<train_model>
pred3= rfc1.predict(test3)
 print(pred3 )
<import_modules><EOS> 
pred_test = pred3
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': pred_test})
 output.to_csv('./submission.csv', index=False )