Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import Perceptron
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix
from sklearn.ensemble import VotingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import seaborn as sns
from pandas.tools.plotting import scatter_matrix
tokeniser = TreebankWordTokenizer()
p['tokens'] = p['clean_2'].apply(tokeniser.tokenize)
 p.head()<string_transform>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_view = pd.read_csv(".. /input/train.csv" )
stemmer = SnowballStemmer('english')
 def mbti_stemmer(words, stemmer):
return [stemmer.stem(word)for word in words]<feature_engineering>
train_view['Age'].loc[train_view['Age'] <= 15] = 1
train_view['Age'].loc[(train_view['Age'] > 15)&(train_view['Age'] <= 25)] = 2
train_view['Age'].loc[(train_view['Age'] > 25)&(train_view['Age'] <= 35)] = 3
train_view['Age'].loc[(train_view['Age'] > 35)&(train_view['Age'] <= 48)] = 4
train_view['Age'].loc[(train_view['Age'] > 48)&(train_view['Age'] <= 65)] = 5
train_view['Age'].loc[(train_view['Age'] > 65)&(train_view['Age'] <= 75)] = 6
train_view['Age'].loc[train_view['Age'] > 75] = 7
p['stem'] = p['tokens'].apply(mbti_stemmer, args=(stemmer,))
p.head()<string_transform>
train_view['Fare'].loc[train_view['Fare'] <= 14] = 1
train_view['Fare'].loc[(train_view['Fare'] > 14)&(train_view['Fare'] <= 20)] = 2
train_view['Fare'].loc[(train_view['Fare'] > 20)&(train_view['Fare'] <= 40)] = 3
train_view['Fare'].loc[(train_view['Fare'] > 40)&(train_view['Fare'] <= 60)] = 4
train_view['Fare'].loc[(train_view['Fare'] > 60)&(train_view['Fare'] <= 80)] = 5
train_view['Fare'].loc[(train_view['Fare'] > 80)&(train_view['Fare'] <= 100)] = 6
train_view['Fare'].loc[train_view['Fare'] > 100] = 7
Stops = set(stopwords.words('english'))<feature_engineering>
train_view["Title"] = train_view["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train_view["Title"] = train_view["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train_view["Title"] = train_view["Title"].astype(int )
p['no_stop'] = p['stem'].apply(lambda x: [word for word in list(x)if word not in Stops])
 p.head()<feature_engineering>
data_df = train_df.append(test_df )
p['no_stop'] = p['no_stop'].apply(lambda x: ' '.join(x))
p.head()<categorify>
data_df['Title'] = data_df['Name']
for name_string in data_df['Name']:
data_df['Title'] = data_df['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
for title in titles:
age_to_impute = data_df.groupby('Title')['Age'].median() [titles.index(title)]
data_df.loc[(data_df['Age'].isnull())&(data_df['Title'] == title), 'Age'] = age_to_impute
train_df['Age'] = data_df['Age'][:891]
test_df['Age'] = data_df['Age'][891:]
data_df.drop('Title', axis = 1, inplace = True )
unique_type = list(p['type'].unique())
 encoder = LabelEncoder().fit(unique_type)
 codes = []
for i in range(0, len(p)) :
codes.append(p['type'][i])
 coder = encoder.transform(codes )<categorify>
data_df['Fare'].fillna(data_df['Fare'].median() , inplace = True)
 data_df['FareBin'] = pd.qcut(data_df['Fare'], 5)
 label = LabelEncoder()
data_df['FareBin_Code'] = label.fit_transform(data_df['FareBin'])
 train_df['FareBin_Code'] = data_df['FareBin_Code'][:891]
test_df['FareBin_Code'] = data_df['FareBin_Code'][891:]
train_df.drop(['Fare'], 1, inplace=True)
 test_df.drop(['Fare'], 1, inplace=True )
p['codes'] = coder
p.head()<prepare_x_and_y>
data_df['AgeBin'] = pd.qcut(data_df['Age'], 4)
 label = LabelEncoder()
data_df['AgeBin_Code'] = label.fit_transform(data_df['AgeBin'])
 train_df['AgeBin_Code'] = data_df['AgeBin_Code'][:891]
test_df['AgeBin_Code'] = data_df['AgeBin_Code'][891:]
train_df.drop(['Age'], 1, inplace=True)
 test_df.drop(['Age'], 1, inplace=True )
X = p.clean
y = p.codes<split>
data_df['Family_Size'] = data_df['Parch'] + data_df['SibSp']
train_df['Family_Size'] = data_df['Family_Size'][:891]
test_df['Family_Size'] = data_df['Family_Size'][891:]
X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.2, random_state=14 )<choose_model_class>
data_df["Alone"] = np.where(data_df['SibSp'] + data_df['Parch'] + 1 == 1, 1,0)
 data_df["Alone"] = np.where(data_df['SibSp'] + data_df['Parch'] + 1 == 1, 1,0)
 train_df['Alone'] = data_df['Alone'][:891]
test_df['Alone'] = data_df['Alone'][891:]
Vect = CountVectorizer(ngram_range=(1, 1), stop_words='english', lowercase = True, max_features = 5000 )<choose_model_class>
data_df['Last_Name'] = data_df['Name'].apply(lambda x: str.split(x, ",")[0])
 data_df['Fare'].fillna(data_df['Fare'].mean() , inplace=True)
 DEFAULT_SURVIVAL_VALUE = 0.5
data_df['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data_df[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data_df.loc[data_df['Family_Survival']!=0.5].shape[0] )
LRmodel = LogisticRegression(class_weight="balanced", C=0.005, penalty = "l2")
 pipe = Pipeline([('vec', Vect),('model', LRmodel)] )<train_model>
for _, grp_df in data_df.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data_df[data_df['Family_Survival']!=0.5].shape[0]))
train_df['Family_Survival'] = data_df['Family_Survival'][:891]
test_df['Family_Survival'] = data_df['Family_Survival'][891:]
pipe.fit(X_train, y_train )<predict_on_test>
label = LabelEncoder()
data_df['Embarked'] = data_df['Embarked'].fillna('None')
 data_df['Embarked_code'] = label.fit_transform(data_df['Embarked'])
 train_df['Embarked_code'] = data_df['Embarked_code'][:891]
test_df['Embarked_code'] = data_df['Embarked_code'][891:]
y_pred_train= pipe.predict(X_train )<compute_test_metric>
train_df['Sex'].replace(['male','female'],[0,1],inplace=True)
 test_df['Sex'].replace(['male','female'],[0,1],inplace=True )
print('Accuracy: '+ str(metrics.accuracy_score(y_train, y_pred_train)))
 print('Precision: '+ str(metrics.precision_score(y_train, y_pred_train, average='macro')))
 print('Recall: '+ str(metrics.recall_score(y_train, y_pred_train, average='macro')))
 print('F1_Score: '+ str(metrics.f1_score(y_train, y_pred_train, average='macro')) )<predict_on_test>
train_df = train_df.drop(columns=['PassengerId','Pclass','Name','SibSp','Parch','Ticket','Cabin','Embarked'])
 test_df = test_df.drop(columns=['PassengerId','Pclass','Name','SibSp','Parch','Ticket','Cabin','Embarked'] )
y_pred_test = pipe.predict(X_test )<compute_test_metric>
train_df = train_df.replace(np.inf, np.nan)
 train_df_na =(train_df.isnull().sum() / len(train_df))
train_df_na = train_df_na.drop(train_df_na[train_df_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :train_df_na})
 missing_data.head()
print('Accuracy: '+ str(metrics.accuracy_score(y_test, y_pred_test)))
 print('Precision: '+ str(metrics.precision_score(y_test, y_pred_test, average='macro')))
 print('Recall: '+ str(metrics.recall_score(y_test, y_pred_test, average='macro')))
 print('F1_Score: '+ str(metrics.f1_score(y_test, y_pred_test, average='macro')) )<create_dataframe>
test_df = test_df.replace(np.inf, np.nan)
 test_df_na =(test_df.isnull().sum() / len(test_df))
test_df_na = test_df_na.drop(test_df_na[test_df_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :test_df_na})
 missing_data.head()
cm_logistic_reg = np.array(metrics.confusion_matrix(y_test, y_pred_test))
cm_logistic = pd.DataFrame(cm_logistic_reg, index=['ENFJ', 'ENFP', 'ENTJ', 'ENTP', 'ESFJ', 'ESFP',
'ESTJ', 'ESTP', 'INFJ', 'INFP', 'INTJ', 'INTP',
'ISFJ', 'ISFP', 'ISTJ', 'ISTP'],
columns=['predict_ENFJ','predict_ENFP','predict_ENTJ',
'predict_ENTP','predict_ESFJ','predict_ESFP',
'predict_ESTJ','predict_ESTP','predict_INFJ',
'predict_INFP','predict_INTJ','predict_INTP',
'predict_ISFJ','predict_ISFP','predict_ISTJ',
'predict_ISTP'])
 cm_logistic<compute_test_metric>
X_train = train_df.drop('Survived', 1)
 Y_train = train_df['Survived']
X_test = test_df.copy()
print(metrics.classification_report(y_test, y_pred_test, target_names=unique_type))<predict_on_test>
std_scaler = StandardScaler()
X_train = std_scaler.fit_transform(X_train)
 X_test = std_scaler.transform(X_test )
test_preds = pipe.predict(z['clean'] )<categorify>
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from xgboost import XGBClassifier
z['type'] = encoder.inverse_transform(test_preds)
 z.head()<data_type_conversions>
xgb = XGBClassifier(booster='gbtree',
gamma=0.1,
learning_rate = 0.01,
max_depth = 6,
min_child_weight= 1,
n_estimators= 100,
subsample= 0.5 )
z['E or I'] = z.apply(lambda x: x['type'][0], axis = 1)
 z['N or S'] = z.apply(lambda x: x['type'][1], axis = 1)
 z['T or F'] = z.apply(lambda x: x['type'][2], axis = 1)
 z['J or P'] = z.apply(lambda x: x['type'][3], axis = 1)
 mind = z['E or I'].astype(str ).apply(lambda x: x[0] == 'E' ).astype('int')
 energy = z['N or S'].astype(str ).apply(lambda x: x[0] == 'N' ).astype('int')
 nature = z['T or F'].astype(str ).apply(lambda x: x[0] == 'T' ).astype('int')
 tactics = z['J or P'].astype(str ).apply(lambda x: x[0] == 'J' ).astype('int')
 z.head()<prepare_output>
rf=RandomForestClassifier(random_state=42,
min_samples_split=2,
max_leaf_nodes=10,
max_features='auto',
n_estimators= 500,
max_depth=5,
criterion='gini' )
df_LogReg = pd.DataFrame({"id":test_df['id'], "mind":mind, "energy":energy, "nature":nature, 'tactics':tactics})
 df_LogReg.head()<save_to_csv>
ada = AdaBoostClassifier(random_state=40,
algorithm = 'SAMME.R',
learning_rate= 0.01,
n_estimators = 200
)
df_LogReg.to_csv('EDSA_Team_8_Classification1.csv', index = False )<choose_model_class>
dt = DecisionTreeClassifier(random_state=40,
criterion = 'gini',
max_depth= 5,
max_features= 'auto',
max_leaf_nodes= 12,
min_samples_split= 2 )
LRmodel2 = LogisticRegression(class_weight="balanced", C=0.004, penalty = "l2")
 pipe2 = Pipeline([('vec', Vect),('model', LRmodel2)] )<train_model>
knn = KNeighborsClassifier(algorithm='auto',
leaf_size=26,
metric='minkowski',
metric_params=None,
n_jobs=-1,
n_neighbors=18,
p=2,
weights='uniform' )
pipe2.fit(X_train, y_train )<predict_on_test>
optimal = VotingClassifier(estimators=[('knn',knn),('ada',ada),('rf',rf),('xgb',xgb),('dt',dt)],voting='hard')
 optimal
y_pred_train2= pipe2.predict(X_train )<compute_test_metric>
optimal.fit(X_train, Y_train)
 Y_pred=optimal.predict(X_test )
<predict_on_test><EOS> 
sub = pd.DataFrame(pd.read_csv(".. /input/test.csv")['PassengerId'])
 sub['Survived'] = Y_pred
sub.to_csv(".. /working/submission.csv", index = False )