Kaggle competition: Titanic - Machine Learning from Disaster
data.isnull().sum() 
<count_values>
train_data = pd.read_csv(".. /input/titanic/train.csv")
 train_data.head()
data.Name.value_counts()<feature_engineering>
test_data = pd.read_csv('.. /input/titanic/test.csv')
 test_data.head()
temp = data.copy()
temp['Initial'] = 0
temp['Initial'] = data.Name.str.extract('([A-Za-z]+)\.')
<count_values>
print("Missing data counts in Training Data : ")
 print(train_data.isnull().sum())
 print("Missing data counts in Test Data : ")
 print(test_data.isnull().sum())
temp['Initial'].value_counts()<groupby>
print("Percentage of data missing Training Data: ")
 print(train_data.isnull().sum() /train_data.shape[0])
 print("Percentage of data missing Test Data: ")
 print(test_data.isnull().sum() /test_data.shape[0] )
def survpct(col):
return temp.groupby(col)['Survived'].mean()
survpct('Initial' )<filter>
columns_to_drop = []
columns_to_drop.append('Cabin' )
temp.loc[temp['Initial'] == 'Dona']<feature_engineering>
test_data[test_data['Fare'].isnull() ]
temp.loc[temp['Initial'] == 'Dona', 'Initial'] = 'Mrs'<feature_engineering>
test_data[test_data['Ticket']=='3701']
temp = temp.reset_index(drop=True)
 temp['Age'] = temp.groupby('Initial')['Age'].apply(lambda x: x.fillna(x.mean()))
temp[31:50]<categorify>
class_3_data = test_data[test_data['Pclass'] == 3]
class_3_S = class_3_data[class_3_data['Embarked'] == 'S']
class_3_S[class_3_S['Age']>40]
temp['Initial'].replace(['Capt', 'Col', 'Countess', 'Don', 'Dona' , 'Dr', 'Jonkheer', 'Lady', 'Major', 'Master', 'Miss' ,'Mlle', 'Mme', 'Mr', 'Mrs', 'Ms', 'Rev', 'Sir'], ['Sacrificed', 'Respected', 'Nobles', 'Mr', 'Mrs', 'Respected', 'Mr', 'Nobles', 'Respected', 'Kids', 'Miss', 'Nobles', 'Nobles', 'Mr', 'Mrs', 'Nobles', 'Sacrificed', 'Nobles'],inplace=True)
 temp['Initial'].replace(['Kids', 'Miss', 'Mr', 'Mrs', 'Nobles', 'Respected', 'Sacrificed'], [4, 4, 2, 5, 6, 3, 1], inplace=True )<feature_engineering>
test_data[test_data["Fare"].isnull() ]
temp['Age_Range'] = pd.qcut(temp['Age'], 10)
 survpct('Age_Range')
<feature_engineering>
test_data.iloc[152,-3]= 14
temp['Agroup'] = 0
temp.loc[temp['Age'] < 1.0, 'Agroup'] = 1
temp.loc[(temp['Age'] >= 1.0)&(temp['Age'] <= 3.0), 'Agroup'] = 2
temp.loc[(temp['Age'] > 3.0)&(temp['Age'] < 11.0), 'Agroup'] = 7
temp.loc[(temp['Age'] >= 11.0)&(temp['Age'] < 15.0), 'Agroup'] = 13
temp.loc[(temp['Age'] >= 15.0)&(temp['Age'] < 18.0), 'Agroup'] = 16
temp.loc[(temp['Age'] >= 18.0)&(temp['Age'] <= 20.0), 'Agroup'] = 18
temp.loc[(temp['Age'] > 20.0)&(temp['Age'] <= 22.0), 'Agroup'] = 21
temp.loc[(temp['Age'] > 22.0)&(temp['Age'] <= 26.0), 'Agroup'] = 24
temp.loc[(temp['Age'] > 26.0)&(temp['Age'] <= 30.0), 'Agroup'] = 28
temp.loc[(temp['Age'] > 30.0)&(temp['Age'] <= 32.0), 'Agroup'] = 31
temp.loc[(temp['Age'] > 32.0)&(temp['Age'] <= 34.0), 'Agroup'] = 33
temp.loc[(temp['Age'] > 34.0)&(temp['Age'] <= 38.0), 'Agroup'] = 36
temp.loc[(temp['Age'] > 38.0)&(temp['Age'] <= 52.0), 'Agroup'] = 45
temp.loc[(temp['Age'] > 52.0)&(temp['Age'] <= 75.0), 'Agroup'] = 60
temp.loc[temp['Age'] > 75.0, 'Agroup'] = 78
temp.head()<feature_engineering>
train_data[train_data['Age'].isnull() ]
temp.loc[(temp['Sex'] == 'male'), 'Sex'] = 1
temp.loc[(temp['Sex'] == 'female'), 'Sex'] = 2
temp.loc[(temp['Age'] < 1), 'Sex'] = 3
survpct('Sex' )<feature_engineering>
def extract_titles(df):
pos = df.columns.get_loc('Name')
 titles = set({})
 for row in df.values:
title = row[pos].split(',')[1].split('.')[0] + '.'.strip()
titles.add(title)
 return titles
temp.loc[(temp['SibSp'] == 0)&(temp['Parch'] == 0), 'Alone'] = 1
temp['Family'] = temp['Parch'] + temp['SibSp'] + 1
temp.head(n=10 )<drop_column>
def add_titles_to_df(df):
titles = extract_titles(df)
 pos = df.columns.get_loc('Name')
 title_list = []
for row in df.values:
for title in titles:
if title in row[pos]:
title_list.append(title)
 break
df['Title'] = title_list
return df
bag('Parch', 'Survived', 'Survived per Parch', 'Parch Survived vs Not Survived' )<filter>
train_data = add_titles_to_df(train_data )
temp.loc[(temp.Embarked.isnull())]<filter>
test_data = add_titles_to_df(test_data )
temp.loc[(temp.Ticket == '113572')]<sort_values>
train_data['Title'].value_counts()
temp.sort_values(['Ticket'], ascending=True)[55:70]<feature_engineering>
test_data['Title'].value_counts()
temp.loc[(temp.Embarked.isnull()), 'Embarked'] = 'S'
temp.loc[(temp.Embarked.isnull())]<feature_engineering>
male_titles = [' Col.',' Major.',' Capt.',' Jonkheer.',' Don.',' Sir.']
female_titles = [' Lady.',' Mme.',' the Countess.',' Dona.',' Mlle.']
temp['Embarked'] = temp['Embarked'].factorize() [0]
temp[11:20]<feature_engineering>
def replace_uncommon_titles(df,new_title,title_list):
pos = df.columns.get_loc('Title')
 for title in title_list:
for i in range(0,df.shape[0]):
if df.iloc[i,pos] == title:
print(title)
 df.iloc[i,pos] = new_title
return df
train_data = replace_uncommon_titles(train_data,' Mr.',male_titles)
 train_data = replace_uncommon_titles(train_data," Miss.",female_titles)
 test_data = replace_uncommon_titles(test_data," Mr.",male_titles)
 test_data = replace_uncommon_titles(test_data," Miss.",female_titles )
temp['Priority'] = 0
temp.loc[(temp['Initial'] == 6), 'Priority'] = 1
temp.loc[(temp['Pclass'] == 1)&(temp['Sex'] == 2), 'Priority'] = 2
temp.loc[(temp['Age'] < 1), 'Priority'] = 3
temp.loc[(temp['Pclass'] == 1)&(temp['Age'] <= 17), 'Priority'] = 4
temp.loc[(temp['Pclass'] == 2)&(temp['Age'] <= 17), 'Priority'] = 5<count_values>
age_mean = train_data.groupby("Title" ).mean() ['Age']
temp.Priority.value_counts()<feature_engineering>
def fill_age_na(df,age_mean):
rows_with_age_missing = df[df['Age'].isnull() ]
pos = df.columns.get_loc("Age")
 for title in age_mean.index:
passengerIds = rows_with_age_missing[rows_with_age_missing['Title'] == title]["PassengerId"]
for Id in passengerIds:
df.iloc[df[df['PassengerId'] == Id].index.values,pos] = age_mean[title]
return df
temp['F1'] = temp['Priority']
temp['F2'] = temp['Initial']
temp['F3'] = temp['NumName']
temp['F4'] = temp['Family']
temp['F5'] = temp['Embarked']
temp['F6'] = temp['Sex']
temp['F7'] = temp['Pclass']<categorify>
train_data = fill_age_na(train_data,age_mean)
 test_data = fill_age_na(test_data,age_mean)
 train_data[train_data['Age'].isnull() ]
dfl = pd.DataFrame()
good_columns = ['F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7']
dfl[good_columns] = temp[good_columns]
dfh = dfl.copy()
dfl_enc = dfl.apply(LabelEncoder().fit_transform)
 dfl_enc.head()<categorify>
test_data[test_data['Age'].isnull() ]
one_hot_cols = dfh.columns.tolist()
dfh_enc = pd.get_dummies(dfh, columns=one_hot_cols)
 dfh_enc.head()<split>
train_data.isnull().sum()
train = dfh_enc[:ntrain]
test = dfh_enc[ntrain:]<prepare_x_and_y>
test_data.isnull().sum()
X_test = test
X_train = train<normalization>
train_data['Embarked']= train_data['Embarked'].fillna(value='S',axis=0 )
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<define_search_model>
columns_to_drop.extend(["Ticket"] )
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier()
gbc = GradientBoostingClassifier()
svc = SVC(probability=True)
 ext = ExtraTreesClassifier()
ada = AdaBoostClassifier()
gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier()
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
model_names = ['Random Forest', 'K Nearest Neighbor', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier']
scores = {}
for ind, mod in enumerate(models):
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring="accuracy", cv=10)
 scores[model_names[ind]] = acc<prepare_output>
def drop_columns(df,list_of_columns):
return df.drop(list_of_columns,axis=1)
results = pd.DataFrame(scores ).T
results['mean'] = results.mean(1)
 result_df = results.sort_values(by='mean', ascending=False)
 result_df.head(11 )<create_dataframe>
train_data = drop_columns(train_data,columns_to_drop)
 test_data = drop_columns(test_data,columns_to_drop )
gbc_imp = pd.DataFrame({'Feature':train.columns, 'gbc importance':gbc.feature_importances_})
 xgb_imp = pd.DataFrame({'Feature':train.columns, 'xgb importance':xgb.feature_importances_})
 ran_imp = pd.DataFrame({'Feature':train.columns, 'ran importance':ran.feature_importances_})
 ext_imp = pd.DataFrame({'Feature':train.columns, 'ext importance':ext.feature_importances_})
 ada_imp = pd.DataFrame({'Feature':train.columns, 'ada importance':ada.feature_importances_})
 importances = gbc_imp.merge(xgb_imp, on='Feature' ).merge(ran_imp, on='Feature' ).merge(ext_imp, on='Feature' ).merge(ada_imp, on='Feature')
 importances['Average'] = importances.mean(axis=1)
 importances = importances.sort_values(by='Average', ascending=False ).reset_index(drop=True )<define_variables>
labels = train_data['Survived']
train_data = train_data.drop('Survived',axis=1 )
mylist = list(importance1.index )<prepare_output>
cleaned_train_data = train_data
cleaned_test_data = test_data
train1 = pd.DataFrame()
test1 = pd.DataFrame()
for i in mylist:
train1[i] = train[i]
test1[i] = test[i]
train1.head()<normalization>
categorical_columns = ['Pclass','Sex','Embarked','Title']
numerical_columns = ['Age','Fare','SibSp','Parch']
train = train1
test = test1
X_train = train
X_test = test
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<choose_model_class>
def preprocess_data(df):
scaler = StandardScaler()
numerical_data = df[numerical_columns]
categorical_data = df[categorical_columns]
std_data_numerical = scaler.fit_transform(numerical_data)
 df_numerical = pd.DataFrame(std_data_numerical,columns=numerical_columns,index=df.index)
 std_data_categorical = pd.get_dummies(categorical_data,columns=categorical_columns)
 final_data = df_numerical.join(std_data_categorical,how='inner')
 return final_data
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier(random_state=1)
 gbc = GradientBoostingClassifier(random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(random_state=1)
 ada = AdaBoostClassifier(random_state=1)
 gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
model_names = ['Random Forest', 'K Nearest Neighbor', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier']
scores2 = {}
for ind, mod in enumerate(models):
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores2[model_names[ind]] = acc<prepare_output>
preprocessed_train_data = preprocess_data(cleaned_train_data )
results = pd.DataFrame(scores2 ).T
results['mean'] = results.mean(1)
 result_df = results.sort_values(by='mean', ascending=False)
 result_df.head(11 )<train_on_grid>
preprocessed_test_data = preprocess_data(cleaned_test_data )
Cs = [0.01, 0.1, 1, 5, 10, 15, 20, 50]
gammas = [0.001, 0.01, 0.1]
hyperparams = {'C': Cs, 'gamma': gammas}
gd = GridSearchCV(estimator=SVC(probability=True), param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
train_X, val_X, train_y, val_y = train_test_split(preprocessed_train_data,labels,random_state=1 )
learning_rate = [0.01, 0.05, 0.1, 0.2, 0.5]
n_estimators = [100, 1000, 2000]
max_depth = [3, 5, 10, 15]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd = GridSearchCV(estimator=GradientBoostingClassifier() , param_grid = hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
estimators = [1, 2, 4, 8, 16, 32, 64, 100, 200]
for num in estimators:
model = RandomForestClassifier(n_estimators=num)
 model.fit(train_X,train_y)
 preds = model.predict(val_X)
 print("Accuracy for {} estimators is {}".format(num,accuracy_score(val_y,preds,normalize=True)) )
penalty = ['l1', 'l2']
C = np.logspace(0, 4, 10)
 hyperparams = {'penalty': penalty, 'C': C}
gd = GridSearchCV(estimator=LogisticRegression() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
n_est = [5,10,20,40,100,200]
for num in n_est:
xgb_model = xgb.XGBClassifier(n_estimators=num)
 xgb_model.fit(train_X,train_y)
 preds = xgb_model.predict(val_X)
 print("Accuracy lr {} is {}".format(num,accuracy_score(val_y,preds,normalize=True)) )
learning_rate = [0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [10, 50, 100, 250, 500, 1000]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd = GridSearchCV(estimator = XGBClassifier() , param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
iters = [50,100,150,200,300,500]
for num in iters:
model = LogisticRegression(penalty='l2',max_iter=num,random_state=1,verbose=3)
 model.fit(train_X,train_y)
 preds = model.predict(val_X)
 print("Accuracy for {} iterations is {}".format(num,accuracy_score(val_y,preds,normalize=True)) )
max_depth = [3, 4, 5, 6, 7, 8, 9, 10]
min_child_weight = [1, 2, 3, 4, 5, 6]
hyperparams = {'max_depth': max_depth, 'min_child_weight': min_child_weight}
gd = GridSearchCV(estimator=XGBClassifier(learning_rate=0.2, n_estimators=50),
param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
model = LogisticRegression(penalty='l2',solver='lbfgs',random_state=1)
 model.fit(preprocessed_train_data,labels)
 preds = model.predict(preprocessed_test_data )
gamma = [i*0.1 for i in range(0,5)]
hyperparams = {'gamma': gamma}
gd = GridSearchCV(estimator= XGBClassifier(learning_rate=0.2, n_estimators=50, max_depth=8, min_child_weight=2),
param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
output = pd.DataFrame({'PassengerId': test_data.PassengerId,
'Survived': preds})
 output.to_csv('submission.csv', index=False)
 output
subsample = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
colsample_bytree = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
hyperparams = {'subsample': subsample, 'colsample_bytree': colsample_bytree}
gd = GridSearchCV(estimator=XGBClassifier(learning_rate=0.2, n_estimators=50, max_depth=8, min_child_weight=2, gamma=0),
param_grid=hyperparams, verbose=True, cv=5, scoring="accuracy", n_jobs=-1)
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_params_ )<train_on_grid>
%matplotlib inline