Kaggle competition: Titanic - Machine Learning from Disaster
year = []
month = []
day = []
for i in train_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 data = pd.concat([train, test])
 train.shape
train_df['Year'] = year
train_df['Month'] = month
train_df['Day'] = day
del train_df['Date']<feature_engineering>
sex_label = LabelEncoder()
cabin_label = LabelEncoder()
embarked_label = LabelEncoder()
family_name_label = LabelEncoder()
title_label = LabelEncoder()
title_remap_label = LabelEncoder()
data['Sex_Code'] = sex_label.fit_transform(data.Sex)
 data['Cabin_Prefix'] = data.Cabin.str.get(0 ).fillna('Z')
 data['Cabin_Code'] = cabin_label.fit_transform(data.Cabin.str.get(0 ).fillna('Z'))
data['Has_Cabin'] =(data.Cabin.str.get(0 ).fillna('Z')!= 'Z' ).astype('int32')
 data['Embarked_fillZ'] = data.Embarked.fillna('Z')
 data['Embarked_Code'] = embarked_label.fit_transform(data.Embarked.fillna('S'))
data['FamilySize'] = data.Parch + data.SibSp + 1
data['BigFamily'] = data.FamilySize.apply(lambda s: s if s < 5 else 5)
 data['IsAlone'] = data.FamilySize == 1
data['FamilyName'] = data.Name.str.extract('(\w+),', expand=False)
 data['FamilyName_Code'] = family_name_label.fit_transform(data.FamilyName)
 data['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 mapping = {
'Mlle': 'Miss',
'Ms': 'Miss',
'Dona': 'Mrs',
'Mme': 'Miss',
'Lady': 'Mrs',
'Capt': 'Honorable',
'Countess': 'Honorable',
'Major': 'Honorable',
'Col': 'Honorable',
'Sir': 'Honorable',
'Don': 'Honorable',
'Jonkheer': 'Honorable',
'Rev': 'Honorable',
'Dr': 'Honorable'
}
data['Title_Remap'] = data.Title.replace(mapping)
 data['Title_Code'] = title_label.fit_transform(data.Title)
 data['Title_Remap_Code'] = title_remap_label.fit_transform(data.Title_Remap)
 data.Age = data.Age.fillna(data.Age.median())
 data.Fare = data.Fare.fillna(data.Fare.median() )
year = []
month = []
day = []
for i in test_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )<feature_engineering>
m = data[['FamilyName', 'Survived']].groupby('FamilyName' ).max()
c = data[['FamilyName', 'PassengerId']].groupby('FamilyName' ).count()
m = m.rename(columns={'Survived': 'FamilySurvived'})
 c = c.rename(columns={'PassengerId': 'FamilyMemberCount'})
 m = m.where(m.join(c ).FamilyMemberCount > 1, other=-1, axis=1 ).fillna(-1 ).join(c)
 m.FamilySurvived = m.FamilySurvived.astype('int32')
 joined_data = data.join(m, on='FamilyName' )
test_df['Year'] = year
test_df['Month'] = month
test_df['Day'] = day
del test_df['Date']
del train_df['Id']
del test_df['ForecastId']<drop_column>
optimal_d = 20
Z = linkage(ticket.reshape(data.shape[0], 1), 'single')
 clusters = fcluster(Z, optimal_d, criterion='distance')
 joined_data['Ticket_Code'] = clusters
del train_df['Year']
del test_df['Year']<data_type_conversions>
joined_data[['FamilyName',
'Name',
'Age',
'Fare',
'BigFamily',
'Pclass',
'Has_Cabin',
'Embarked',
'Sex',
'Title',
'Ticket_Code',
'Ticket_Code_Remap',
'Survived']][joined_data.Ticket_Code==89].sort_values(by='FamilyName' )
train_df['ConfirmedCases'] = train_df['ConfirmedCases'].apply(int)
 train_df['Fatalities'] = train_df['Fatalities'].apply(int )<drop_column>
joined_data[['FamilyName',
'Name',
'Age',
'Fare',
'BigFamily',
'Pclass',
'Has_Cabin',
'Embarked',
'Sex',
'Title',
'Ticket_Code',
'Ticket_Code_Remap',
'Survived']][joined_data.Ticket_Code==186].sort_values(by='FamilyName' )
cases = train_df.ConfirmedCases
fatalities = train_df.Fatalities
del train_df['ConfirmedCases']
del train_df['Fatalities']<categorify>
joined_data[['FamilyName',
'Name',
'Age',
'Fare',
'BigFamily',
'Pclass',
'Has_Cabin',
'Embarked',
'Sex',
'Title',
'Ticket_Code',
'Ticket_Code_Remap',
'Survived']][joined_data.Ticket_Code==127].sort_values(by='FamilyName' )
lb = LabelEncoder()
train_df['Country_Region'] = lb.fit_transform(train_df['Country_Region'])
 test_df['Country_Region'] = lb.transform(test_df['Country_Region'] )<normalization>
selected_features = ['Age',
'Fare',
'BigFamily',
'Pclass',
'Has_Cabin',
'Embarked_Code',
'Sex_Code',
'Title_Remap_Code',
'Ticket_Code_Remap',
'FamilySurvived',
]
one_hot_features = ['Pclass',
'BigFamily',
'FamilySurvived',
'Embarked_Code',
'Title_Remap_Code',
'Ticket_Code_Remap',
]
selected_data = joined_data[selected_features]
print('Does the following feature contain any NaN? ')
 for f in selected_features:
print('%s: %s' %(f, repr(selected_data[f].isna().any())) )
scaler = MinMaxScaler()
x_train = scaler.fit_transform(train_df.values)
 x_test = scaler.transform(test_df.values )<import_modules>
selected_data_one_hot = pd.get_dummies(selected_data,
columns = one_hot_features)
 rescaling_features = ['Age', 'Fare']
std_scaler = StandardScaler()
for f in rescaling_features:
selected_data_one_hot[f] = std_scaler.fit_transform(selected_data_one_hot[f].values.reshape(-1, 1))
train_x = selected_data[:train.shape[0]]
test_x = selected_data[train.shape[0]:]
train_x_one_hot = selected_data_one_hot[:train.shape[0]]
test_x_one_hot = selected_data_one_hot[train.shape[0]:]
train_y = data[:train.shape[0]].Survived
from xgboost import XGBRegressor<train_model>
parameters = {'n_estimators': [10,50,100,200],
'learning_rate': [0.05, 0.1],
'max_depth': [2,3,4],
'min_samples_leaf': [2,3],
'verbose': [0]}
grid_obj = GridSearchCV(GradientBoostingClassifier() , parameters, scoring = 'roc_auc', cv = 4, n_jobs = 4, verbose = 1)
 grid_obj = grid_obj.fit(train_x, train_y)
 gb = grid_obj.best_estimator_
gb
rf = XGBRegressor(n_estimators = 2500 , random_state = 0 , max_depth = 27)
 rf.fit(x_train,cases )<predict_on_test>
model = gb.fit(train_x, train_y)
 pred_y = gb.predict(train_x)
 f1 = f1_score(train_y, pred_y)
 acc = accuracy_score(train_y, pred_y )
cases_pred = rf.predict(x_test)
 cases_pred<feature_engineering>
test_y = pd.Series(gb.predict(test_x), name="Survived", dtype='int32')
 results = pd.concat([data[train.shape[0]:].PassengerId, test_y], axis=1 )
cases_pred = np.around(cases_pred,decimals = 0)
 cases_pred<concatenate>
results.to_csv("gbdt_csv_to_submit.csv",index=False )
x_train_cas = []
for i in range(len(x_train)) :
x = list(x_train[i])
 x.append(cases[i])
 x_train_cas.append(x)
 x_train_cas[0]<prepare_x_and_y>
training_config = {
'gbdt': {
'clf': GradientBoostingClassifier() ,
'parameters': {
'n_estimators': [10,50,100,200],
'learning_rate': [0.05, 0.1],
'max_depth': [2,3,4],
'min_samples_leaf': [2,3],
},
'n_jobs': 4,
'one_hot': False
},
'logit' : {
'clf': LogisticRegression() ,
'parameters': {
'penalty': ['l1', 'l2'],
'C': list(np.arange(0.5, 8.0, 0.1))
}
},
'svm': {
'clf': LinearSVC() ,
'parameters': {
'penalty': ['l2'],
'loss': ['hinge', 'squared_hinge'],
'C': list(np.arange(0.5, 8.0, 0.1))
}
},
'rf': {
'clf': RandomForestClassifier() ,
'parameters': {
'n_estimators': [10,50,100,200],
'criterion': ['gini', 'entropy'],
'max_depth': [2,3,4],
'min_samples_leaf': [2,3],
},
'n_jobs': 4,
'one_hot': False
},
'ada': {
'clf': AdaBoostClassifier() ,
'parameters': {
'n_estimators': [10,50,100,200],
'learning_rate': [0.05, 0.1, 0.5, 1.0, 2.0],
},
'n_jobs': 4,
'one_hot': False
}
}
exp_to_run = training_config.keys()
x_train_cas = np.array(x_train_cas )<train_model>
results = { 'name': [], 'f1': [], 'accuracy': [] }
train_pred = {}
test_pred = {}
for name in exp_to_run:
conf = training_config[name]
clf = conf['clf']
parameters = conf['parameters']
n_jobs = conf.get('n_jobs', 1)
 one_hot = conf.get('one_hot', True)
 print('=' * 20)
 print('Starting training:', name)
 grid_obj = GridSearchCV(clf, parameters, scoring = 'roc_auc', cv = 4, n_jobs = n_jobs, verbose = 1)
 train_X = train_x_one_hot if one_hot else train_x
print('Number of Features:', train_X.columns.shape[0])
 grid_obj = grid_obj.fit(train_X, train_y)
 best_clf = grid_obj.best_estimator_
print('Best classifier:', repr(best_clf))
model = best_clf.fit(train_X, train_y)
 pred_y = best_clf.predict(train_X)
 train_pred[name] = pred_y
f1 = f1_score(train_y, pred_y)
 acc = accuracy_score(train_y, pred_y)
 results['name'].append(name)
 results['f1'].append(f1)
 results['accuracy'].append(acc)
 test_X = test_x_one_hot if one_hot else test_x
test_y = pd.Series(best_clf.predict(test_X), name="Survived", dtype='int32')
 test_pred[name] = test_y
output = pd.concat([test.PassengerId, test_y], axis=1)
 output_filename = name + "_csv_to_submit.csv"
print('Writing submission file:', output_filename)
 output.to_csv(output_filename, index=False)
rf = XGBRegressor(n_estimators = 2500 , random_state = 0 , max_depth = 27)
 rf.fit(x_train_cas,fatalities )<concatenate>
pred_y = pd.DataFrame.from_dict(train_pred ).mean(axis=1)> 0.5
f1 = f1_score(train_y, pred_y)
 acc = accuracy_score(train_y, pred_y)
 results['name'].append('voting')
 results['f1'].append(f1)
 results['accuracy'].append(acc)
 test_y = pd.Series(pd.DataFrame.from_dict(test_pred ).mean(axis=1)> 0.5,
name="Survived",
dtype='int32')
 output = pd.concat([test.PassengerId, test_y], axis=1)
 output_filename = 'voting_csv_to_submit.csv'
print('Writing submission file:', output_filename)
 output.to_csv(output_filename, index=False )
<predict_on_test><EOS> 
pd.DataFrame.from_dict(results )