Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<predict_on_test>
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42
train_preds = alg.predict(train[feats] )<compute_test_metric>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['Survived'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )
mean_squared_error(train['nota_mat'], train_preds)**(1/2 )<predict_on_test>
def display_missing(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in dfs:
print('{}'.format(df.name))
display_missing(df )
valid_preds = alg.predict(valid[feats] )<compute_test_metric>
age_by_pclass_sex = df_all.groupby(['Sex', 'Pclass'] ).median() ['Age']
for pclass in range(1, 4):
for sex in ['female', 'male']:
print('Median age of Pclass {} {}s: {}'.format(pclass, sex, age_by_pclass_sex[sex][pclass]))
print('Median age of all passengers: {}'.format(df_all['Age'].median()))
df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
mean_squared_error(valid['nota_mat'], valid_preds)**(1/2 )<predict_on_test>
df_all[df_all['Embarked'].isnull() ]
test['nota_mat'] = np.exp(alg.predict(test[feats]))<save_to_csv>
df_all['Embarked'] = df_all['Embarked'].fillna('S' )
test[['codigo_mun', 'nota_mat']].to_csv('Edison_Martins_13.csv', index=False )<load_from_csv>
med_fare = df_all.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med_fare )
test = pd.read_csv('.. /input/test.csv',low_memory=False)
 df = pd.read_csv('.. /input/train.csv',low_memory=False )<feature_engineering>
idx = df_all[df_all['Deck'] == 'T'].index
df_all.loc[idx, 'Deck'] = 'A'
test['nota_mat'] = -2<concatenate>
df_all_decks_survived = df_all.groupby(['Deck', 'Survived'] ).count().drop(columns=['Sex', 'Age', 'SibSp', 'Parch', 'Fare',
'Embarked', 'Pclass', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
def get_survived_dist(df):
surv_counts = {'A':{}, 'B':{}, 'C':{}, 'D':{}, 'E':{}, 'F':{}, 'G':{}, 'M':{}}
decks = df.columns.levels[0]
for deck in decks:
for survive in range(0, 2):
surv_counts[deck][survive] = df[deck][survive][0]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
return surv_counts, surv_percentages
def display_surv_dist(percentages):
df_survived_percentages = pd.DataFrame(percentages ).transpose()
deck_names =('A', 'B', 'C', 'D', 'E', 'F', 'G', 'M')
 bar_count = np.arange(len(deck_names))
bar_width = 0.85
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(20, 10))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Deck', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, deck_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage in Decks', size=18, y=1.05)
 plt.show()
all_surv_count, all_surv_per = get_survived_dist(df_all_decks_survived)
 display_surv_dist(all_surv_per )
df = df.append(test )<count_missing_values>
df_all['Deck'] = df_all['Deck'].replace(['A', 'B', 'C'], 'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D', 'E'], 'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F', 'G'], 'FG')
 df_all['Deck'].value_counts()
df.isnull().sum()<feature_engineering>
df_all.drop(['Cabin'], inplace=True, axis=1)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train, df_test]
for df in dfs:
display_missing(df )
df['codigo_mun'] = df['codigo_mun'].apply(lambda x: x.replace('ID_ID_', ''))<data_type_conversions>
corr = df_train_corr_nd['Correlation Coefficient'] > 0.1
df_train_corr_nd[corr]
for col in df.columns:
if df[col].dtype == 'object' and col != 'codigo_mun':
df[col] = df[col].astype('category' ).cat.codes<define_variables>
corr = df_test_corr_nd['Correlation Coefficient'] > 0.1
df_test_corr_nd[corr]
removed_cols = ['Unnamed: 0', 'codigo_mun','municipio', 'nota_mat']
feats = [c for c in df.columns if c not in removed_cols]<filter>
df_all = concat_df(df_train, df_test)
 df_all.head()
test = df[df['nota_mat'] == -2]
df = df[~(df['nota_mat'] == -2)]<split>
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
df_all['Family'] = extract_surname(df_all['Name'])
 df_train = df_all.loc[:890]
df_test = df_all.loc[891:]
dfs = [df_train, df_test]
train, valid = train_test_split(df, test_size=0.3333, random_state=42 )<choose_model_class>
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA
rf = RandomForestRegressor(n_estimators=200, min_samples_split=5, max_depth=4, random_state=42 )<train_model>
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2
rf.fit(train[feats], train['nota_mat'] )<compute_train_metric>
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )
valid_preds = rf.predict(valid[feats])
 mean_squared_error(( valid['nota_mat']), valid_preds)**(1/2 )<define_variables>
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )
removed_cols = ['exp_anos_estudo', 'densidade_dem','gasto_pc_saude', 'hab_p_medico','servidores',
'area','comissionados','comissionados_por_servidor','indice_governanca',
'ranking_igm','porte','capital']
feats = [c for c in df.columns if c not in removed_cols]<train_model>
df_all = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
df_all.drop(columns=drop_cols, inplace=True)
 df_all.head()
rf.fit(train[feats], train['nota_mat'] )<predict_on_test>
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
X_train = pd.DataFrame(X_train)
 y_train = pd.DataFrame(y_train)
 X_test = pd.DataFrame(X_test)
 X_train.columns = feature_stores
X_test.columns = feature_stores
X_train.head()
valid_preds = rf.predict(valid[feats])
 mean_squared_error(( valid['nota_mat']), valid_preds)**(1/2 )<predict_on_test>
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from xgboost import XGBClassifier
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
test['nota_mat'] = rf.predict(valid[feats] )<save_to_csv>
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 old_leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=0)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=20,
max_depth=3,
min_samples_split=2,
min_samples_leaf=1,
max_features='auto',
oob_score=True,
random_state=42,
n_jobs=-1,
verbose=0)
 cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0 )
test[['codigo_mun','nota_mat']].to_csv('rf_iesb_2019.csv', index=False )<load_from_csv>
X_rfe = None
df = pd.read_csv('.. /input/train.csv')
 ts = pd.read_csv('.. /input/test.csv' )<concatenate>
df = df.append(ts )<feature_engineering>
FEATURES_pipeline = np.array(['Deck_1', 'Deck_4', 'Embarked_2', 'Family_Size_Grouped_1'
'Family_Size_Grouped_3', 'Fare', 'Pclass_1', 'Survival_Rate_NA'
'Ticket_Frequency', 'Title_4'] )
df['codigo_mun'] = df['codigo_mun'].str.replace('ID_ID_', '' )<define_variables>
myrun_bestresults = np.array(['Age', 'Deck_1', 'Deck_2', 'Deck_4', 'Embarked_1', 'Embarked_2',
'Embarked_3', 'Family_Size_Grouped_1', 'Family_Size_Grouped_3',
'Family_Size_Grouped_4', 'Fare', 'Is_Married', 'Pclass_1',
'Pclass_2', 'Pclass_3', 'Sex_1', 'Survival_Rate',
'Survival_Rate_NA', 'Ticket_Frequency', 'Title_2', 'Title_3',
'Title_4'] )
lista = ['estado','municipio','porte','populacao','pib','anos_estudo_empreendedor','jornada_trabalho','gasto_pc_educacao','exp_anos_estudo','nota_mat']<data_type_conversions>
choose = [FEATURES,X_rfe,FEATURES_pipeline,myrun_bestresults]
FEATURES = choose[3]
for i in df[lista].select_dtypes(include=['object']):
df[i] = df[i].astype('category' ).cat.codes<filter>
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
leaderboard_model,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = pd.DataFrame()
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, X_train[FEATURES], y_train, cv = cv_split, return_train_score=True)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(X_train[FEATURES], y_train)
 MLA_predict[MLA_name] = alg.predict(X_train[FEATURES])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare
ts = df[df['nota_mat']==-2]
df = df[df['nota_mat']!=-2]<split>
grid_n_estimator = [250,500,1000, 10, 20, 40,50, 100, 1750, 2000, 1500]
grid_ratio = [.1,.25,.5,.75, 1.0]
grid_learn = [.01,.03,.05, 0.75,.1,1.25,.25]
grid_max_depth = [1,2,3, 4,5,6,7,8,9,10]
grid_min_samples = [5, 10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
min_sample_split = [6,4,8]
min_sample_leaf = [6,4,8]
SEED=np.arange(1,100,1)
 vote_est = [ 
('rfc', ensemble.RandomForestClassifier()),
]
grid_param = [
[{
'n_estimators': [20],
'criterion': ["gini"],
'max_depth': [3],
'min_samples_split': [2],
'min_samples_leaf': [1],
'oob_score': [True],
'random_state': [42],
'n_jobs': [-1]
}]
]
start_total = time.perf_counter()
for clf, param in zip(vote_est, grid_param):
start = time.perf_counter()
best_search = model_selection.GridSearchCV(estimator = clf[1], param_grid = param, cv = cv_split, scoring = 'roc_auc')
 best_search.fit(X_train[FEATURES], y_train)
 run = time.perf_counter() - start
best_param = best_search.best_params_
print('The best parameter for {} is {} with a runtime of {:.2f} seconds.'.format(clf[1].__class__.__name__, best_param, run))
clf[1].set_params(**best_param)
 run_total = time.perf_counter() - start_total
print('Total optimization time was {:.2f} minutes.'.format(run_total/60))
print('-'*10 )
train, valid = train_test_split(df[lista], random_state=42)
 train.shape, valid.shape<train_model>
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train[FEATURES], y_train, cv = cv_split, return_train_score=True)
 grid_hard.fit(X_train[FEATURES], y_train)
 print("Hard Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train[FEATURES], y_train, cv = cv_split, return_train_score=True)
 grid_soft.fit(X_train[FEATURES], y_train)
 print("Soft Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_soft_cv['test_score'].std() *100*3))
print('-'*10)
<compute_train_metric><EOS> 
submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = grid_hard.predict(X_test[FEATURES])
 submission_df["Survived"] = submission_df["Survived"].astype(int)
 submission_df.to_csv('submission.csv', header=True, index=False)
 submission_df.head(10 )