Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
import numpy as np
import pandas as pd
import scipy.special
import matplotlib.pyplot as plt
import os
import random
def prepare_pca(matches_combined_teams, in_components=None):
print(len(matches_combined_teams[1].drop(['winPlacePerc'],axis=1 ).columns))
ipca = IncrementalPCA(n_components=in_components)
 for i in matches_combined_teams:
ipca.partial_fit(i.drop(['winPlacePerc'],axis=1 ).values)
 print(ipca.explained_variance_ratio_)
 print(ipca.n_components_)
 return ipca
def prepare_pca_input(df_train, df_test, dfeatures=None):
df_train_in=[]
df_train_groupids=[]
for i in df_train:
df_train_groupids.append(i[['groupId','maxPlace']])
 if len(dfeatures)>0:
df_train_in.append(i.drop(['groupId','maxPlace'],axis=1 ).drop(dfeatures,axis=1 ).reindex(sorted(i.drop(['groupId','maxPlace'],axis=1 ).drop(dfeatures,axis=1 ).columns), axis=1))
else:
df_train_in.append(i.drop(['groupId','maxPlace'],axis=1 ).reindex(sorted(i.drop(['groupId','maxPlace'],axis=1 ).columns), axis=1))
df_test_in=[]
df_test_groupids=[]
for df, dic in df_test:
df_test_groupids.append([df[['groupId','maxPlace']],dic])
 if len(dfeatures)>0:
df_test_in.append(df.drop(['groupId','maxPlace'],axis=1 ).drop(dfeatures,axis=1 ).reindex(sorted(df.drop(['groupId','maxPlace'],axis=1 ).drop(dfeatures,axis=1 ).columns), axis=1))
else:
df_test_in.append(df.drop(['groupId','maxPlace'],axis=1 ).reindex(sorted(df.drop(['groupId','maxPlace'],axis=1 ).columns), axis=1))
return df_train_in, df_train_groupids, df_test_in, df_test_groupids
def pca_transform_data(matches_combined_teams, matches_combined_teams_test, ipca):
x_train_pca = []
for i in matches_combined_teams:
x_train_pca.append(ipca.transform(i.drop('winPlacePerc',axis=1 ).values))
y_train_pca=[]
for i in matches_combined_teams:
y_train_pca.append(i['winPlacePerc'].values.reshape(-1, 1))
x_test_pca = []
for i in matches_combined_teams_test:
x_test_pca.append(ipca.transform(i.values))
return x_train_pca, y_train_pca, x_test_pca<normalization>
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
from sklearn.metrics import accuracy_score
from xgboost import XGBClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
def scale_data(x_train_pca, x_test_pca):
scaler = MinMaxScaler()
for i in x_train_pca:
scaler.partial_fit(i)
 x_train_scaled=[]
for i in x_train_pca:
x_train_scaled.append(scaler.transform(i))
x_test_scaled=[]
for i in x_test_pca:
x_test_scaled.append(scaler.transform(i))
return x_train_scaled, x_test_scaled<train_model>
warnings.filterwarnings("ignore" )
def mlp_evaluate(x_train, y_train, df_train_groupids, training_level=100, verbose=False):
working_sample=np.random.choice(len(x_train), training_level*5)
 mlp=MLPRegressor(solver='adam',hidden_layer_sizes=(50,50,))
train_counter=0
lenxtrain=len(x_train)
 kfold_score=[]
for k in range(training_level):
for i in range(4):
mlp.partial_fit(x_train[working_sample[train_counter]], y_train[working_sample[train_counter]].ravel())
 train_counter+=1
y_pred_kfold=pd.DataFrame(mlp.predict(x_train[working_sample[train_counter]]), columns=['ranks'])
 y_pred_kfold=y_pred_kfold.join(df_train_groupids[working_sample[train_counter]] ).sort_values(by='ranks', ascending=False)
 y_test=pd.DataFrame(y_train[working_sample[train_counter]], columns=['winPlacePerc'] ).join(df_train_groupids[working_sample[train_counter]] ).sort_values(by='groupId', ascending=True)
 tmp_winPlacePerc=[]
for i in range(1,len(y_pred_kfold)+1):
tmp_winPlacePerc.append([y_pred_kfold['groupId'].values[i-1],(len(y_pred_kfold)-i)/(len(y_pred_kfold)-1)])
 tmp_winPlacePerc=pd.DataFrame(tmp_winPlacePerc, columns=['groupId','winPlacePerc'] ).sort_values(by='groupId', ascending=True)
 kfold_score.append(mean_absolute_error(y_test['winPlacePerc'],tmp_winPlacePerc['winPlacePerc']))
train_counter+=1
if verbose:
print("Evaluating "+str(k)+"/"+str(training_level))
print("Current mean bsolut error: "+str(np.mean(kfold_score)))
 print("Average mean absolute error: "+str(np.mean(kfold_score)))
 return kfold_score<train_model>
train_data = pd.read_csv(path_in+'train.csv', index_col=0)
 test_data = pd.read_csv(path_in+'test.csv', index_col=0)
 samp_subm = pd.read_csv(path_in+'gender_submission.csv', index_col=0 )
def mlp_train(x_train, y_train):
mlp=MLPRegressor(solver='adam',hidden_layer_sizes=(50,50,))
lenxtrain=len(x_train)
 for i in range(lenxtrain):
mlp.partial_fit(x_train[i], y_train[i].ravel())
 return mlp<predict_on_test>
print('number train samples: ', len(train_data.index))
print('number test samples: ', len(test_data.index))
print('number features: ', len(train_data.columns)-1 )
def predict_winPlacePerc(x_test, df_test_groupids, mlp, verbose=False):
winPlacePerc=[]
num_matches=len(x_test)
 for k in range(num_matches):
if verbose and k%100==0:
print("Predicting match "+str(k)+"/"+str(num_matches))
y_pred=pd.DataFrame(mlp.predict(x_test[k]), columns=['ranks'])
 y_pred=y_pred.join(df_test_groupids[k][0] ).sort_values(by='ranks', ascending=False)
 tmp_winPlacePerc=[]
for i in range(1,len(y_pred)+1):
tmp_winPlacePerc.append([y_pred['groupId'].values[i-1],(len(y_pred)-i)/(len(y_pred)-1)])
 winPlacePerc.append(pd.DataFrame(tmp_winPlacePerc, columns=['groupId','winPlacePerc'] ).sort_values(by='groupId', ascending=True))
result_list=[]
counter=0
for df, el in df_test_groupids:
match_results=[]
for i, gid in el.items() :
wpp=winPlacePerc[counter][winPlacePerc[counter]['groupId']==gid]['winPlacePerc'].values
match_results.append([i,wpp])
 result_list.append(pd.DataFrame(match_results, columns=['Id','winPlacePerc']))
counter+=1
winPlacePerc_tot=pd.concat(result_list)
 winPlacePerc_tot['winPlacePerc']=winPlacePerc_tot['winPlacePerc'].str[0]
return winPlacePerc_tot<define_variables>
train_data.isnull().sum()
def predict_one_team_matches(df):
result_list=[]
counter=0
for df, dic in df:
match_results=[]
for i, gid in dic.items() :
match_results.append([i,0.0])
 result_list.append(pd.DataFrame(match_results, columns=['Id','winPlacePerc']))
counter+=1
if len(result_list)>0:
return pd.concat(result_list)
 else:
return []
def predict_two_team_matches(df):
result_list=[]
counter=0
for df, dic in df:
match_results=[]
killpointsMax=np.max([df['killPlace'].values[0],df['killPlace'].values[1]])
 maxKillpointsGId=df[df['killPlace']==killpointsMax]['groupId'].values[0]
for i, gid in dic.items() :
if maxKillpointsGId in gid:
match_results.append([i,0.0])
 else:
match_results.append([i,1.0])
 result_list.append(pd.DataFrame(match_results, columns=['Id','winPlacePerc']))
counter+=1
if len(result_list)>0:
return pd.concat(result_list)
 else:
return []<concatenate>
cols_with_missing_train = [col for col in train_data.columns if train_data[col].isnull().any() ]
cols_with_missing_test = [col for col in test_data.columns if test_data[col].isnull().any() ]
df_red_squad=combine_training_teams(df_squad )<normalization>
print('train missing values: {:0.2f}%'.format(100*train_data['Age'].isna().sum() /len(train_data)))
 print('test missing values: {:0.2f}%'.format(100*test_data['Age'].isna().sum() /len(test_data)) )
df_red_squad_1, df_red_squad_2, df_red_squad=filter_small_teams(df_red_squad )<concatenate>
def fill_age(s):
if np.isnan(s)== False:
return s
else:
return random.randrange(age_mean-age_std, age_mean+age_std )
df_test_per_match_squad = combine_testing_teams(df_test_squad )<split>
train_data['Age'] = train_data['Age'].apply(fill_age)
 test_data['Age'] = test_data['Age'].apply(fill_age )
df_test_per_match_squad_1, df_test_per_match_squad_2, df_test_per_match_squad=filter_small_teams_test(df_test_per_match_squad )<set_options>
print('train missing values: {:0.2f}%'.format(100*train_data['Cabin'].isna().sum() /len(train_data)))
 print('test missing values: {:0.2f}%'.format(100*test_data['Cabin'].isna().sum() /len(test_data)) )
df_squad.drop(['killPoints','maxPlace','roadKills','teamKills','vehicleDestroys','winPoints','swimDistance'],axis=1 ).corr().style.format("{:.2}" ).background_gradient(cmap=plt.get_cmap('coolwarm'), axis=1 )<drop_column>
train_data['Cabin'] = train_data['Cabin'].fillna('Unknown', inplace=False)
 test_data['Cabin'] = test_data['Cabin'].fillna('Unknown', inplace=False )
del df_squad
del df_test_squad<define_variables>
print('train missing values:', train_data['Embarked'].isna().sum() )
drop_features_squad=['killPoints','roadKills','teamKills','vehicleDestroys','winPoints']<split>
train_data['Embarked'] = train_data['Embarked'].fillna('Unknown', inplace=False )
df_train_in_squad, df_train_groupids_squad, df_test_in_squad, df_test_groupids_squad = prepare_pca_input(df_red_squad, df_test_per_match_squad, drop_features_squad )<concatenate>
print('test missing values:', test_data['Fare'].isna().sum() )
pca_squad=prepare_pca(df_train_in_squad, in_components=None )<prepare_x_and_y>
mean = test_data['Fare'].mean()
test_data['Fare'] = test_data['Fare'].fillna(mean, inplace=False )
x_train_pca_squad, y_train_pca_squad, x_test_pca_squad=pca_transform_data(df_train_in_squad, df_test_in_squad, pca_squad )<drop_column>
train_data['FamilySize'] = train_data['SibSp'] + train_data['Parch'] + 1
test_data['FamilySize'] = test_data['SibSp'] + test_data['Parch'] + 1
del df_red_squad
del df_test_in_squad
del df_train_in_squad<normalization>
def family_group(s):
if(s >= 2)&(s <= 4):
return 2
elif(( s > 4)&(s <= 7)) |(s == 1):
return 1
elif(s > 7):
return 0
x_train_scaled_squad, x_test_scaled_squad=scale_data(x_train_pca_squad, x_test_pca_squad )<compute_test_metric>
train_data['FamilyGroup'] = train_data['FamilySize'].apply(family_group)
 test_data['FamilyGroup'] = test_data['FamilySize'].apply(family_group )
eval_score_squad=mlp_evaluate(x_train_scaled_squad, y_train_pca_squad, df_train_groupids_squad, training_level=10000, verbose=False )<train_model>
def IsAlone(s):
if s==1:
return 1
else:
return 0
mlp_squad=mlp_train(x_train_scaled_squad, y_train_pca_squad )<predict_on_test>
train_data['IsAlone'] = train_data['FamilySize'].apply(IsAlone)
 test_data['IsAlone'] = test_data['FamilySize'].apply(IsAlone )
winPlacePerc_one_team_squad=predict_one_team_matches(df_test_per_match_squad_1)
 winPlacePerc_two_teams_squad=predict_two_team_matches(df_test_per_match_squad_2 )<predict_on_test>
ticket_group = dict(( train_data['Ticket'].append(test_data['Ticket'])).value_counts())
 train_data['TicketGroup'] = train_data['Ticket'].apply(lambda x: ticket_group[x])
 test_data['TicketGroup'] = test_data['Ticket'].apply(lambda x: ticket_group[x] )
winPlacePerc_squad = predict_winPlacePerc(x_test_scaled_squad, df_test_groupids_squad, mlp_squad, verbose=False)
 winPlacePerc_squad = winPlacePerc_squad.append(winPlacePerc_two_teams_squad )<concatenate>
train_data['Title'] = train_data['Name'].apply(lambda x:x.split(',')[1].split('.')[0].strip())
 test_data['Title'] = test_data['Name'].apply(lambda x:x.split(',')[1].split('.')[0].strip())
 title_dict = {}
title_dict.update(dict.fromkeys(['Capt', 'Col', 'Major', 'Dr', 'Rev'], 'Officer'))
title_dict.update(dict.fromkeys(['Don', 'Sir', 'the Countess', 'Dona', 'Lady'], 'Royalty'))
title_dict.update(dict.fromkeys(['Mme', 'Ms', 'Mrs'], 'Mrs'))
title_dict.update(dict.fromkeys(['Mlle', 'Miss'], 'Miss'))
title_dict.update(dict.fromkeys(['Mr'], 'Mr'))
title_dict.update(dict.fromkeys(['Master','Jonkheer'], 'Master'))
train_data['Title'] = train_data['Title'].map(title_dict)
 test_data['Title'] = test_data['Title'].map(title_dict )
df_red_duo=combine_training_teams(df_duo )<normalization>
def age_group(s):
if s == 0:
return -1
elif(s > 0)&(s <= 13):
return 1
elif(s > 13)&(s <= 20):
return 2
elif(s > 20)&(s <= 30):
return 3
elif(s > 30)&(s <= 40):
return 4
elif(s > 40)&(s <= 50):
return 5
elif(s > 50)&(s <= 60):
return 6
elif(s > 60)&(s <= 70):
return 7
elif(s > 70)&(s <= 80):
return 8
df_red_duo_1, df_red_duo_2, df_red_duo=filter_small_teams(df_red_duo )<concatenate>
train_data['AgeGroup'] = train_data['Age'].apply(age_group)
 test_data['AgeGroup'] = test_data['Age'].apply(age_group )
df_test_per_match_duo = combine_testing_teams(df_test_duo )<split>
train_data['FareGroup'] = train_data['Fare'].apply(fare_group)
 test_data['FareGroup'] = test_data['Fare'].apply(fare_group )
df_test_per_match_duo_1, df_test_per_match_duo_2, df_test_per_match_duo=filter_small_teams_test(df_test_per_match_duo )<set_options>
train_data['LastName'] = train_data['Name'].apply(last_name)
 test_data['LastName'] = test_data['Name'].apply(last_name )
df_duo.drop(['killPoints','roadKills','teamKills','vehicleDestroys','winPoints'],axis=1 ).corr().style.format("{:.2}" ).background_gradient(cmap=plt.get_cmap('coolwarm'), axis=1 )<drop_column>
for row in train_data.index:
if train_data.loc[row, 'Sex']=='female' or train_data.loc[row, 'Age']<14:
train_data.loc[row, 'WomenOrChild'] = 1
else:
train_data.loc[row, 'WomenOrChild'] = 0
for row in test_data.index:
if test_data.loc[row, 'Sex']=='female' or test_data.loc[row, 'Age']<14:
test_data.loc[row, 'WomenOrChild'] = 1
else:
test_data.loc[row, 'WomenOrChild'] = 0
del df_duo
del df_test_duo<define_variables>
train_data['FarePerPerson']=train_data['Fare']/(train_data['FamilySize']+1)
 test_data['FarePerPerson']=test_data['Fare']/(test_data['FamilySize']+1 )
drop_features_duo=['killPoints','roadKills','teamKills','vehicleDestroys','winPoints']<split>
train_data['AgePclass'] = train_data['Age']*train_data['Pclass']
test_data['AgePclass'] = test_data['Age']*test_data['Pclass']
df_train_in_duo, df_train_groupids_duo, df_test_in_duo, df_test_groupids_duo = prepare_pca_input(df_red_duo, df_test_per_match_duo, dfeatures=drop_features_duo )<concatenate>
train_data['Cabin'] = train_data['Cabin'].str[0]
test_data['Cabin'] = test_data['Cabin'].str[0]
pca_duo=prepare_pca(df_train_in_duo, in_components=None )<prepare_x_and_y>
features_cat = ['Sex', 'Cabin', 'Embarked', 'Title']
le = LabelEncoder()
for col in features_cat:
le.fit(train_data[col])
 train_data[col] = le.transform(train_data[col])
 test_data[col] = le.transform(test_data[col] )
x_train_pca_duo, y_train_pca_duo, x_test_pca_duo=pca_transform_data(df_train_in_duo, df_test_in_duo, pca_duo )<drop_column>
train_data_dummy = pd.get_dummies(train_data['Pclass'], prefix = 'Pclass')
 train_data = pd.concat([train_data, train_data_dummy], axis=1)
 del train_data['Pclass']
test_data_dummy = pd.get_dummies(test_data['Pclass'], prefix = 'Pclass')
 test_data = pd.concat([test_data, test_data_dummy], axis=1)
 del test_data['Pclass']
del df_red_duo
del df_test_in_duo
del df_train_in_duo<normalization>
no_features = ['Survived', 'Name', 'Ticket', 'LastName']
x_train_scaled_duo, x_test_scaled_duo=scale_data(x_train_pca_duo, x_test_pca_duo )<compute_test_metric>
X_train = train_data[train_data.columns.difference(no_features)].copy(deep=False)
 y_train = train_data['Survived']
X_test = test_data[test_data.columns.difference(no_features)].copy(deep=False )
eval_score_duo=mlp_evaluate(x_train_scaled_duo, y_train_pca_duo, df_train_groupids_duo, training_level=10000, verbose=False )<train_model>
min_max = MinMaxScaler()
X_train_scaled = min_max.fit_transform(X_train)
 X_test_scaled = min_max.transform(X_test )
mlp_duo=mlp_train(x_train_scaled_duo, y_train_pca_duo )<predict_on_test>
param_grid = {'criterion': ['gini'],
'max_features': [None, 'auto', 'sqrt', 'log2'],
'max_depth': [i for i in range(1, 6)],
'class_weight': [None, 'balanced'],
'min_samples_split': [2, 4, 6, 8, 10 ,12],
'min_samples_leaf': [1, 2, 3, 4],
'random_state': [2020]}
grid = GridSearchCV(DecisionTreeClassifier() , param_grid=param_grid, cv=5, scoring='accuracy')
 grid.fit(X_train_scaled, y_train)
 best_params = grid.best_params_
print('Best score of cross validation: {:.2f}'.format(grid.best_score_))
print('Best parameters:', best_params )
winPlacePerc_one_team_duo=predict_one_team_matches(df_test_per_match_duo_1)
 winPlacePerc_two_teams_duo=predict_two_team_matches(df_test_per_match_duo_2 )<predict_on_test>
model = DecisionTreeClassifier()
model.set_params(**best_params )
winPlacePerc_duo = predict_winPlacePerc(x_test_scaled_duo, df_test_groupids_duo, mlp_duo, verbose=False)
 winPlacePerc_duo=winPlacePerc_duo.append(winPlacePerc_one_team_duo )<set_options>
model.fit(X_train_scaled, y_train )
df_solo['winPlacePerc'].fillna(0.0, inplace=True )<concatenate>
y_test = model.predict(X_test_scaled )
df_red_solo=combine_training_teams(df_solo )<normalization>
output = pd.DataFrame({'PassengerId': samp_subm.index,
'Survived': y_test})
 output.to_csv('submission.csv', index=False )
<concatenate><EOS> 
output['Survived'].value_counts()