Kaggle competition: Titanic - Machine Learning from Disaster
test_new = import_data('.. /input/test.csv')
 test_new['winPlacePercPred'] = pred
aux = test_new.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test_new = test_new.merge(aux, how='left', on=['matchId','groupId'])
 submission = import_data('.. /input/sample_submission.csv')
 submission = test_new[['Id','winPlacePerc']]
submission.to_csv('sample_submission.csv', index=False )<set_options>
warnings.filterwarnings('ignore')
warnings.filterwarnings("ignore")
 gc.enable()
print(os.listdir(".. /input"))<load_from_csv>
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams['figure.figsize'] = 12,8
def feature_engineering(is_train=True,debug=True):
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 print("remove some columns")
 target = 'winPlacePerc'
print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = df.groupby(['matchId','groupId'])[target].agg('mean')
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
del df, df_out, agg, agg_rank
gc.collect()
return X, y<prepare_x_and_y>
data_raw = pd.read_csv('.. /input/titanic/train.csv')
 data_val = pd.read_csv('.. /input/titanic/test.csv')
 data1 = data_raw.copy(deep = True)
 data_cleaner = [data1, data_val]
print(data_raw.info())
 data_raw.sample(10 )
x_train, y_train = feature_engineering(True,False)
 x_test, _ = feature_engineering(False,True )<drop_column>
women = data_raw.loc[data_raw.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )
x_train = reduce_mem_usage(x_train)
 x_test = reduce_mem_usage(x_test )<load_from_csv>
men = data_raw.loc[data_raw.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )
def post_rst(pred_test):
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission_adjusted.csv", index=False )<choose_model_class>
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
data1.drop(drop_column, axis=1, inplace = True)
 print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
def nn_model(input_shape=x_train.shape[1], hidden_size=64):
model = Sequential()
model.add(Dense(hidden_size, input_dim=input_shape, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mae'])
 return model<split>
for dataset in data_cleaner:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['Title'] = dataset['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 stat_min = 10
title_names =(data1['Title'].value_counts() < stat_min)
 data1['Title'] = data1['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 print(data1['Title'].value_counts())
 print("-"*10)
 data1.info()
data_val.info()
data1.sample(10 )
def train_val_split(x_train, y_train):
train_index = round(int(x_train.shape[0]*0.8))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
del x_train, y_train
gc.collect() ;
return dev_X, val_X, dev_y, val_y<split>
label = LabelEncoder()
for dataset in data_cleaner:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['Title_Code'] = label.fit_transform(dataset['Title'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
data1_x = ['Sex','Pclass', 'Embarked', 'Title','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
data1_x_calc = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code','SibSp', 'Parch', 'Age', 'Fare']
data1_xy = Target + data1_x
print('Original X Y: ', data1_xy, '
')
 data1_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
data1_xy_bin = Target + data1_x_bin
print('Bin X Y: ', data1_xy_bin, '
')
 data1_dummy = pd.get_dummies(data1[data1_x])
 data1_x_dummy = data1_dummy.columns.tolist()
data1_xy_dummy = Target + data1_x_dummy
print('Dummy X Y: ', data1_xy_dummy, '
')
 data1_dummy.head()
dev_X, val_X, dev_y, val_y = train_val_split(x_train, y_train)
 def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':5000, 'early_stopping_rounds':200,
"num_leaves" : 150, "learning_rate" : 0.05, "bagging_fraction" : 0.5,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.5
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y
pred_test_lgb = run_lgb(dev_X, dev_y, val_X, val_y, x_test)
 pred_test_lgb = pred_test_lgb.reshape(-1,1)
 del dev_X, val_X, dev_y, val_y
gc.collect()<train_model>
train1_x, test1_x, train1_y, test1_y = model_selection.train_test_split(data1[data1_x_calc], data1[Target], random_state = 0)
 train1_x_bin, test1_x_bin, train1_y_bin, test1_y_bin = model_selection.train_test_split(data1[data1_x_bin], data1[Target] , random_state = 0)
 train1_x_dummy, test1_x_dummy, train1_y_dummy, test1_y_dummy = model_selection.train_test_split(data1_dummy[data1_x_dummy], data1[Target], random_state = 0)
 print("Data1 Shape: {}".format(data1.shape))
print("Train1 Shape: {}".format(train1_x.shape))
print("Test1 Shape: {}".format(test1_x.shape))
train1_x_bin.head()
np.random.seed(42)
 std_scaler = StandardScaler().fit(x_train)
 x_train = std_scaler.transform(x_train)
 x_test = std_scaler.transform(x_test)
 mlp_model = LinearRegression()
mlp_model.fit(x_train, y_train, batch_size=128, epochs=8, verbose=1, validation_split=0.1, shuffle=True)
 pred_test_mlp = mlp_model.predict(x_test)
 del mlp_model
gc.collect()<drop_column>
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data1[Target]
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, data1[data1_x_bin], data1[Target], cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(data1[data1_x_bin], data1[Target])
 MLA_predict[MLA_name] = alg.predict(data1[data1_x_bin])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare
del x_train, y_train, x_test
gc.collect()<define_variables>
pivot_female = data1[data1.Sex=='female'].groupby(['Sex','Pclass', 'Embarked','FareBin'])['Survived'].mean()
print('Survival Decision Tree w/Female Node:
',pivot_female)
 pivot_male = data1[data1.Sex=='male'].groupby(['Sex','Title'])['Survived'].mean()
print('
Survival Decision Tree w/Male Node:
',pivot_male )
pred_test =(pred_test_mlp + pred_test_lgb)/ 2.0<set_options>
def mytree(df):
Model = pd.DataFrame(data = {'Predict':[]})
 male_title = ['Master']
for index, row in df.iterrows() :
Model.loc[index, 'Predict'] = 0
if(df.loc[index, 'Sex'] == 'female'):
Model.loc[index, 'Predict'] = 1
if(( df.loc[index, 'Sex'] == 'female')& 
(df.loc[index, 'Pclass'] == 3)& 
(df.loc[index, 'Embarked'] == 'S')& 
(df.loc[index, 'Fare'] > 8)
):
Model.loc[index, 'Predict'] = 0
if(( df.loc[index, 'Sex'] == 'male')& 
(df.loc[index, 'Title'] in male_title)
):
Model.loc[index, 'Predict'] = 1
return Model
Tree_Predict = mytree(data1)
 print('Decision Tree Model Accuracy/Precision Score: {:.2f}%
'.format(metrics.accuracy_score(data1['Survived'], Tree_Predict)*100))
print(metrics.classification_report(data1['Survived'], Tree_Predict))
%load_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
dtree = tree.DecisionTreeClassifier(random_state = 0)
 base_results = model_selection.cross_validate(dtree, data1[data1_x_bin], data1[Target], cv = cv_split)
 dtree.fit(data1[data1_x_bin], data1[Target])
 print('BEFORE DT Parameters: ', dtree.get_params())
 print("BEFORE DT Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 param_grid = {'criterion': ['gini', 'entropy'],
'max_depth': [2,4,6,8,10,None],
'random_state': [0]
}
tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 tune_model.fit(data1[data1_x_bin], data1[Target])
 print('AFTER DT Parameters: ', tune_model.best_params_)
 print("AFTER DT Test w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score 3*std: +/- {:.2f}".format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10 )
from fastai.imports import *
from fastai.structured import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics<load_from_csv>
print('BEFORE DT RFE Training Shape Old: ', data1[data1_x_bin].shape)
 print('BEFORE DT RFE Training Columns Old: ', data1[data1_x_bin].columns.values)
 print("BEFORE DT RFE Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT RFE Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 dtree_rfe = feature_selection.RFECV(dtree, step = 1, scoring = 'accuracy', cv = cv_split)
 dtree_rfe.fit(data1[data1_x_bin], data1[Target])
 X_rfe = data1[data1_x_bin].columns.values[dtree_rfe.get_support() ]
rfe_results = model_selection.cross_validate(dtree, data1[X_rfe], data1[Target], cv = cv_split)
 print('AFTER DT RFE Training Shape New: ', data1[X_rfe].shape)
 print('AFTER DT RFE Training Columns New: ', X_rfe)
 print("AFTER DT RFE Test w/bin score mean: {:.2f}".format(rfe_results['test_score'].mean() *100))
print("AFTER DT RFE Test w/bin score 3*std: +/- {:.2f}".format(rfe_results['test_score'].std() *100*3))
print('-'*10)
 rfe_tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 rfe_tune_model.fit(data1[X_rfe], data1[Target])
 print('AFTER DT RFE Tuned Parameters: ', rfe_tune_model.best_params_)
 print("AFTER DT RFE Tuned Test w/bin score mean: {:.2f}".format(rfe_tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT RFE Tuned Test w/bin score 3*std: +/- {:.2f}".format(rfe_tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10 )
df_raw = pd.read_csv('.. /input/train_V2.csv' )<correct_missing_values>
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
vote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 vote_hard_cv = model_selection.cross_validate(vote_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting Test w/bin score mean: {:.2f}".format(vote_hard_cv['test_score'].mean() *100))
print("Hard Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_hard_cv['test_score'].std() *100*3))
print('-'*10)
 vote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 vote_soft_cv = model_selection.cross_validate(vote_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting Test w/bin score mean: {:.2f}".format(vote_soft_cv['test_score'].mean() *100))
print("Soft Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_soft_cv['test_score'].std() *100*3))
print('-'*10 )
df_raw = df_raw.dropna()<feature_engineering>
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_soft_cv['test_score'].std() *100*3))
print('-'*10 )
<sort_values><EOS> 
print(data_val.info())
 print("-"*10)
 data_val['Survived'] = mytree(data_val ).astype(int)
 data_val['Survived'] = grid_hard.predict(data_val[data1_x_bin])
 submit = data_val[['PassengerId','Survived']]
submit.to_csv(".. /working/submit.csv", index=False)
 print('Validation Data Distribution:
', data_val['Survived'].value_counts(normalize = True))
submit.sample(10 )