Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
sns.set_style('whitegrid')
 sns.set_context('talk')
 params = {'legend.fontsize': 'x-large',
'figure.figsize':(30, 10),
'axes.labelsize': 'x-large',
'axes.titlesize':'x-large',
'xtick.labelsize':'x-large',
'ytick.labelsize':'x-large'}
plt.rcParams.update(params)
folds = pd.read_csv('.. /input/atposfolds/folds.csv' )<define_variables>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print(train_df.shape)
 print(test_df.shape )
fold_num = 1
val_idxs = folds[folds['folds'] == fold_num].index.values<define_variables>
test_df['Survived'] = 0
full = pd.concat([train_df, test_df],
axis=0,
ignore_index=True,
sort=False )
bs = 64
sz=256<categorify>
full.rename(columns={'Pclass':'TicketClass',
'SibSp':'Sibling_Spouse',
'Parch':'Parent_Children',
'Fare':'TicketFare'},
inplace=True )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=360, max_warp=0,
max_zoom=1.1, max_lighting=0.1, p_lighting=0.5)
 src =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_idx(val_idxs)
.label_from_df(cols='diagnosis', label_cls=FloatList)
)
 data=(src.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
 )<compute_test_metric>
full[full.Name.isin(['Connolly, Miss.Kate','Kelly, Mr.James'])].sort_values(by='Name' )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )<choose_model_class>
full.loc[:891,['TicketClass', 'Survived']]\ 
.groupby(['TicketClass'], as_index=False)\ 
.mean().sort_values(by='Survived',
ascending=False )
learn = cnn_learner(data, base_arch=models.resnet50, metrics = [quadratic_kappa], pretrained=False)
 learn.load('resnet50' )<choose_model_class>
full.loc[:891,['Sex', 'Survived']]\ 
.groupby(['Sex'], as_index=False ).mean() \ 
.sort_values(by='Survived',
ascending=False )
learn_2 = cnn_learner(data, base_arch=models.resnet152, metrics = [quadratic_kappa], pretrained=False)
 learn_2.load('resnet152' )<find_best_params>
full.loc[:891,['Embarked', 'Survived']]\ 
.groupby(['Embarked'], as_index=False ).mean() \ 
.sort_values(by='Survived',
ascending=False )
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )<import_modules>
full.drop(['Cabin','Ticket','PassengerId'],
axis=1,
inplace=True )
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json<compute_test_metric>
full.drop(['Name'], axis=1, inplace=True )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<choose_model_class>
full.isnull().sum()
optR = OptimizedRounder()<load_from_csv>
full.loc[full['Embarked'].isnull() ,'Embarked'] = \
full['Embarked'].dropna().mode() [0]
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()<define_variables>
full.loc[full['TicketFare'].isnull() ,'TicketFare'] = \
full['TicketFare'].dropna().mean()
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))<predict_on_test>
full.Age.isnull().sum()
preds,y = learn.get_preds(ds_type=DatasetType.Test )<define_variables>
age_estimator = full[['Age','TicketClass','Sibling_Spouse']]\ 
.groupby(['TicketClass','Sibling_Spouse'] ).agg(['mean','std'])
 age_nulls = full.loc[full.Age.isnull() ,:]
for idx,rec in age_nulls.iterrows() :
mean = age_estimator.loc[(rec['TicketClass'],rec['Sibling_Spouse']),('Age','mean')]
std = age_estimator.loc[(rec['TicketClass'],rec['Sibling_Spouse']),('Age','std')]
gen_age = random.uniform(mean-std, mean+std)
 full.loc[idx,'Age'] = gen_age if gen_age >= 1 else 1
full['Age'] = full['Age'].apply(math.ceil )
learn_2.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))<predict_on_test>
full.Age.isnull().sum()
preds_2, y_2 = learn_2.get_preds(ds_type=DatasetType.Test )<predict_on_test>
quantile_label = ['Cheap','Regular','Premium']
full['TicketFare'] = pd.qcut(full['TicketFare'],
q=quantile_list,
labels=quantile_label)
 full.TicketFare.value_counts().sort_values()
preds_avg =(preds * 0.6 + preds_2 * 0.4)
 test_predictions = optR.predict(preds_avg, coefficients )<data_type_conversions>
full['FamilySize'] = full['Sibling_Spouse'] + \
full['Parent_Children'] + \
1
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()<save_to_csv>
full.loc[full.FamilySize > 1, 'IsAlone'] = 0
full.loc[full.FamilySize <= 1, 'IsAlone'] = 1
full['IsAlone'] = full['IsAlone'].astype(int )
sample_df.to_csv('submission.csv',index=False )<count_values>
full.loc[:891,['FamilySize','Survived']]\ 
.groupby(['FamilySize'],as_index=False)\ 
.mean().sort_values(by='Survived', ascending=False )
sample_df.diagnosis.value_counts()<import_modules>
full.loc[:891,['IsAlone','Survived']].groupby(['IsAlone'],as_index=False)\ 
.mean().sort_values(by='Survived', ascending=False )
import gc
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout
from keras import optimizers
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
import lightgbm as lgb<define_variables>
full.drop(['Sibling_Spouse','Parent_Children'], axis=1, inplace=True )
INPUT_DIR = ".. /input/"
LABEL = 'winPlacePerc'<load_from_csv>
for nom_feature in ['Sex','Embarked','Title']:
gle = LabelEncoder()
labels = gle.fit_transform(full[nom_feature])
 report = {index: label for index,label in enumerate(gle.classes_)}
full[nom_feature] = labels
print(nom_feature,':',report,'
','-'*50 )
df_train = pd.read_csv(INPUT_DIR+'train_V2.csv' )<set_options>
age_ord_map = {'infant':0, 'kid':1, 'young':2, 'mid-age':3, 'old':4}
full.Age = full.Age.map(age_ord_map)
 tf_ord_map = {'Cheap':0, 'Regular':1, 'Premium':2}
full.TicketFare = full.TicketFare.map(tf_ord_map )
df_train = reduce_mem_usage(df_train)
 gc.collect()<count_missing_values>
list_category_features = ['TicketClass','Sex','Age','TicketFare','Embarked','Title']
dummy_features = pd.get_dummies(full[list_category_features], columns=list_category_features)
 full.drop(list_category_features, axis=1,inplace=True)
 full = pd.concat([full, dummy_features], axis=1)
 full.sample(10 )
df_train.isnull().any()<correct_missing_values>
train_df_new = full.iloc[:891]
y = train_df_new['Survived']
X = train_df_new.drop(['Survived'], axis=1)
 test_df_new = full.iloc[891:]
test_df_new = test_df_new.drop(['Survived'], axis=1)
 X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=42)
 print(X_train.shape, X_test.shape )
df_train = df_train.dropna()<filter>
logistic = LogisticRegression()
logistic.fit(X_train, y_train)
 y_pred = logistic.predict(X_test)
 print('Logistic Regression model score:',
np.round(logistic.score(X_test, y_test), 3))
df_train = df_train[df_train['maxPlace'] > 1]<drop_column>
def model_report(y_test, y_pred):
print('Confusion Matrix:
',
metrics.confusion_matrix(y_true=y_test,
y_pred=y_pred,
labels=[0, 1]))
print('{:-^30}'.format('|'))
print('{:15}{:.3f}'.format('Accuracy:',
metrics.accuracy_score(y_test,
y_pred)))
 print('{:-^30}'.format('|'))
print('{:15}0:{:.3f}|1:{:.3f}'.format('Precision:',
metrics.precision_score(y_test,y_pred,average=None)[0],
metrics.precision_score(y_test,y_pred,average=None)[1]))
print('{:-^30}'.format('|'))
print('{:15}0:{:.3f}|1:{:.3f}'.format('Recall:',
metrics.recall_score(y_test,y_pred,average=None)[0],
metrics.recall_score(y_test,y_pred,average=None)[1]))
print('{:-^30}'.format('|'))
print('{:15}0:{:.3f}|1:{:.3f}'.format('f1-score:',
metrics.f1_score(y_test,y_pred,average=None)[0],
metrics.f1_score(y_test,y_pred,average=None)[1]))
print('{:-^30}'.format('|'))
model_report(y_test, y_pred )
def FE(df,train=True):
LABEL = 'winPlacePerc'
if train:
df_y = df.groupby(['matchId','groupId'])[LABEL].agg('mean')
 df = df.drop([LABEL],axis=1)
 else:
df_ids = df[['Id','matchId','groupId']]
df = df.drop('Id',axis=1)
 MATCH_FEATURE_part = ['numGroups','matchDuration','matchType','maxPlace']
GROUP_FEATURE = df.columns.tolist()
GROUP_FEATURE.remove('groupId')
 GROUP_FEATURE.remove('matchId')
 for fe in MATCH_FEATURE_part:
GROUP_FEATURE.remove(fe)
 MATCH_FEATURE = MATCH_FEATURE_part + GROUP_FEATURE
MATCH_FEATURE.remove('matchType')
 dm = df.groupby(['matchId','groupId'])
 df_X = dm.size().to_frame(name='groupSize')
 gp = dm[GROUP_FEATURE].agg('max')
 gp_rank = gp.reset_index().groupby(['matchId'])[GROUP_FEATURE].rank(pct=True ).set_index(df_X.index)
 df_X = df_X.join(gp ).join(gp_rank,rsuffix='_max_rank')
 gp = dm[GROUP_FEATURE].agg('min')
 gp_rank = gp.reset_index().groupby(['matchId'])[GROUP_FEATURE].rank(pct=True ).set_index(df_X.index)
 df_X = df_X.join(gp,rsuffix='_min' ).join(gp_rank,rsuffix='_min_rank')
 gp = dm[GROUP_FEATURE].agg('mean')
 gp_rank = gp.reset_index().groupby(['matchId'])[GROUP_FEATURE].rank(pct=True ).set_index(df_X.index)
 df_X = df_X.join(gp,rsuffix='_mean' ).join(gp_rank,rsuffix='_mean_rank')
 df_X = df_X.drop(columns = ['killPlace_min_rank','killPlace_max_rank','killPlace_mean_rank'])
 dm = df.groupby(['matchId'])
 df_X = df_X.join(dm[MATCH_FEATURE].agg('mean'),lsuffix='_max',rsuffix='_mean_match')
 df_X = df_X.join(pd.concat([pd.get_dummies(df.matchType),df[['matchId','groupId']]],axis=1 ).groupby(['matchId','groupId'] ).agg('min'))
if not train:
df_X_index = df_X.reset_index() [['matchId','groupId']]
lst_features = list(df_X.columns)
 del df,dm,gp,gp_rank
gc.collect()
if train:
return df_X,df_y,lst_features
else:
return df_X, df_X_index, df_ids 
<prepare_x_and_y>
model = LogisticRegression()
model.fit(X_train, y_train)
 predicted = model.predict(X_test)
 report = classification_report(y_test, predicted, digits=3)
 print(report )
X_train,y_train,lst_features = FE(df_train )<split>
kfold = model_selection.KFold(n_splits=10, random_state=0)
 model = LogisticRegression()
scoring = 'accuracy'
results = model_selection.cross_val_score(model,
X,
y,
cv=kfold,
scoring=scoring)
 print("Average Accuracy: {:.3f}".format(results.mean()))
<import_modules>
names = ["Nearest Neighbors",
"Linear SVM",
"RBF SVM",
"Gaussian Process",
"Decision Tree",
"Random Forest",
"Neural Net",
"AdaBoost",
"Naive Bayes",
"Logistic Regression"]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0)) ,
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
MLPClassifier(alpha=1),
AdaBoostClassifier() ,
GaussianNB() ,
LogisticRegression() ]
results = {}
for name, clf in zip(names, classifiers):
results[name] = {}
results[name]['Score'] = model_selection.cross_val_score(clf,
X,
y,
cv=kfold,
scoring=scoring ).mean()
results[name]['Confusion_matrix'] = metrics.confusion_matrix(y,
model_selection.cross_val_predict(clf,
X,
y,
cv=kfold))
<set_options>
for clf_name in sorted(names, key=lambda x:(results[x]['Score'])) :
print('{:19} :{:.3f}'.format(clf_name, results[clf_name]['Score']))
del df_train
gc.collect()<load_from_csv>
hyper_params={'max_depth':range(5,21,5),
'min_samples_split':range(2,9,2),
'min_samples_leaf':range(1,6,1),
'max_leaf_nodes':range(2,11,1)}
grid = GridSearchCV(RandomForestClassifier(random_state=1),
param_grid=hyper_params)
 grid.fit(X,y)
 print('Best Score: {:.4f}'.format(grid.best_score_))
print('Best Parameters setting:',grid.best_params_ )
df_test = pd.read_csv(INPUT_DIR+'test_V2.csv')
 df_test = reduce_mem_usage(df_test)
 gc.collect()<prepare_x_and_y>
ranfor_model = RandomForestClassifier(random_state=1,
max_depth=10,
max_leaf_nodes=10,
min_samples_leaf=2,
min_samples_split=2)
 ranfor_model.fit(X,y)
 y_pred = ranfor_model.predict(test_df_new )
<create_dataframe><EOS> 
submission = pd.DataFrame({
"PassengerId": range(892,1310),
"Survived": y_pred
})
 submission.to_csv('titanic.csv', index=False )