Kaggle competition: Titanic - Machine Learning from Disaster
print("--- Preparing text data ---")
 train_text = bag_of_words(x_train)
 submission_text = bag_of_words(x_test)
 prep_train_text = bag_of_words(new_x_train)
 prep_submission_text = bag_of_words(new_x_test)
 prep_train_text_underscore = concatenated_words(new_x_train)
 prep_submission_text_underscore = concatenated_words(new_x_test)
<categorify>
warnings.filterwarnings('ignore')
 %matplotlib inline
pio.templates.default = 'ggplot2'
tfidf_enc = TfidfVectorizer()
def tf_idf_features(text, flag):

 if flag == "train":
x = tfidf_enc.fit_transform(text)
 else:
x = tfidf_enc.transform(text)
 x = x.astype('float16')
 return x<feature_engineering>
dpath = Path('/kaggle/input/titanic')
 train_raw = pd.read_csv(dpath / 'train.csv')
 test_raw = pd.read_csv(dpath / 'test.csv' )
train_text_features = tf_idf_features(train_text, flag="train")
 submission_text_features = tf_idf_features(submission_text, flag="submission")
 prep_train_text_features = tf_idf_features(prep_train_text, flag="train")
 prep_submission_text_features = tf_idf_features(prep_submission_text, flag="submission")
 prep_train_text_underscore_features = tf_idf_features(prep_train_text_underscore, flag="train")
 prep_submission_text_underscore_features = tf_idf_features(prep_submission_text_underscore, flag="submission" )<compute_train_metric>
full = pd.concat([train_raw.copy(deep=True), test_raw.copy(deep=True)], ignore_index=True )
RANDOM_SEED = 1
names = ['DecisionTree','RandomForestClassifier','Logistic_Regression', "SVC"]
regressors = [DecisionTreeClassifier(random_state = RANDOM_SEED),
RandomForestClassifier(n_estimators = 10, random_state = RANDOM_SEED),
LogisticRegression(random_state = RANDOM_SEED),
SVC(C=10, gamma = 1, decision_function_shape=None, random_state = RANDOM_SEED),
]
def kfold_cross_validation(X,y, col_names):
N_FOLDS = 3
cv_results = np.zeros(( N_FOLDS, len(names)))
 kf = KFold(n_splits = N_FOLDS, shuffle = False, random_state = RANDOM_SEED)
 index_for_fold = 0
for train_index, test_index in kf.split(X,y):
print('
Fold index:', index_for_fold + 1,
'------------------------------------------')
 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
 print('
Shape of input data for this fold:',
'
Data Set:(Observations, Variables)')
 print('X_train:', X_train.shape)
 print('X_test:',X_test.shape)
 print('y_train:', y_train.shape)
 print('y_test:',y_test.shape)
 index_for_method = 0
for name, reg_model in zip(names, regressors):
print('
Regression model evaluation for:', name)
 print(' Scikit Learn method:', reg_model)
 reg_model.fit(X_train, y_train)
 y_test_predict = reg_model.predict(X_test)
 fold_method_result = reg_model.score(X_test, y_test)
 cv_results[index_for_fold, index_for_method] = fold_method_result
index_for_method += 1
index_for_fold += 1
cv_results_df = pd.DataFrame(cv_results)
 cv_results_df.columns = col_names
return cv_results_df 
<compute_train_metric>
def split_data(df):

 df = df.reset_index(drop=True)
 passengerId = df['PassengerId']
df = df.drop(columns=['PassengerId'])
 if 'Survived' in df.columns:
df['Survived'] = df['Survived'].astype('category')
 return(df, passengerId )
starttime = time.monotonic()
print("--- Performing 3-fold cross validation ---
")
 print("---------Standard Results---------")
 train_text_cv = kfold_cross_validation(train_text_features,
y_enc_labels,
col_names = ["DecisionTree",
"RandomForest",
"LogisticRegression",
"SVM"])
 print("
---------Removing Special Characters & Punctuations Results---------")
 prep_train_text_cv = kfold_cross_validation(prep_train_text_features,
y_enc_labels,
col_names = ["DecisionTree",
"RandomForest",
"LogisticRegression",
"SVM"])
 print("
---------Concatenated Words Results---------")
 prep_train_text_underscore_cv = kfold_cross_validation(prep_train_text_underscore_features,
y_enc_labels,
col_names = ["DecisionTree",
'RandomForest',
'LogisticRegression',
"SVM"])
 print("
That took ",(time.monotonic() -starttime)/60, " minutes" )<create_dataframe>
def remove_wordy_cols(df):

 df = df.drop(columns=['Name', 'Ticket'])
 return df
baseline_models = pd.DataFrame(prep_train_text_cv.mean() ,columns=["Avg Accuracy"])
 baseline_models = baseline_models.reset_index()
baseline_models.columns = ["Model","Baseline Score"]<train_on_grid>
def fill_fare_mean(df):
df['Fare'] = df['Fare'].fillna(df['Fare'].mean())
 return df
def grid_search_function(func_X_train, func_X_test, func_y_train, func_y_test, parameters, model):
model = model(random_state=RANDOM_SEED)
 grid_search = GridSearchCV(model, parameters)
 classifier= grid_search.fit(func_X_train,func_y_train)
 return classifier<split>
def apply_all(df, funs, debug=False):

 for fun in funs:
if debug:
print(f'Apply {fun.__name__}')
 df = fun(df)
 return df
def train_test_split_function(X,y, test_size_percent):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size_percent, random_state=RANDOM_SEED)
 return X_train, X_test, y_train, y_test<split>
prep_nn_1 = lambda x : apply_all(x, [remove_wordy_cols, fill_fare_mean, split_data])
 train = train_raw.copy(deep=True)
 train, train_pId = prep_nn_1(train)
 display(train.head(3))
display(train_pId.head(3 ).to_frame() )
starttime = time.monotonic()
parameters = {'n_estimators':[10,100,300]}
X_train, X_test, y_train, y_test = train_test_split_function(prep_train_text_features,
y_enc_labels,
test_size_percent = 0.20)
 rf_classifier = grid_search_function(X_train, X_test, y_train, y_test,
parameters,
model = RandomForestClassifier)
 print("
That took ",(time.monotonic() -starttime)/60, " minutes" )<find_best_params>
dls.show_batch() , learn.loss_func
rf_classifier.best_estimator_<train_model>
learn.lr_find()
def training_info_and_classification_report(X,y,test_size_percent, model):

 starttime = time.monotonic()
X_train, X_test, y_train, y_test = train_test_split_function(X,y, test_size_percent)
 print("
--- Fitting Model ---")
 model.fit(X_train, y_train)
 print("
--- Predicting Cuisines ---")
 y_predict = model.predict(X_test)
 print("
--- Scoring Model ---")
 model_training_score = model.score(X_train,y_train)
 model_testing_score = model.score(X_test,y_test)
 print("
--- Creating Classification Report ---")
 clf_report = classification_report(y_test,
y_predict,
target_names = lb_enc.inverse_transform(model.classes_ ).tolist())
 array = confusion_matrix(y_test, y_predict)
 row_sums = array.sum(axis=1,keepdims=True)
 norm_conf_mx = array/row_sums
error_matrix_df = pd.DataFrame(norm_conf_mx, index = [i for i in lb_enc.inverse_transform(model.classes_ ).tolist() ],
columns = [i for i in lb_enc.inverse_transform(model.classes_ ).tolist() ])
 print("
That took ",(time.monotonic() -starttime)/60, " minutes")
 return model_training_score, model_testing_score, clf_report, array, norm_conf_mx, error_matrix_df<train_model>
learn.fit_one_cycle(5, lr_max=slice(0.004, 5e-2, 0.005))
learn.fit_one_cycle(25, lr_max=slice(0.004, 1e-2, 0.005), cbs=callbacks )
training_score_rf,testing_score_rf,rf_classification_report, rf_array, rf_norm_conf_mx, rf_error_matrix_df = training_info_and_classification_report(prep_train_text_features,
y_enc_labels,
test_size_percent = 0.20,
model = rf_classifier.best_estimator_ )<compute_test_metric>
test, test_pId = prep_nn_1(test_raw.copy(deep=True))
test_dl = learn.dls.test_dl(test)
 preds, _ = learn.get_preds(dl=test_dl)
 preds = np.argmax(preds, 1 ).numpy()
submission = pd.DataFrame(
 {'PassengerId': test_pId,
'Survived': preds} 
)
 submission.to_csv('submission_trivial.csv', index=False )
print(" Random Forest Classification Report ")
 print(rf_classification_report )<split>
full['Title'] = full['Name'].apply(title)
 full['Title'] = full['Title'].replace({'Ms': 'Miss', 'Mlle': 'Miss', 'Mme': 'Mrs'})
 RARE_TITLE = list(( full['Title'].value_counts() < 8 ).replace({False: np.nan} ).dropna().index)
 full['Title'] = full['Title'].replace(RARE_TITLE, 'Rare')
 def extract_title(df, reduce=True):
df['Title'] = df['Name'].apply(title)
 if reduce:
df['Title'] = df['Title'].replace({'Ms': 'Miss', 'Mlle': 'Miss', 'Mme': 'Mrs'})
 df['Title'] = df['Title'].replace(RARE_TITLE, 'Rare')
 df['Title'] = df['Title'].fillna('Unknown')
 return df
starttime = time.monotonic()
parameters = {'C':[1,5,10,100,1000]}
X_train, X_test, y_train, y_test = train_test_split_function(prep_train_text_features,
y_enc_labels,
test_size_percent = 0.20)
 logreg_classifier = grid_search_function(X_train, X_test, y_train, y_test, parameters, model = LogisticRegression)
 print("
That took ",(time.monotonic() -starttime)/60, " minutes" )<find_best_params>
def extract_name_length(df):
df['Name_Length'] = df['Name'].str.len()
return df
logreg_classifier.best_estimator_<train_model>
AGE_TITLE = full.groupby(['Title'])['Age'].agg(['mean', 'std'])
 AGE_TITLE.columns = [f'AGE_{i}' for i in AGE_TITLE.columns]
def fill_age_with_normal_rand(df):
df = df.merge(AGE_TITLE,
how='left',
left_on='Title',
right_index=True)
 df['RAND_AGE'] = df.apply(lambda x: np.random.normal(x['AGE_mean'], x['AGE_std']/2.0), axis=1)
 df['Age'] = df['Age'].fillna(df['RAND_AGE'])
 df = df.drop(columns=['RAND_AGE', 'AGE_mean', 'AGE_std'])
 return df
training_score_lgr,testing_score_lgr,lgr_classification_report, lgr_array, lgr_norm_conf_mx, lgr_error_matrix_df = training_info_and_classification_report(prep_train_text_features,
y_enc_labels,
test_size_percent = 0.20,
model = logreg_classifier.best_estimator_ )<prepare_x_and_y>
def add_family_size(df):
df['FamilySize'] = df['Parch'] + df['SibSp'] + 1
df['Single'] =(df['FamilySize'] == 1 ).astype(int)
 df['FamilySize'] = df['FamilySize'].astype(str)
 return df
That took ",(time.monotonic() -starttime)/60, " minutes")
<choose_model_class>
full['HasCabin'] = full['Cabin'].notna()
full.groupby('HasCabin')['Survived'].mean()
svm_clf = SVC(C=10, cache_size=200, class_weight=None, coef0=0.0,
decision_function_shape=None, degree=3, gamma=1, kernel='rbf',
max_iter=-1, probability=False, random_state=1, shrinking=True,
tol=0.001, verbose=False)
 svm_ovr = OneVsRestClassifier(svm_clf )<split>
def cabin_information(df):
df['HasCabin'] = df['Cabin'].notna().astype(str)
 df['CabinType'] = df['Cabin'].str.split(r'(^[A-Z])',expand = True)[1]
df['CabinType'] = df['CabinType'].fillna('Unknown')
 df = df.drop(columns='Cabin')
 return df
def get_training_info(model,full_X_train, full_y_train):

 X_train, X_test, y_train, y_test = train_test_split_function(full_X_train,
full_y_train,
test_size_percent = 0.20)
 print("
--- Fitting Model ---")
 model.fit(X_train, y_train)
 print("
--- Scoring Model ---")
 model_training_score = model.score(X_train,y_train)
 model_testing_score = model.score(X_test,y_test)
 return model_training_score,model_testing_score 
<find_best_model_class>
EMBARKED_MODE = full['Embarked'].mode() [0]
def fill_embarked(df):
df['Embarked'] = df['Embarked'].fillna(EMBARKED_MODE)
 return df
def get_classification_report(X,y,test_size_percent, model):

 X_train, X_test, y_train, y_test = train_test_split_function(X,y, test_size_percent)
 model.fit(X_train, y_train)
 y_predict = model.predict(X_test)
 clf_report = classification_report(y_test,
y_predict,
target_names = lb_enc.inverse_transform(model.classes_ ).tolist())
 array = confusion_matrix(y_test, y_predict)
 row_sums = array.sum(axis=1,keepdims=True)
 norm_conf_mx = array/row_sums
error_matrix_df = pd.DataFrame(norm_conf_mx, index = [i for i in lb_enc.inverse_transform(model.classes_ ).tolist() ],
columns = [i for i in lb_enc.inverse_transform(model.classes_ ).tolist() ])
 return clf_report, array, norm_conf_mx, error_matrix_df<split>
FARE_MEDIAN = full['Fare'].median()
def fill_fare(df):
df['Fare'] = df['Fare'].fillna(FARE_MEDIAN)
 return df
training_score_svm, testing_score_svm = get_training_info(svm_ovr,prep_train_text_features,y_enc_labels )<choose_model_class>
full['TicketCat'] = full['Ticket'].str[0:2]
RARE_TICKET = list(( full['Ticket'].value_counts() < 20 ).replace({False: np.nan} ).dropna().index )
p_svm_clf = SVC(C=10, cache_size=200, class_weight=None, coef0=0.0,
decision_function_shape=None, degree=3, gamma=1, kernel='rbf',
max_iter=-1, probability=True, random_state=1, shrinking=True,
tol=0.001, verbose=False )<compute_test_metric>
def ticket_cat(df, reduce=True):
df['TicketCat'] = df['Ticket'].str[0:2]
df['TicketCat'] = df['TicketCat'].replace(RARE_TICKET, 'Rare')
 df['TicketCat'] = df['TicketCat'].fillna('Unknown')
 return df
print(" SVM Classification Report ")
 print(svm_classification_report )<find_best_params>
torch.device('cuda')
 preps = [extract_title,
extract_name_length,
fill_age_with_normal_rand,
add_family_size,
cabin_information,
fill_embarked,
fill_fare,
ticket_cat,
remove_wordy_cols,
split_data]
prep_nn_2 = lambda x : apply_all(x, preps)
 train = train_raw.copy(deep=True)
 train, train_pId = prep_nn_2(train)
 cont_names = ['Pclass', 'Age', 'Name_Length', 'Parch', 'SibSp']
cat_names = ['Sex', 'TicketCat', 'CabinType', 'HasCabin', 'Single', 'Title', 'Embarked', 'Fare', 'FamilySize']
procs = [Categorify, Normalize]
dep_var = 'Survived'
splits = RandomSplitter(valid_pct=0.2, seed=42 )(train.index)
 cfg = tabular_config(embed_p=0.25, ps=0.05)
 dls = TabularPandas(train,
cont_names=cont_names,
cat_names=cat_names,
procs=procs,
y_names=dep_var,
splits=splits ).dataloaders(bs=64)
 callbacks = [SaveModelCallback(min_delta=0.005, monitor='accuracy', fname='model_feat01_best')]
learn = tabular_learner(dls, layers=[500,200,100], metrics=[accuracy], config=cfg)
 learn.lr_find()
rf_clf = rf_classifier.best_estimator_
lgr_clf = logreg_classifier.best_estimator_
print(rf_clf,"
")
 print(lgr_clf,"
")
 print(svm_clf,"
" )<define_variables>
learn.fit_one_cycle(5, lr_max=slice(0.004, 5e-2, 0.05))
learn.fit_one_cycle(45, lr_max=slice(0.005, 1e-2, 0.005), cbs=callbacks )
model_names = [
"Random Forest Classifier",
"Logistic Regression",
"SVM",
"VotingClassifier"]
best_model_testing_score = [
testing_score_rf,
testing_score_lgr,
testing_score_svm,
"-"
]
best_model_training_score = [
training_score_rf,
training_score_lgr,
training_score_svm,
"-"
]<save_to_csv>
test, test_pId = prep_nn_2(test_raw.copy(deep=True))
test_dl = learn.dls.test_dl(test)
 preds, _ = learn.get_preds(dl=test_dl)
 preds = np.argmax(preds, 1 ).numpy()
submission = pd.DataFrame(
 {'PassengerId': test_pId,
'Survived': preds} 
)
 submission.to_csv('submission_feature_eng.csv', index=False )
def model_to_submission_file(model, full_X_train, full_y_train, full_X_test ,nameOfcsvfile):
print("
--- Fitting Model ---")
 starttime = time.monotonic()
model_clf = model.fit(full_X_train,full_y_train)
 print("
--- Predicting Cuisines ---")
 submission_pred = model_clf.predict(full_X_test)
 test_cuisine = lb_enc.inverse_transform(submission_pred)
 test_id = [recipe['id'] for recipe in test]
submission_df = pd.DataFrame({'id':test_id, 'cuisine':test_cuisine},columns = ['id','cuisine'])
 submission_df.to_csv('{}'.format(nameOfcsvfile), index= False)
 print("
--- Results have been saved ---")
 print("
That took ",(time.monotonic() -starttime)/60, " minutes" )<save_to_csv>
splits = RandomSplitter(valid_pct=0.01, seed=42 )(train.index)
 cfg = tabular_config(embed_p=0.25, ps=0.05)
 dls = TabularPandas(train,
cont_names=cont_names,
cat_names=cat_names,
procs=procs,
y_names=dep_var,
splits=splits ).dataloaders(bs=64)
 learn = tabular_learner(dls, layers=[500,200,100], metrics=[accuracy], config=cfg )
ovr_svm = OneVsRestClassifier(svm_clf, n_jobs = 4)
 model_to_submission_file(model=ovr_svm,
full_X_train=prep_train_text_features,
full_y_train=y_enc_labels,
full_X_test=prep_submission_text_features,
nameOfcsvfile="svm_model.csv")
<create_dataframe>
learn.fit_one_cycle(5, lr_max=slice(0.004, 5e-2, 0.05))
learn.fit_one_cycle(15, lr_max=slice(0.005, 1e-2, 0.005))
model_testing_score = [0.75663, 0.78751, 0.81999, 0.81989]
pd.DataFrame({"Model":model_names,
"Training Score":best_model_training_score,
"Testing Score": best_model_testing_score,
"Submission Score": model_testing_score},
columns=["Model","Training Score", "Testing Score", "Submission Score"] )<set_options>
test, test_pId = prep_nn_2(test_raw.copy(deep=True))
test_dl = learn.dls.test_dl(test)
 preds, _ = learn.get_preds(dl=test_dl)
 preds = np.argmax(preds, 1 ).numpy()
submission = pd.DataFrame(
 {'PassengerId': test_pId,
'Survived': preds} 
)
 submission.to_csv('submission_feature_eng_full.csv', index=False )