Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<find_best_model_class>
import numpy as np
import pandas as pd
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.naive_bayes import ComplementNB
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import CategoricalNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import Perceptron
from sklearn.svm import SVC
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
knn=KNeighborsClassifier(n_neighbors=9)
 scores = cross_val_score(knn,X_num,Y_num,cv=10)
 knn.fit(X_num,Y_num)
 Y_testpredict=knn.predict(X_test)
 scores<data_type_conversions>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv' )
convert_cols=['workclass','education','marital.status','occupation','race','relationship',
'sex','native.country']
test_adult[convert_cols] = test_adult[convert_cols].astype(str )<categorify>
missing_values = train_data.isna().any()
print('Columns which have missing values:
{0}'.format(missing_values[missing_values == True].index.tolist()))
Xencode_adult= na_adult.iloc[:,:-1].apply(LabelEncoder().fit_transform)
 Xencode_test_adult = test_adult.apply(LabelEncoder().fit_transform)
 X_adult = Xencode_adult
X_test = Xencode_test_adult<categorify>
print("Percentage of missing values in `Age` column: {0:.2f}".format(100.*(train_data.Age.isna().sum() /len(train_data))))
print("Percentage of missing values in `Cabin` column: {0:.2f}".format(100.*(train_data.Cabin.isna().sum() /len(train_data))))
print("Percentage of missing values in `Embarked` column: {0:.2f}".format(100.*(train_data.Embarked.isna().sum() /len(train_data))))
Yfit_adult= LabelEncoder().fit(na_adult["income"])
 Y_adult = Yfit_adult.transform(na_adult["income"] )<train_on_grid>
duplicates = train_data.duplicated().sum()
print('Duplicates in train data: {0}'.format(duplicates))
knn =KNeighborsClassifier(n_neighbors=10)
 scores = cross_val_score(knn,X_adult,Y_adult,cv=10)
 knn.fit(X_adult,Y_adult)
 scores<predict_on_test>
categorical = train_data.nunique().sort_values(ascending=True)
 print('Categorical variables in train data:
{0}'.format(categorical))
Ytest_predict= knn.predict(X_test)
 print(Ytest_predict )<prepare_x_and_y>
def clean_data(data):
data.drop(['Cabin'], axis=1, inplace=True)
 data.drop(['Name', 'Ticket', 'Fare', 'Embarked'], axis=1, inplace=True)
 return data
train_data = clean_data(train_data)
 test_data = clean_data(test_data )
atributos=atributos=["age","workclass","education.num","occupation","sex","marital.status","capital.gain","capital.loss"]
X_adult = Xencode_adult[atributos]
X_test = Xencode_test_adult[atributos]<find_best_params>
train_data['Sex'].replace({'male':0, 'female':1}, inplace=True)
 test_data['Sex'].replace({'male':0, 'female':1}, inplace=True)
 all_data = pd.concat([train_data, test_data])
 average = all_data.Age.median()
print("Average Age: {0}".format(average))
train_data.fillna(value={'Age': average}, inplace=True)
 test_data.fillna(value={'Age': average}, inplace=True )
k_range=list(range(1,35))
k_scores = []
for k in k_range:
knn = KNeighborsClassifier(n_neighbors=k)
 scores = cross_val_score(knn, X_adult, Y_adult, cv=10, scoring='accuracy')
 k_scores.append(scores.mean())
 print(k_scores )<train_model>
X = train_data.drop(['Survived', 'PassengerId'], axis=1)
 y = train_data['Survived']
test_X = test_data.drop(['PassengerId'], axis=1 )
knn =KNeighborsClassifier(n_neighbors=27)
 knn.fit(X_adult,Y_adult)
 scores<predict_on_test>
best_models = {}
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 def print_best_parameters(hyperparameters, best_parameters):
value = "Best parameters: "
for key in hyperparameters:
value += str(key)+ ": " + str(best_parameters[key])+ ", "
if hyperparameters:
print(value[:-2])
 def get_best_model(estimator, hyperparameters, fit_params={}):
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
 grid_search = GridSearchCV(estimator=estimator, param_grid=hyperparameters, n_jobs=-1, cv=cv, scoring="accuracy")
 best_model = grid_search.fit(train_X, train_y, **fit_params)
 best_parameters = best_model.best_estimator_.get_params()
print_best_parameters(hyperparameters, best_parameters)
 return best_model
def evaluate_model(model, name):
print("Accuracy score:", accuracy_score(train_y, model.predict(train_X)))
 best_models[name] = model
Ytest_predict= knn.predict(X_test )<save_to_csv>
hyperparameters = {
'solver' : ['newton-cg', 'lbfgs', 'liblinear'],
'penalty' : ['l2'],
'C' : [100, 10, 1.0, 0.1, 0.01]
}
estimator = LogisticRegression(random_state=1)
 best_model_logistic = get_best_model(estimator, hyperparameters )
label_out = Yfit_adult.inverse_transform(Ytest_predict)
 df_out = pd.DataFrame({'Id': X_test.index,'income':label_out})
 df_out.to_csv('submission_adult.csv',index=False )<load_from_csv>
evaluate_model(best_model_logistic.best_estimator_, 'logistic' )
pd.read_csv("submission_adult.csv" )<set_options>
hyperparameters = {
'var_smoothing': np.logspace(0, -9, num=100)
 }
estimator = GaussianNB()
best_model_gaussian_nb = get_best_model(estimator, hyperparameters )
%matplotlib inline 
<load_from_csv>
evaluate_model(best_model_gaussian_nb.best_estimator_, 'gaussian_nb' )
adult = pd.read_csv(".. /input/mydata/train_data.csv",
names=[
"Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country", "Target"],
skiprows=1,
sep=r'\s*,\s*',
engine='python',
na_values="?" )<count_values>
hyperparameters = {
'alpha' : [0.5, 1.0, 1.5, 2.0, 5],
'fit_prior' : [True, False],
}
estimator = MultinomialNB()
best_model_multinominal_nb = get_best_model(estimator, hyperparameters )
adult["Country"].value_counts()<correct_missing_values>
evaluate_model(best_model_multinominal_nb.best_estimator_, 'multinominal_nb' )
nadult = adult.dropna()<load_from_csv>
hyperparameters = {
'alpha' : [0.5, 1.0, 1.5, 2.0, 5],
'fit_prior' : [True, False],
'norm' : [True, False]
}
estimator = ComplementNB()
best_model_complement_nb = get_best_model(estimator, hyperparameters )
testAdult = pd.read_csv(".. /input/mydata/test_data.csv",
names=[
"ID","Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country"],
skiprows=1,
index_col=0,
sep=r'\s*,\s*',
engine='python',
na_values="?" )<prepare_x_and_y>
evaluate_model(best_model_complement_nb.best_estimator_, 'complement_nb' )
Yadult = nadult.Target<import_modules>
hyperparameters = {
'alpha' : [0.5, 1.0, 1.5, 2.0, 5],
'fit_prior' : [True, False],
}
estimator = BernoulliNB()
best_model_bernoulli_nb = get_best_model(estimator, hyperparameters )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
evaluate_model(best_model_bernoulli_nb.best_estimator_, 'bernoulli_nb' )
from sklearn.neighbors import KNeighborsClassifier<import_modules>
hyperparameters = {
'n_neighbors' : list(range(1,5)) ,
'weights' : ['uniform', 'distance'],
'algorithm' : ['auto', 'ball_tree', 'kd_tree', 'brute'],
'leaf_size' : list(range(1,10)) ,
'p' : [1,2]
}
estimator = KNeighborsClassifier()
best_model_kneighbors = get_best_model(estimator, hyperparameters )
from sklearn.neighbors import KNeighborsClassifier<choose_model_class>
evaluate_model(best_model_kneighbors.best_estimator_, 'kneighbors' )
knn = KNeighborsClassifier(n_neighbors=3 )<import_modules>
hyperparameters = {
'penalty' : ['l1', 'l2', 'elasticnet'],
'eta0' : [0.0001, 0.001, 0.01, 0.1, 1.0],
'max_iter' : list(range(50, 200, 50))
}
estimator = Perceptron(random_state=1)
 best_model_perceptron = get_best_model(estimator, hyperparameters )
from sklearn.model_selection import cross_val_score<compute_train_metric>
evaluate_model(best_model_perceptron.best_estimator_, 'perceptron' )
scores = cross_val_score(knn, Xadult, Yadult, cv=10 )<train_model>
hyperparameters = {
'C' : [0.1, 1, 10, 100],
'gamma' : [0.0001, 0.001, 0.01, 0.1, 1],
'kernel' : ['rbf']
}
estimator = SVC(random_state=1)
 best_model_svc = get_best_model(estimator, hyperparameters )
knn.fit(Xadult,Yadult )<predict_on_test>
evaluate_model(best_model_svc.best_estimator_, 'svc' )
YtestPred = knn.predict(XtestAdult )<import_modules>
hyperparameters = {
'loss' : ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron'],
'penalty' : ['l1', 'l2', 'elasticnet'],
'alpha' : [0.01, 0.1, 1, 10]
}
estimator = SGDClassifier(random_state=1, early_stopping=True)
 best_model_sgd = get_best_model(estimator, hyperparameters )
from sklearn.metrics import accuracy_score<load_from_csv>
evaluate_model(best_model_sgd.best_estimator_, 'sgd' )
UCITest = pd.read_csv(".. /input/mydata/adult.test",
names=[
"Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
"Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
"Hours per week", "Country", "Target"],
sep=r'\s*,\s*',
engine='python',
na_values="?" )<correct_missing_values>
hyperparameters = {
'loss' : ['deviance', 'exponential'],
'learning_rate' : [0.01, 0.1, 0.2, 0.3],
'n_estimators' : [50, 100, 200],
'subsample' : [0.1, 0.2, 0.5, 1.0],
'max_depth' : [2, 3, 4, 5]
}
estimator = GradientBoostingClassifier(random_state=1)
 best_model_gbc = get_best_model(estimator, hyperparameters )
nUCITest = UCITest.dropna()<define_variables>
evaluate_model(best_model_gbc.best_estimator_, 'gbc' )
XUCITest = nUCITest[["Age","Education-Num","Capital Gain", "Capital Loss", "Hours per week"]]<prepare_x_and_y>
hyperparameters = {
'n_estimators' : [10, 50, 100, 500],
'learning_rate' : [0.001, 0.01, 0.1, 1.0]
}
estimator = AdaBoostClassifier(random_state=1)
 best_model_adaboost = get_best_model(estimator, hyperparameters )
YUCITest = nUCITest.Target<predict_on_test>
evaluate_model(best_model_adaboost.best_estimator_, 'adaboost' )
YUCIPred = knn.predict(XUCITest)
 YUCIPred<compute_test_metric>
hyperparameters = {
'criterion' : ['gini', 'entropy'],
'splitter' : ['best', 'random'],
'max_depth' : [None, 1, 2, 3, 4, 5],
'min_samples_split' : list(range(2,5)) ,
'min_samples_leaf' : list(range(1,5))
}
estimator = DecisionTreeClassifier(random_state=1)
 best_model_decision_tree = get_best_model(estimator, hyperparameters )
accuracy_score(YUCIPred, YUCITest )<choose_model_class>
evaluate_model(best_model_decision_tree.best_estimator_, 'decision_tree' )
knn = KNeighborsClassifier(n_neighbors=30 )<train_model>
hyperparameters = {
'n_estimators' : list(range(10, 50, 10)) ,
'max_features' : ['auto', 'sqrt', 'log2'],
'criterion' : ['gini', 'entropy'],
'max_depth' : [None, 1, 2, 3, 4, 5],
'min_samples_split' : list(range(2,5)) ,
'min_samples_leaf' : list(range(1,5))
}
estimator = RandomForestClassifier(random_state=1)
 best_model_random_forest = get_best_model(estimator, hyperparameters )
knn.fit(Xadult, Yadult )<train_model>
evaluate_model(best_model_random_forest.best_estimator_, 'random_forest' )
knn.fit(Xadult, Yadult )<compute_train_metric>
hyperparameters = {
'learning_rate' : [0.3, 0.4, 0.5],
'gamma' : [0, 0.4, 0.8],
'max_depth' : [2, 3, 4],
'reg_lambda' : [0, 0.1, 1],
'reg_alpha' : [0.1, 1]
}
fit_params = {
'verbose' : False,
'early_stopping_rounds' : 40,
'eval_metric' : 'logloss',
'eval_set' : [(val_X, val_y)]
}
estimator = XGBClassifier(seed=1, tree_method='gpu_hist', predictor='gpu_predictor', use_label_encoder=False)
 best_model_xgb = get_best_model(estimator, hyperparameters, fit_params )
scores = cross_val_score(knn, Xadult, Yadult, cv=10 )<predict_on_test>
evaluate_model(best_model_xgb.best_estimator_, 'xgb' )
YUCIPred = knn.predict(XUCITest )<compute_test_metric>
hyperparameters = {
'boosting_type' : ['gbdt', 'dart', 'goss'],
'num_leaves' : [4, 8, 16, 32],
'learning_rate' : [0.01, 0.1, 1],
'n_estimators' : [25, 50, 100],
'reg_alpha' : [0, 0.1, 1],
'reg_lambda' : [0, 0.1, 1],
}
estimator = LGBMClassifier(random_state=1, device='gpu')
 best_model_lgbm = get_best_model(estimator, hyperparameters )
accuracy_score(YUCITest, YUCIPred )<find_best_params>
evaluate_model(best_model_lgbm.best_estimator_, 'lgbm' )
<choose_model_class><EOS> 
for model in best_models:
predictions = best_models[model].predict(test_X)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission_' + model + '.csv', index=False )