Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
%matplotlib inline
warnings.filterwarnings('ignore' )
def has_field_transformer(column_name, new_column_name=None, is_missing_func=pd.notna)-> TransformerMixin:
return DataFrameColumnMapper(column_name=column_name,
mapping_func=lambda name: np.int(is_missing_func(name)) ,
drop_original=True,
new_column_name=new_column_name if new_column_name is not None else column_name)
 def value_matches_transformer(column_name, new_column_name=None, matches=pd.notna)-> TransformerMixin:
return DataFrameColumnMapper(column_name=column_name,
mapping_func=lambda value: np.int(matches(value)) ,
drop_original=False,
new_column_name=new_column_name if new_column_name is not None else column_name)
 def map_categories(column_name, mapping_dict)-> TransformerMixin:
return DataFrameColumnMapper(column_name=column_name,
mapping_func=lambda x: mapping_dict[x])
 def onehot_encode(columns)-> TransformerMixin:
return CategoricalToOneHotEncoder(columns=columns)
 def truncate_categorical(column_name, n_values_to_keep=10):
return CategoricalTruncator(column_name=column_name, n_values_to_keep=n_values_to_keep)
 ONEHOT_ENCODED_COLUMNS = ["Type", "Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3", "Health",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "State", "RescuerID"]
def build_preprocessing_pipeline() -> Pipeline:
return Pipeline([ 
('add_has_name', has_field_transformer(column_name="Name", new_column_name="hasName")) , 
('add_is_free', value_matches_transformer(column_name="Fee", new_column_name="isFree",
matches=lambda value: value < 1)) , 
('map_type_to_species', map_categories(column_name="Type", mapping_dict={1: 'dog', 2: 'cat'})) , 
('map_gender_to_names', map_categories(column_name="Gender", mapping_dict={1: 'male', 2: 'female', 3: 'mixed'})) , 
('truncate_breed1', truncate_categorical(column_name="Breed1", n_values_to_keep=10)) , 
('truncate_breed2', truncate_categorical(column_name="Breed2", n_values_to_keep=10)) , 
('truncate_state', truncate_categorical(column_name="State", n_values_to_keep=10)) , 
('truncate_rescuer_id', truncate_categorical(column_name="RescuerID", n_values_to_keep=10)) , 
('onehot_encode', CategoricalToOneHotEncoder(columns=ONEHOT_ENCODED_COLUMNS)) , 
('drop_unused_columns', DataFrameColumnDropper(
 column_names=['PetID', 'Description', 'Type_dog'
])) , 
('pick_columns_by_importance', ColumnByFeatureImportancePicker(n_features=None))
])
 preprocessing_pipeline = build_preprocessing_pipeline()
X_train_preprocessed = preprocessing_pipeline.fit_transform(X_train, y_train)
 X_val_preprocessed = preprocessing_pipeline.transform(X_val)
 X_train_preprocessed.head(10 )<count_values>
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train.describe(include="all" )
print("Number of features:", len(list(X_train_preprocessed)))
 print("")
 print("Numerical columns:", list(X_train_preprocessed.select_dtypes(include="number")))
 print("")
 print("Non-numerical columns:", list(X_train_preprocessed.select_dtypes(exclude="number")) )<categorify>
ids = test['PassengerId']
def build_preparation_pipeline() :
return Pipeline([ 
('to_numpy', DataFrameToValuesTransformer()), 
('scaler', StandardScaler())
 ])
 def build_full_pipeline(classifier=None):
preprocessing_pipeline = build_preprocessing_pipeline()
preparation_pipeline = build_preparation_pipeline()
return Pipeline([ 
('preprocessing', preprocessing_pipeline), 
('preparation', preparation_pipeline), 
('classifier', classifier)
 ] )<compute_train_metric>
print(pd.isnull(train ).sum() )
rf_classifier = RandomForestClassifier(n_estimators=100)
 rf_pipeline = build_full_pipeline(classifier=rf_classifier)
 cross_val_score(rf_pipeline, X_train, y_train, cv=5, scoring=make_scorer(cohen_kappa_score))<train_on_grid>
all_data['Embarked'].fillna(all_data['Embarked'].mode() [0], inplace = True)
 all_data['Fare'].fillna(all_data['Fare'].median() , inplace = True)
def build_search(pipeline, param_distributions, n_iter=10):
return RandomizedSearchCV(pipeline, param_distributions=param_distributions,
cv=5, return_train_score=True, refit='cohen_kappa',
n_iter=n_iter,
scoring={
'accuracy': make_scorer(accuracy_score),
'cohen_kappa': make_scorer(cohen_kappa_score)
 },
verbose=1, random_state=42)
 def pretty_cv_results(cv_results,
sort_by='rank_test_cohen_kappa',
sort_ascending=True,
n_rows=20):
df = pd.DataFrame(cv_results)
 cols_of_interest = [key for key in df.keys() if key.startswith('param_')
 or key.startswith('mean_train')
 or key.startswith('mean_test_')
 or key.startswith('rank')]
return df.loc[:, cols_of_interest].sort_values(by=sort_by, ascending=sort_ascending ).head(n_rows)
 def run_search(search):
search.fit(X_train, y_train)
 print('Best score is:', search.best_score_)
 return pretty_cv_results(search.cv_results_ )<find_best_params>
all_data['Title'] = all_data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 all_data['Title'].value_counts()
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None, 50],
'classifier': [RandomForestClassifier(n_estimators=250, random_state=42, max_depth=10)],
'classifier__max_depth': [10, None]
}
rf_feature_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=4)
 rf_feature_cv_results = run_search(search=rf_feature_search)
 rf_feature_cv_results<choose_model_class>
frequent_titles = all_data['Title'].value_counts() [:5].index.tolist()
frequent_titles
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [RandomForestClassifier(n_estimators=500, random_state=42)],
'classifier__n_estimators': [500],
'classifier__max_features': ['auto', 'log2'],
'classifier__max_depth': [None, 10],
'classifier__bootstrap': [False, True],
'classifier__min_samples_leaf': [1, 5, 10],
'classifier__min_samples_split': [2, 5, 10],
'classifier__criterion': ['gini', 'entropy'],
}
rf_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=50)
 rf_cv_results = run_search(search=rf_search)
 rf_cv_results<define_search_space>
all_data['Title'] = all_data['Title'].apply(lambda x: x if x in frequent_titles else 'Other')
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [ExtraTreesClassifier(n_estimators=500, random_state=42)],
'classifier__n_estimators': [500],
'classifier__max_features': ['auto', 'log2'],
'classifier__max_depth': [None],
'classifier__min_samples_split': [2, 5, 10],
'classifier__criterion': ['gini', 'entropy'],
}
et_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=50)
 et_cv_results = run_search(search=et_search)
 et_cv_results<find_best_params>
median_ages = {}
for title in frequent_titles:
median_ages[title] = all_data.loc[all_data['Title'] == title]['Age'].median()
median_ages['Other'] = all_data['Age'].median()
all_data.loc[all_data['Age'].isnull() , 'Age'] = all_data[all_data['Age'].isnull() ]['Title'].map(median_ages )
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None, 50],
'classifier': [LogisticRegression(solver='lbfgs', random_state=42)],
'classifier__multi_class': ['ovr', 'multinomial'],
'classifier__C': np.logspace(-3, 0, 4),
}
logistic_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=20)
 logistic_cv_results = run_search(search=logistic_search)
 logistic_cv_results<define_search_space>
Cat_Features = ['Sex', 'Embarked', 'Title']
for feature in Cat_Features:
label = LabelEncoder()
all_data[feature] = label.fit_transform(all_data[feature] )
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None, 50],
'preparation__scaler': [MinMaxScaler() ],
'classifier': [MLPClassifier(hidden_layer_sizes=(100,), random_state=42)],
'classifier__hidden_layer_sizes': [[10], [10, 10,], [10, 10, 10]],
'classifier__alpha': np.logspace(-4, -2, 3),
'classifier__solver': ['adam'],
'classifier__tol': np.logspace(-4, -2, 3),
'classifier__learning_rate_init': np.logspace(-3, -1, 3),
'classifier__activation': ['relu', 'tanh'],
}
mlp_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=5)
 mlp_cv_results = run_search(search=mlp_search)
 mlp_cv_results<choose_model_class>
Cont_Features = ['Age', 'Fare']
num_bins = 5
for feature in Cont_Features:
bin_feature = feature + 'Bin'
all_data[bin_feature] = pd.qcut(all_data[feature], num_bins)
 label = LabelEncoder()
all_data[bin_feature] = label.fit_transform(all_data[bin_feature] )
<define_search_space>
all_data['Surname'] = all_data.Name.str.extract(r'([A-Za-z]+),', expand=False)
 all_data['TicketPrefix'] = all_data.Ticket.str.extract(r' (.*\d)', expand=False)
 all_data['Surname_Ticket'] = all_data['Surname'] + all_data['TicketPrefix']
all_data['IsFamily'] = all_data.Surname_Ticket.duplicated(keep=False ).astype(int )
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [ GradientBoostingClassifier(random_state=42)],
'classifier__loss': ['deviance'],
'classifier__n_estimators': [100, 300],
'classifier__max_features': ['log2', None],
'classifier__max_depth': [5, 10],
'classifier__min_samples_leaf': [1, 5, 10],
'classifier__min_samples_split': [2, 5, 10],
'classifier__learning_rate': [0.1, 0.2],
'classifier__subsample': [0.75, 0.90, 1.0]
}
gb_search = build_search(pipeline=build_full_pipeline() , param_distributions=param_distributions, n_iter=20)
 gb_cv_results = run_search(search=gb_search)
 gb_cv_results<init_hyperparams>
all_data['Child'] = all_data.Age.map(lambda x: 1 if x <=16 else 0)
 FamilyWithChild = all_data[(all_data.IsFamily==1)&(all_data.Child==1)]['Surname_Ticket'].unique()
len(FamilyWithChild )
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [ lgb.sklearn.LGBMClassifier(random_state=42, objective='multiclass')],
'classifier__boosting_type': ['gbdt', 'dart'],
'classifier__num_leaves': [20, 31, 50],
'classifier__max_depth': [-1],
'classifier__learning_rate': [0.1, 0.2],
'classifier__n_estimators': [100, 300],
'classifier__subsample': [1.0, 0.9],
'classifier__reg_alpha': [0.0, *np.logspace(-3, -2, 2)],
'classifier__reg_lambda': [0.0],
}
lgbm_search = build_search(pipeline=build_full_pipeline() , param_distributions=param_distributions, n_iter=20)
 lgbm_cv_results = run_search(search=lgbm_search)
 lgbm_cv_results<find_best_model_class>
all_data['FamilyId'] = 0
for ind, identifier in enumerate(FamilyWithChild):
all_data.loc[all_data.Surname_Ticket==identifier, ['FamilyId']] = ind + 1
def cross_val_predictions(classifiers, X, y):

 return np.hstack([cross_val_predict(classifier, X, y, cv=5, method='predict_proba')for classifier in classifiers])
 def first_level_predictions(classifiers, X):

 return np.hstack([classifier.predict_proba(X)for classifier in classifiers])
 best_estimators = [
rf_search.best_estimator_,
et_search.best_estimator_,
gb_search.best_estimator_,
lgbm_search.best_estimator_,
xgb_search.best_estimator_
]
X_train_second_level = cross_val_predictions(classifiers=best_estimators, X=X_train, y=y_train)
 stacking_classifier = GradientBoostingClassifier(random_state=42)
 stacking_classifier.fit(X_train_second_level, y_train )<predict_on_test>
X_train = train[['Pclass', 'Sex', 'Parch', 'Embarked', 'CabinBool', 'Title', 'AgeBin', 'FareBin', 'FamilySurvival']]
y_train = train['Survived']
for estimator in best_estimators:
estimator.fit(X_train, y_train)
 X_val_second_level = first_level_predictions(best_estimators, X_val)
 y_val_pred = stacking_classifier.predict(X_val_second_level)
 print("Performance of stacking classifier on the hold-out set:", cohen_kappa_score(y_val, y_val_pred))<find_best_model_class>
model = CatBoostClassifier(verbose=False )
X_train_val_second_level = cross_val_predictions(best_estimators, X=X_train_val, y=y_train_val)
 stacking_classifier.fit(X_train_val_second_level, y_train_val)
 for estimator in best_estimators:
estimator.fit(X_train_val, y_train_val)
 X_test_second_level = first_level_predictions(best_estimators, X=X_test )<predict_on_test>
main_features = ['Sex', 'FamilySurvival', 'FareBin', 'Pclass', 'Title']
X_test = test[main_features]
X_train = train[main_features]
def get_predictions(estimator, X):
predictions = estimator.predict(X)
 indices = X_test.loc[:, 'PetID']
as_dict = [{'PetID': index, 'AdoptionSpeed': prediction} for index, prediction in zip(indices, predictions)]
df = pd.DataFrame.from_dict(as_dict)
 df = df.reindex(['PetID', 'AdoptionSpeed'], axis=1)
 return df
predictions = get_predictions(stacking_classifier, X=X_test_second_level )<save_to_csv>
cross_val_score(estimator=model, X=X_train, y=y_train, cv=5 ).mean()
def write_submission(predictions):
submission_folder = '.'
dest_file = os.path.join(submission_folder, 'submission.csv')
 predictions.to_csv(dest_file, index=False)
 print("Wrote to {}".format(dest_file))
write_submission(predictions )<set_options>
ensemble = [CatBoostClassifier(verbose=False), RandomForestClassifier() , svm.NuSVC(probability=True), neighbors.KNeighborsClassifier() ]
classifiers_with_names = []
_ = [classifiers_with_names.append(( clf.__class__.__name__, clf)) for clf in ensemble]
voting = VotingClassifier(classifiers_with_names, voting='hard')
 cv_results = cross_validate(voting, X_train, y_train, cv=5)
 print(cv_results['test_score'].mean())
 voting.fit(X_train, y_train)
 predictions = voting.predict(X_test)
<install_modules><EOS> 
output = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': predictions.astype(int)})
 output.to_csv('submission_new_session.csv', index=False )