Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
import numpy as np
import pandas as pd
import os
from typing import Tuple
count_m = []
for recipe in features_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed training sample: " + str(len(count_m)) )<define_variables>
pd.options.mode.chained_assignment = None
count_m = []
for recipe in features_test_processed:
if not recipe:
count_m.append([recipe])
 else: pass
print("Empty instances in the preprocessed test sample: " + str(len(count_m)) )<choose_model_class>
class DataHelper() :

 def __init__(self, directory: str ='.. /input/', train_file: str ='train.csv', test_file: str ='test.csv'):
self.directory =directory
self.test_file = test_file
self.train_file = train_file
pass
def __read_csv(self, filename: str)-> pd.DataFrame:
return pd.read_csv(self.directory + filename)
 def get_train_data(self, y_columns)-> Tuple[pd.DataFrame, pd.DataFrame]:

 train_data = self.__read_csv(self.train_file)
 return train_data.drop(y_columns, axis=1), train_data[y_columns]
def get_test_data(self)-> pd.DataFrame:

 return self.__read_csv(self.test_file)
 def __format_prediction_before_save(self, index, prediction)-> pd.DataFrame:
reshaped = np.array(prediction ).reshape(418,1)
 return pd.DataFrame(reshaped, index=index, columns=['Survived'])
 def save_prediction(self, index, prediction: pd.DataFrame, filename: str = 'submission.csv'):
self.__format_prediction_before_save(index, prediction ).to_csv(path_or_buf=filename, sep=',', header=True )
vectorizer = CountVectorizer(analyzer = "word",
ngram_range =(1,1),
binary = True,
tokenizer = None,
preprocessor = None,
stop_words = None,
max_df = 0.99 )<feature_engineering>
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import OneHotEncoder
from sklearn.linear_model import SGDClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
train_X = vectorizer.fit_transform([str(i)for i in features_processed])
 test_X = vectorizer.transform([str(i)for i in features_test_processed] )<prepare_x_and_y>
class CabinExtractor(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def __get_deck(self, cabin)-> int:
return 0 if cabin == 0 else 1
def transform(self, X, y=None):
X.Cabin.fillna(0, inplace=True)
 X.Cabin = X.Cabin.apply(self.__get_deck)
 return X
target = train_data['cuisine']<categorify>
class TitleExtractor(BaseEstimator, TransformerMixin):

 def __init__(self, regex: str):
self.regex = regex
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X['Title'] = X.Name.str.extract(self.regex)[0].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 return X.drop('Name', axis=1 )
lb = LabelEncoder()
train_Y = lb.fit_transform(target )<split>
class DataFrameSelector(BaseEstimator, TransformerMixin):

 def __init__(self, column_names):
self.column_names = column_names
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return X[self.column_names]
X_train, X_test, y_train, y_test = train_test_split(train_X, train_Y , random_state = 0 )<train_on_grid>
class DataFrameToValueConverter(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
return X.values
clfs = []
rfc = RandomForestClassifier(n_estimators=50, random_state=42, n_jobs=-1)
 rfc.fit(X_train, y_train)
 print('RFC LogLoss {score}'.format(score=log_loss(y_test, rfc.predict_proba(X_test))))
clfs.append(rfc)
 logreg = LogisticRegression(random_state = 42)
 logreg.fit(X_train, y_train)
 print('LogisticRegression LogLoss {score}'.format(score=log_loss(y_test, logreg.predict_proba(X_test))))
clfs.append(logreg)
 svc = SVC(random_state=42,probability=True, kernel='linear')
 svc.fit(X_train, y_train)
 print('SVC LogLoss {score}'.format(score=log_loss(y_test, svc.predict_proba(X_test))))
clfs.append(svc)
<find_best_params>
class FamilyAttributeCombiner(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def get_val(self, amount):
if amount > 0:
return 1
else:
return 0
def transform(self, X, y=None):
X['HasFamily'] = X[['SibSp','Parch']].sum(axis=1 ).apply(self.get_val)
 return X.drop(['SibSp','Parch'], axis=1 )
predictions = []
for clf in clfs:
predictions.append(clf.predict_proba(X_test))
def log_loss_func(weights):

 final_prediction = 0
for weight, prediction in zip(weights, predictions):
final_prediction += weight*prediction
return log_loss(y_test, final_prediction)
 starting_values = [0.5]*len(predictions)
 cons =({'type':'eq','fun':lambda w: 1-sum(w)})
 bounds = [(0,1)]*len(predictions)
 res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons)
 print('Ensamble Score: {best_score}'.format(best_score=res['fun']))
print('Best Weights: {weights}'.format(weights=res['x']))<choose_model_class>
class GridSearchHelper() :
def __init__(self, preprocesing_pipeline: Pipeline, X: pd.DataFrame, y: pd.DataFrame):
self.preprocesing_pipeline = preprocesing_pipeline
self.X = X
self.y = y
pass
def __get_grid_search(self, clf, param_grid: list, kwargs)-> GridSearchCV:
return GridSearchCV(clf, param_grid, **kwargs)
 def __get_best_estimator(self, clf, param_grid: list, kwargs)-> BaseEstimator:
grid_search = self.__get_grid_search(clf, param_grid, kwargs)
 grid_search.fit(self.preprocesing_pipeline.fit_transform(self.X), self.y)
 return grid_search.best_estimator_
def get_best_classifier(self, clf, param_grid, **kwargs):
return self.__get_best_estimator(clf, param_grid, kwargs )
vclf=VotingClassifier(estimators=[('clf1',RandomForestClassifier(n_estimators = 50,random_state = 42)) , 
('clf2',LogisticRegression(random_state = 42)) , 
('clf3',SVC(kernel='linear',random_state = 42,probability=True))
],
voting='soft', weights = [0.05607363, 0.70759724, 0.23632913])
 vclf.fit(train_X, train_Y )<compute_train_metric>
not_needed_attribs = ['PassengerId', 'Name', 'Ticket','Cabin', 'Embarked']
cat_attribs = ['Sex', 'Pclass']
num_attribs = ['SibSp', 'Parch', 'Age', 'Fare']
kfold = model_selection.KFold(n_splits=5, random_state=42)
 valscores = model_selection.cross_val_score(vclf, train_X, train_Y, cv=kfold)
 print('Mean accuracy on 5-fold cross validation: ' + str(np.mean(valscores)) )<save_to_csv>
title_regex = '([A-Za-z]+)\.'
predictions = vclf.predict(test_X)
 predictions = lb.inverse_transform(predictions)
 predictions_final = pd.DataFrame({'cuisine' : predictions , 'id' : test_data.id }, columns=['id', 'cuisine'])
 predictions_final.to_csv('Final_submission.csv', index = False )<set_options>
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_attribs)) , 
('df_to_value_converter',DataFrameToValueConverter()), 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('std_scaler', StandardScaler()),
] )
%matplotlib inline
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(message)s')
 warnings.filterwarnings('ignore' )<set_options>
cat_pipeline = Pipeline([ 
('selector', DataFrameSelector(cat_attribs)) , 
('df_to_value_converter',DataFrameToValueConverter()), 
('one_hot_encoder', OneHotEncoder()),
] )
np.set_printoptions(precision=6 )<define_variables>
feature_union = FeatureUnion(transformer_list=[ 
('num_pipeline', num_pipeline), 
('cat_pipeline', cat_pipeline),
] )
DATA_DIR='.. /input'
TRAIN_FILE=DATA_DIR + '/train.json'
TEST_FILE=DATA_DIR + '/test.json'
SUBMIT_FILE= 'sample_submission.csv' 
<categorify>
preprocessing_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_attribs + cat_attribs)) , 
('feature_union', feature_union),
] )
def clean_data(X):
X['ingredients'] = X['ingredients'].map(lambda l: [x.lower() for x in l])
 X['ingredients'] = X['ingredients'].map(lambda l: [re.sub(r'\(\s*[^\s]*\s+oz\.\s*\)\s*', '', x ).strip() for x in l])
 X['ingredients'] = X['ingredients'].map(lambda l: [x.replace("-", " ")for x in l])
 X['ingredients'] = X['ingredients'].map(lambda l: [x.replace("half & half", "half milk cream")for x in l])
 X['ingredients'] = X['ingredients'].map(lambda l: [x.replace(" & ", " ")for x in l])
 X['ingredients'] = X['ingredients'].map(lambda l: [re.sub(r'\d+%\s+less\s+sodium ', '', x ).strip() for x in l])
 lemmatizer = WordNetLemmatizer()
X['ingredients'] = X['ingredients'].map(lambda l: [lemmatizer.lemmatize(x)for x in l])
 return X<categorify>
X_train, y_train = dataHelper.get_train_data(['Survived'])
 X_test = dataHelper.get_test_data()
def words_to_text(list_of_words, add_separate_words=True):
if isinstance(list_of_words, list):
l = list_of_words
else:
l = eval(list_of_words)
 s = ''
for i, w_0 in enumerate(l):
if i > 0:
s = s + ' '
w_0 = w_0.strip()
w_1 = w_0.replace(' ', '_' ).replace(',','_vir_')
 s = s + w_1
if add_separate_words and ' ' in w_0:
s = s + ' ' + w_0
return s
def vectorize(X_words, sublinear_tf=True, max_df=0.75):
print("Extracting vectorized feats from the training data using a sparse vectorizer")
 t0 = time()
vectorizer = TfidfVectorizer(sublinear_tf=True,
max_df=max_df,
stop_words='english')
 X_vectorized = vectorizer.fit_transform(X_words)
 duration = time() - t0
print("done in %fs " %(duration))
print("n_samples: %d, n_features: %d" % X_vectorized.shape)
 print()
return X_vectorized, vectorizer, duration 
<compute_train_metric>
preprocessing_pipeline.fit(X_train)
 gridSearchHelper = GridSearchHelper(preprocessing_pipeline, X_train, y_train.values.ravel() )
def benchmark(clf, X_train, y_train, X_test, y_test):
print('_' * 80)
 print("Training {}: ".format(str(type(clf))))
print(clf)
 t0 = time()
clf.fit(X_train, y_train)
 train_time = time() - t0
print("train time: %0.3fs" % train_time)
 t0 = time()
y_pred = clf.predict(X_test)
 test_time = time() - t0
print("test time: %0.3fs" % test_time)
 score = metrics.accuracy_score(y_test, y_pred)
 print("accuracy score: %0.3f" % score)

cm = metrics.confusion_matrix(y_test, y_pred)
 clf_descr = str(clf ).split('(')[0]
ret = {
'classifier': clf,
'accuracy_score': score,
'confusion_matrix': cm,
'train_time': train_time,
'test_time': test_time,
'y_pred': y_pred
}
return ret<load_from_disk>
def get_estimator_and_print_score(clf, param_grid):
X_prepared = preprocessing_pipeline.transform(X_train)
 estimator = gridSearchHelper.get_best_classifier(clf, param_grid, cv=2, scoring='accuracy', refit=True)
 estimator.fit(X_prepared, y_train.values.ravel())
 scores = cross_val_score(estimator, X_prepared, y_train.values.ravel() , cv=2, scoring='accuracy')
 print(type(clf ).__name__)
 print('Crossval:', scores.mean())
 print('Accuracy:',accuracy_score(y_train, estimator.predict(X_prepared)))
 print('F1-score:', f1_score(y_train, estimator.predict(X_prepared)))
 return estimator
df = pd.read_json(TRAIN_FILE, encoding='iso-8859-1')
 clean_data(df)
 df.head(3 )<categorify>
svc_param_grid = [
{
'C':[0.9,1.0,1.1],
'gamma':[1]
},
]
X_words = df['ingredients'].map(words_to_text)
 X_words.head(3 )<feature_engineering>
rfc_param_grid = [
{
'max_depth' : [10], 'max_features' : ['log2'], 'max_leaf_nodes' : [None],
'min_impurity_decrease' : [0.0], 'min_impurity_split' : [None],
'min_samples_leaf' : [1], 'min_samples_split' : [30],
'min_weight_fraction_leaf' : [0.0], 'n_estimators' : [100], 'n_jobs' : [1],
'oob_score' : [False], 'random_state' : [None], 'verbose' : [0], 'warm_start' : [False]
}
]
X_vects, Vectorizer_inst, Vectorizer_duration = vectorize(X_words)
 X_column_names = Vectorizer_inst.get_feature_names()<categorify>
dtc_param_grid = [
{
'max_depth': [4, 5],
}
]
encoder = LabelEncoder()
y_encoded = encoder.fit_transform(df['cuisine'])
 y_encoded_classes_names = encoder.inverse_transform(range(20))<split>
sgdc_param_grid = [
{
'shuffle': [True],
'max_iter':[np.ceil(10**6 / len(X_train)) ,],
},
]
Xs_train, Xs_test, y_train, y_test = train_test_split(X_vects, y_encoded, random_state=0 )<choose_model_class>
knc_param_grid = [
{
'n_neighbors':[2,3,4,5,7],
'weights':['uniform','distance'],
},
{
}
]
BEST_PARAMS = {
'C': 80,
'kernel': 'rbf',
'gamma': 1.7,
'coef0': 1,
'cache_size': 500
}
BEST_ESTIMATOR = OneVsRestClassifier(
 SVC(**BEST_PARAMS))<compute_test_metric>
final_pipeline = Pipeline([ 
('preprocessing', preprocessing_pipeline), 
('estimator', current_estimator)
 ] )
RESULT = benchmark(BEST_ESTIMATOR, Xs_train, y_train, Xs_test, y_test )<categorify>
final_pipeline.fit(X_train, y_train.values.ravel() )
<filter><EOS> 
dataHelper.save_prediction(X_test['PassengerId'], final_pipeline.predict(X_test))