Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_train_metric>
import numpy as np
import pandas as pd
from mlens.ensemble import SuperLearner
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.linear_model import RidgeClassifier, Perceptron, PassiveAggressiveClassifier
import optuna
from optuna.samplers import TPESampler
import matplotlib.pyplot as plt
import plotly.express as px
import warnings
from sklearn.exceptions import ConvergenceWarning
learn.fit(10, 5e-2)
 pred = learn.get_preds(ds_type=DatasetType.Train)
 y_pred = [int(np.argmax(row)) for row in pred[0]]
print('QWK insample', cohen_kappa_score(y_pred, pred[1], weights='quadratic'))<compute_test_metric>
optuna.logging.set_verbosity(optuna.logging.WARNING)
 warnings.filterwarnings(action='ignore', category=ConvergenceWarning )
print('RMSE insample', np.sqrt(np.mean(( np.array(y_pred)- np.array(pred[1])) **2)) )<count_unique_values>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
np.unique(y_pred, return_counts=True )<save_to_csv>
for col in train.columns:
print(col, str(round(100* train[col].isnull().sum() / len(train), 2)) + '%' )
pred = learn.get_preds(ds_type=DatasetType.Test)
 y_pred = [int(np.argmax(row)) for row in pred[0]]
test_csv['AdoptionSpeed'] = y_pred
test_csv[['PetID', 'AdoptionSpeed']].to_csv('submission.csv', index=False )<set_options>
train['LastName'] = train['Name'].str.split(',', expand=True)[0]
test['LastName'] = test['Name'].str.split(',', expand=True)[0]
ds = pd.concat([train, test])
 sur = list()
died = list()
for index, row in ds.iterrows() :
s = ds[(ds['LastName']==row['LastName'])&(ds['Survived']==1)]
d = ds[(ds['LastName']==row['LastName'])&(ds['Survived']==0)]
s=len(s)
 if row['Survived'] == 1:
s-=1
d=len(d)
 if row['Survived'] == 0:
d-=1
sur.append(s)
 died.append(d)
 ds['FamilySurvived'] = sur
ds['FamilyDied'] = died
ds['FamilySize'] = ds['SibSp'] + ds['Parch'] + 1
ds['IsAlone'] = 0
ds.loc[ds['FamilySize'] == 1, 'IsAlone'] = 1
ds['Fare'] = ds['Fare'].fillna(train['Fare'].median())
 ds['Embarked'] = ds['Embarked'].fillna('Q')
 train = ds[ds['Survived'].notnull() ]
test = ds[ds['Survived'].isnull() ]
test = test.drop(['Survived'], axis=1)
 train['rich_woman'] = 0
test['rich_woman'] = 0
train['men_3'] = 0
test['men_3'] = 0
train.loc[(train['Pclass']<=2)&(train['Sex']=='female'), 'rich_woman'] = 1
test.loc[(test['Pclass']<=2)&(test['Sex']=='female'), 'rich_woman'] = 1
train.loc[(train['Pclass']==3)&(train['Sex']=='male'), 'men_3'] = 1
test.loc[(test['Pclass']==3)&(test['Sex']=='male'), 'men_3'] = 1
train['rich_woman'] = train['rich_woman'].astype(np.int8)
 test['rich_woman'] = test['rich_woman'].astype(np.int8)
 train["Cabin"] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in train['Cabin']])
 test['Cabin'] = pd.Series([i[0] if not pd.isnull(i)else 'X' for i in test['Cabin']])
 for cat in ['Pclass', 'Sex', 'Embarked', 'Cabin']:
train = pd.concat([train, pd.get_dummies(train[cat], prefix=cat)], axis=1)
 train = train.drop([cat], axis=1)
 test = pd.concat([test, pd.get_dummies(test[cat], prefix=cat)], axis=1)
 test = test.drop([cat], axis=1)
 train = train.drop(['PassengerId', 'Ticket', 'LastName', 'SibSp', 'Parch', 'Sex_male', 'Name'], axis=1)
 test = test.drop(['PassengerId', 'Ticket', 'LastName', 'SibSp', 'Parch', 'Sex_male', 'Name'], axis=1)
 train = train.fillna(-1)
 test = test.fillna(-1)
 train.head()
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128
plt.rcParams['figure.figsize'] =(12, 9)
 plt.style.use('ggplot' )<define_variables>
y = train['Survived']
X = train.drop(['Survived', 'Cabin_T'], axis=1)
 X_test = test.copy()
X, X_val, y, y_val = train_test_split(X, y, random_state=0, test_size=0.2, shuffle=False )
Run_Tabular = True
Run_Metadata = False
Run_Sentiment = False
Run_Metadata_AvgSum = False
Run_Sentiment_AvgSum = False
Run_Breed_Map = False
Run_Rescuer_Cnt = True
Run_Text_Features = False<load_from_csv>
class Optimizer:
def __init__(self, metric, trials=30):
self.metric = metric
self.trials = trials
self.sampler = TPESampler(seed=666)
 def objective(self, trial):
model = create_model(trial)
 model.fit(X, y)
 preds = model.predict(X_val)
 if self.metric == 'acc':
return accuracy_score(y_val, preds)
 else:
return f1_score(y_val, preds)
 def optimize(self):
study = optuna.create_study(direction="maximize", sampler=self.sampler)
 study.optimize(self.objective, n_trials=self.trials)
 return study.best_params
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sample_submission = pd.read_csv('.. /input/test/sample_submission.csv' )<load_from_csv>
rf = RandomForestClassifier(random_state=666)
 rf.fit(X, y)
 preds = rf.predict(X_val)
 print('Random Forest accuracy: ', accuracy_score(y_val, preds))
print('Random Forest f1-score: ', f1_score(y_val, preds))
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 n_estimators = trial.suggest_int("n_estimators", 2, 150)
 min_samples_leaf = trial.suggest_int("min_samples_leaf", 1, 10)
 model = RandomForestClassifier(
 min_samples_leaf=min_samples_leaf,
n_estimators=n_estimators,
max_depth=max_depth,
random_state=666 
)
 return model
optimizer = Optimizer('f1')
 rf_f1_params = optimizer.optimize()
rf_f1_params['random_state'] = 666
rf_f1 = RandomForestClassifier(**rf_f1_params)
 rf_f1.fit(X, y)
 preds = rf_f1.predict(X_val)
 print('Optimized on F1 score')
 print('Optimized Random Forest: ', accuracy_score(y_val, preds))
print('Optimized Random Forest f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 rf_acc_params = optimizer.optimize()
rf_acc_params['random_state'] = 666
rf_acc = RandomForestClassifier(**rf_acc_params)
 rf_acc.fit(X, y)
 preds = rf_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized Random Forest: ', accuracy_score(y_val, preds))
print('Optimized Random Forest f1-score: ', f1_score(y_val, preds))
labels_breed = pd.read_csv('.. /input/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/color_labels.csv')
 labels_color = pd.read_csv('.. /input/state_labels.csv' )<define_variables>
xgb = XGBClassifier(random_state=666)
 xgb.fit(X, y)
 preds = xgb.predict(X_val)
 print('XGBoost accuracy: ', accuracy_score(y_val, preds))
print('XGBoost f1-score: ', f1_score(y_val, preds))
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 n_estimators = trial.suggest_int("n_estimators", 1, 150)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0000001, 1)
 gamma = trial.suggest_uniform('gamma', 0.0000001, 1)
 model = XGBClassifier(
 learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth=max_depth,
gamma=gamma,
random_state=666 
)
 return model
optimizer = Optimizer('f1')
 xgb_f1_params = optimizer.optimize()
xgb_f1_params['random_state'] = 666
xgb_f1 = XGBClassifier(**xgb_f1_params)
 xgb_f1.fit(X, y)
 preds = xgb_f1.predict(X_val)
 print('Optimized on F1 score')
 print('Optimized XGBoost accuracy: ', accuracy_score(y_val, preds))
print('Optimized XGBoost f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 xgb_acc_params = optimizer.optimize()
xgb_acc_params['random_state'] = 666
xgb_acc = XGBClassifier(**xgb_acc_params)
 xgb_acc.fit(X, y)
 preds = xgb_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized XGBoost accuracy: ', accuracy_score(y_val, preds))
print('Optimized XGBoost f1-score: ', f1_score(y_val, preds))
train_image_files = sorted(glob.glob('.. /input/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )<count_unique_values>
lgb = LGBMClassifier(random_state=666)
 lgb.fit(X, y)
 preds = lgb.predict(X_val)
 print('LightGBM accuracy: ', accuracy_score(y_val, preds))
print('LightGBM f1-score: ', f1_score(y_val, preds))
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 n_estimators = trial.suggest_int("n_estimators", 1, 150)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0000001, 1)
 num_leaves = trial.suggest_int("num_leaves", 2, 3000)
 min_child_samples = trial.suggest_int('min_child_samples', 3, 200)
 model = LGBMClassifier(
 learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth=max_depth,
num_leaves=num_leaves,
min_child_samples=min_child_samples,
random_state=666 
)
 return model
optimizer = Optimizer('f1')
 lgb_f1_params = optimizer.optimize()
lgb_f1_params['random_state'] = 666
lgb_f1 = LGBMClassifier(**lgb_f1_params)
 lgb_f1.fit(X, y)
 preds = lgb_f1.predict(X_val)
 print('Optimized on F1-score')
 print('Optimized LightGBM accuracy: ', accuracy_score(y_val, preds))
print('Optimized LightGBM f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 lgb_acc_params = optimizer.optimize()
lgb_acc_params['random_state'] = 666
lgb_acc = LGBMClassifier(**lgb_acc_params)
 lgb_acc.fit(X, y)
 preds = lgb_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized LightGBM accuracy: ', accuracy_score(y_val, preds))
print('Optimized LightGBM f1-score: ', f1_score(y_val, preds))
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )<load_pretrained>
lr = LogisticRegression(random_state=666)
 lr.fit(X, y)
 preds = lr.predict(X_val)
 print('Logistic Regression: ', accuracy_score(y_val, preds))
print('Logistic Regression f1-score: ', f1_score(y_val, preds))
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r', encoding="utf8")as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r', encoding="utf8")as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
df_sentiment = []
if Run_Sentiment == True:
sentiment_filename = '.. /input/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
if Run_Metadata == True:
metadata_filenames = sorted(glob.glob('.. /input/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<merge>
dt = DecisionTreeClassifier(random_state=666)
 dt.fit(X, y)
 preds = dt.predict(X_val)
 print('Decision Tree accuracy: ', accuracy_score(y_val, preds))
print('Decision Tree f1-score: ', f1_score(y_val, preds))
def create_model(trial):
max_depth = trial.suggest_int("max_depth", 2, 6)
 min_samples_split = trial.suggest_int('min_samples_split', 2, 16)
 min_weight_fraction_leaf = trial.suggest_uniform('min_weight_fraction_leaf', 0.0, 0.5)
 min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10)
 model = DecisionTreeClassifier(
 min_samples_split=min_samples_split,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_depth=max_depth,
min_samples_leaf=min_samples_leaf,
random_state=666 
)
 return model
optimizer = Optimizer('f1')
 dt_f1_params = optimizer.optimize()
dt_f1_params['random_state'] = 666
dt_f1 = DecisionTreeClassifier(**dt_f1_params)
 dt_f1.fit(X, y)
 preds = dt_f1.predict(X_val)
 print('Optimized on F1-score')
 print('Optimized Decision Tree accuracy: ', accuracy_score(y_val, preds))
print('Optimized Decision Tree f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 dt_acc_params = optimizer.optimize()
dt_acc_params['random_state'] = 666
dt_acc = DecisionTreeClassifier(**dt_acc_params)
 dt_acc.fit(X, y)
 preds = dt_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized Decision Tree accuracy: ', accuracy_score(y_val, preds))
print('Optimized Decision Tree f1-score: ', f1_score(y_val, preds))
train_proc = train.copy()
if Run_Sentiment_AvgSum == True:
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 if Run_Metadata_AvgSum == True:
train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 test_proc = test.copy()
if Run_Sentiment_AvgSum == True:
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 if Run_Metadata_AvgSum == True:
test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
bc = BaggingClassifier(random_state=666)
 bc.fit(X, y)
 preds = bc.predict(X_val)
 print('Bagging Classifier accuracy: ', accuracy_score(y_val, preds))
print('Bagging Classifier f1-score: ', f1_score(y_val, preds))
def create_model(trial):
n_estimators = trial.suggest_int('n_estimators', 2, 200)
 max_samples = trial.suggest_int('max_samples', 1, 100)
 model = BaggingClassifier(
 n_estimators=n_estimators,
max_samples=max_samples,
random_state=666 
)
 return model
optimizer = Optimizer('f1')
 bc_f1_params = optimizer.optimize()
bc_f1_params['random_state'] = 666
bc_f1 = BaggingClassifier(**bc_f1_params)
 bc_f1.fit(X, y)
 preds = bc_f1.predict(X_val)
 print('Optimized on F1-score')
 print('Optimized Bagging Classifier accuracy: ', accuracy_score(y_val, preds))
print('Optimized Bagging Classifier f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 bc_acc_params = optimizer.optimize()
bc_acc_params['random_state'] = 666
bc_acc = BaggingClassifier(**bc_acc_params)
 bc_acc.fit(X, y)
 preds = bc_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized Bagging Classifier accuracy: ', accuracy_score(y_val, preds))
print('Optimized Bagging Classifier f1-score: ', f1_score(y_val, preds))
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 if Run_Breed_Map == True:
train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 if Run_Breed_Map == True:
test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
knn = KNeighborsClassifier()
knn.fit(X, y)
 preds = knn.predict(X_val)
 print('KNN accuracy: ', accuracy_score(y_val, preds))
print('KNN f1-score: ', f1_score(y_val, preds))
sampler = TPESampler(seed=0)
 def create_model(trial):
n_neighbors = trial.suggest_int("n_neighbors", 2, 25)
 model = KNeighborsClassifier(n_neighbors=n_neighbors)
 return model
optimizer = Optimizer('f1')
 knn_f1_params = optimizer.optimize()
knn_f1 = KNeighborsClassifier(**knn_f1_params)
 knn_f1.fit(X, y)
 preds = knn_f1.predict(X_val)
 print('Optimized on F1-score')
 print('Optimized KNN accuracy: ', accuracy_score(y_val, preds))
print('Optimized KNN f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 knn_acc_params = optimizer.optimize()
knn_acc = KNeighborsClassifier(**knn_acc_params)
 knn_acc.fit(X, y)
 preds = knn_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized KNN accuracy: ', accuracy_score(y_val, preds))
print('Optimized KNN f1-score: ', f1_score(y_val, preds))
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))<define_variables>
abc = AdaBoostClassifier(random_state=666)
 abc.fit(X, y)
 preds = abc.predict(X_val)
 print('AdaBoost accuracy: ', accuracy_score(y_val, preds))
print('AdaBoost f1-score: ', f1_score(y_val, preds))
def create_model(trial):
n_estimators = trial.suggest_int("n_estimators", 2, 150)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0005, 1.0)
 model = AdaBoostClassifier(
 n_estimators=n_estimators,
learning_rate=learning_rate,
random_state=666 
)
 return model
optimizer = Optimizer('f1')
 abc_f1_params = optimizer.optimize()
abc_f1_params['random_state'] = 666
abc_f1 = AdaBoostClassifier(**abc_f1_params)
 abc_f1.fit(X, y)
 preds = abc_f1.predict(X_val)
 print('Optimized on F1-score')
 print('Optimized AdaBoost accuracy: ', accuracy_score(y_val, preds))
print('Optimized AdaBoost f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 abc_acc_params = optimizer.optimize()
abc_acc_params['random_state'] = 666
abc_acc = AdaBoostClassifier(**abc_acc_params)
 abc_acc.fit(X, y)
 preds = abc_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized AdaBoost accuracy: ', accuracy_score(y_val, preds))
print('Optimized AdaBoost f1-score: ', f1_score(y_val, preds))
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))<define_variables>
et = ExtraTreesClassifier(random_state=666)
 et.fit(X, y)
 preds = et.predict(X_val)
 print('ExtraTreesClassifier accuracy: ', accuracy_score(y_val, preds))
print('ExtraTreesClassifier f1-score: ', f1_score(y_val, preds))
def create_model(trial):
n_estimators = trial.suggest_int("n_estimators", 2, 150)
 max_depth = trial.suggest_int("max_depth", 2, 6)
 model = ExtraTreesClassifier(
 n_estimators=n_estimators,
max_depth=max_depth,
random_state=0 
)
 return model
optimizer = Optimizer('f1')
 et_f1_params = optimizer.optimize()
et_f1_params['random_state'] = 666
et_f1 = ExtraTreesClassifier(**et_f1_params)
 et_f1.fit(X, y)
 preds = et_f1.predict(X_val)
 print('Optimized on F1-score')
 print('Optimized ExtraTreesClassifier accuracy: ', accuracy_score(y_val, preds))
print('Optimized ExtraTreesClassifier f1-score: ', f1_score(y_val, preds))
optimizer = Optimizer('acc')
 et_acc_params = optimizer.optimize()
et_acc_params['random_state'] = 666
et_acc = ExtraTreesClassifier(**et_acc_params)
 et_acc.fit(X, y)
 preds = et_acc.predict(X_val)
 print('Optimized on accuracy')
 print('Optimized ExtraTreesClassifier accuracy: ', accuracy_score(y_val, preds))
print('Optimized ExtraTreesClassifier f1-score: ', f1_score(y_val, preds))
X_temp = X.copy()
text_columns = ['Description']
if Run_Metadata_AvgSum==True:
text_columns += ['metadata_annots_top_desc']
if Run_Sentiment_AvgSum==True:
text_columns += ['sentiment_entities']
categorical_columns = []
if Run_Breed_Map == True:
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
print(text_columns)
 print(categorical_columns )<merge>
model = SuperLearner(
 folds=5,
random_state=666 
)
 model.add(
 [
bc,
lgb,
xgb,
rf,
dt,
knn
] 
)
 model.add_meta(
 LogisticRegression() 
)
 model.fit(X, y)
 preds = model.predict(X_val)
 print('SuperLearner accuracy: ', accuracy_score(y_val, preds))
print('SuperLearner f1-score: ', f1_score(y_val, preds))
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
if Run_Rescuer_Cnt == True:
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<normalization>
mdict = {
'RF': RandomForestClassifier(random_state=666),
'XGB': XGBClassifier(random_state=666),
'LGBM': LGBMClassifier(random_state=666),
'DT': DecisionTreeClassifier(random_state=666),
'KNN': KNeighborsClassifier() ,
'BC': BaggingClassifier(random_state=666),
'OARF': RandomForestClassifier(**rf_acc_params),
'OFRF': RandomForestClassifier(**rf_f1_params),
'OAXGB': XGBClassifier(**xgb_acc_params),
'OFXGB': XGBClassifier(**xgb_f1_params),
'OALGBM': LGBMClassifier(**lgb_acc_params),
'OFLGBM': LGBMClassifier(**lgb_f1_params),
'OADT': DecisionTreeClassifier(**dt_acc_params),
'OFDT': DecisionTreeClassifier(**dt_f1_params),
'OAKNN': KNeighborsClassifier(**knn_acc_params),
'OFKNN': KNeighborsClassifier(**knn_f1_params),
'OABC': BaggingClassifier(**bc_acc_params),
'OFBC': BaggingClassifier(**bc_f1_params),
'OAABC': AdaBoostClassifier(**abc_acc_params),
'OFABC': AdaBoostClassifier(**abc_f1_params),
'OAET': ExtraTreesClassifier(**et_acc_params),
'OFET': ExtraTreesClassifier(**et_f1_params),
'LR': LogisticRegression(random_state=666),
'ABC': AdaBoostClassifier(random_state=666),
'SGD': SGDClassifier(random_state=666),
'ET': ExtraTreesClassifier(random_state=666),
'MLP': MLPClassifier(random_state=666),
'GB': GradientBoostingClassifier(random_state=666),
'RDG': RidgeClassifier(random_state=666),
'PCP': Perceptron(random_state=666),
'PAC': PassiveAggressiveClassifier(random_state=666)
 }
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<data_type_conversions>
def create_model(trial):
model_names = list()
models_list = [
'RF', 'XGB', 'LGBM', 'DT',
'KNN', 'BC', 'OARF', 'OFRF',
'OAXGB', 'OFXGB', 'OALGBM',
'OFLGBM', 'OADT', 'OFDT',
'OAKNN', 'OFKNN', 'OABC',
'OFBC', 'OAABC', 'OFABC',
'OAET', 'OFET', 'LR',
'ABC', 'SGD', 'ET',
'MLP', 'GB', 'RDG',
'PCP', 'PAC'
]
head_list = [
'RF', 'XGB', 'LGBM', 'DT',
'KNN', 'BC', 'LR', 'ABC',
'SGD', 'ET', 'MLP', 'GB',
'RDG', 'PCP', 'PAC'
]
n_models = trial.suggest_int("n_models", 2, 6)
 for i in range(n_models):
model_item = trial.suggest_categorical('model_{}'.format(i), models_list)
 if model_item not in model_names:
model_names.append(model_item)
 folds = trial.suggest_int("folds", 2, 6)
 model = SuperLearner(
 folds=folds,
random_state=666 
)
 models = [
mdict[item] for item in model_names
]
model.add(models)
 head = trial.suggest_categorical('head', head_list)
 model.add_meta(
 mdict[head] 
)
 return model
def objective(trial):
model = create_model(trial)
 model.fit(X, y)
 preds = model.predict(X_val)
 score = accuracy_score(y_val, preds)
 return score
study = optuna.create_study(
 direction="maximize",
sampler=sampler 
)
 study.optimize(
 objective,
n_trials=50
)
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )<feature_engineering>
model = SuperLearner(
 folds=folds,
random_state=666 
)
 models = [
mdict[item] for item in result
]
model.add(models)
 model.add_meta(mdict[head])
 model.fit(X, y)
 preds = model.predict(X_val)
 print('Optimized SuperLearner accuracy: ', accuracy_score(y_val, preds))
print('Optimized SuperLearner f1-score: ', f1_score(y_val, preds))
n_components = 7
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 if Run_Text_Features == True:
X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<count_missing_values>
preds = model.predict(X_test)
 preds = preds.astype(np.int16 )
np.sum(pd.isnull(X_train))<count_missing_values>
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission['Survived'] = preds
submission.to_csv('submission.csv', index=False )