Kaggle competition: Titanic - Machine Learning from Disaster
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def create_nn_model(output_len, model_type):
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 if model_type == 'BERT':
config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
elif model_type == 'GPT2':
config = GPT2Config()
gpt2_model = TFGPT2Model.from_pretrained(GPT2_PATH+'gpt2-tf_model.h5', config=config)
 q_embedding = gpt2_model(q_id)[0]
elif model_type == 'XLNET':
config = XLNetConfig.from_dict(xlnetcfg)
 xlnet_model = TFXLNetModel.from_pretrained(XLNET_PATH+'xlnet-tf_model.h5', config=config)
 q_embedding = xlnet_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
x = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 x = tf.keras.layers.Dense(output_len, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn], outputs=x)
 return model<define_variables>
warnings.filterwarnings('ignore' )
question_only = ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written']<define_variables>
%matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams['figure.figsize'] = 12,8
answer_and_question = ['answer_level_of_information', 'answer_helpful','answer_plausible','answer_relevance','answer_satisfaction']<define_variables>
data_raw = pd.read_csv('.. /input/train.csv')
 data_val = pd.read_csv('.. /input/test.csv')
 data1 = data_raw.copy(deep = True)
 data_cleaner = [data1, data_val]
print(data_raw.info())
 data_raw.sample(10 )
answer_only = ['answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']<define_variables>
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 drop_column = ['PassengerId','Cabin', 'Ticket']
data1.drop(drop_column, axis=1, inplace = True)
 print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
AQ_and_AO = answer_and_question + answer_only<split>
for dataset in data_cleaner:
dataset['FamilySize'] = dataset ['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 1
dataset['IsAlone'].loc[dataset['FamilySize'] > 1] = 0
dataset['Title'] = dataset['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'].astype(int), 5)
 stat_min = 10
title_names =(data1['Title'].value_counts() < stat_min)
 data1['Title'] = data1['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 print(data1['Title'].value_counts())
 print("-"*10)
 data1.info()
data_val.info()
data1.sample(10 )
gkf10 = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 gkf5 = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 common_validation_idx = []
val10 = []
val5 = []
val10_fold0 = None
for fold,(train_idx, valid_idx)in enumerate(gkf10):
if fold in selected_folds:
val10 += list(valid_idx)
 if fold == 0:
val10_fold0 = valid_idx
for fold,(train_idx, valid_idx)in enumerate(gkf5):
if fold in selected_folds:
val5 += list(valid_idx)
 common_validation_idx = np.array(list(set(val5 ).intersection(set(val10))))<predict_on_test>
label = LabelEncoder()
for dataset in data_cleaner:
dataset['Sex_Code'] = label.fit_transform(dataset['Sex'])
 dataset['Embarked_Code'] = label.fit_transform(dataset['Embarked'])
 dataset['Title_Code'] = label.fit_transform(dataset['Title'])
 dataset['AgeBin_Code'] = label.fit_transform(dataset['AgeBin'])
 dataset['FareBin_Code'] = label.fit_transform(dataset['FareBin'])
 Target = ['Survived']
data1_x = ['Sex','Pclass', 'Embarked', 'Title','SibSp', 'Parch', 'Age', 'Fare', 'FamilySize', 'IsAlone']
data1_x_calc = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code','SibSp', 'Parch', 'Age', 'Fare']
data1_xy = Target + data1_x
print('Original X Y: ', data1_xy, '
')
 data1_x_bin = ['Sex_Code','Pclass', 'Embarked_Code', 'Title_Code', 'FamilySize', 'AgeBin_Code', 'FareBin_Code']
data1_xy_bin = Target + data1_x_bin
print('Bin X Y: ', data1_xy_bin, '
')
 data1_dummy = pd.get_dummies(data1[data1_x])
 data1_x_dummy = data1_dummy.columns.tolist()
data1_xy_dummy = Target + data1_x_dummy
print('Dummy X Y: ', data1_xy_dummy, '
')
 data1_dummy.head()
def predict_nn(train_data, valid_data, test_data, weights, model_type):
K.clear_session()
model = create_nn_model(train_data[1].shape[1], model_type)
 model.load_weights(weights)
 trn_preds = np.zeros(train_data[1].shape)
 val_preds = model.predict(valid_data[0])
 print(f'Lengths of test list is {len(test_data)}')
 test_preds = model.predict(test_data)if test_data is not None else None
rho_trn = compute_spearmanr_ignore_nan(train_data[1], trn_preds)
 rho_val = compute_spearmanr_ignore_nan(valid_data[1], val_preds)
 print(f'Score train {rho_trn}; Score validation {rho_val}')
 return trn_preds, val_preds, test_preds
def get_cross_fold_preds_nn(input_idx, model_type):
all_fold_trn_preds, all_fold_val_preds, all_fold_test_preds = [],[],[]
all_fold_trn_outputs, all_fold_val_outputs = [], []
gkf = GroupKFold(n_splits=n_splits ).split(X=df_train.question_body, groups=df_train.question_body)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
if(USE_ONLY_SELECTED_FOLDS &(fold in selected_folds)) or(not USE_ONLY_SELECTED_FOLDS):
if MOD_DATA_STUCTURE == '2 split':
if input_idx == [0,1,2]: output_idx = [i for i, z in enumerate(output_categories)if z in question_only]
if input_idx == [3,4,5]: output_idx = [i for i, z in enumerate(output_categories)if z in AQ_and_AO]
if MOD_DATA_STUCTURE == '3 split':
if input_idx == [0,1,2]: output_idx = [i for i, z in enumerate(output_categories)if z in question_only]
if input_idx == [3,4,5]: output_idx = [i for i, z in enumerate(output_categories)if z in answer_and_question]
if input_idx == [6,7,8]: output_idx = [i for i, z in enumerate(output_categories)if z in answer_only]
train_inputs = [inputs[i][train_idx] for i in input_idx]
train_outputs = outputs[np.array(train_idx)[:,None], output_idx]
all_fold_trn_outputs.append(train_outputs)
 valid_inputs = [inputs[i][valid_idx] for i in input_idx]
valid_outputs = outputs[np.array(valid_idx)[:,None], output_idx]
all_fold_val_outputs.append(valid_outputs)
 current_test_input = [test_inputs[i] for i in input_idx]
print(f'Fold {fold}')
 input_type = None
if(input_idx == [3,4,5])&(model_type == 'XLNET'):
weights_path = f".. /input/gq-xlnet-pretrained/XLNET_question_only_fold_{fold}.h5"
input_type = 'question and answer'
elif(input_idx == [0,1,2])&(model_type == 'XLNET'):
weights_path = f".. /input/gq-xlnet-pretrained/XLNET_question_answer_fold_{fold}.h5"
input_type = 'question only'
if(input_idx == [3,4,5])&(model_type != 'XLNET'):
print(f'Using weights for BERT fold {fold}, question and answer modification')
 weights_path = f".. /input/{model_roor_dir}/{model_type}_question_answer_fold_{fold}.h5"
input_type = 'question and answer'
elif(input_idx == [0,1,2])&(model_type != 'XLNET'):
weights_path = f".. /input/{model_roor_dir}/{model_type}_question_only_fold_{fold}.h5"
input_type = 'question only'
elif(input_idx == [6,7,8])&(model_type != 'XLNET'):
weights_path = f".. /input/{model_roor_dir}/{model_type}_answer_only_fold_{fold}.h5"
input_type = 'answer only'
trn_preds, val_preds, test_preds = predict_nn(( train_inputs, train_outputs),(valid_inputs, valid_outputs), current_test_input, weights_path, model_type)
 all_fold_trn_preds.append(trn_preds)
 all_fold_val_preds.append(val_preds)
 all_fold_test_preds.append(test_preds)
 trn_preds, val_preds = np.concatenate(all_fold_trn_preds), np.concatenate(all_fold_val_preds)
 trn_out, val_out = np.concatenate(all_fold_trn_outputs), np.concatenate(all_fold_val_outputs)
 test_preds = np.stack(all_fold_test_preds, axis=2)
 test_preds = np.mean(test_preds, axis=2)
 print(f'Finished all folds for {model_type} {input_type}')
 print(test_preds.shape, trn_out.shape, val_preds.shape, val_out.shape, test_preds.shape)
 return(trn_preds, trn_out),(val_preds, val_out), test_preds
def get_nn_all_outputs(model_type):
print('Getting all folds for QUESTION ONLY')
 qonly_trn, qonly_val, qonly_tst = get_cross_fold_preds_nn([0,1,2], model_type)
 print('Getting all folds for QUESTION ANSWER')
 qa_trn, qa_val, qa_tst = get_cross_fold_preds_nn([3,4,5], model_type)
 if MOD_DATA_STUCTURE == '3 split':
print('Getting all folds for ANSWER ONLY')
 ao_trn, ao_val, ao_tst = get_cross_fold_preds_nn([6,7,8], model_type)
 trn =(np.concatenate(( qonly_trn[0], qa_trn[0], ao_trn[0]), axis=1), np.concatenate(( qonly_trn[1], qa_trn[1], ao_trn[1]), axis=1))
val =(np.concatenate(( qonly_val[0], qa_val[0], ao_val[0]), axis=1), np.concatenate(( qonly_val[1], qa_val[1], ao_val[1]), axis=1))
tst = np.concatenate(( qonly_tst, qa_tst, ao_tst), axis=1)
 if MOD_DATA_STUCTURE == '2 split':
trn =(np.concatenate(( qonly_trn[0], qa_trn[0]), axis=1), np.concatenate(( qonly_trn[1], qa_trn[1]), axis=1))
val =(np.concatenate(( qonly_val[0], qa_val[0]), axis=1), np.concatenate(( qonly_val[1], qa_val[1]), axis=1))
tst = np.concatenate(( qonly_tst, qa_tst), axis=1)
 print(f'Finsihed entire dataset(qonly and qa)for {model_type}')
 print(trn[0].shape, trn[1].shape, val[0].shape, val[1].shape, tst.shape)
 save_file(trn, f'{model_roor_dir}_trn')
 save_file(val, f'{model_roor_dir}_val')
 save_file(tst, f'{model_roor_dir}_tst')
 return trn, val, tst<load_pretrained>
train1_x, test1_x, train1_y, test1_y = model_selection.train_test_split(data1[data1_x_calc], data1[Target], random_state = 0)
 train1_x_bin, test1_x_bin, train1_y_bin, test1_y_bin = model_selection.train_test_split(data1[data1_x_bin], data1[Target] , random_state = 0)
 train1_x_dummy, test1_x_dummy, train1_y_dummy, test1_y_dummy = model_selection.train_test_split(data1_dummy[data1_x_dummy], data1[Target], random_state = 0)
 print("Data1 Shape: {}".format(data1.shape))
print("Train1 Shape: {}".format(train1_x.shape))
print("Test1 Shape: {}".format(test1_x.shape))
train1_x_bin.head()
tokenizer = XLNetTokenizer.from_pretrained('.. /input/gq-manual-uploads/xlnet tokenizer from colab/')
 USING_PAD_TOKEN = False
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 save_file(outputs, 'XLNET_outputs')
 save_file(inputs, 'XLNET_inputs')
 save_file(test_inputs, 'XLNET_test_inputs' )<define_variables>
plt.figure(figsize=[16,12])
 plt.subplot(231)
 plt.boxplot(x=data1['Fare'], showmeans = True, meanline = True)
 plt.title('Fare Boxplot')
 plt.ylabel('Fare($)')
 plt.subplot(232)
 plt.boxplot(data1['Age'], showmeans = True, meanline = True)
 plt.title('Age Boxplot')
 plt.ylabel('Age(Years)')
 plt.subplot(233)
 plt.boxplot(data1['FamilySize'], showmeans = True, meanline = True)
 plt.title('Family Size Boxplot')
 plt.ylabel('Family Size(
 plt.subplot(234)
 plt.hist(x = [data1[data1['Survived']==1]['Fare'], data1[data1['Survived']==0]['Fare']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Fare Histogram by Survival')
 plt.xlabel('Fare($)')
 plt.ylabel('
plt.legend()
plt.subplot(235)
 plt.hist(x = [data1[data1['Survived']==1]['Age'], data1[data1['Survived']==0]['Age']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Age Histogram by Survival')
 plt.xlabel('Age(Years)')
 plt.ylabel('
plt.legend()
plt.subplot(236)
 plt.hist(x = [data1[data1['Survived']==1]['FamilySize'], data1[data1['Survived']==0]['FamilySize']],
stacked=True, color = ['g','r'],label = ['Survived','Dead'])
 plt.title('Family Size Histogram by Survival')
 plt.xlabel('Family Size(
 plt.ylabel('
plt.legend()
model_roor_dir = 'gq-xlnet-pretrained'
MOD_DATA_STUCTURE = '2 split'
n_splits = 10
xlnet_trn, xlnet_val, xlnet_tst = get_nn_all_outputs('XLNET' )<load_pretrained>
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.BaggingClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.PassiveAggressiveClassifier() ,
linear_model.RidgeClassifierCV() ,
linear_model.SGDClassifier() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
discriminant_analysis.LinearDiscriminantAnalysis() ,
discriminant_analysis.QuadraticDiscriminantAnalysis() ,
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits = 10, test_size =.3, train_size =.6, random_state = 0)
 MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD' ,'MLA Time']
MLA_compare = pd.DataFrame(columns = MLA_columns)
 MLA_predict = data1[Target]
row_index = 0
for alg in MLA:
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
 cv_results = model_selection.cross_validate(alg, data1[data1_x_bin], data1[Target], cv = cv_split)
 MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std() *3
alg.fit(data1[data1_x_bin], data1[Target])
 MLA_predict[MLA_name] = alg.predict(data1[data1_x_bin])
 row_index+=1
MLA_compare.sort_values(by = ['MLA Test Accuracy Mean'], ascending = False, inplace = True)
 MLA_compare
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 USING_PAD_TOKEN = True
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 save_file(outputs, 'BERT_outputs')
 save_file(inputs, 'BERT_inputs')
 save_file(test_inputs, 'BERT_test_inputs' )<define_variables>
for index, row in data1.iterrows() :
if random.random() >.5:
data1.set_value(index, 'Random_Predict', 1)
 else:
data1.set_value(index, 'Random_Predict', 0)
 data1['Random_Score'] = 0
data1.loc[(data1['Survived'] == data1['Random_Predict']), 'Random_Score'] = 1
print('Coin Flip Model Accuracy: {:.2f}%'.format(data1['Random_Score'].mean() *100))
print('Coin Flip Model Accuracy w/SciKit: {:.2f}%'.format(metrics.accuracy_score(data1['Survived'], data1['Random_Predict'])*100))
model_roor_dir = 'gq-bert-pretrained'
MOD_DATA_STUCTURE = '2 split'
n_splits = 10
bert_trn, bert_val, bert_tst = get_nn_all_outputs('BERT' )<define_variables>
pivot_female = data1[data1.Sex=='female'].groupby(['Sex','Pclass', 'Embarked','FareBin'])['Survived'].mean()
print('Survival Decision Tree w/Female Node:
',pivot_female)
 pivot_male = data1[data1.Sex=='male'].groupby(['Sex','Title'])['Survived'].mean()
print('
Survival Decision Tree w/Male Node:
',pivot_male )
model_roor_dir = '3rd-training-2nd-gen-bert-download-from-gdrive'
MOD_DATA_STUCTURE = '3 split'
n_splits = 10
selected_folds = [0]
bert_trn_3split, bert_val_3split, bert_tst_3split = get_nn_all_outputs('BERT')
 selected_folds = [0,1,2]<define_variables>
def mytree(df):
Model = pd.DataFrame(data = {'Predict':[]})
 male_title = ['Master']
for index, row in df.iterrows() :
Model.loc[index, 'Predict'] = 0
if(df.loc[index, 'Sex'] == 'female'):
Model.loc[index, 'Predict'] = 1
if(( df.loc[index, 'Sex'] == 'female')& 
(df.loc[index, 'Pclass'] == 3)& 
(df.loc[index, 'Embarked'] == 'S')& 
(df.loc[index, 'Fare'] > 8)
):
Model.loc[index, 'Predict'] = 0
if(( df.loc[index, 'Sex'] == 'male')& 
(df.loc[index, 'Title'] in male_title)
):
Model.loc[index, 'Predict'] = 1
return Model
Tree_Predict = mytree(data1)
 print('Decision Tree Model Accuracy/Precision Score: {:.2f}%
'.format(metrics.accuracy_score(data1['Survived'], Tree_Predict)*100))
print(metrics.classification_report(data1['Survived'], Tree_Predict))
model_roor_dir = '2nd-training-1st-gen-bert-download-from-gdrive'
MOD_DATA_STUCTURE = '2 split'
n_splits = 5
bert_trn_5fold, bert_val_5fold, bert_tst_5fold = get_nn_all_outputs('BERT' )<load_pretrained>
dtree = tree.DecisionTreeClassifier(random_state = 0)
 base_results = model_selection.cross_validate(dtree, data1[data1_x_bin], data1[Target], cv = cv_split)
 dtree.fit(data1[data1_x_bin], data1[Target])
 print('BEFORE DT Parameters: ', dtree.get_params())
 print("BEFORE DT Training w/bin score mean: {:.2f}".format(base_results['train_score'].mean() *100))
print("BEFORE DT Test w/bin score mean: {:.2f}".format(base_results['test_score'].mean() *100))
print("BEFORE DT Test w/bin score 3*std: +/- {:.2f}".format(base_results['test_score'].std() *100*3))
print('-'*10)
 param_grid = {'criterion': ['gini', 'entropy'],
'max_depth': [2,4,6,8,10,None],
'random_state': [0]
}
tune_model = model_selection.GridSearchCV(tree.DecisionTreeClassifier() , param_grid=param_grid, scoring = 'roc_auc', cv = cv_split)
 tune_model.fit(data1[data1_x_bin], data1[Target])
 print('AFTER DT Parameters: ', tune_model.best_params_)
 print("AFTER DT Training w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_train_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score mean: {:.2f}".format(tune_model.cv_results_['mean_test_score'][tune_model.best_index_]*100))
print("AFTER DT Test w/bin score 3*std: +/- {:.2f}".format(tune_model.cv_results_['std_test_score'][tune_model.best_index_]*100*3))
print('-'*10)
tokenizer = GPT2Tokenizer.from_pretrained('.. /input/gq-manual-uploads/gpt2 config from colab/')
 USING_PAD_TOKEN = False
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 save_file(outputs, 'GPT2_outputs')
 save_file(inputs, 'GPT2_inputs')
 save_file(test_inputs, 'GPT2_test_inputs' )<define_variables>
dot_data = tree.export_graphviz(dtree, out_file=None,
feature_names = data1_x_bin, class_names = True,
filled = True, rounded = True)
 graph = graphviz.Source(dot_data)
 graph
model_roor_dir = 'gq-gpt2-pretrained'
MOD_DATA_STUCTURE = '2 split'
n_splits = 10
gpt2_trn, gpt2_val, gpt2_tst = get_nn_all_outputs('GPT2' )<split>
vote_est = [ 
('ada', ensemble.AdaBoostClassifier()), 
('bc', ensemble.BaggingClassifier()), 
('etc',ensemble.ExtraTreesClassifier()), 
('gbc', ensemble.GradientBoostingClassifier()), 
('rfc', ensemble.RandomForestClassifier()), 
('gpc', gaussian_process.GaussianProcessClassifier()), 
('lr', linear_model.LogisticRegressionCV()), 
('bnb', naive_bayes.BernoulliNB()), 
('gnb', naive_bayes.GaussianNB()), 
('knn', neighbors.KNeighborsClassifier()), 
('svc', svm.SVC(probability=True)) , 
('xgb', XGBClassifier())
 ]
vote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 vote_hard_cv = model_selection.cross_validate(vote_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_hard.fit(data1[data1_x_bin], data1[Target])
 print("Hard Voting Training w/bin score mean: {:.2f}".format(vote_hard_cv['train_score'].mean() *100))
print("Hard Voting Test w/bin score mean: {:.2f}".format(vote_hard_cv['test_score'].mean() *100))
print("Hard Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_hard_cv['test_score'].std() *100*3))
print('-'*10)
 vote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 vote_soft_cv = model_selection.cross_validate(vote_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 vote_soft.fit(data1[data1_x_bin], data1[Target])
 print("Soft Voting Training w/bin score mean: {:.2f}".format(vote_soft_cv['train_score'].mean() *100))
print("Soft Voting Test w/bin score mean: {:.2f}".format(vote_soft_cv['test_score'].mean() *100))
print("Soft Voting Test w/bin score 3*std: +/- {:.2f}".format(vote_soft_cv['test_score'].std() *100*3))
print('-'*10 )
model_roor_dir = '2nd-training-1st-gen-gpt-download-from-gdrive'
MOD_DATA_STUCTURE = '2 split'
n_splits = 5
gpt2_trn_5fold, gpt2_val_5fold, gpt2_tst_5fold = get_nn_all_outputs('GPT2' )<feature_engineering>
grid_n_estimator = [10, 50, 100, 300]
grid_ratio = [.1,.25,.5,.75, 1.0]
grid_learn = [.01,.03,.05,.1,.25]
grid_max_depth = [2, 4, 6, 8, 10, None]
grid_min_samples = [5, 10,.03,.05,.10]
grid_criterion = ['gini', 'entropy']
grid_bool = [True, False]
grid_seed = [0]
grid_param = [
[{
'n_estimators': grid_n_estimator,
'learning_rate': grid_learn,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'max_samples': grid_ratio,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'learning_rate': [.05],
'n_estimators': [300],
'max_depth': grid_max_depth,
'random_state': grid_seed
}],
[{
'n_estimators': grid_n_estimator,
'criterion': grid_criterion,
'max_depth': grid_max_depth,
'oob_score': [True],
'random_state': grid_seed
}],
[{
'max_iter_predict': grid_n_estimator,
'random_state': grid_seed
}],
[{
'fit_intercept': grid_bool,
'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'random_state': grid_seed
}],
[{
'alpha': grid_ratio,
}],
[{}],
[{
'n_neighbors': [1,2,3,4,5,6,7],
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']
}],
[{
'C': [1,2,3,4,5],
'gamma': grid_ratio,
'decision_function_shape': ['ovo', 'ovr'],
'probability': [True],
'random_state': grid_seed
}],
[{
'learning_rate': grid_learn,
'max_depth': [1,2,4,6,8,10],
'n_estimators': grid_n_estimator,
'seed': grid_seed
}]
]
start_total = time.perf_counter()
for clf, param in zip(vote_est, grid_param):
start = time.perf_counter()
best_search = model_selection.GridSearchCV(estimator = clf[1], param_grid = param, cv = cv_split, scoring = 'roc_auc')
 best_search.fit(data1[data1_x_bin], data1[Target])
 run = time.perf_counter() - start
best_param = best_search.best_params_
print('The best parameter for {} is {} with a runtime of {:.2f} seconds.'.format(clf[1].__class__.__name__, best_param, run))
clf[1].set_params(**best_param)
 run_total = time.perf_counter() - start_total
print('Total optimization time was {:.2f} minutes.'.format(run_total/60))
print('-'*10 )
if USE_LGB:
def remove_articles(df):
for i in ['question_title', 'question_body', 'answer']:
df.loc[:,f'{i}_orig'] = df.loc[:,i]
for i in ['question_title', 'question_body', 'answer']:
df.loc[:,i] = df.loc[:,i].apply(lambda x: x.replace(' the ',' ' ).replace(' a ',' ' ).replace(' an ',' '))
return df
df_train = remove_articles(df_train)
 df_test = remove_articles(df_test )<feature_engineering>
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_hard.fit(data1[data1_x_bin], data1[Target])
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, data1[data1_x_bin], data1[Target], cv = cv_split)
 grid_soft.fit(data1[data1_x_bin], data1[Target])
 data_val['Survived'] = mytree(data_val ).astype(int)
 submit = data_val[['PassengerId','Survived']]
if USE_LGB:
df_train.loc[:,'q_users_host'] = df_train.apply(lambda x: x.question_user_name + x.host, axis=1)
 df_train.loc[:,'a_users_host'] = df_train.apply(lambda x: x.answer_user_name + x.host, axis=1)
 df_test.loc[:,'q_users_host'] = df_test.apply(lambda x: x.question_user_name + x.host, axis=1)
 df_test.loc[:,'a_users_host'] = df_test.apply(lambda x: x.answer_user_name + x.host, axis=1)
 q_users_train = dict(df_train.q_users_host.value_counts())
 q_users_test = dict(df_test.q_users_host.value_counts())
 a_users_train = dict(df_train.a_users_host.value_counts())
 a_users_test = dict(df_test.a_users_host.value_counts())
 q_users = q_users_train
for i in q_users:
if i in q_users_test:
q_users_train[i] += q_users_test[i]
for i in q_users_test:
if i not in q_users:
q_users[i] = q_users_test[i]
a_users = a_users_train
for i in a_users:
if i in a_users_test:
a_users_train[i] += a_users_test[i]
for i in a_users_test:
if i not in a_users:
a_users[i] = a_users_test[i]<feature_engineering>
submit.to_csv("titanic_submission.csv", index = False )