Kaggle competition: Titanic - Machine Learning from Disaster
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )<define_variables>
train_dir = ".. /input/train.csv"
test_dir = ".. /input/test.csv"
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )<merge>
df = pd.read_csv(train_dir)
 test_df = pd.read_csv(test_dir)
 print("Total number of instance : ",len(df))
df.isna().sum()
df_train=pd.merge(df_train, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id')
 df_test = pd.merge(df_test, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id' )<categorify>
df.drop(["Cabin"], axis = 1, inplace = True)
 test_df.drop(["Cabin"], axis = 1, inplace = True)
 df.drop(["Ticket"], axis = 1, inplace = True)
 test_df.drop(["Ticket"], axis = 1, inplace = True)
 df.info()
df.head(10 )
<categorify>
df.drop("PassengerId", axis = 1, inplace = True)
 test_df.drop("PassengerId", axis = 1, inplace = True )
<drop_column>
df["Sex"].replace("male", 0, inplace = True)
 test_df["Sex"].replace("male", 0, inplace = True)
 df["Sex"].replace("female", 1, inplace = True)
 test_df["Sex"].replace("female", 1, inplace = True)
 df["Embarked"].replace(["S","C","Q"],[0,1,2], inplace = True)
 test_df["Embarked"].replace(["S","C","Q"],[0,1,2], inplace = True )
df_train.drop(['host', 'category'], inplace=True, axis=1)
 df_test.drop(['host', 'category'], inplace=True, axis=1 )<string_transform>
def create_family_ranges(df):
familysize = []
for members in df["n_fam_mem"]:
if members == 0:
familysize.append(0)
 elif members > 0 and members <=4:
familysize.append(1)
 elif members > 4:
familysize.append(2)
 return familysize
famsize = create_family_ranges(df)
 df["familysize"] = famsize
test_famsize = create_family_ranges(test_df)
 test_df["familysize"] = test_famsize
def _get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+[0]*(max_seq_length-len(tokens))
def _get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments=[]
first_sep=True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == '[SEP]':
current_segment_id=1
return segments+[0]*(max_seq_length-len(tokens))
def _trim_input_questions(title, question, max_sequence_length, t_max_len=30, q_max_len=479):
t = tokenizer2.tokenize(title)
 q = tokenizer2.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if q_max_len > q_len:
q_new_len = q_len
else:
q_new_len = q_max_len
if t_new_len+q_new_len+3 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+q_new_len+3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0]*(max_seq_length - len(token_ids))
return input_ids
def _convert_to_bert_inputs_questions(title, question,tokenizer, max_seq_length):
stoken = ['[CLS]']+title+['[SEP]']+question+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_questions(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
t, q = col['treated_question_title'], col['treated_question_body']
t,q = _trim_input_questions(t,q, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_questions(t,q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def _trim_input_answers(answer, max_sequence_length, a_max_len=510):
a = tokenizer2.tokenize(answer)
 a_len = len(a)
 if(a_len+2)> max_sequence_length:
if a_max_len > a_len:
a_new_len = a_len
else:
a_new_len = a_max_len
if a_new_len+2 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(a_new_len+2)))
 a = a[:a_new_len]
return a
def _convert_to_bert_inputs_answers(answer, tokenizer, max_seq_length):
stoken = ['[CLS]']+answer+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
a = col['treated_answer']
a = _trim_input_answers(a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
 def compute_spearmanr(trues, preds):
rhos=[]
for t, p in zip(trues.T, preds.T):
rhos.append(
 spearmanr(t,p ).correlation 
)
 return np.nanmean(rhos)
 test_predictions=[]
class CustomCallback(keras.callbacks.Callback):
def __init__(self, valid_data, test_data, test_predictions=test_predictions, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.test_predictions = test_predictions
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions=[]
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f"
validation rho: {round(rho_val,4)}")
 if(epoch+1)%4==0:
self.model.save_weights(f'/kaggle/working/bert-base-{fold}-{epoch}.hdf5')
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))<load_pretrained>
def age_to_int(df):
agelist = df["Age"].values.tolist()
for i in range(len(agelist)) :
if agelist[i] < 18 and agelist[i] >= 0:
agelist[i] = 0
elif agelist[i] >= 18 and agelist[i] < 60:
agelist[i] = 1
elif agelist[i]>=60 and agelist[i]<200:
agelist[i] = 2
else:
agelist[i] = -1
ageint = pd.DataFrame(agelist)
 return ageint
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True)
 def bertModel() :
input_ids_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_q')
 input_mask_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_q')
 input_segments_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_q')
 input_categories = keras.layers.Input(( len([i for i in df.columns if i.startswith('host_')or i.startswith('cat_')])) ,dtype=tf.float32, name='input_categorical')
 input_new_features = keras.layers.Input(( 9,), dtype=tf.float32, name='input_new_features')
 input_ids_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_a')
 input_mask_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_a')
 input_segments_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_a')
 bert_model_q = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 bert_model_a = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _, _, hidden_states_q = bert_model_q([input_ids_q,input_mask_q, input_segments_a])
 _, _, hidden_states_a = bert_model_a([input_ids_a,input_mask_a, input_segments_a])
 h12_q = tf.reshape(hidden_states_q[-1][:,0],(-1,1,768))
h11_q = tf.reshape(hidden_states_q[-2][:,0],(-1,1,768))
concat_hidden_q = keras.layers.Concatenate(axis=2 )([h12_q, h11_q])
 h12_a = tf.reshape(hidden_states_a[-1][:,0],(-1,1,768))
h11_a = tf.reshape(hidden_states_a[-2][:,0],(-1,1,768))
concat_hidden_a = keras.layers.Concatenate(axis=2 )([h12_a, h11_a])
 x = keras.layers.GlobalAveragePooling1D()(concat_hidden_q)
 x_a = keras.layers.GlobalAveragePooling1D()(concat_hidden_a)
 x = keras.layers.Concatenate()([x, input_new_features, input_categories])
 x_a = keras.layers.Concatenate()([x_a,x, input_new_features, input_categories])
 out = keras.layers.Dense(len([i for i in output_categories if i.startswith('question')]), activation='sigmoid', name='final_dense_output_q' )(x)
 out_a = keras.layers.Dense(len([i for i in output_categories if i.startswith('answer')]), activation='sigmoid', name='final_dense_output_a' )(x_a)
 out = keras.layers.Concatenate()([out,out_a])
 model = keras.models.Model(inputs=[input_ids_q, input_mask_q, input_segments_q,input_categories, input_new_features, input_ids_a, input_mask_a, input_segments_a]
, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=3e-5))
return model<split>
ageint = age_to_int(df)
 df["Ageint"] = ageint
df.drop("Age", axis = 1, inplace = True)
 test_ageint = age_to_int(test_df)
 test_df["Ageint"] = test_ageint
test_df.drop("Age", axis = 1, inplace = True)
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body )<randomize_order>
def conv_fare_ranges(df):
fare_ranges = []
for fare in df.actual_fare:
if fare < 7:
fare_ranges.append(0)
 elif fare >=7 and fare < 14:
fare_ranges.append(1)
 elif fare >=14 and fare < 30:
fare_ranges.append(2)
 elif fare >=30 and fare < 50:
fare_ranges.append(3)
 elif fare >=50:
fare_ranges.append(4)
 return fare_ranges
fare_ranges = conv_fare_ranges(df)
 df["fare_ranges"] = fare_ranges
test_fare_ranges = conv_fare_ranges(test_df)
 test_df["fare_ranges"] = test_fare_ranges
outputs = compute_output_arrays(df_train, output_categories)
 inputs_q = compute_input_array_questions(df_train, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 inputs_a = compute_input_array_answers(df_train, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_q = compute_input_array_questions(df_test, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_a = compute_input_array_answers(df_test, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH )<define_variables>
def name_to_int(df):
name = df["Name"].values.tolist()
namelist = []
for i in name:
index = 1
inew = i.split()
if inew[0].endswith(","):
index = 1
elif inew[1].endswith(","):
index = 2
elif inew[2].endswith(","):
index = 3
namelist.append(inew[index])
 print(set(namelist))
titlelist = []
for i in range(len(namelist)) :
if namelist[i] == "Lady.":
titlelist.append("Lady.")
 elif namelist[i] == "Ms.":
titlelist.append("Ms.")
 elif namelist[i] == "Miss.":
titlelist.append("Miss.")
 elif namelist[i] == "Dr.":
titlelist.append("Dr.")
 elif namelist[i] == "Mr.":
titlelist.append("Mr.")
 elif namelist[i] == "Jonkheer.":
titlelist.append("Jonkheer.")
 elif namelist[i] == "Col.":
titlelist.append("Col.")
 elif namelist[i] == "Mrs.":
titlelist.append("Mrs")
 elif namelist[i] == "Sir.":
titlelist.append("Sir.")
 elif namelist[i] == "Mlle.":
titlelist.append("Mlle.")
 elif namelist[i] == "Capt.":
titlelist.append("Capt.")
 elif namelist[i] == "the":
titlelist.append("the")
 elif namelist[i] == "Don.":
titlelist.append("Don.")
 elif namelist[i] == "Master.":
titlelist.append("Master.")
 elif namelist[i] == "Rev.":
titlelist.append("Rev.")
 elif namelist[i] == "Mme.":
titlelist.append("Mme.")
 elif namelist[i] == "Major.":
titlelist.append("Major.")
 else:
titlelist.append("sometitle")
 print(set(namelist))
return titlelist
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold<2:
keras.backend.clear_session()
model = bertModel()
train_inputs_q = [inputs_q[i][train_idx] for i in range(3)]
train_inputs_a = [inputs_a[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs_q = [inputs_q[i][valid_idx] for i in range(3)]
valid_inputs_a = [inputs_a[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
custom_callback = CustomCallback(valid_data=(valid_inputs_q + [df_train[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].iloc[valid_idx,:].values]+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[valid_idx,:].values]+ valid_inputs_a,
valid_outputs),
test_data=test_inputs_q + [df_test[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].values]+
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values]+test_inputs_a,
batch_size=8, fold=fold)
 H = model.fit(train_inputs_q+
[df_train[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].iloc[train_idx,:].values]+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[train_idx,:].values] + train_inputs_a,
train_outputs, batch_size=4, epochs=3, callbacks=[custom_callback])
 histories.append(H )<normalization>
titlelist = name_to_int(df)
 df["titles"] = titlelist
df["titles"].value_counts()
testtitlelist = name_to_int(test_df)
 test_df["titles"] = testtitlelist
def _get_masks_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_google_qa(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input_google_qa(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer_google_qa.tokenize(title)
 q = tokenizer_google_qa.tokenize(question)
 a = tokenizer_google_qa.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_google_qa(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_google_qa(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks_google_qa(stoken, max_sequence_length)
 input_segments = _get_segments_google_qa(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_google_qa(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance['treated_question_title'], instance['treated_question_body'], instance['treated_answer']
t, q, a = _trim_input_google_qa(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_google_qa(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays_google_qa(df, columns):
return np.asarray(df[columns])
 def bert_model_google_qa(learning_rate=4e-5,loss_function='binary_crossentropy'):
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[[input_word_ids, input_masks, input_segments]], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 return model 
<split>
df["titles"].replace(["Ms.","Jonkheer.","the","Don.","Capt.","Sir.","Lady.","Mme.","Col.","Major."],"sometitle", inplace = True)
 test_df["titles"].replace(["Ms.","Jonkheer.","the","Don.","Capt.","Sir.","Lady.","Mme.","Col.","Major."],"sometitle", inplace = True)
 df["titles"].replace("Mlle.","Miss.", inplace = True)
 test_df["titles"].replace("Mlle.","Miss.", inplace = True)
 test_df["titles"].value_counts()
gkf_google_qa = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs_google_qa = compute_output_arrays_google_qa(df_train, output_categories)
 inputs_google_qa = compute_input_arays_google_qa(df_train, ['treated_question_title','treated_question_body','treated_answer'], tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
 test_inputs_google_qa = compute_input_arays_google_qa(df_test, ['treated_question_title','treated_question_body','treated_answer'],tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
<choose_model_class>
df["titles"].replace(["Mr.", "Miss.", "Mrs", "Master.", "Dr.", "Rev.", "sometitle"],[0,1,2,3,4,5,6], inplace = True)
 df["titles"].astype("int64")
 test_df["titles"].replace(["Mr.", "Miss.", "Mrs", "Master.", "Dr.", "Rev.", "sometitle"],[0,1,2,3,4,5,6], inplace = True)
 test_df["titles"].astype("int64")
 df.drop(["Name"], axis = 1, inplace = True)
 test_df.drop(["Name"], axis = 1, inplace = True )
test_predictions_google_qa=[]
for fold,(train_idx, valid_idx)in enumerate(gkf_google_qa):
if fold<3:
keras.backend.clear_session()
model_qa = bert_model_google_qa()
print(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert-base-{fold}-4.hdf5')
 model_qa.load_weights(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert-base-{fold}-4.hdf5')
 preds_qa = model_qa.predict(test_inputs_google_qa , batch_size=8, verbose=1)
 test_predictions_google_qa.append(preds_qa)
 print(f'Fold:{fold} Completed' )<normalization>
df.drop(["SibSp","Parch","Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True)
 test_df.drop(["SibSp","Parch","Fare","n_fam_mem","actual_fare"], axis = 1, inplace = True )
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape<compute_test_metric>
labels = df["Survived"]
data = df.drop("Survived", axis = 1)
 X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1 )
final_preds = np.average(np.array([test_preds, test_preds_google_qa]),axis=0)
 final_preds.shape<save_to_csv>
final_clf = None
clf_names = ["Logistic Regression", "KNN(3)", "KNN(5)", "Random forest classifier", "Decision Tree Classifier",
"Gradient Boosting Classifier", "Support Vector Machine"]
classifiers = []
scores = []
df_sub.iloc[:, 1:] = final_preds
df_sub.to_csv('submission.csv', index=False )<set_options>
bestknn5 = None
bestknn3 = None
bestrf = None
bestgb = None
bestcvm = None
bestlr = None
bestdt = None
for i in range(10):
X_train, X_test, Y_train, Y_test = train_test_split(data, labels, test_size = 0.1)
 tempscores = []
lr_clf = LogisticRegression()
lr_clf.fit(X_train, Y_train)
 tempscores.append(( lr_clf.score(X_test, Y_test)) *100)
 if lr_clf.score(X_test, Y_test)*100 > 85:
bestlr = lr_clf
knn3_clf = KNeighborsClassifier()
knn3_clf.fit(X_train, Y_train)
 tempscores.append(( knn3_clf.score(X_test, Y_test)) *100)
 if knn3_clf.score(X_test, Y_test)*100 > 85:
bestknn3 = knn3_clf
knn5_clf = KNeighborsClassifier()
knn5_clf.fit(X_train, Y_train)
 tempscores.append(( knn5_clf.score(X_test, Y_test)) *100)
 if knn5_clf.score(X_test, Y_test)*100 > 85:
bestknn5 = knn5_clf
rf_clf = RandomForestClassifier(n_estimators = 100)
 rf_clf.fit(X_train, Y_train)
 tempscores.append(( rf_clf.score(X_test, Y_test)) *100)
 if rf_clf.score(X_test, Y_test)*100 > 85:
bestrf = rf_clf
dt_clf = DecisionTreeClassifier()
dt_clf.fit(X_train, Y_train)
 tempscores.append(( dt_clf.score(X_test, Y_test)) *100)
 if dt_clf.score(X_test, Y_test)*100 > 85:
bestdt = dt_clf
gb_clf = GradientBoostingClassifier()
gb_clf.fit(X_train, Y_train)
 tempscores.append(( gb_clf.score(X_test, Y_test)) *100)
 if gb_clf.score(X_test, Y_test)*100 > 85:
bestgb = lr_clf
svm_clf = SVC(gamma = "scale")
 svm_clf.fit(X_train, Y_train)
 tempscores.append(( svm_clf.score(X_test, Y_test)) *100)
 if svm_clf.score(X_test, Y_test)*100 > 85:
bestsvm = svm_clf
scores.append(tempscores )
np.set_printoptions(suppress=True)
 tf.random.set_seed(42)
 random.seed(42 )<load_from_csv>
scores = np.array(scores)
 clfs = pd.DataFrame({"Classifier":clf_names})
 clfs["iteration0"] = scores[0].T
clfs["iteration1"] = scores[1].T
clfs["iteration2"] = scores[2].T
clfs["iteration3"] = scores[3].T
clfs["iteration4"] = scores[4].T
clfs["iteration5"] = scores[5].T
clfs["iteration6"] = scores[6].T
clfs["iteration7"] = scores[7].T
clfs["iteration8"] = scores[8].T
clfs["iteration9"] = scores[9].T
means = clfs.mean(axis = 1)
 means = means.values.tolist()
clfs["Average"] = means
PATH = '.. /input/google-quest-challenge/'
VOCAB_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = tokenization.FullTokenizer(VOCAB_PATH + '/assets/vocab.txt', do_lower_case=True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
final_clf = bestsvm
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _get_category_onehot(df, columns):
return pd.get_dummies(df[columns] ).values
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 input_ids_array = np.asarray(input_ids, dtype=np.int32)
 input_masks_array = np.asarray(input_masks, dtype=np.int32)
 input_segments_array = np.asarray(input_segments, dtype=np.int32)
 for i in range(5):
input_ids_array = np.delete(input_ids_array, -1, 1)
 input_masks_array = np.delete(input_masks_array, -1, 1)
 input_segments_array = np.delete(input_segments_array, -1, 1)
 one_hot_array = _get_category_onehot(df, 'category')
 input_ids_array_512 = np.concatenate(( input_ids_array, one_hot_array), axis=1)
 input_masks_array_512 = np.concatenate(( input_masks_array, one_hot_array), axis=1)
 input_segments_array_512 = np.concatenate(( input_segments_array, one_hot_array), axis=1)
 return [input_ids_array_512,
input_masks_array_512,
input_segments_array_512]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
test_data = test_df
predictions = final_clf.predict(test_data)
 print(len(predictions))
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(VOCAB_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 y = tf.keras.layers.BatchNormalization(epsilon=1e-06, momentum=0.9, weights=None)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<split>
final_csv = []
csv_title = ['PassengerId', 'Survived']
final_csv.append(csv_title)
 for i in range(len(predictions)) :
passengerid = i + 892
survived = predictions[i]
temp = [passengerid, survived]
final_csv.append(temp)
 print(len(final_csv))
with open('submission_csv.csv', 'w')as file:
writer = csv.writer(file)
 writer.writerows(final_csv)
 file.close()