Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<init_hyperparams>
%matplotlib inline
warnings.filterwarnings("ignore" )
tf.keras.backend.clear_session()
lr_sched = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-3*(0.83 **(( epoch+1)/ 2)))
 gkf = GroupKFold(n_splits = 5 ).split(X=train_df.url, groups = train_df.url)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
prev_weights = None
if fold != most_common:
continue
train_inputs =(( data['train_question_title'][train_idx], data['train_answer'][train_idx], data['train_question_title_use'][train_idx],
data['train_answer_use'][train_idx]),(final_outputs[train_idx]))
valid_inputs =(
(
([data['train_question_title'][valid_idx]], [data['train_answer'][valid_idx]], [data['train_question_title_use'][valid_idx]], [data['train_answer_use'][valid_idx]]), 
(None)
)
)
 BATCH_SIZE = 32
train_dataset = get_generator(train_inputs)
 valid_dataset = tf.data.Dataset.from_tensor_slices(valid_inputs)
 model = create_model()
optimizer = tf.keras.optimizers.Adam()
model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_dataset, epochs = 7, steps_per_epoch = train_idx.shape[0]//BATCH_SIZE,
callbacks=[PredictCallback(valid_dataset, final_outputs[valid_idx]), lr_sched, EWA() ])
 break 
<drop_column>
train_data_orig = pd.read_csv('.. /input/train.csv')
 test_data_orig = pd.read_csv('.. /input/test.csv' )
del train_inputs, valid_inputs, valid_dataset, train_dataset
gc.collect()<predict_on_test>
train_data = train_data_orig.drop(['id'], axis = 1)
 test_data = test_data_orig.drop(['id'], axis = 1 )
inputs =(
(
([data['train_question_title']], [data['train_answer']], [data['train_question_title_use']], [data['train_answer_use']]), 
(None)
)
)
 dataset = tf.data.Dataset.from_tensor_slices(inputs)
 predictions = model.predict(dataset )<compute_test_metric>
color_le = preprocessing.LabelEncoder()
color_le.fit(train_data['color'])
 train_data['color_int'] = color_le.transform(train_data['color'])
 _ = sns.pairplot(train_data.drop('color', axis = 1), hue = 'type', palette = 'muted', diag_kind='kde')
 train_data.drop('color_int', axis = 1, inplace = True )
train_y = final_outputs[train_idx]
valid_y = final_outputs[valid_idx]
train_preds = predictions[train_idx]
valid_preds = predictions[valid_idx]
opt = Optimize()
opt.fit(train_y, train_preds)
 post_valid_preds = opt.transform(valid_preds)
 print(f"Validation Score(Before){SpearmanCorrCoeff(valid_y, valid_preds)}")
 print(f"Validation Score(After){SpearmanCorrCoeff(valid_y, post_valid_preds)}" )<create_dataframe>
df = pd.get_dummies(train_data.drop('type', axis = 1))
X_train, X_test, y_train, y_test = train_test_split(df, train_data['type'], test_size = 0.25, random_state = 0)
 dt_clf = DecisionTreeClassifier(random_state = 0)
 dt_clf.fit(X_train, y_train)
 y_pred = dt_clf.predict(X_test)
 print(metrics.classification_report(y_test, y_pred))
print("
Accuracy Score is: " + str(metrics.accuracy_score(y_test, y_pred)) )
test_inputs =(
(
([data['test_question_title']], [data['test_answer']], [data['test_question_title_use']], [data['test_answer_use']]), 
(None)
)
)
 test_dataset = tf.data.Dataset.from_tensor_slices(test_inputs )<save_to_csv>
accuracy_scorer = metrics.make_scorer(metrics.accuracy_score )
test_preds = model.predict(test_dataset)
 post_test_preds = opt.transform(test_preds)
 submission = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 submission.iloc[:,1:] = post_test_preds
submission.to_csv("submission.csv", index = False)
 submission.head()<import_modules>
X_train = pd.get_dummies(train_data.drop('type', axis = 1))
y_train = train_data['type']
X_test = pd.get_dummies(test_data )
import pandas as pd
import numpy as np
import gc
import os
from math import floor, ceil
from tqdm.notebook import tqdm
from sklearn.model_selection import GroupKFold
from scipy.stats import spearmanr
import tensorflow as tf
import tensorflow_hub as hub
import bert_tokenization as tokenization
import tensorflow.keras.backend as K<load_from_csv>
params = {'n_estimators':[10, 20, 50, 100], 'criterion':['gini', 'entropy'], 'max_depth':[None, 5, 10, 25, 50]}
rf = RandomForestClassifier(random_state = 0)
 clf = GridSearchCV(rf, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
rf_best = RandomForestClassifier(n_estimators = 10, random_state = 0 )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = tokenization.FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('Train Shape =', df_train.shape)
 print('Test Shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
Output Categories:
\t', output_categories)
 print('
Input Categories:
\t', input_categories)
<categorify>
params = {'n_estimators':[10, 25, 50, 100], 'max_samples':[1, 3, 5, 10]}
bag = BaggingClassifier(random_state = 0)
 clf = GridSearchCV(bag, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
bag_best = BaggingClassifier(max_samples = 5, n_estimators = 25, random_state = 0 )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
<compute_train_metric>
params = {'learning_rate':[0.05, 0.1, 0.5], 'n_estimators':[100, 200, 500], 'max_depth':[2, 3, 5, 10]}
gbc = GradientBoostingClassifier(random_state = 0)
 clf = GridSearchCV(gbc, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
gbc_best = GradientBoostingClassifier(learning_rate = 0.1, max_depth = 5, n_estimators = 100, random_state = 0 )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
 )<choose_model_class>
params = {'n_neighbors':[3, 5, 10, 20], 'leaf_size':[20, 30, 50], 'p':[1, 2, 5], 'weights':['uniform', 'distance']}
knc = KNeighborsClassifier()
clf = GridSearchCV(knc, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
knc_best = KNeighborsClassifier(n_neighbors = 10 )
def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True, name = 'BERT_Layer')
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model<train_model>
params = {'penalty':['l1', 'l2'], 'C':[1, 2, 3, 5, 10]}
lr = LogisticRegression(random_state = 0)
 clf = GridSearchCV(lr, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
lr_best = LogisticRegression(penalty = 'l1', C = 1, random_state = 0 )
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
<choose_model_class>
params = {'kernel':['linear', 'rbf'], 'C':[1, 3, 5, 10], 'degree':[3, 5, 10]}
svc = SVC(probability = True, random_state = 0)
 clf = GridSearchCV(svc, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
svc_best = SVC(C = 10, degree = 3, kernel = 'linear', probability = True, random_state = 0 )
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=5, batch_size=8,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history)
<save_to_csv>
voting_clf = VotingClassifier(estimators=[('rf', rf_best),('bag', bag_best),('gbc', gbc_best),('lr', lr_best),('svc', svc_best)]
, voting='hard')
 voting_clf.fit(X_train, y_train)
 y_pred = voting_clf.predict(X_test)
 print("
Accuracy Score for VotingClassifier is: " + str(voting_clf.score(X_train, y_train)) )
<install_modules><EOS> 
submission = pd.DataFrame({'id':test_data_orig['id'], 'type':y_pred})
 submission.to_csv('.. /working/submission.csv', index=False )