Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<set_options>
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsOneClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
import seaborn as sns
np.set_printoptions(suppress=True )<define_variables>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True )<define_variables>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
MAX_SEQUENCE_LENGTH = 512<string_transform>
types = train["type"]
indexes_test = test["id"]
colors_train = train.color
colors_test = test.color
ids = test.id
train.drop(["type","id", 'color'], axis=1, inplace=True)
 test.drop(["id", 'color'], axis=1, inplace=True )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<load_pretrained>
train_eng = train.copy(deep=True)
 train_eng['hairXsoul'] = train_eng.hair_length * train_eng.has_soul
train_eng['hairXbone'] = train_eng.hair_length * train_eng.bone_length
train_eng['boneXsoul'] = train_eng.bone_length * train_eng.has_soul
test_eng = test.copy(deep=True)
 test_eng['hairXsoul'] = test_eng.hair_length * test_eng.has_soul
test_eng['hairXbone'] = test_eng.hair_length * test_eng.bone_length
test_eng['boneXsoul'] = test_eng.bone_length * test_eng.has_soul
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True )<compute_test_metric>
dummies = pd.get_dummies(types)
 y = np.argmax(dummies.as_matrix() , 1)
 labels = pd.DataFrame(y, columns=['monster'] )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=8, fold=3):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if(self.fold is not None)and(epoch == 4):
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 print('Model Saved')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_model = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _,_, sequence_output = bert_model([input_word_ids, input_masks, input_segments])
 h12_q = tf.reshape(sequence_output[-1][:,0],(-1,1,768))
h11_q = tf.reshape(sequence_output[-2][:,0],(-1,1,768))
h10_q = tf.reshape(sequence_output[-3][:,0],(-1,1,768))
h09_q = tf.reshape(sequence_output[-4][:,0],(-1,1,768))
h08_q = tf.reshape(sequence_output[-5][:,0],(-1,1,768))
h07_q = tf.reshape(sequence_output[-6][:,0],(-1,1,768))
h06_q = tf.reshape(sequence_output[-7][:,0],(-1,1,768))
h05_q = tf.reshape(sequence_output[-8][:,0],(-1,1,768))
h04_q = tf.reshape(sequence_output[-9][:,0],(-1,1,768))
concat_hidden = tf.keras.layers.Concatenate(axis=2 )([h12_q, h11_q, h10_q, h09_q,h08_q,h07_q,h06_q,h05_q,h04_q])
 x = tf.keras.layers.GlobalAveragePooling1D()(concat_hidden)
 x = tf.keras.layers.Dropout(0.1 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 model.summary()
return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=3)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<split>
models = []
scores = []
predictions = []
estimators = [LogisticRegression(penalty='l2',
tol=0.01,
C=1000.0,
solver='liblinear',
max_iter=10000,
multi_class='ovr'),
LogisticRegression() ,
RandomForestClassifier(n_estimators=500),
MLPClassifier(max_iter=1000,
solver='lbfgs',
hidden_layer_sizes=[16])
 ]
for estimator in estimators:
print(estimator)
 for color in colors_train.unique() :
X = train_eng[colors_train!=color].values
y = labels[colors_train!=color]
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.3)
 clf = OneVsOneClassifier(estimator)
 clf.fit(X_train, y_train.values.ravel())
 score = accuracy_score(y_test,
clf.predict(X_test))
scores.append(score)
 print(score)
 predictions.append(clf.predict(test_eng))
print('
')
 print("{m} +/- {sd}".format(m=np.mean(scores), sd=np.std(scores)*2))
gkf = GroupKFold(n_splits=10 ).split(X=train.question_body, groups=train.question_body)
 outputs = compute_output_arrays(train, target_cols)
 inputs = compute_input_arays(train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<choose_model_class>
predictions_top = np.array(predictions)[scores.argsort() [-top_n:][::-1]]
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=4, batch_size=4,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history )<save_to_csv>
for x in range(0, len(ids)) :
a = predictions_top[:,x]
v, c = np.unique(a,return_counts=True)
 if max(c)>=(top_n/2):
continue
print(a, v, c )
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 sample_submission.iloc[:, 1:] = test_predictions
sample_submission.to_csv('submission.csv', index=False )<import_modules>
voted = []
for x in range(0, len(ids)) :
a = np.array(predictions)[:,x] 
(values, counts)= np.unique(a,return_counts=True)
 if max(c)>=(top_n/2):
pred = values[counts==max(counts)][0]
else:
pred = int(a[np.argmax(scores)])
 voted.append(pred)
<import_modules><EOS> 
Y = pd.DataFrame()
Y["id"] = ids
Y["type"] = dummies.columns[voted]
Y.to_csv("submission_voter.csv",index=False )