Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
ghosts_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 ghosts_df.head()
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
sns.set_style('whitegrid')
 sns.pairplot(ghosts_df, hue="type" )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<load_pretrained>
ghosts_df = ghosts_df.drop("id",axis=1)
 def remap_as_int(dataframe, indice, value, newValue):
print("value={} reassigned to id={}".format(value, newValue))
dataframe.loc[dataframe[indice] == value, indice] = newValue
def enumerate_list_unique(dataframe, indice):
return list(enumerate(np.unique(dataframe[indice])))
 monsters_list = enumerate_list_unique(ghosts_df, 'type')
 monsters_list = [(0, 'Ghost'),(1, 'Ghoul'),(2, 'Goblin')]
print("All known types of monsters = {}".format(np.unique(ghosts_df['type'])))
 for index, monster in monsters_list:
remap_as_int(ghosts_df, 'type', monster, index)
 colors_list = [(0, 'black'),(0.2, 'blood'),(0.4, 'blue'),(0.6, 'clear'),(0.8, 'green'),(1, 'white')]
print("All known colors of monsters = {}".format(np.unique(ghosts_df['color'])))
 for index, color in colors_list:
remap_as_int(ghosts_df, 'color', color, index )
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
remove_indexes = ["color", "hair_length", "has_soul", "rotting_flesh"]
variant = 'all'
test = False
show_test_result = False
alphas = [0.00001, 0.01, 0.1, 0.5, 1, 10]
if test:
X_train, X_test, y_train, y_test = train_test_split(ghosts_df.drop(remove_indexes + ["type"],axis=1), ghosts_df["type"], test_size=0.2, random_state=0)
 variant = 'all'
else:
X_train = ghosts_df.drop(remove_indexes + ["type"],axis=1)
 y_train = ghosts_df["type"]
create_features(test_df)
 def print_results(grid_search):
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
if show_test_result:
y_pred = grid_search.predict(X_test)
 print('Test-accuracy={}'.format(accuracy_score(y_test, y_pred)))
 for index, color in colors_list:
remap_as_int(test_df, 'color', color, index)
 test_df['color'] = test_df['color'].astype(float)
 test_df = test_df.drop(remove_indexes, axis=1)
 X_submission = test_df.drop("id",axis=1 ).copy()
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path )<concatenate>
if variant is 'lr' or variant is 'all':
logreg = LogisticRegression()
parameter_grid = {'C': alphas,
'solver' : ['newton-cg', 'lbfgs', 'sag'],
'multi_class' : ['multinomial', 'ovr']}
grid_search_logit = GridSearchCV(logreg, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_logit.fit(X_train, y_train)
 print_results(grid_search_logit )
models.append(model )<define_variables>
if variant is 'svc' or variant is 'all':
svc = SVC()
parameter_grid = {'C': alphas,
'kernel' : ['linear', 'rbf', 'sigmoid'],
'gamma' : alphas}
grid_search_svc = GridSearchCV(svc, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_svc.fit(X_train, y_train)
 print_results(grid_search_svc )
test_predictions = []<predict_on_test>
if variant is 'rf' or variant is 'all':
random_forest = RandomForestClassifier(random_state=1)
 parameter_grid = {'min_samples_leaf': [1, 5, 10],
'n_estimators': [10, 30, 50, 100],
'min_samples_split': [2,3]}
grid_search_rf = GridSearchCV(random_forest, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_rf.fit(X_train, y_train)
 print_results(grid_search_rf )
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))<prepare_output>
if variant is 'knn' or variant is 'all':
knn = KNeighborsClassifier()
parameter_grid = {'n_neighbors': [3, 5, 10],
'leaf_size': [5, 10, 30, 50],
'metric': ['minkowski', 'euclidean']}
grid_search_knn = GridSearchCV(knn, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_knn.fit(X_train, y_train)
 print_results(grid_search_knn )
final_predictions = np.mean(test_predictions, axis=0 )<save_to_csv>
if variant is 'nn' or variant is 'all':
neural_net = MLPClassifier(solver='lbfgs', random_state=1)
 parameter_grid = {'hidden_layer_sizes': [(15,1),(20,1),(15,3),(20,3),(15,6),(20,6)],
'alpha': alphas}
grid_search_nn = GridSearchCV(neural_net, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_nn.fit(X_train, y_train)
 print_results(grid_search_nn )
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )<import_modules>
Y_pred = grid_search_svc.predict(X_submission)
 Y_pred = Y_pred.astype(object)
 Y_pred[Y_pred == monsters_list[0][0]] = monsters_list[0][1]
Y_pred[Y_pred == monsters_list[1][0]] = monsters_list[1][1]
Y_pred[Y_pred == monsters_list[2][0]] = monsters_list[2][1]
pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )<load_from_csv>
submission = pd.DataFrame({
"id": test_df["id"],
"type": Y_pred
})
 submission.to_csv('new.csv', index=False )