Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<normalization>
%matplotlib inline
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)> max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<string_transform>
monster_train = pd.read_csv('.. /input/train.csv', index_col='id')
 monster_test = pd.read_csv('.. /input/test.csv', index_col='id' )
def _trim_input_answers(title, question, answer, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + a_new_len + q_new_len + 4)))
 q_len_head = round(q_new_len/2)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/2)
 a_len_tail = -1*(a_new_len -a_len_head)
 t = t[:t_new_len]
if HEAD_TAIL :
q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
else:
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_answers(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q, a = instance.question_title,instance.question_body, instance.answer
t, q, a = _trim_input_answers(t, q, a, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(t,q, a,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<string_transform>
msk = np.random.rand(len(monster_train)) < 0.8
monster_train_A = monster_train[msk]
monster_train_B = monster_train[~msk]
print('%d monsters in training subset A' % len(monster_train_A))
print('%d monsters in training subset B' % len(monster_train_B))
def _trim_input(title, question, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=479):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len +(t_max_len - t_len)
 q_new_len = q_max_len
else:
t_new_len = t_max_len
q_new_len = q_max_len
if q_max_len > q_len:
q_new_len = q_len
if t_new_len+q_new_len+3 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + q_new_len + 3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _convert_to_bert_inputs(title, question, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q = instance.question_title, instance.question_body
t, q = _trim_input(t, q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]<create_dataframe>
features = ['bone_length', 'rotting_flesh', 'hair_length', 'has_soul']
def how_accurate(classifier, n=5):
accuracy = []
for i in range(n):
classifier.fit(monster_train_A[features], monster_train_A['type'])
 preds = classifier.predict(monster_train_B[features])
 accuracy.append(np.where(preds==monster_train_B['type'], 1, 0 ).sum() / float(len(monster_train_B)))
 return np.mean(accuracy )
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, usefeatures,question_data, answer_data, title_data, lengths, labels = None):
self.inputs = inputs
self.usefeatures = usefeatures
self.question_data = question_data
self.answer_data = answer_data
self.title_data = title_data
if labels is not None:
self.labels = labels
else:
self.labels = None
self.lengths = lengths
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
lengths = self.lengths[idx]
usef = self.usefeatures[idx,:]
question_data = self.question_data[idx,:]
answer_data = self.answer_data[idx,:]
title_data = self.title_data[idx,:]
if self.labels is not None:
labels = self.labels[idx]
return input_ids, input_masks, input_segments, usef, question_data, answer_data, title_data, labels, lengths
return input_ids, input_masks, input_segments, usef, question_data,answer_data, title_data, lengths
def __len__(self):
return len(self.inputs[0] )<predict_on_test>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.neural_network import MLPClassifier
def predict_result_question(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 21))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device),
question_data = x_batch[4].to(device),
answer_data = x_batch[5].to(device),
title_data = x_batch[6].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output
def predict_result_answer(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 9))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device),
question_data = x_batch[4].to(device),
answer_data = x_batch[5].to(device),
title_data = x_batch[6].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output<load_pretrained>
knn_clf = KNeighborsClassifier(n_neighbors=6)
 tree_clf = DecisionTreeClassifier(max_depth=5)
 forest_clf = RandomForestClassifier(max_depth=2, n_estimators=100, max_features=1)
 gaussian_clf = GaussianProcessClassifier(4.0 * RBF(1.0), warm_start=True, n_jobs=-1)
 neural_clf = MLPClassifier(alpha=1, max_iter=400)
 clfs = {'K Nearest Neighbors': knn_clf,
'Decision Tree': tree_clf,
'Random Forest': forest_clf,
'Gassian Process': gaussian_clf,
'Neural Network': neural_clf}
set_seeds(SEED)
 question_tokenizer = BertTokenizer.from_pretrained(".. /input/bert-vocab1/vocab.txt")
 input_categories_q = list(train.columns[[1,2]]);
bert_model_config_question = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config_q = BertConfig.from_json_file(bert_model_config_question)
 bert_config_q.output_hidden_states = True
bert_config_q.num_labels = 21
answer_tokenizer = BertTokenizer.from_pretrained(".. /input/bert-vocab1/vocab.txt")
 input_categories_a = list(train.columns[[1,2,5]]);
bert_model_config_answer = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config_a = BertConfig.from_json_file(bert_model_config_answer)
 bert_config_a.output_hidden_states = True
bert_config_a.num_labels = 9
device = 'cuda' if torch.cuda.is_available() else 'cpu'
test_inputs_q = compute_input_arays(df = test, columns = input_categories_q, tokenizer = question_tokenizer, max_sequence_length=512)
 lengths_test_q = np.argmax(test_inputs_q[0] == 0, axis=1)
 lengths_test_q[lengths_test_q == 0] = test_inputs_q[0].shape[1]
test_inputs_a = compute_input_arays_answers(df = test, columns = input_categories_a, tokenizer = answer_tokenizer, max_sequence_length=512)
 lengths_test_a = np.argmax(test_inputs_a[0] == 0, axis=1)
 lengths_test_a[lengths_test_a == 0] = test_inputs_a[0].shape[1]<create_dataframe>
accuracy = {clf: how_accurate(clfs[clf], n=20)for clf in clfs}
NUM_FOLDS = 5
BATCH_SIZE = 8
epochs = 4
test_predictions_q = []
test_predictions_a = []
test_predictions = []
test_set_q = QuestDataset(inputs=test_inputs_q,
usefeatures=torch.FloatTensor(X_test),
question_data = torch.LongTensor(test_question_tokenized),
answer_data = torch.LongTensor(test_answer_tokenized),
title_data = torch.LongTensor(test_title_tokenized),
lengths=lengths_test_q, labels=None)
 test_loader_q = DataLoader(test_set_q, batch_size=32, shuffle=False)
 result_q = np.zeros(( len(test), 21))
test_set_a = QuestDataset(inputs=test_inputs_a,
usefeatures=torch.FloatTensor(X_test),
question_data = torch.LongTensor(test_question_tokenized),
answer_data = torch.LongTensor(test_answer_tokenized),
title_data = torch.LongTensor(test_title_tokenized),
lengths=lengths_test_a, labels=None)
 test_loader_a = DataLoader(test_set_a, batch_size=32, shuffle=False)
 result_a = np.zeros(( len(test), 9))
set_seeds(SEED )<define_variables>
upper_quantile = 0.98
lower_quantile =.02
<define_variables><EOS> 
for col in monster_train_A.columns.tolist() [:4]:
upper_limit = monster_train_A[col].quantile(upper_quantile)
 lower_limit = monster_train_A[col].quantile(lower_quantile)
 monster_train_A.loc[monster_train_A[col]>upper_limit, col] = np.nan
monster_train_A.loc[monster_train_A[col]<lower_limit, col] = np.nan
monster_train_A = monster_train_A.dropna()