Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
test_df = pd.read_csv(os.path.join(DATA_DIR,"test.csv"))<feature_engineering>
import tensorflow as tf
eng_stopwords = set(stopwords.words("english"))
def add_external_features(df):
eng_stopwords = set(stopwords.words("english"))
df['question_body'] = df['question_body'].apply(lambda x:str(x))
df['question_num_words'] = df.question_body.str.count('\S+')
 df['question_title_num_words'] = df.question_title.str.count('\S+')
 df['answer'] = df['answer'].apply(lambda x:str(x))
df['answer_num_words'] = df.answer.str.count('\S+')
 df["question_title_num_unique_words"] = df["question_title"].apply(lambda x: len(set(str(x ).split())))
 df["question_body_num_unique_words"] = df["question_body"].apply(lambda x: len(set(str(x ).split())))
 df["answer_num_unique_words"] = df["answer"].apply(lambda x: len(set(str(x ).split())))
 df["question_title_num_chars"] = df["question_title"].apply(lambda x: len(str(x)))
 df["question_body_num_chars"] = df["question_body"].apply(lambda x: len(str(x)))
 df["answer_num_chars"] = df["answer"].apply(lambda x: len(str(x)))
 df['qt_words'] = df['question_title'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['q_words'] = df['question_body'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['a_words'] = df['answer'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['qa_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['q_words'], s['a_words'])) , axis = 1)
 df['qt_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['qt_words'], s['a_words'])) , axis = 1)
 df['qa_word_overlap_norm'] = df.apply(lambda s: s['qa_word_overlap']/(len(s['a_words'])+ len(s['q_words'])- s['qa_word_overlap']), axis = 1)
 df['qta_word_overlap_norm'] = df.apply(lambda s: s['qt_word_overlap']/(len(s['a_words'])+ len(s['qt_words'])- s['qt_word_overlap']), axis = 1)
 df.drop(['q_words', 'a_words', 'qt_words'], axis = 1, inplace = True)
 return df<feature_engineering>
with ZipFile('/kaggle/input/ghouls-goblins-and-ghosts-boo/test.csv.zip', 'r')as test:
test.extractall()
test_df = add_external_features(test_df )<normalization>
with ZipFile('/kaggle/input/ghouls-goblins-and-ghosts-boo/train.csv.zip', 'r')as train:
train.extractall()
with ZipFile('/kaggle/input/ghouls-goblins-and-ghosts-boo/sample_submission.csv.zip', 'r')as sample_submission:
sample_submission.extractall()
handmade_cols = ["question_body_num_unique_words", 'question_num_words',
"question_title_num_unique_words", "question_title_num_words",
"answer_num_unique_words", "answer_num_words"]
with open('/kaggle/input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/C_scaler.pickle', mode='rb')as f:
num_words_scaler = pickle.load(f)
 test_df[handmade_cols]= num_words_scaler.transform(test_df[handmade_cols].values)
 train_handmade_features= test_df[handmade_cols + ['qa_word_overlap_norm', 'qta_word_overlap_norm']].values<categorify>
sample = pd.read_csv("sample_submission.csv" )
def label_encoder(x, dict_reverse):
try:
return dict_reverse[x]
except:
return 0
with open('/kaggle/input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/C_category.pickle', mode='rb')as f:
category_dict_reverse = pickle.load(f)
 test_df['category'] = test_df['category'].apply(lambda x: label_encoder(x, category_dict_reverse))
with open('/kaggle/input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/C_host.pickle', mode='rb')as f:
host_dict_reverse = pickle.load(f)
 test_df['host'] = test_df['host'].apply(lambda x: label_encoder(x, host_dict_reverse))
n_cat = len(category_dict_reverse)+ 1
cat_emb = 256
n_host = len(host_dict_reverse)+ 1
host_emb = 256<load_pretrained>
test_data = pd.read_csv("test.csv" )
bert_config = transformers.XLNetConfig.from_json_file(BERT_CONFIG_PATH)
 bert_config.output_hidden_states = True
tokenizer = transformers.XLNetTokenizer(BERT_VOCAB_PATH )<string_transform>
train_data = pd.read_csv("train.csv" )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_segments2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q_len_head = round(q_new_len/3)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/3)
 a_len_tail = -1*(a_new_len -a_len_head)
 q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
return t, q, a
def convert_lines(question, answer, max_sequence_length, tokenizer):
all_tokens = []
all_masks = []
all_segments = []
longer = 0
for q, a in tqdm(zip(question, answer)) :
tokens_q = tokenizer.tokenize(q)
 tokens_a = tokenizer.tokenize(a)
 q_len = len(tokens_q)
 a_len = len(tokens_a)
 if(q_len+a_len+3)> max_sequence_length:
longer += 1
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(np.ceil(new_q_len)) , int(np.floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
tokens_q = tokens_q[:new_q_len]
tokens_a = tokens_a[:new_a_len]
stoken = ["[CLS]"] + tokens_q + ["[SEP]"] + tokens_a + ["[SEP]"]
token_ids = tokenizer.encode(stoken, add_special_tokens=Ture)
 input_ids = token_ids + [0] *(max_sequence_length-len(token_ids))
attention_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 all_tokens.append(input_ids)
 all_masks.append(attention_masks)
 all_segments.append(input_segments)
 print(longer)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<categorify>
train =train_data.drop(["id",'color','type'], axis=1)
 train
def convert_lines2(title, question, answer, max_sequence_length, tokenizer, t_max_len_seq=30, q_max_len_seq=239, a_max_len_seq=239):
all_tokens = []
all_masks = []
all_segments = []
longer = 0
for t, q, a in tqdm(zip(title, question, answer)) :
tokens_t, tokens_q, tokens_a = _trim_input(t, q, a, max_sequence_length=max_sequence_length)
 stoken = tokens_t + ["[SEP]"] + tokens_q + ["[SEP]"] + tokens_a + ["[SEP]"] + ["[CLS]"]
token_ids = tokenizer.convert_tokens_to_ids(stoken)
 input_ids = token_ids + [0] *(max_sequence_length-len(token_ids))
attention_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 all_tokens.append(input_ids)
 all_masks.append(attention_masks)
 all_segments.append(input_segments)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<categorify>
test = test_data.drop(["id",'color'], axis=1)
 test
test_tokens, test_masks, test_segments = convert_lines2(test_df["question_title"],
test_df["question_body"],
test_df["answer"],
max_sequence_length=512,
tokenizer=tokenizer )<categorify>
categorical=train_data.columns[train_data.dtypes=='object']
categorical
class QuestDataset_test(Dataset):
def __init__(self, token_ids, masks, segments, hosts, categories, handmade_features):
self.token_ids = token_ids
self.masks = masks
self.segments = segments
self.hosts = hosts
self.categories = categories
self.handmades = handmade_features
def __len__(self):
return self.token_ids.shape[0]
def __getitem__(self, idx):
token_id = self.token_ids[idx]
mask = self.masks[idx]
segment = self.segments[idx]
host = self.hosts[idx]
category = self.categories[idx]
handmade = self.handmades[idx]
return [token_id, mask, segment, host, category, handmade]<categorify>
dummies=pd.get_dummies(train_data['color'])
 dummies.columns=str('color')+'_'+dummies.columns
train=pd.concat([train,dummies],axis=1)
 train
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, torch.nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))<normalization>
dummies=pd.get_dummies(test_data['color'])
 dummies.columns=str('color')+'_'+dummies.columns
test=pd.concat([test,dummies],axis=1)
 test
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
pd.get_dummies(train_data['type'] )
class QuestModel6(nn.Module):
def __init__(self, n_cat, cat_emb, n_host, host_emb, num_labels):
super().__init__()
BERT_DIMS = 1024
LSTM_UNITS = 512
model_path = os.path.join(BERT_MODEL_PATH)
 self.bert_model = transformers.XLNetModel.from_pretrained(model_path,from_tf=True, config=bert_config)
 set_trainable(self.bert_model, False)
 self.category_embedding = nn.Embedding(n_cat, cat_emb)
 self.host_embedding = nn.Embedding(n_host, host_emb)
 self.embedding_dropout = SpatialDropout(0.5)
 self.lstm1 = nn.LSTM(BERT_DIMS*2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.GRU(LSTM_UNITS*2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.atten1 = Attention(LSTM_UNITS*2, 512)
 self.atten2 = Attention(LSTM_UNITS*2, 512)
 self.dropout1 = nn.Dropout(0.2)
 self.fc1 = nn.Linear(LSTM_UNITS*6 + int(cat_emb)+ int(host_emb)+ 8, 21)
 self.fc2 = nn.Linear(LSTM_UNITS*6 + int(cat_emb)+ int(host_emb)+ 8, 9)
 self._init_weights(self.category_embedding)
 self._init_weights(self.host_embedding)
 self._init_weights(self.fc1)
 self._init_weights(self.fc2)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
print("initailize weight")
 module.weight.data.normal_(mean=0.0, std=0.02)
 if isinstance(module, nn.Linear)and module.bias is not None:
print("initailize bias")
 module.bias.data.zero_()
def forward(self, token_ids, masks, segment, hosts, categories, handmades):
category_embed = self.category_embedding(categories)
 host_embed = self.host_embedding(hosts)
 external_features = torch.cat(( category_embed, host_embed, handmades), 1)
 seq_output, hidden_states = self.bert_model(input_ids=token_ids,
attention_mask=masks,
token_type_ids=segment)
 last_four_layer = torch.cat(( hidden_states[-5], hidden_states[-6]), 2)
 lstm_input = self.embedding_dropout(last_four_layer)
 lstm1_output, _ = self.lstm1(lstm_input)
 lstm2_output, _ = self.lstm2(lstm1_output)
 meanpooled_output = torch.mean(lstm2_output, 1)
 maxpooled_output, _ = torch.max(lstm2_output, 1)
 attention_output_q = self.atten1(lstm2_output)
 attention_output_a = self.atten2(lstm2_output)
 pooled_output_q = torch.cat(( meanpooled_output, maxpooled_output, attention_output_q, external_features), 1)
 pooled_output_q = self.dropout1(pooled_output_q)
 pooled_output_a = torch.cat(( meanpooled_output, maxpooled_output, attention_output_a, external_features), 1)
 pooled_output_a = self.dropout1(pooled_output_a)
 q_results = self.fc1(pooled_output_q)
 a_results = self.fc2(pooled_output_a)
 results = torch.cat(( q_results, a_results), 1)
 return results<predict_on_test>
Y_train = np.array(pd.get_dummies(train_data['type']))
def predict_result(model, test_loader, batch_size=batch_size):
output = np.zeros(( len(test_set), 30))
model.eval()
with torch.no_grad() :
for idx, inputs in enumerate(test_loader):
start_index = idx * batch_size
end_index = min(start_index + batch_size, len(test_set))
token_ids, masks, segments, hosts, categories, handmades = inputs
token_ids = token_ids.long().cuda()
masks = masks.long().cuda()
segments = segments.long().cuda()
hosts = hosts.long().cuda()
categories = categories.long().cuda()
handmades = handmades.float().cuda()
predictions = model(token_ids, masks, segments, hosts, categories, handmades)
 predictions = torch.sigmoid(predictions)
 output[start_index:end_index, :] = predictions.detach().cpu().numpy()
return output<define_variables>
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten
pretrain_weighted_1 = glob.glob('.. /input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-1/*.pt')
 pretrain_weighted_2 = glob.glob('.. /input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/*.pt')
 pretrain_weighted = pretrain_weighted_1 + pretrain_weighted_2<install_modules>
model = Sequential()
model.add(Dense(100, input_shape=(10,)))
 model.add(Activation('relu'))
model.add(Dense(100))
model.add(Activation('relu'))
model.add(Dense(100))
model.add(Activation('relu'))
model.add(Dense(3))
model.add(Activation('softmax'))
model.summary()
!nvidia-smi<install_modules>
model.compile(loss='categorical_crossentropy',
optimizer = 'adam',
metrics = ['accuracy'] )
!nvidia-smi<create_dataframe>
model.fit(X_train,Y_train,
batch_size=12,
epochs = 32,
verbose=2 )
test_set = QuestDataset_test(test_tokens, test_masks, test_segments,
test_df['host'].values,
test_df['category'].values,
train_handmade_features)
 test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False )<choose_model_class>
pred = model.predict(X_test ).argmax(axis =1 )
model = QuestModel6(n_cat, cat_emb, n_host, host_emb, num_labels=30)
 model.cuda()<predict_on_test>
submit = pd.DataFrame({'id':test_data['id'],
'type': pred})
 submit
for i, weight in tqdm(enumerate(pretrain_weighted)) :
model.load_state_dict(torch.load(weight))
results.append(predict_result(model, test_loader))<define_variables>
submission=submission.replace(1,'Ghoul')
 submission=submission.replace(2,'Goblin')
 submission
<set_options><EOS> 
submission.to_csv('submission.csv',index = False )