Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<drop_column>
!pip install.. /input/apex01cp37cp37mlinux-x86-64whl/apex-0.1-cp37-cp37m-linux_x86_64.whl
train = reduce_mem_usage(train )<create_dataframe>
stats = []
for col in train.columns:
stats.append(( col, train[col].nunique() , train[col].isnull().sum() * 100 / train.shape[0], train[col].value_counts(normalize=True, dropna=False ).values[0] * 100, train[col].dtype))
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])
 stats_df.sort_values('Percentage of missing values', ascending=False )<count_values>
def bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len):
class TFQADataset(Dataset):
def __init__(self, id_list):
self.id_list=id_list
def __len__(self):
return len(self.id_list)
 def __getitem__(self, index):
return self.id_list[index]
class Collator(object):
def __init__(self, data_dict, new_token_dict, tokenizer, max_seq_len=384, max_question_len=64):
self.data_dict = data_dict
self.new_token_dict = new_token_dict
self.tokenizer = tokenizer
self.max_seq_len = max_seq_len
self.max_question_len = max_question_len
def _get_input_ids(self, doc_id, candidate_index):
data = self.data_dict[doc_id]
question_tokens = self.tokenizer.tokenize(data['question_text'])[:self.max_question_len]
doc_words = data['document_text'].split()
max_answer_tokens = self.max_seq_len-len(question_tokens)-3
candidate = data['long_answer_candidates'][candidate_index]
candidate_start = candidate['start_token']
candidate_end = candidate['end_token']
candidate_words = doc_words[candidate_start:candidate_end]
for i, word in enumerate(candidate_words):
if re.match(r'<.+>', word):
if word in self.new_token_dict:
candidate_words[i] = self.new_token_dict[word]
else:
candidate_words[i] = '<'
words_to_tokens_index = []
candidate_tokens = []
for i, word in enumerate(candidate_words):
words_to_tokens_index.append(len(candidate_tokens))
tokens = self.tokenizer.tokenize(word)
 if len(candidate_tokens)+len(tokens)> max_answer_tokens:
break
for token in tokens:
candidate_tokens.append(token)
 input_tokens = ['[CLS]'] + question_tokens + ['[SEP]'] + candidate_tokens + ['[SEP]']
input_ids = self.tokenizer.convert_tokens_to_ids(input_tokens)
 return input_ids, words_to_tokens_index, len(input_ids), len(question_tokens)+2
def __call__(self, batch_ids):
batch_size = len(batch_ids)
 batch_input_ids_temp = []
batch_seq_len = []
batch_offset = []
batch_words_to_tokens_index = []
for i,(doc_id, candidate_index)in enumerate(batch_ids):
input_ids, words_to_tokens_index, seq_len, offset = self._get_input_ids(doc_id, candidate_index)
 batch_input_ids_temp.append(input_ids)
 batch_seq_len.append(seq_len)
 batch_offset.append(offset)
 batch_words_to_tokens_index.append(words_to_tokens_index)
 batch_max_seq_len = max(batch_seq_len)
 batch_input_ids = np.zeros(( batch_size, batch_max_seq_len), dtype=np.int64)
 batch_token_type_ids = np.ones(( batch_size, batch_max_seq_len), dtype=np.int64)
 for i in range(batch_size):
input_ids = batch_input_ids_temp[i]
batch_input_ids[i, :len(input_ids)] = input_ids
batch_token_type_ids[i, :len(input_ids)] = [0 if k<=input_ids.index(102)else 1 for k in range(len(input_ids)) ]
batch_attention_mask = batch_input_ids > 0
return torch.from_numpy(batch_input_ids), torch.from_numpy(batch_attention_mask), torch.from_numpy(batch_token_type_ids), batch_words_to_tokens_index, batch_offset, batch_max_seq_len
class BertForQuestionAnswering(BertPreTrainedModel):
def __init__(self, config):
super(BertForQuestionAnswering, self ).__init__(config)
 self.bert = BertModel(config)
 self.qa_outputs = nn.Linear(config.hidden_size, 2)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.init_weights()
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 sequence_output = outputs[0]
pooled_output = outputs[1]
qa_logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = qa_logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 pooled_output = self.dropout(pooled_output)
 classifier_logits = self.classifier(pooled_output)
 return start_logits, end_logits, classifier_logits
max_seq_len = 360
max_question_len = 64
batch_size = int(384/4)
 model_path = '.. /models/models/bert/'
config = BertConfig.from_pretrained(model_path)
 config.num_labels = 5
config.vocab_size = 30531
tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = BertForQuestionAnswering.from_pretrained(model_dir, config=config)
 new_token_dict = {
'<P>':'qw1',
'<Table>':'qw2',
'<Tr>':'qw3',
'<Ul>':'qw4',
'<Ol>':'qw5',
'<Fl>':'qw6',
'<Li>':'qw7',
'<Dd>':'qw8',
'<Dt>':'qw9',
}
new_token_list = [
'qw1',
'qw2',
'qw3',
'qw4',
'qw5',
'qw6',
'qw7',
'qw8',
'qw9',
]
num_added_toks = tokenizer.add_tokens(new_token_list)
 model.resize_token_embeddings(len(tokenizer))
model.cuda()
optimizer = optim.Adam(model.parameters() , lr=1e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)
 if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
 test_datagen = TFQADataset(id_list=id_candidate_list_sorted)
 test_collate = Collator(data_dict=data_dict,
new_token_dict=new_token_dict,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
max_question_len=max_question_len)
 test_generator = DataLoader(dataset=test_datagen,
collate_fn=test_collate,
batch_size=batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
 model.eval()
list_offset = []
list_words_to_tokens_index = []
test_prob1 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob2 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob3 = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 for j,(batch_input_ids, batch_attention_mask, batch_token_type_ids, batch_words_to_tokens_index, batch_offset, batch_max_seq_len)in tqdm(enumerate(test_generator)) :
with torch.no_grad() :
start = j*batch_size
end = start+batch_size
if j == len(test_generator)-1:
end = len(test_generator.dataset)
 batch_input_ids = batch_input_ids.cuda()
batch_attention_mask = batch_attention_mask.cuda()
batch_token_type_ids = batch_token_type_ids.cuda()
logits1, logits2, logits3 = model(batch_input_ids, batch_attention_mask, batch_token_type_ids)
 test_prob1[start:end, :batch_max_seq_len] += F.softmax(logits1,dim=1 ).cpu().data.numpy()
test_prob2[start:end, :batch_max_seq_len] += F.softmax(logits2,dim=1 ).cpu().data.numpy()
test_prob3[start:end] += F.softmax(logits3,dim=1 ).cpu().data.numpy()
list_words_to_tokens_index += batch_words_to_tokens_index
list_offset += batch_offset
test_word_prob1 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 test_word_prob2 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 for i in range(len(id_candidate_list_sorted)) :
for j in range(len(list_words_to_tokens_index[i])) :
test_word_prob1[i,j] = test_prob1[i, list_words_to_tokens_index[i][j]+list_offset[i]]
test_word_prob2[i,j] = test_prob2[i, list_words_to_tokens_index[i][j]+list_offset[i]]
return test_word_prob1, test_word_prob2, test_prob3
good_cols = list(train.columns)
 for col in train.columns:
rate = train[col].value_counts(normalize=True, dropna=False ).values[0]
if rate > 0.9:
good_cols.remove(col )<drop_column>
start_time = time.time()
data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted = reduce1(n_candidate=10, th_candidate=0.2)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
id_candidate_list_sorted = reduce2(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, n_candidate=4, th_candidate=0.35)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
word_len = 360
start_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 end_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 start_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 end_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 class_prob = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 print("before bert predicting")
 weight_albert = 0.7
model_dir = '.. /models/models/bert/'
test_prob1, test_prob2, test_prob3 = bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob +=(1-weight_albert)*test_prob1
end_prob +=(1-weight_albert)*test_prob2
class_prob +=(1-weight_albert)*test_prob3
print("before albert predicting")
 model_dir = '.. /models/models/albert/'
test_prob1, test_prob2, test_prob3 = albert_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob += weight_albert*test_prob1
end_prob += weight_albert*test_prob2
class_prob += weight_albert*test_prob3
start_label = np.argmax(start_prob, axis=1)
 end_label = np.argmax(end_prob, axis=1)
 temp_dict = {}
for doc_id in id_list:
temp_dict[doc_id] = {
'long_answer': {'start_token': -1, 'end_token': -1},
'long_answer_score': -1.0,
'short_answers': [{'start_token': -1, 'end_token': -1}],
'short_answers_score': -1.0,
'yes_no_answer': 'NONE'
}
for i,(doc_id, candidate_index)in tqdm(enumerate(id_candidate_list_sorted)) :
long_answer_score = 1.0 - class_prob[i,0]
if long_answer_score > temp_dict[doc_id]['long_answer_score']:
temp_dict[doc_id]['long_answer_score'] = long_answer_score
temp_dict[doc_id]['long_answer']['start_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
temp_dict[doc_id]['long_answer']['end_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['end_token']
short_answer_score = 1.0 - class_prob[i,0] - class_prob[i,1]
temp_dict[doc_id]['short_answers_score'] = short_answer_score
temp_dict[doc_id]['short_answers'][0]['start_token'] = -1
temp_dict[doc_id]['short_answers'][0]['end_token'] = -1
temp_dict[doc_id]['yes_no_answer'] = 'NONE'
if max([class_prob[i,3], class_prob[i,4]])> class_prob[i,2]:
if class_prob[i,3] > class_prob[i,4]:
temp_dict[doc_id]['yes_no_answer'] = 'NO'
else:
temp_dict[doc_id]['yes_no_answer'] = 'YES'
else:
short_start_word = int(start_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
short_end_word = int(end_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
if short_end_word > short_start_word:
temp_dict[doc_id]['short_answers'][0]['start_token'] = short_start_word
temp_dict[doc_id]['short_answers'][0]['end_token'] = short_end_word
final_dict = {}
final_dict['predictions'] = []
for doc_id in id_list:
prediction_dict = {
'example_id': doc_id,
'long_answer': {'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['long_answer']['start_token'], 'end_token': temp_dict[doc_id]['long_answer']['end_token']},
'long_answer_score': temp_dict[doc_id]['long_answer_score'],
'short_answers': [{'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['short_answers'][0]['start_token'], 'end_token': temp_dict[doc_id]['short_answers'][0]['end_token']}],
'short_answers_score': temp_dict[doc_id]['short_answers_score'],
'yes_no_answer': temp_dict[doc_id]['yes_no_answer']
}
final_dict['predictions'].append(prediction_dict)
 print("--- %s seconds ---" %(time.time() - start_time))
train = train[good_cols]<load_from_csv>
answers_df = pd.DataFrame.from_dict(final_dict)
 long_best_threshold = 0.7
short_best_threshold = 0.7
def df_long_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['long_answer_score'] > long_best_threshold:
index['start'] = df["predictions"][i]['long_answer']['start_token']
index['end'] = df["predictions"][i]['long_answer']['end_token']
index['score'] = df["predictions"][i]['long_answer_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_short_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['short_answers_score'] > short_best_threshold and df["predictions"][i]['short_answers'][0]['start_token'] != -1:
index['start'] = df["predictions"][i]['short_answers'][0]['start_token']
index['end'] = df["predictions"][i]['short_answers'][0]['end_token']
index['score'] = df["predictions"][i]['short_answers_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_example_id(df):
return df['example_id']
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':' + str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
test_dtypes = {k: v for k, v in dtypes.items() if k in good_cols}
test = pd.read_csv('.. /input/test.csv', dtype=test_dtypes, usecols=good_cols[:-1])
 test.loc[6529507, 'OsBuildLab'] = '17134.1.amd64fre.rs4_release.180410-1804'
test['OsBuildLab'] = test['OsBuildLab'].fillna('17134.1.amd64fre.rs4_release.180410-1804')
 test = reduce_mem_usage(test )<count_values>
answers_df['long_indexes_and_scores'] = df_long_index_score(answers_df)
 answers_df['short_indexes_and_scores'] = df_short_index_score(answers_df)
 answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(
 create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(
 create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df = answers_df.drop(
 ['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 sample_submission = pd.read_csv(
 ".. /.. /.. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains(
 "_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains(
 "_short"), "PredictionString"] = short_prediction_strings
train['HasDetections'].value_counts()<data_type_conversions>
torch.cuda.empty_cache()
train['AVProductStatesIdentifier'] = train['AVProductStatesIdentifier'].astype('category')
 test['AVProductStatesIdentifier'] = test['AVProductStatesIdentifier'].astype('category' )<feature_engineering>
del BertTokenizer, BertModel, BertPreTrainedModel, BertConfig, AlbertTokenizer, AlbertModel, AlbertPreTrainedModel, AlbertConfig
<data_type_conversions>
!pip install --upgrade.. /input/tokenizers0011cp37/tokenizers-0.0.11-cp37-cp37m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl
train['AVProductsInstalled'] = train['AVProductsInstalled'].astype('category')
 test['AVProductsInstalled'] = test['AVProductsInstalled'].astype('category' )<data_type_conversions>


logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',
train['CountryIdentifier'] = train['CountryIdentifier'].astype('category')
 test['CountryIdentifier'] = test['CountryIdentifier'].astype('category' )<data_type_conversions>
submission_no2 = pd.read_csv("submission1.csv")
 for i in range(len(submission_no2)) :
if submission_no2["PredictionString"][i] is np.nan and sample_submission["PredictionString"][i] is not np.nan:
submission_no2["PredictionString"][i] = sample_submission["PredictionString"][i]
train['CityIdentifier'] = train['CityIdentifier'].astype('category')
 test['CityIdentifier'] = test['CityIdentifier'].astype('category' )<data_type_conversions>
path = 'submission1.csv'
if os.path.exists(path):
os.remove(path )
<data_type_conversions><EOS> 
submission_no2.to_csv('submission.csv', index=False)