Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<compute_test_metric>
!python --version
best_score = np.min(result['score'])
 best_lag = np.argmin(result['score'])
 print('best_score', best_score, '
best_lag', best_lag )<feature_engineering>
!nvidia-smi
def rewrite_compiled_leak(leak_df, lag):
leak_df["compiled_leak"] = 0
for i in range(lag):
c = "leaked_target_"+str(i)
 zeroleak = leak_df["compiled_leak"]==0
leak_df.loc[zeroleak, "compiled_leak"] = leak_df.loc[zeroleak, c]
return leak_df<categorify>
%%time
!pip install /kaggle/input/tf-115-dependencies/tensorboard-1.15.0-py3-none-any.whl
!pip install /kaggle/input/tf-115-dependencies/tensorflow_estimator-1.15.1-py2.py3-none-any.whl
!pip install /kaggle/input/tf-115/tensorflow_gpu-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl
!pip install /kaggle/input/tf-115-dependencies/bert_tensorflow-1.0.1-py2.py3-none-any.whl
leaky_cols = [c for c in train_leak.columns if 'leaked_target_' in c]
train_leak = rewrite_compiled_leak(train_leak, best_lag)
 train_leak[['ID']+leaky_cols+['compiled_leak']].head()<feature_engineering>
tf.__version__
def compiled_leak_result_test(max_nlags):
test_leak = test[["ID", "target"] + cols]
test_leak["compiled_leak"] = 0
test_leak["nonzero_mean"] = test[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 test_leak[c] = _get_leak(test, cols,ss, i)
 leaky_cols.append(c)
 test_leak = test.join(
 test_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = test_leak["compiled_leak"]==0
test_leak.loc[zeroleak, "compiled_leak"] = test_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(test_leak["compiled_leak"] > 0))
print("Leak values found in test", leaky_value_counts[-1])
 result = dict(
 leaky_count=leaky_value_counts, 
)
 return test_leak, result<categorify>
flags = tf.flags
FLAGS = flags.FLAGS
flags.DEFINE_string(
 "bert_config_file", None,
"The config json file corresponding to the pre-trained BERT model."
"This specifies the model architecture.")
 flags.DEFINE_string("vocab_file", None,
"The vocabulary file that the BERT model was trained on.")
 flags.DEFINE_string(
 "output_dir", None,
"The output directory where the model checkpoints will be written.")
 flags.DEFINE_string("train_file", None,
"SQuAD json for training.E.g., train-v1.1.json")
 flags.DEFINE_string(
 "predict_file", None,
"SQuAD json for predictions.E.g., dev-v1.1.json or test-v1.1.json")
 flags.DEFINE_string(
 "init_checkpoint", None,
"Initial checkpoint(usually from a pre-trained BERT model ).")
 flags.DEFINE_bool(
 "do_lower_case", True,
"Whether to lower case the input text.Should be True for uncased "
"models and False for cased models.")
 flags.DEFINE_integer(
 "max_seq_length", 384,
"The maximum total input sequence length after WordPiece tokenization."
"Sequences longer than this will be truncated, and sequences shorter "
"than this will be padded.")
 flags.DEFINE_integer(
 "doc_stride", 128,
"When splitting up a long document into chunks, how much stride to "
"take between chunks.")
 flags.DEFINE_integer(
 "max_query_length", 64,
"The maximum number of tokens for the question.Questions longer than "
"this will be truncated to this length.")
 flags.DEFINE_bool("do_train", False, "Whether to run training.")
 flags.DEFINE_bool("do_predict", False, "Whether to run eval on the dev set.")
 flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")
 flags.DEFINE_integer("predict_batch_size", 8,
"Total batch size for predictions.")
 flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.")
 flags.DEFINE_float("num_train_epochs", 3.0,
"Total number of training epochs to perform.")
 flags.DEFINE_float(
 "warmup_proportion", 0.1,
"Proportion of training to perform linear learning rate warmup for."
"E.g., 0.1 = 10% of training.")
 flags.DEFINE_integer("save_checkpoints_steps", 1000,
"How often to save the model checkpoint.")
 flags.DEFINE_integer("iterations_per_loop", 1000,
"How many steps to make in each estimator call.")
 flags.DEFINE_integer(
 "n_best_size", 20,
"The total number of n-best predictions to generate in the "
"nbest_predictions.json output file.")
 flags.DEFINE_integer(
 "max_answer_length", 30,
"The maximum length of an answer that can be generated.This is needed "
"because the start and end predictions are not conditioned on one another.")
 flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.")
 tf.flags.DEFINE_string(
 "tpu_name", None,
"The Cloud TPU to use for training.This should be either the name "
"used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
"url.")
 tf.flags.DEFINE_string(
 "tpu_zone", None,
"[Optional] GCE zone where the Cloud TPU is located in.If not "
"specified, we will attempt to automatically detect the GCE project from "
"metadata.")
 tf.flags.DEFINE_string(
 "gcp_project", None,
"[Optional] Project name for the Cloud TPU-enabled project.If not "
"specified, we will attempt to automatically detect the GCE project from "
"metadata.")
 tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.")
 flags.DEFINE_integer(
 "num_tpu_cores", 8,
"Only used if `use_tpu` is True.Total number of TPU cores to use.")
 flags.DEFINE_bool(
 "verbose_logging", False,
"If true, all of the warnings related to data processing will be printed."
"A number of warnings are expected for a normal SQuAD evaluation.")
 flags.DEFINE_bool(
 "version_2_with_negative", False,
"If true, the SQuAD examples contain some that do not have an answer.")
 flags.DEFINE_float(
 "null_score_diff_threshold", 0.0,
"If null_score - best_non_null is greater than the threshold predict null.")
 flags.DEFINE_string('f', '', 'kernel')
 flags.DEFINE_string('HistoryManager.hist_file', '', 'kernel')
 flags.DEFINE_string(
 "output_prediction_file", 'bert_model_output/predictions_tiny_dev.json',
"Where to print predictions in NQ prediction format, to be passed to"
"natural_questions.nq_eval.")
 flags.DEFINE_boolean(
 "skip_nested_contexts", True,
"Completely ignore context that are not top level nodes in the page.")
 flags.DEFINE_integer("task_id", 0,
"Train and dev shard to read from and write to.")
 flags.DEFINE_integer("max_contexts", 48,
"Maximum number of contexts to output for an example.")
 flags.DEFINE_integer(
 "max_position", 50,
"Maximum context position for which to generate special tokens.")
 TextSpan = collections.namedtuple("TextSpan", "token_positions text" )
test_leak, test_result = compiled_leak_result_test(max_nlags=38)
<create_dataframe>
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx"])
 class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def my_compute_predictions(example):

 predictions = []
n_best_size = 10
max_answer_length = 30
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = example.features[unique_id]["token_map"].int64_list.value
start_indexes = get_best_indexes(result["start_logits"], n_best_size)
 end_indexes = get_best_indexes(result["end_logits"], n_best_size)
 for start_index in start_indexes:
for end_index in end_indexes:
if end_index < start_index:
continue
if token_map[start_index] == -1:
continue
if token_map[end_index] == -1:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
summary = ScoreSummary()
summary.short_span_score =(
 result["start_logits"][start_index] +
result["end_logits"][end_index])
 summary.cls_token_score =(
 result["start_logits"][0] + result["end_logits"][0])
 summary.answer_type_logits = result["answer_type_logits"]
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, summary, start_span, end_span))
score = -10000.0
short_span = Span(-1, -1)
 long_span = Span(-1, -1)
 summary = ScoreSummary()
nbest_summary = []
seen_labels = set()
nbest = 20
if predictions:
for i in range(min(nbest,len(predictions))):
score, summary, start_span, end_span = sorted(predictions, key=lambda x:x[0],reverse=True)[i]
short_span = Span(start_span, end_span)
 for c in example.candidates:
start = short_span.start_token_idx
end = short_span.end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span = Span(c["start_token"], c["end_token"])
 break
summary.predicted_label = {
"example_id": example.example_id,
"long_answer": {
"start_token": long_span.start_token_idx,
"end_token": long_span.end_token_idx,
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": score,
"short_answers": [{
"start_token": short_span.start_token_idx,
"end_token": short_span.end_token_idx,
"start_byte": -1,
"end_byte": -1
}],
"short_answers_score": score,
"yes_no_answer": "NONE"
}
if(long_span.start_token_idx,long_span.end_token_idx,\
short_span.start_token_idx,short_span.end_token_idx)not in seen_labels:
nbest_summary.append(summary)
 seen_labels.add(( long_span.start_token_idx,long_span.end_token_idx,\
short_span.start_token_idx,short_span.end_token_idx))
if not predictions:
summary.predicted_label = {
"example_id": example.example_id,
"long_answer": {
"start_token": long_span.start_token_idx,
"end_token": long_span.end_token_idx,
"start_byte": -1,
"end_byte": -1
},
"long_answer_score": score,
"short_answers": [{
"start_token": short_span.start_token_idx,
"end_token": short_span.end_token_idx,
"start_byte": -1,
"end_byte": -1
}],
"short_answers_score": score,
"yes_no_answer": "NONE"
}
nbest_summary.append(summary)
 return nbest_summary
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}
def compute_pred_dict(candidates_dict, dev_features, raw_results):

 raw_results_by_id = [(int(res["unique_id"] + 1), res)for res in raw_results]
sess = tf.Session()
all_candidates = candidates_dict.items()
example_ids = tf.to_int32(np.array([int(k)for k, _ in all_candidates
])).eval(session=sess)
 examples_by_id = list(zip(example_ids, all_candidates))
feature_ids = []
features = []
for f in dev_features:
feature_ids.append(f.features.feature["unique_ids"].int64_list.value[0] + 1)
 features.append(f.features.feature)
 feature_ids = tf.to_int32(np.array(feature_ids)).eval(session=sess)
 features_by_id = list(zip(feature_ids, features))
examples = []
merged = sorted(examples_by_id + raw_results_by_id + features_by_id,key=lambda x:x[0])
 for idx, datum in merged:
if isinstance(datum, tuple):
examples.append(EvalExample(datum[0], datum[1]))
elif "token_map" in datum:
examples[-1].features[idx] = datum
else:
examples[-1].results[idx] = datum
tf.logging.info("Computing predictions...")
 nbest_summary_dict = {}
summary_dict = {}
nq_pred_dict = {}
for e in examples:
nbest_summary = my_compute_predictions(e)
 nbest_summary_dict[e.example_id] = nbest_summary
summary_dict[e.example_id] = nbest_summary[0]
nq_pred_dict[e.example_id] = nbest_summary[0].predicted_label
if len(nq_pred_dict)% 100 == 0:
tf.logging.info("Examples processed: %d", len(nq_pred_dict))
tf.logging.info("Done computing predictions.")
 return nq_pred_dict, nbest_summary_dict
def read_candidates_from_one_split(input_path):
candidates_dict = {}
try:
with gzip.open(input_path)as input_file:
tf.logging.info("Reading examples from: %s", input_path)
 for line in input_file:
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
except:
with tf.io.gfile.GFile(input_path, "r")as input_file:
tf.logging.info("Reading examples from: %s", input_path)
 for line in input_file:
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict
def read_candidates(input_pattern):

 input_paths = tf.gfile.Glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict
RawResult = collections.namedtuple(
 "RawResult",
["unique_id", "start_logits", "end_logits", "answer_type_logits"])
 def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
use_one_hot_embeddings):

 model = modeling.BertModel(
 config=bert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=segment_ids,
use_one_hot_embeddings=use_one_hot_embeddings)
 final_hidden = model.get_sequence_output()
final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3)
 batch_size = final_hidden_shape[0]
seq_length = final_hidden_shape[1]
hidden_size = final_hidden_shape[2]
output_weights = tf.get_variable(
 "cls/nq/output_weights", [2, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.get_variable(
 "cls/nq/output_bias", [2], initializer=tf.zeros_initializer())
 final_hidden_matrix = tf.reshape(final_hidden,
[batch_size * seq_length, hidden_size])
 logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True)
 logits = tf.nn.bias_add(logits, output_bias)
 logits = tf.reshape(logits, [batch_size, seq_length, 2])
 logits = tf.transpose(logits, [2, 0, 1])
 unstacked_logits = tf.unstack(logits, axis=0)
(start_logits, end_logits)=(unstacked_logits[0], unstacked_logits[1])
 answer_type_output_layer = model.get_pooled_output()
answer_type_hidden_size = answer_type_output_layer.shape[-1].value
num_answer_types = 5
answer_type_output_weights = tf.get_variable(
 "answer_type_output_weights", [num_answer_types, answer_type_hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
answer_type_output_bias = tf.get_variable(
 "answer_type_output_bias", [num_answer_types],
initializer=tf.zeros_initializer())
 answer_type_logits = tf.matmul(
 answer_type_output_layer, answer_type_output_weights, transpose_b=True)
 answer_type_logits = tf.nn.bias_add(answer_type_logits,
answer_type_output_bias)
 return(start_logits, end_logits, answer_type_logits)
 def validate_flags_or_throw(bert_config):

 if not FLAGS.do_train and not FLAGS.do_predict:
raise ValueError("At least one of `{do_train,do_predict}` must be True.")
 if FLAGS.do_train:
if not FLAGS.train_precomputed_file:
raise ValueError("If `do_train` is True, then `train_precomputed_file` "
"must be specified.")
 if not FLAGS.train_num_precomputed:
raise ValueError("If `do_train` is True, then `train_num_precomputed` "
"must be specified.")
 if FLAGS.do_predict:
if not FLAGS.predict_file:
raise ValueError(
 "If `do_predict` is True, then `predict_file` must be specified.")
 if FLAGS.max_seq_length > bert_config.max_position_embeddings:
raise ValueError(
 "Cannot use sequence length %d because the BERT model "
"was only trained up to sequence length %d" % 
(FLAGS.max_seq_length, bert_config.max_position_embeddings))
if FLAGS.max_seq_length <= FLAGS.max_query_length + 3:
raise ValueError(
 "The max_seq_length(%d)must be greater than max_query_length "
"(%d)+ 3" %(FLAGS.max_seq_length, FLAGS.max_query_length))
class AnswerType(enum.IntEnum):

 UNKNOWN = 0
YES = 1
NO = 2
SHORT = 3
LONG = 4
class Answer(collections.namedtuple("Answer", ["type", "text", "offset"])) :

 def __new__(cls, type_, text=None, offset=None):
return super(Answer, cls ).__new__(cls, type_, text, offset)
 class InputFeatures(object):

 def __init__(self,
unique_id,
example_index,
doc_span_index,
token_to_orig_map,
input_ids,
input_mask,
segment_ids,
start_position=None,
end_position=None,
answer_text="",
answer_type=AnswerType.SHORT):
self.unique_id = unique_id
self.example_index = example_index
self.doc_span_index = doc_span_index
self.token_to_orig_map = token_to_orig_map
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.start_position = start_position
self.end_position = end_position
self.answer_text = answer_text
self.answer_type = answer_type
_SPECIAL_TOKENS_RE = re.compile(r"^\[[^ ]*\]$", re.UNICODE)
 def tokenize(tokenizer, text, apply_basic_tokenization=False):

 tokenize_fn = tokenizer.tokenize
if apply_basic_tokenization:
tokenize_fn = tokenizer.basic_tokenizer.tokenize
tokens = []
for token in text.split(" "):
if _SPECIAL_TOKENS_RE.match(token):
if token in tokenizer.vocab:
tokens.append(token)
 else:
tokens.append(tokenizer.wordpiece_tokenizer.unk_token)
 else:
tokens.extend(tokenize_fn(token))
return tokens
def convert_single_example(example, tokenizer, is_training):

 tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
features = []
for(i, token)in enumerate(example.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = tokenize(tokenizer, token)
 tok_to_orig_index.extend([i] * len(sub_tokens))
all_doc_tokens.extend(sub_tokens)
 if example.doc_tokens_map:
tok_to_orig_index = [
example.doc_tokens_map[index] for index in tok_to_orig_index
]
query_tokens = []
query_tokens.append("[Q]")
 query_tokens.extend(tokenize(tokenizer, example.questions[-1]))
if len(query_tokens)> FLAGS.max_query_length:
query_tokens = query_tokens[-FLAGS.max_query_length:]
tok_start_position = 0
tok_end_position = 0
if is_training:
tok_start_position = orig_to_tok_index[example.start_position]
if example.end_position < len(example.doc_tokens)- 1:
tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
max_tokens_for_doc = FLAGS.max_seq_length - len(query_tokens)- 3
_DocSpan = collections.namedtuple(
 "DocSpan", ["start", "length"])
 doc_spans = []
start_offset = 0
while start_offset < len(all_doc_tokens):
length = len(all_doc_tokens)- start_offset
length = min(length, max_tokens_for_doc)
 doc_spans.append(_DocSpan(start=start_offset, length=length))
if start_offset + length == len(all_doc_tokens):
break
start_offset += min(length, FLAGS.doc_stride)
 for(doc_span_index, doc_span)in enumerate(doc_spans):
tokens = []
token_to_orig_map = {}
segment_ids = []
tokens.append("[CLS]")
 segment_ids.append(0)
 tokens.extend(query_tokens)
 segment_ids.extend([0] * len(query_tokens))
tokens.append("[SEP]")
 segment_ids.append(0)
 for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]
tokens.append(all_doc_tokens[split_token_index])
 segment_ids.append(1)
 tokens.append("[SEP]")
 segment_ids.append(1)
 assert len(tokens)== len(segment_ids)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_mask = [1] * len(input_ids)
 padding = [0] *(FLAGS.max_seq_length - len(input_ids))
input_ids.extend(padding)
 input_mask.extend(padding)
 segment_ids.extend(padding)
 assert len(input_ids)== FLAGS.max_seq_length
assert len(input_mask)== FLAGS.max_seq_length
assert len(segment_ids)== FLAGS.max_seq_length
start_position = None
end_position = None
answer_type = None
answer_text = ""
if is_training:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
contains_an_annotation =(
 tok_start_position >= doc_start and tok_end_position <= doc_end)
 if(( not contains_an_annotation)or
example.answer.type == AnswerType.UNKNOWN):
if(FLAGS.include_unknowns < 0 or
random.random() > FLAGS.include_unknowns):
continue
start_position = 0
end_position = 0
answer_type = AnswerType.UNKNOWN
else:
doc_offset = len(query_tokens)+ 2
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
answer_type = example.answer.type
answer_text = " ".join(tokens[start_position:(end_position + 1)])
 feature = InputFeatures(
 unique_id=-1,
example_index=-1,
doc_span_index=doc_span_index,
token_to_orig_map=token_to_orig_map,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
start_position=start_position,
end_position=end_position,
answer_text=answer_text,
answer_type=answer_type)
 features.append(feature)
 return features
def convert_examples_to_features(examples, tokenizer, is_training, output_fn):

 num_spans_to_ids = collections.defaultdict(list)
 for example in tqdm(examples):
example_index = example.example_id
features = convert_single_example(example, tokenizer, is_training)
 num_spans_to_ids[len(features)].append(example.qas_id)
 for feature in features:
feature.example_index = example_index
feature.unique_id = feature.example_index + feature.doc_span_index
output_fn(feature)
 return num_spans_to_ids
def read_nq_entry(entry, is_training):

 def is_whitespace(c):
return c in " \t\r
" or ord(c)== 0x202F
examples = []
contexts_id = entry["id"]
contexts = entry["contexts"]
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in contexts:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 questions = []
for i, question in enumerate(entry["questions"]):
qas_id = "{}".format(contexts_id)
 question_text = question["input_text"]
start_position = None
end_position = None
answer = None
if is_training:
answer_dict = entry["answers"][i]
answer = make_nq_answer(contexts, answer_dict)
 if answer is None or answer.offset is None:
continue
start_position = char_to_word_offset[answer.offset]
end_position = char_to_word_offset[answer.offset + len(answer.text)- 1]
actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
 cleaned_answer_text = " ".join(
 tokenization.whitespace_tokenize(answer.text))
if actual_text.find(cleaned_answer_text)== -1:
tf.logging.warning("Could not find answer: '%s' vs.'%s'", actual_text,
cleaned_answer_text)
 continue
questions.append(question_text)
 example = NqExample(
 example_id=int(contexts_id),
qas_id=qas_id,
questions=questions[:],
doc_tokens=doc_tokens,
doc_tokens_map=entry.get("contexts_map", None),
answer=answer,
start_position=start_position,
end_position=end_position)
 examples.append(example)
 return examples
class NqExample(object):

 def __init__(self,
example_id,
qas_id,
questions,
doc_tokens,
doc_tokens_map=None,
answer=None,
start_position=None,
end_position=None):
self.example_id = example_id
self.qas_id = qas_id
self.questions = questions
self.doc_tokens = doc_tokens
self.doc_tokens_map = doc_tokens_map
self.answer = answer
self.start_position = start_position
self.end_position = end_position
def has_long_answer(a):
return(a["long_answer"]["start_token"] >= 0 and
a["long_answer"]["end_token"] >= 0)
 def should_skip_context(e, idx):
if(FLAGS.skip_nested_contexts and
not e["long_answer_candidates"][idx]["top_level"]):
return True
elif not get_candidate_text(e, idx ).text.strip() :
return True
else:
return False
def get_first_annotation(e):

 positive_annotations = sorted(
 [a for a in e["annotations"] if has_long_answer(a)],
key=lambda a: a["long_answer"]["candidate_index"])
 for a in positive_annotations:
if a["short_answers"]:
idx = a["long_answer"]["candidate_index"]
start_token = a["short_answers"][0]["start_token"]
end_token = a["short_answers"][-1]["end_token"]
return a, idx,(token_to_char_offset(e, idx, start_token),
token_to_char_offset(e, idx, end_token)- 1)
 for a in positive_annotations:
idx = a["long_answer"]["candidate_index"]
return a, idx,(-1, -1)
 return None, -1,(-1, -1)
 def get_text_span(example, span):

 token_positions = []
tokens = []
for i in range(span["start_token"], span["end_token"]):
t = example["document_tokens"][i]
if not t["html_token"]:
token_positions.append(i)
 token = t["token"].replace(" ", "")
 tokens.append(token)
 return TextSpan(token_positions, " ".join(tokens))
def token_to_char_offset(e, candidate_idx, token_idx):

 c = e["long_answer_candidates"][candidate_idx]
char_offset = 0
for i in range(c["start_token"], token_idx):
t = e["document_tokens"][i]
if not t["html_token"]:
token = t["token"].replace(" ", "")
 char_offset += len(token)+ 1
return char_offset
def get_candidate_type(e, idx):

 c = e["long_answer_candidates"][idx]
first_token = e["document_tokens"][c["start_token"]]["token"]
if first_token == "<Table>":
return "Table"
elif first_token == "<P>":
return "Paragraph"
elif first_token in("<Ul>", "<Dl>", "<Ol>"):
return "List"
elif first_token in("<Tr>", "<Li>", "<Dd>", "<Dt>"):
return "Other"
else:
tf.logging.warning("Unknoww candidate type found: %s", first_token)
 return "Other"
def add_candidate_types_and_positions(e):

 counts = collections.defaultdict(int)
 for idx, c in candidates_iter(e):
context_type = get_candidate_type(e, idx)
 if counts[context_type] < FLAGS.max_position:
counts[context_type] += 1
c["type_and_position"] = "[%s=%d]" %(context_type, counts[context_type])
 def get_candidate_type_and_position(e, idx):

 if idx == -1:
return "[NoLongAnswer]"
else:
return e["long_answer_candidates"][idx]["type_and_position"]
def get_candidate_text(e, idx):

 if idx < 0 or idx >= len(e["long_answer_candidates"]):
return TextSpan([], "")
 return get_text_span(e, e["long_answer_candidates"][idx])
 def candidates_iter(e):

 for idx, c in enumerate(e["long_answer_candidates"]):
if should_skip_context(e, idx):
continue
yield idx, c
def create_example_from_jsonl(line):

 e = json.loads(line, object_pairs_hook=collections.OrderedDict)
 add_candidate_types_and_positions(e)
 annotation, annotated_idx, annotated_sa = get_first_annotation(e)
 question = {"input_text": e["question_text"]}
answer = {
"candidate_id": annotated_idx,
"span_text": "",
"span_start": -1,
"span_end": -1,
"input_text": "long",
}
if annotation is not None:
assert annotation["yes_no_answer"] in("YES", "NO", "NONE")
 if annotation["yes_no_answer"] in("YES", "NO"):
answer["input_text"] = annotation["yes_no_answer"].lower()
if annotated_sa !=(-1, -1):
answer["input_text"] = "short"
span_text = get_candidate_text(e, annotated_idx ).text
answer["span_text"] = span_text[annotated_sa[0]:annotated_sa[1]]
answer["span_start"] = annotated_sa[0]
answer["span_end"] = annotated_sa[1]
expected_answer_text = get_text_span(
 e, {
"start_token": annotation["short_answers"][0]["start_token"],
"end_token": annotation["short_answers"][-1]["end_token"],
} ).text
assert expected_answer_text == answer["span_text"],(expected_answer_text,
answer["span_text"])
 elif annotation and annotation["long_answer"]["candidate_index"] >= 0:
answer["span_text"] = get_candidate_text(e, annotated_idx ).text
answer["span_start"] = 0
answer["span_end"] = len(answer["span_text"])
 context_idxs = [-1]
context_list = [{"id": -1, "type": get_candidate_type_and_position(e, -1)}]
context_list[-1]["text_map"], context_list[-1]["text"] =(
 get_candidate_text(e, -1))
for idx, _ in candidates_iter(e):
context = {"id": idx, "type": get_candidate_type_and_position(e, idx)}
context["text_map"], context["text"] = get_candidate_text(e, idx)
 context_idxs.append(idx)
 context_list.append(context)
 if len(context_list)>= FLAGS.max_contexts:
break
example = {
"name": e["document_title"],
"id": str(e["example_id"]),
"questions": [question],
"answers": [answer],
"has_correct_context": annotated_idx in context_idxs
}
single_map = []
single_context = []
offset = 0
for context in context_list:
single_map.extend([-1, -1])
 single_context.append("[ContextId=%d] %s" % 
(context["id"], context["type"]))
offset += len(single_context[-1])+ 1
if context["id"] == annotated_idx:
answer["span_start"] += offset
answer["span_end"] += offset
if context["text"]:
single_map.extend(context["text_map"])
 single_context.append(context["text"])
 offset += len(single_context[-1])+ 1
example["contexts"] = " ".join(single_context)
 example["contexts_map"] = single_map
if annotated_idx in context_idxs:
expected = example["contexts"][answer["span_start"]:answer["span_end"]]
assert expected == answer["span_text"],(expected, answer["span_text"])
 return example
def create_example_from_jsonl_simple(line):

 e = json.loads(line, object_pairs_hook=collections.OrderedDict)
 document_tokens = e["document_text"].split(" ")
 e["document_tokens"] = []
for token in document_tokens:
e["document_tokens"].append({"token":token, "start_byte":-1, "end_byte":-1, "html_token":"<" in token})
 add_candidate_types_and_positions(e)
 try:
annotation, annotated_idx, annotated_sa = get_first_annotation(e)
 except:
annotation, annotated_idx, annotated_sa = None, -1,(-1, -1)
 question = {"input_text": e["question_text"]}
answer = {
"candidate_id": annotated_idx,
"span_text": "",
"span_start": -1,
"span_end": -1,
"input_text": "long",
}
if annotation is not None:
assert annotation["yes_no_answer"] in("YES", "NO", "NONE")
 if annotation["yes_no_answer"] in("YES", "NO"):
answer["input_text"] = annotation["yes_no_answer"].lower()
if annotated_sa !=(-1, -1):
answer["input_text"] = "short"
span_text = get_candidate_text(e, annotated_idx ).text
answer["span_text"] = span_text[annotated_sa[0]:annotated_sa[1]]
answer["span_start"] = annotated_sa[0]
answer["span_end"] = annotated_sa[1]
expected_answer_text = get_text_span(
 e, {
"start_token": annotation["short_answers"][0]["start_token"],
"end_token": annotation["short_answers"][-1]["end_token"],
} ).text
assert expected_answer_text == answer["span_text"],(expected_answer_text,
answer["span_text"])
 elif annotation and annotation["long_answer"]["candidate_index"] >= 0:
answer["span_text"] = get_candidate_text(e, annotated_idx ).text
answer["span_start"] = 0
answer["span_end"] = len(answer["span_text"])
 context_idxs = [-1]
context_list = [{"id": -1, "type": get_candidate_type_and_position(e, -1)}]
context_list[-1]["text_map"], context_list[-1]["text"] =(
 get_candidate_text(e, -1))
for idx, _ in candidates_iter(e):
context = {"id": idx, "type": get_candidate_type_and_position(e, idx)}
context["text_map"], context["text"] = get_candidate_text(e, idx)
 context_idxs.append(idx)
 context_list.append(context)
 if len(context_list)>= FLAGS.max_contexts:
break
if "document_title" not in e:
e["document_title"] = e["example_id"]
example = {
"name": e["document_title"],
"id": str(e["example_id"]),
"questions": [question],
"answers": [answer],
"has_correct_context": annotated_idx in context_idxs
}
single_map = []
single_context = []
offset = 0
for context in context_list:
single_map.extend([-1, -1])
 single_context.append("[ContextId=%d] %s" % 
(context["id"], context["type"]))
offset += len(single_context[-1])+ 1
if context["id"] == annotated_idx:
answer["span_start"] += offset
answer["span_end"] += offset
if context["text"]:
single_map.extend(context["text_map"])
 single_context.append(context["text"])
 offset += len(single_context[-1])+ 1
example["contexts"] = " ".join(single_context)
 example["contexts_map"] = single_map
if annotated_idx in context_idxs:
expected = example["contexts"][answer["span_start"]:answer["span_end"]]
assert expected == answer["span_text"],(expected, answer["span_text"])
 return example
def read_nq_examples(input_file, is_training):

 input_paths = tf.gfile.Glob(input_file)
 input_data = []
def _open(path):
if path.endswith(".gz"):
return gzip.open(path, "r")
 else:
return tf.gfile.Open(path, "r")
 for path in input_paths:
tf.logging.info("Reading: %s", path)
 with _open(path)as input_file:
for line in input_file:
try:
input_data.append(create_example_from_jsonl(line))
except:
input_data.append(create_example_from_jsonl_simple(line))
examples = []
for entry in input_data:
examples.extend(read_nq_entry(entry, is_training))
return examples
def model_fn_builder(bert_config, init_checkpoint, learning_rate,
num_train_steps, num_warmup_steps, use_tpu,
use_one_hot_embeddings):

 def model_fn(features, labels, mode, params):

 tf.logging.info("*** Features ***")
 for name in sorted(features.keys()):
tf.logging.info(" name = %s, shape = %s" %(name, features[name].shape))
unique_ids = features["unique_ids"]
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
is_training =(mode == tf.estimator.ModeKeys.TRAIN)
(start_logits, end_logits, answer_type_logits)= create_model(
 bert_config=bert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
use_one_hot_embeddings=use_one_hot_embeddings)
 tvars = tf.trainable_variables()
initialized_variable_names = {}
scaffold_fn = None
if init_checkpoint: 
(assignment_map, initialized_variable_names 
)= modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)
 if use_tpu:
def tpu_scaffold() :
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
 return tf.train.Scaffold()
scaffold_fn = tpu_scaffold
else:
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
 tf.logging.info("**** Trainable Variables ****")
 for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape,
init_string)
 output_spec = None
if mode == tf.estimator.ModeKeys.TRAIN:
seq_length = modeling.get_shape_list(input_ids)[1]
def compute_loss(logits, positions):
one_hot_positions = tf.one_hot(
 positions, depth=seq_length, dtype=tf.float32)
 log_probs = tf.nn.log_softmax(logits, axis=-1)
 loss = -tf.reduce_mean(
 tf.reduce_sum(one_hot_positions * log_probs, axis=-1))
return loss
def compute_label_loss(logits, labels):
one_hot_labels = tf.one_hot(
 labels, depth=len(AnswerType), dtype=tf.float32)
 log_probs = tf.nn.log_softmax(logits, axis=-1)
 loss = -tf.reduce_mean(
 tf.reduce_sum(one_hot_labels * log_probs, axis=-1))
return loss
start_positions = features["start_positions"]
end_positions = features["end_positions"]
answer_types = features["answer_types"]
start_loss = compute_loss(start_logits, start_positions)
 end_loss = compute_loss(end_logits, end_positions)
 answer_type_loss = compute_label_loss(answer_type_logits, answer_types)
 total_loss =(start_loss + end_loss + answer_type_loss)/ 3.0
train_op = optimization.create_optimizer(total_loss, learning_rate,
num_train_steps,
num_warmup_steps, use_tpu)
 output_spec = contrib_tpu.TPUEstimatorSpec(
 mode=mode,
loss=total_loss,
train_op=train_op,
scaffold_fn=scaffold_fn)
 elif mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
"unique_ids": unique_ids,
"start_logits": start_logits,
"end_logits": end_logits,
"answer_type_logits": answer_type_logits,
}
output_spec = contrib_tpu.TPUEstimatorSpec(
 mode=mode, predictions=predictions, scaffold_fn=scaffold_fn)
 else:
raise ValueError("Only TRAIN and PREDICT modes are supported: %s" % 
(mode))
return output_spec
return model_fn
def input_fn_builder(input_file, seq_length, is_training, drop_remainder):

 name_to_features = {
"unique_ids": tf.FixedLenFeature([], tf.int64),
"input_ids": tf.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.FixedLenFeature([seq_length], tf.int64),
}
if is_training:
name_to_features["start_positions"] = tf.FixedLenFeature([], tf.int64)
 name_to_features["end_positions"] = tf.FixedLenFeature([], tf.int64)
 name_to_features["answer_types"] = tf.FixedLenFeature([], tf.int64)
 def _decode_record(record, name_to_features):

 example = tf.parse_single_example(record, name_to_features)
 for name in list(example.keys()):
t = example[name]
if t.dtype == tf.int64:
t = tf.to_int32(t)
 example[name] = t
return example
def input_fn(params):

 batch_size = params["batch_size"]
d = tf.data.TFRecordDataset(input_file)
 if is_training:
d = d.repeat()
d = d.shuffle(buffer_size=100)
 d = d.apply(
 tf.data.experimental.map_and_batch(
 lambda record: _decode_record(record, name_to_features),
batch_size=batch_size,
drop_remainder=drop_remainder))
return d
return input_fn
class FeatureWriter(object):

 def __init__(self, filename, is_training):
self.filename = filename
self.is_training = is_training
self.num_features = 0
self._writer = tf.python_io.TFRecordWriter(filename)
 def process_feature(self, feature):

 self.num_features += 1
def create_int_feature(values):
feature = tf.train.Feature(
 int64_list=tf.train.Int64List(value=list(values)))
 return feature
features = collections.OrderedDict()
features["unique_ids"] = create_int_feature([feature.unique_id])
 features["input_ids"] = create_int_feature(feature.input_ids)
 features["input_mask"] = create_int_feature(feature.input_mask)
 features["segment_ids"] = create_int_feature(feature.segment_ids)
 if self.is_training:
features["start_positions"] = create_int_feature([feature.start_position])
 features["end_positions"] = create_int_feature([feature.end_position])
 features["answer_types"] = create_int_feature([feature.answer_type])
 else:
token_map = [-1] * len(feature.input_ids)
 for k, v in feature.token_to_orig_map.items() :
token_map[k] = v
features["token_map"] = create_int_feature(token_map)
 tf_example = tf.train.Example(features=tf.train.Features(feature=features))
self._writer.write(tf_example.SerializeToString())
 def close(self):
self._writer.close()
test_result = pd.DataFrame.from_dict(test_result, orient='columns')
 test_result.T<save_to_csv>
FLAGS.vocab_file = '/kaggle/input/bert-config-vocab/vocab-nq.txt'
FLAGS.bert_config_file = '/kaggle/input/bert-config-vocab/bert_config.json'
FLAGS.do_train=False
FLAGS.do_predict=True
FLAGS.predict_batch_size=32
FLAGS.max_seq_length=512
FLAGS.doc_stride=256
FLAGS.output_dir='tfrecords'
FLAGS.use_tpu=False
FLAGS.max_contexts = 80
eval_set = 'test'
FLAGS.predict_file = '/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl'
test_result.to_csv('test_leaky_stat.csv', index=False )<choose_model_class>
%%time
eval_examples = read_nq_examples(input_file=FLAGS.predict_file, is_training=False)
 print(len(eval_examples))
best_lag = 37<categorify>
%%time
candidates_dict = read_candidates(FLAGS.predict_file)
 print(len(candidates_dict))
test_leak = rewrite_compiled_leak(test_leak, best_lag)
 test_leak[['ID']+leaky_cols+['compiled_leak']].head()<save_to_csv>
def main_pred() :
tf.logging.set_verbosity(tf.logging.INFO)
 bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
 validate_flags_or_throw(bert_config)
 tf.gfile.MakeDirs(FLAGS.output_dir)
 tokenizer = tokenization.FullTokenizer(
 vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
 FLAGS.use_tpu = False
tpu_cluster_resolver = None
is_per_host = contrib_tpu.InputPipelineConfig.PER_HOST_V2
run_config = contrib_tpu.RunConfig(
 cluster=tpu_cluster_resolver,
master=FLAGS.master,
model_dir=FLAGS.output_dir,
keep_checkpoint_max=100,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
tpu_config=contrib_tpu.TPUConfig(
 iterations_per_loop=FLAGS.iterations_per_loop,
num_shards=FLAGS.num_tpu_cores,
per_host_input_for_training=is_per_host))
num_train_steps = None
num_warmup_steps = None
model_fn = model_fn_builder(
 bert_config=bert_config,
init_checkpoint=FLAGS.init_checkpoint,
learning_rate=FLAGS.learning_rate,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
use_tpu=FLAGS.use_tpu,
use_one_hot_embeddings=FLAGS.use_tpu)
 estimator = tf.contrib.tpu.TPUEstimator(
 use_tpu=FLAGS.use_tpu,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.train_batch_size,
predict_batch_size=FLAGS.predict_batch_size)
 if FLAGS.do_lower_case:
tfrecord_file = os.path.join(f"{eval_set}.tf_record")
 else:
tfrecord_file = os.path.join(f"{eval_set}_cased.tf_record")
 if not os.path.exists(tfrecord_file):
eval_writer = FeatureWriter(
 filename=tfrecord_file,
is_training=False)
 eval_features = []
def append_feature(feature):
eval_features.append(feature)
 eval_writer.process_feature(feature)
 convert_examples_to_features(
 examples=eval_examples,
tokenizer=tokenizer,
is_training=False,
output_fn=append_feature)
 eval_writer.close()
tf.logging.info("***** Running predictions *****")
 tf.logging.info(" Num orig examples = %d", len(eval_examples))
tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size)
 predict_input_fn = input_fn_builder(
 input_file=tfrecord_file,
seq_length=FLAGS.max_seq_length,
is_training=False,
drop_remainder=False)
 eval_features = [
tf.train.Example.FromString(r)
 for r in tf.python_io.tf_record_iterator(tfrecord_file)
 ]
all_results = []
for result in tqdm(estimator.predict(
 predict_input_fn, yield_single_examples=True),total=len(eval_features)) :
if len(all_results)% 1000 == 0:
tf.logging.info("Processing example: %d" %(len(all_results)))
 unique_id = int(result["unique_ids"])
 start_logits = [float(x)for x in result["start_logits"].flat]
end_logits = [float(x)for x in result["end_logits"].flat]
answer_type_logits = [float(x)for x in result["answer_type_logits"].flat]
all_results.append(
 RawResult(
 unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits,
answer_type_logits=answer_type_logits))
nq_pred_dict,nbest_summary_dict = compute_pred_dict(candidates_dict, eval_features,
[r._asdict() for r in all_results])
 predictions_json = {"predictions": list(nq_pred_dict.values())}
with open(FLAGS.output_prediction_file, "w")as f:
json.dump(predictions_json, f, indent=4)
 pickle.dump(nbest_summary_dict,open(
 FLAGS.output_prediction_file.replace('pred_jsons/pred','nbest_pkl/nbest_dict' ).replace('.json','.pkl'),
'wb'))
test_res = test_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 test_res.to_csv('test_leak.csv', index=False )<feature_engineering>
ckpts = ['/kaggle/input/model-l/_wwm_stride_192_neg_0.01_0.04-64-2.00E-05/model.ckpt-7000',
'/kaggle/input/model-o/model.ckpt-9500',
'/kaggle/input/model-r2/model.ckpt-8500'
]
ensemble_wts = [1, 1.1, 0.8]
long_thr = 2.3972
short_thr = 2.895
yes_thr = 5.5
no_thr = 5.5
pred_files = [x.split('/')[-2] for x in ckpts]
pred_files
test_leak.loc[test_leak["compiled_leak"]==0, "compiled_leak"] = test_leak.loc[test_leak["compiled_leak"]==0, "nonzero_mean"]<save_to_csv>
%%time
FLAGS.do_lower_case = True
FLAGS.vocab_file = '/kaggle/input/bert-config-vocab/vocab-nq.txt'
FLAGS.bert_config_file = '/kaggle/input/bert-config-vocab/bert_config.json'
for i in range(2):
FLAGS.init_checkpoint = ckpts[i]
FLAGS.output_prediction_file = pred_files[i]+'.json'
main_pred()
FLAGS.do_lower_case = False
FLAGS.vocab_file = '/kaggle/input/bert-config-vocab/vocab_cased-nq.txt'
FLAGS.bert_config_file = '/kaggle/input/bert-config-vocab/bert_config_cased.json'
i=2
FLAGS.init_checkpoint = ckpts[i]
FLAGS.output_prediction_file = pred_files[i]+'.json'
main_pred()
sub = test[["ID"]]
sub["target"] = test_leak["compiled_leak"]
sub.to_csv(f"baseline_sub_lag_{best_lag}.csv", index=False)
 print(f"baseline_sub_lag_{best_lag}.csv saved" )<set_options>
NQLabel = collections.namedtuple(
 'NQLabel',
[
'example_id',
'long_answer_span',
'short_answer_span_list',
'yes_no_answer',
'long_score',
'short_score'
])
 class uitl_Span(object):

 def __init__(self, start_byte, end_byte, start_token_idx, end_token_idx):
if(( start_byte < 0 and end_byte >= 0)or 
(start_byte >= 0 and end_byte < 0)) :
raise ValueError('Inconsistent Null Spans(Byte ).')
 if(( start_token_idx < 0 and end_token_idx >= 0)or 
(start_token_idx >= 0 and end_token_idx < 0)) :
raise ValueError('Inconsistent Null Spans(Token ).')
 if start_byte >= 0 and end_byte >= 0 and start_byte >= end_byte:
raise ValueError('Invalid byte spans(start_byte >= end_byte ).')
 if(( start_token_idx >= 0 and end_token_idx >= 0)and 
(start_token_idx >= end_token_idx)) :
raise ValueError('Invalid token spans(start_token_idx >= end_token_idx)')
 self.start_byte = start_byte
self.end_byte = end_byte
self.start_token_idx = start_token_idx
self.end_token_idx = end_token_idx
def is_null_span(self):

 if(self.start_byte < 0 and self.end_byte < 0 and
self.start_token_idx < 0 and self.end_token_idx < 0):
return True
return False
def __str__(self):
byte_str = 'byte: [' + str(self.start_byte)+ ',' + str(self.end_byte)+ ')'
tok_str =('tok: [' + str(self.start_token_idx)+ ',' + str(
 self.end_token_idx)+ ')')
 return byte_str + ' ' + tok_str
def __repr__(self):
return self.__str__()
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 9999<load_from_csv>
def ensemble_nbest_to_nq_pred_dict(nbest_lst, wts, yes_thr=None, no_thr=None):
def sigmoid(x): return 1 /(1 + np.exp(-x))
assert len(nbest_lst)==len(wts)
 nq_pred_dict = {}
for i in range(1,len(nbest_lst)) : assert set(nbest_lst[0].keys())== set(nbest_lst[i].keys())
 for example_id in nbest_lst[0].keys() :
d_long = {}
d_short = {}
yes_logits,no_logits = 0,0
yes_exception, no_exception = False,False
for i,nbest in enumerate(nbest_lst):
if yes_thr and(not yes_exception):
try:
yes_logits += nbest[example_id][0].answer_type_logits[1] * wts[i]
except:
print("exception yes_logits += nbest[example_id][0].answer_type_logits[1] * wts[i]")
 yes_exception = True
if no_thr and(not no_exception):
try:
no_logits += nbest[example_id][0].answer_type_logits[2] * wts[i]
except:
print("exception no_logits += nbest[example_id][0].answer_type_logits[2] * wts[i]")
 no_exception = True
long_seen = set()
short_seen = set()
for score_summary in nbest[example_id]:
pred_lbl = score_summary.predicted_label
long_ans = pred_lbl['long_answer']
short_ans = pred_lbl['short_answers'][0]
long_key = long_ans['start_token'], long_ans['end_token']
short_key = short_ans['start_token'], short_ans['end_token']
if long_key not in long_seen:
long_seen.add(long_key)
 if long_key not in d_long: d_long[long_key] = sigmoid(pred_lbl['long_answer_score'])* wts[i]
else: d_long[long_key]+= sigmoid(pred_lbl['long_answer_score'])* wts[i]
if short_key not in short_seen:
short_seen.add(short_key)
 if short_key not in d_short: d_short[short_key] = sigmoid(pred_lbl['short_answers_score'])* wts[i]
else: d_short[short_key]+= sigmoid(pred_lbl['short_answers_score'])* wts[i] 
(start_tok_long,end_tok_long),prob_long = sorted(list(d_long.items()),key=lambda x:x[1],reverse=True)[0] 
(start_tok_short,end_tok_short),prob_short = sorted(list(d_short.items()),key=lambda x:x[1],reverse=True)[0]
short_answer_span_list = [uitl_Span(-1,-1,start_tok_short,end_tok_short)]
yes_no_answer='none'
try:
if yes_thr and yes_logits > yes_thr:
yes_no_answer,prob_short = 'yes', 999999
short_answer_span_list = [uitl_Span(-1,-1,-1,-1)]
elif no_thr and no_logits > no_thr:
yes_no_answer,prob_short = 'no', 999999
short_answer_span_list = [uitl_Span(-1,-1,-1,-1)]
except:
print('exception if yes_thr and yes_logits > yes_thr...')
 nq_pred_dict[int(example_id)] = NQLabel(
 example_id=example_id,
long_answer_span = uitl_Span(-1, -1, start_tok_long, end_tok_long),
short_answer_span_list = short_answer_span_list,
yes_no_answer=yes_no_answer,
long_score=prob_long,
short_score=prob_short)
 return nq_pred_dict
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train rows and columns : ", train_df.shape)
 print("Test rows and columns : ", test_df.shape )<count_missing_values>
pkl_lst = [x+'.pkl' for x in pred_files]
pkl_lst
np.sum(train_df.isna().sum() )<groupby>
nbest_test_lst = [pickle.load(open(nbest_file,'rb')) for nbest_file in pkl_lst]
nq_pred_dict_en = ensemble_nbest_to_nq_pred_dict(nbest_test_lst, ensemble_wts, yes_thr, no_thr )
dtype_df = train_df.dtypes.reset_index()
dtype_df.columns = ["Count", "Column Type"]
dtype_df.groupby("Column Type" ).aggregate('count' ).reset_index()<count_unique_values>
def nqlabel_to_dict(nqlabel):
nqlabel.example_id
nqlabel.long_answer_span
nqlabel.short_answer_span_list
nqlabel.yes_no_answer
nqlabel.long_score
nqlabel.short_score
return {'example_id': int(nqlabel.example_id),
'long_answer': {'start_token': nqlabel.long_answer_span.start_token_idx,
'end_token': nqlabel.long_answer_span.end_token_idx,
'start_byte': nqlabel.long_answer_span.start_byte,
'end_byte': nqlabel.long_answer_span.end_byte},
'long_answer_score': nqlabel.long_score,
'short_answers': [{'start_token': x.start_token_idx,
'end_token': x.end_token_idx,
'start_byte': x.start_byte,
'end_byte': x.end_byte} for x in nqlabel.short_answer_span_list],
'short_answers_score': nqlabel.short_score,
'yes_no_answer': nqlabel.yes_no_answer.upper()
}
test_answers_df = pd.Series([nqlabel_to_dict(v)for(k,v)in nq_pred_dict_en.items() ] ).to_frame()
test_answers_df = test_answers_df.rename(columns={0:'predictions'} )
unique_df = train_df.nunique().reset_index()
unique_df.columns = ["col_name", "unique_count"]
constant_df = unique_df[unique_df["unique_count"]==1]
constant_df.shape<filter>
def create_short_answer(entry):
if entry["short_answers_score"] < short_thr:
return ""
answer = []
for short_answer in entry["short_answers"]:
if short_answer["start_token"] > -1:
answer.append(str(short_answer["start_token"])+ ":" + str(short_answer["end_token"]))
if entry["yes_no_answer"] != "NONE":
answer.append(entry["yes_no_answer"])
 return " ".join(answer)
 def create_long_answer(entry):
if entry["long_answer_score"] < long_thr:
return ""
answer = []
if entry["long_answer"]["start_token"] > -1:
answer.append(str(entry["long_answer"]["start_token"])+ ":" + str(entry["long_answer"]["end_token"]))
return " ".join(answer)
 test_answers_df["long_answer_score"] = test_answers_df["predictions"].apply(lambda q: q["long_answer_score"])
 test_answers_df["short_answer_score"] = test_answers_df["predictions"].apply(lambda q: q["short_answers_score"])
 test_answers_df["long_answer"] = test_answers_df["predictions"].apply(create_long_answer)
 test_answers_df["short_answer"] = test_answers_df["predictions"].apply(create_short_answer)
 test_answers_df["example_id"] = test_answers_df["predictions"].apply(lambda q: str(q["example_id"]))
long_answers = dict(zip(test_answers_df["example_id"], test_answers_df["long_answer"]))
short_answers = dict(zip(test_answers_df["example_id"], test_answers_df["short_answer"]))
sample_submission = pd.read_csv('/kaggle/input/tensorflow2-question-answering/sample_submission.csv')
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings
sample_submission.to_csv('submission.csv',index=False )
print(train_df[train_df.columns[~train_df.columns.isin(constant_df.col_name.tolist())]].shape)
 train_df_without_uniques = train_df[train_df.columns[~train_df.columns.isin(constant_df.col_name.tolist())]]<prepare_x_and_y>
pd.read_csv('submission.csv' ).count()
<create_dataframe><EOS> 
( pd.read_csv('submission.csv' ).PredictionString=='YES' ).sum() ,(pd.read_csv('submission.csv' ).PredictionString=='NO' ).sum()