Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<merge>
!pip install --upgrade.. /input/tokenizers-repo/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl
test = pd.merge(test, sub3, how='left', on='ID', )<concatenate>
train = train.replace(0, np.nan)
 test = test.replace(0, np.nan)
 train = pd.concat(( train, test), axis=0, ignore_index=True )<save_to_csv>
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
b1 = sub1.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('submission.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend01.csv', index=False )<save_to_csv>
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend01.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.8)+(b1['dp2'] * 0.2)
 b1[['ID','target']].to_csv('blend02.csv', index=False )<save_to_csv>
def find_closing_tag(tokens, opening_tag):
closing_tag = f'</{opening_tag[1: -1]}>'
index, stack = -1, []
for token_index, token in enumerate(tokens):
if token == opening_tag:
stack.insert(0, opening_tag)
 elif token == closing_tag:
stack.pop()
if len(stack)== 0:
index = token_index
break
return index
b1 = sub2.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend02.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.5)+(b1['dp2'] * 0.5)
 b1[['ID','target']].to_csv('blend03.csv', index=False )<save_to_csv>
def read_candidates(candidate_files, do_cache=True):
assert isinstance(candidate_files,(tuple, list)) , candidate_files
for fn in candidate_files:
assert os.path.exists(fn), f'Missing file {fn}'
cache_fn = 'candidates.pkl'
candidates = {}
if not os.path.exists(cache_fn):
for fn in candidate_files:
with open(fn)as f:
for line in tqdm(f):
entry = json.loads(line)
 example_id = str(entry['example_id'])
 cnds = entry.pop('long_answer_candidates')
 cnds = [LongAnswerCandidate(c['start_token'], c['end_token'],
c['top_level'])for c in cnds]
candidates[example_id] = cnds
if do_cache:
with open(cache_fn, 'wb')as f:
pickle.dump(candidates, f)
 else:
print(f'Loading from cache: {cache_fn}')
 with open(cache_fn, 'rb')as f:
candidates = pickle.load(f)
 return candidates
b1 = sub3.rename(columns={'target':'dp1'})
 b2 = pd.read_csv('blend03.csv' ).rename(columns={'target':'dp2'})
 b1 = pd.merge(b1, b2, how='left', on='ID')
 b1['target'] =(b1['dp1'] * 0.6)+(b1['dp2'] * 0.4)
 b1[['ID','target']].to_csv('timepass.csv', index=False )<load_from_csv>
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" or ord(c)== 0x202F:
return True
return False
!rm submission.csv
!rm blend01.csv
!rm blend02.csv
!rm blend03.csv<load_from_csv>
def read_nq_examples(input_file_or_data, is_training):

 if isinstance(input_file_or_data, str):
with open(input_file_or_data, "r", encoding='utf-8')as f:
input_data = json.load(f)["data"]
else:
input_data = input_file_or_data
for entry_index, entry in enumerate(tqdm(input_data, total=len(input_data))):
assert len(entry["paragraphs"])== 1
paragraph = entry["paragraphs"][0]
paragraph_text = paragraph["context"]
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in paragraph_text:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 assert len(paragraph["qas"])== 1
qa = paragraph["qas"][0]
start_position = None
end_position = None
long_position = None
orig_answer_text = None
short_is_impossible = False
long_is_impossible = False
if is_training:
short_is_impossible = qa["short_is_impossible"]
short_answers = qa["short_answers"]
if len(short_answers)>= 2:
short_answers = sorted(short_answers, key=lambda sa: sa["answer_start"])
 short_answers = short_answers[0: 1]
if not short_is_impossible:
answer = short_answers[0]
orig_answer_text = answer["text"]
answer_offset = answer["answer_start"]
answer_length = len(orig_answer_text)
 start_position = char_to_word_offset[answer_offset]
end_position = char_to_word_offset[
answer_offset + answer_length - 1]
actual_text = " ".join(doc_tokens[start_position:
end_position + 1])
 cleaned_answer_text = " ".join(
 whitespace_tokenize(orig_answer_text))
if actual_text.find(cleaned_answer_text)== -1:
logger.warning(
 "Could not find answer: '%s' vs.'%s'",
actual_text, cleaned_answer_text)
 continue
else:
start_position = -1
end_position = -1
orig_answer_text = ""
long_is_impossible = qa["long_is_impossible"]
long_answers = qa["long_answers"]
if(len(long_answers)!= 1)and not long_is_impossible:
raise ValueError(f"For training, each question"
f" should have exactly 1 long answer.")
 if not long_is_impossible:
long_answer = long_answers[0]
long_answer_offset = long_answer["answer_start"]
long_position = char_to_word_offset[long_answer_offset]
else:
long_position = -1
if not short_is_impossible and not long_is_impossible:
assert long_position <= start_position
if not short_is_impossible and long_is_impossible:
assert False, f'Invalid pair short, long pair'
example = NQExample(
 qas_id=qa["id"],
question_text=qa["question"],
doc_tokens=doc_tokens,
orig_answer_text=orig_answer_text,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible,
crop_start=qa["crop_start"])
 yield example
train = pd.read_csv(".. /input/santander-value-prediction-challenge/train.csv")
 test = pd.read_csv(".. /input/santander-value-prediction-challenge/test.csv")
 transact_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values<feature_engineering>
DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
 def get_spans(doc_stride, max_tokens_for_doc, max_len):
doc_spans = []
start_offset = 0
while start_offset < max_len:
length = max_len - start_offset
if length > max_tokens_for_doc:
length = max_tokens_for_doc
doc_spans.append(DocSpan(start=start_offset, length=length))
if start_offset + length == max_len:
break
start_offset += min(length, doc_stride)
 return doc_spans
test["target"] = train["target"].mean()<concatenate>
def convert_examples_to_crops(examples_gen, tokenizer, max_seq_length,
doc_stride, max_query_length, is_training,
cls_token='[CLS]', sep_token='[SEP]', pad_id=0,
sequence_a_segment_id=0,
sequence_b_segment_id=1,
cls_token_segment_id=0,
pad_token_segment_id=0,
mask_padding_with_zero=True,
p_keep_impossible=None,
sep_token_extra=False):

 assert p_keep_impossible is not None, '`p_keep_impossible` is required'
unique_id = 1000000000
num_short_pos, num_short_neg = 0, 0
num_long_pos, num_long_neg = 0, 0
sub_token_cache = {}
crops = []
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info('Converting %s: short_pos %s short_neg %s'
' long_pos %s long_neg %s',
example_index, num_short_pos, num_short_neg,
num_long_pos, num_long_neg)
 query_tokens = tokenizer.tokenize(example.question_text)
 if len(query_tokens)> max_query_length:
query_tokens = query_tokens[0:max_query_length]
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for i, token in enumerate(example.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = sub_token_cache.get(token)
 if sub_tokens is None:
sub_tokens = tokenizer.tokenize(token)
 sub_token_cache[token] = sub_tokens
tok_to_orig_index.extend([i for _ in range(len(sub_tokens)) ])
 all_doc_tokens.extend(sub_tokens)
 tok_start_position = None
tok_end_position = None
if is_training and example.short_is_impossible:
tok_start_position = -1
tok_end_position = -1
if is_training and not example.short_is_impossible:
tok_start_position = orig_to_tok_index[example.start_position]
if example.end_position < len(example.doc_tokens)- 1:
tok_end_position = orig_to_tok_index[
example.end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
tok_long_position = None
if is_training and example.long_is_impossible:
tok_long_position = -1
if is_training and not example.long_is_impossible:
tok_long_position = orig_to_tok_index[example.long_position]
special_tokens_count = 3
if sep_token_extra:
special_tokens_count += 1
max_tokens_for_doc = max_seq_length - len(query_tokens)- special_tokens_count
assert max_tokens_for_doc > 0
doc_spans = get_spans(doc_stride, max_tokens_for_doc, len(all_doc_tokens))
for doc_span_index, doc_span in enumerate(doc_spans):
tokens = []
token_to_orig_map = UNMAPPED * np.ones(( max_seq_length,), dtype=np.int32)
 token_is_max_context = np.zeros(( max_seq_length,), dtype=np.bool)
 token_type_ids = []
short_is_impossible = example.short_is_impossible
start_position = None
end_position = None
special_tokens_offset = special_tokens_count - 1
doc_offset = len(query_tokens)+ special_tokens_offset
if is_training and not short_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_start_position >= doc_start and tok_end_position <= doc_end):
start_position = 0
end_position = 0
short_is_impossible = True
else:
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
long_is_impossible = example.long_is_impossible
long_position = None
if is_training and not long_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_long_position >= doc_start and tok_long_position <= doc_end):
long_position = 0
long_is_impossible = True
else:
long_position = tok_long_position - doc_start + doc_offset
if long_is_impossible:
if np.random.rand() > p_keep_impossible:
continue
tokens.append(cls_token)
 token_type_ids.append(cls_token_segment_id)
 tokens += query_tokens
token_type_ids += [sequence_a_segment_id] * len(query_tokens)
 tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 if sep_token_extra:
tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[
split_token_index] + example.crop_start
token_is_max_context[len(tokens)] = check_is_max_context(doc_spans,
doc_span_index, split_token_index)
 tokens.append(all_doc_tokens[split_token_index])
 token_type_ids.append(sequence_b_segment_id)
 paragraph_len = doc_span.length
tokens.append(sep_token)
 token_type_ids.append(sequence_b_segment_id)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
 while len(input_ids)< max_seq_length:
input_ids.append(pad_id)
 attention_mask.append(0 if mask_padding_with_zero else 1)
 token_type_ids.append(pad_token_segment_id)
 input_ids = np.array(input_ids, dtype=np.int32)
 attention_mask = np.array(attention_mask, dtype=np.bool)
 token_type_ids = np.array(token_type_ids, dtype=np.uint8)
 if is_training and short_is_impossible:
start_position = CLS_INDEX
end_position = CLS_INDEX
if is_training and long_is_impossible:
long_position = CLS_INDEX
if example_index in(0, 10):
if doc_span_index in(0, 5):
logger.info("*** Example ***")
 logger.info("unique_id: %s" %(unique_id))
logger.info("example_index: %s" %(example_index))
logger.info("doc_span_index: %s" %(doc_span_index))
logger.info("tokens: %s" % " ".join(tokens))
logger.info("input_ids: %s" % input_ids)
 logger.info("attention_mask: %s" % np.uint8(attention_mask))
logger.info("token_type_ids: %s" % token_type_ids)
 if is_training and short_is_impossible:
logger.info("short impossible example")
 if is_training and long_is_impossible:
logger.info("long impossible example")
 if is_training and not short_is_impossible:
answer_text = " ".join(tokens[start_position: end_position + 1])
 logger.info("start_position: %d" %(start_position))
logger.info("end_position: %d" %(end_position))
logger.info("answer: %s" %(answer_text))
if short_is_impossible:
num_short_neg += 1
else:
num_short_pos += 1
if long_is_impossible:
num_long_neg += 1
else:
num_long_pos += 1
crop = Crop(
 unique_id=unique_id,
example_index=example_index,
doc_span_index=doc_span_index,
tokens=tokens,
token_to_orig_map=token_to_orig_map,
token_is_max_context=token_is_max_context,
input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
paragraph_len=paragraph_len,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible)
 crops.append(crop)
 unique_id += 1
return crops
all_df = pd.concat([train, test] ).reset_index(drop=True)
 all_df.columns = all_df.columns.astype(str)
 print(all_df.shape )<define_variables>
PrelimPrediction = collections.namedtuple("PrelimPrediction",
["crop_index", "start_index", "end_index", "start_logit", "end_logit"])
 NbestPrediction = collections.namedtuple("NbestPrediction", [
"text", "start_logit", "end_logit",
"start_index", "end_index",
"orig_doc_start", "orig_doc_end", "crop_index"] )
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f',
'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5',
'58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867',
'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7',
'1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535',
'fc99f9426', '91f701ba2', '0572565c2', '190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98'
] 
<define_variables>
def clean_text(tok_text):
tok_text = tok_text.replace("
tok_text = tok_text.replace("
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
 return tok_text
colgroups = [
['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1', '15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9', 'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b', '62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992', 'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd', '703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a', '6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2', '190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98'],
['266525925', '4b6dfc880', '2cff4bf0c', 'a3382e205', '6488c8200', '547d3135b', 'b46191036', '453128993', '2599a7eb7', '2fc60d4d9', '009319104', 'de14e7687', 'aa31dd768', '2b54cddfd', 'a67d02050', '37aab1168', '939cc02f5', '31f72667c', '6f951302c', '54723be01', '4681de4fd', '8bd53906a', '435f27009', 'f82167572', 'd428161d9', '9015ac21d', 'ec4dc7883', '22c7b00ef', 'd4cc42c3d', '1351bf96e', '1e8801477', 'b7d59d3b5', 'a459b5f7d', '580f5ff06', '39b3c553a', '1eec37deb', '692c44993', 'ce8ce671e', '88ef1d9a8', 'bf042d928'],
['9d5c7cb94', '197cb48af', 'ea4887e6b', 'e1d0e11b5', 'ac30af84a', 'ba4ceabc5', 'd4c1de0e2', '6d2ece683', '9c42bff81', 'cf488d633', '0e1f6696a', 'c8fdf5cbf', 'f14b57b8f', '3a62b36bd', 'aeff360c7', '64534cc93', 'e4159c59e', '429687d5a', 'c671db79e', 'd79736965', '2570e2ba9', '415094079', 'ddea5dc65', 'e43343256', '578eda8e0', 'f9847e9fe', '097c7841e', '018ab6a80', '95aea9233', '7121c40ee', '578b81a77', '96b6bd42b', '44cb9b7c4', '6192f193d', 'ba136ae3f', '8479174c2', '64dd02e44', '4ecc3f505', 'acc4a8e68', '994b946ad'],
['f1eeb56ae', '62ffce458', '497adaff8', 'ed1d5d137', 'faf7285a1', 'd83da5921', '0231f07ed', '7950f4c11', '051410e3d', '39e1796ab', '2e0148f29', '312832f30', '6f113540d', 'f3ee6ba3c', 'd9fc63fa1', '6a0b386ac', '5747a79a9', '64bf3a12a', 'c110ee2b7', '1bf37b3e2', 'fdd07cac1', '0872fe14d', 'ddef5ad30', '42088cf50', '3519bf4a4', 'a79b1f060', '97cc1b416', 'b2790ef54', '1a7de209c', '2a71f4027', 'f118f693a', '15e8a9331', '0c545307d', '363713112', '73e591019', '21af91e9b', '62a915028', '2ab5a56f5', 'a8ee55662', '316b978cd'],
['b26d16167', '930f989bf', 'ca58e6370', 'aebe1ea16', '03c589fd7', '600ea672f', '9509f66b0', '70f4f1129', 'b0095ae64', '1c62e29a7', '32a0342e2', '2fc5bfa65', '09c81e679', '49e68fdb9', '026ca57fd', 'aacffd2f4', '61483a9da', '227ff4085', '29725e10e', '5878b703c', '50a0d7f71', '0d1af7370', '7c1af7bbb', '4bf056f35', '3dd64f4c4', 'b9f75e4aa', '423058dba', '150dc0956', 'adf119b9a', 'a8110109e', '6c4f594e0', 'c44348d76', 'db027dbaf', '1fcba48d0', '8d12d44e1', '8d13d891d', '6ff9b1760', '482715cbd', 'f81c2f1dd', 'dda820122'],
['c928b4b74', '8e4d0fe45', '6c0e0801a', '02861e414', 'aac52d8d9', '041c5d0c9', 'd7875bb6c', 'e7c0cfd0f', 'd48c08bda', '0c9462c08', '57dd44c29', 'a93118262', '850027e38', 'db3839ab0', '27461b158', '32174174c', '9306da53f', '95742c2bf', '5831f4c76', '1e6306c7c', '06393096a', '13bdd610a', 'd7d314edc', '9a07d7b1f', '4d2671746', '822e49b95', '3c8a3ced0', '83635fb67', '1857fbccf', 'c4972742d', 'b6c0969a2', 'e78e3031b', '36a9a8479', 'e79e5f72c', '092271eb3', '74d7f2dc3', '277ef93fc', 'b30e932ba', '8f57141ec', '350473311'],
['06148867b', '4ec3bfda8', 'a9ca6c2f4', 'bb0408d98', '1010d7174', 'f8a437c00', '74a7b9e4a', 'cfd55f2b6', '632fed345', '518b5da24', '60a5b79e4', '3fa0b1c53', 'e769ee40d', '9f5f58e61', '83e3e2e60', '77fa93749', '3c9db4778', '42ed6824a', '761b8e0ec', 'ee7fb1067', '71f5ab59f', '177993dc6', '07df9f30c', 'b1c5346c4', '9a5cd5171', 'b5df42e10', 'c91a4f722', 'd93058147', '20a325694', 'f5e0f4a16', '5edd220bc', 'c901e7df1', 'b02dfb243', 'bca395b73', '1791b43b0', 'f04f0582d', 'e585cbf20', '03055cc36', 'd7f15a3ad', 'ccd9fc164'],
['df838756c', '2cb73ede7', '4dcf81d65', '61c1b7eb6', 'a9f61cf27', '1af4d24fa', 'e13b0c0aa', 'b9ba17eb6', '796c218e8', '37f57824c', 'd1e0f571b', 'f9e3b03b7', 'a3ef69ad5', 'e16a20511', '04b88be38', '99e779ee0', '9f7b782ac', '1dd7bca9f', '2eeadde2b', '6df033973', 'cdfc2b069', '031490e77', '5324862e4', '467bee277', 'a3fb07bfd', '64c6eb1cb', '8618bc1fd', '6b795a2bc', '956d228b9', '949ed0965', 'a4511cb0b', 'b64425521', '2e3c96323', '191e21b5f', 'bee629024', '1977eaf08', '5e645a169', '1d04efde3', '8675bec0b', '8337d1adc'],
['a1cd7b681', '9b490abb3', 'b10f15193', '05f54f417', 'a7ac690a8', 'ed6c300c2', 'd0803e3a1', 'b1bb8eac3', 'bd1c19973', 'a34f8d443', '84ec1e3db', '24018f832', '82e01a220', '4c2064b00', '0397f7c9b', 'ba42e41fa', '22d7ad48d', '9abffd22c', 'dbfa2b77f', '2c6c62b54', '9fa38def3', 'ecb354edf', '9c3154ae6', '2f26d70f4', '53102b93f', 'a36b95f78', '1fa0f78d0', '19915a6d3', 'c944a48b5', '482b04cba', '2ce77a58f', '86558e595', 'c3f400e36', '20305585c', 'f8ccfa064', 'dd771cb8e', '9aa27017e', 'cd7f0affd', '236cc1ff5', 'a3fc511cd'],
['920a04ee2', '93efdb50f', '15ea45005', '78c57d7cd', '91570fb11', 'c5dacc85b', '145c7b018', '590b24ab1', 'c283d4609', 'e8bd579ae', '7298ca1ef', 'ce53d1a35', 'a8f80f111', '2a9fed806', 'feb40ad9f', 'cfd255ee3', '31015eaab', '303572ae2', 'cd15bb515', 'cb5161856', 'a65b73c87', '71d64e3f7', 'ec5fb550f', '4af2493b6', '18b4fa3f5', '3d655b0ed', '5cc9b6615', '88c0ec0a6', '8722f33bb', '5ed0c24d0', '54f26ee08', '04ecdcbb3', 'ade8a5a19', 'd5efae759', 'ac7a97382', 'e1b20c3a6', 'b0fcfeab8', '438b8b599', '43782ef36', 'df69cf626'],
['50603ae3d', '48282f315', '090dfb7e2', '6ccaaf2d7', '1bf2dfd4a', '50b1dd40f', '1604c0735', 'e94c03517', 'f9378f7ef', '65266ad22', 'ac61229b6', 'f5723deba', '1ced7f0b4', 'b9a4f06cd', '8132d18b8', 'df28ac53d', 'ae825156f', '936dc3bc4', '5b233cf72', '95a2e29fc', '882a3da34', '2cb4d123e', '0e1921717', 'c83d6b24d', '90a2428a5', '67e6c62b9', '320931ca8', '900045349', 'bf89fac56', 'da3b0b5bb', 'f06078487', '56896bb36', 'a79522786', '71c2f04c9', '1af96abeb', '4b1a994cc', 'dee843499', '645b47cde', 'a8e15505d', 'cc9c2fc87'],
['b6daeae32', '3bdee45be', '3d6d38290', '5a1589f1a', '961b91fe7', '29c059dd2', 'cfc1ce276', '0a953f97e', '30b3daec2', 'fb5f5836e', 'c7525612c', '6fa35fbba', '72d34a148', 'dcc269cfe', 'bdf773176', '469630e5c', '23db7d793', 'dc10234ae', '5ac278422', '6cf7866c1', 'a39758dae', '45f6d00da', '251d1aa17', '84d9d1228', 'b98f3e0d7', '66146c12d', 'd6470c4ce', '3f4a39818', 'f16a196c6', 'b8f892930', '6f88afe65', 'ed8951a75', '371da7669', '4b9540ab3', '230a025ca', 'f8cd9ae02', 'de4e75360', '540cc3cd1', '7623d805a', 'c2dae3a5a'],
['d0d340214', '34d3715d5', '9c404d218', 'c624e6627', 'a1b169a3a', 'c144a70b1', 'b36a21d49', 'dfcf7c0fa', 'c63b4a070', '43ebb15de', '1f2a670dd', '3f07a4581', '0b1560062', 'e9f588de5', '65d14abf0', '9ed0e6ddb', '0b790ba3a', '9e89978e3', 'ee6264d2b', 'c86c0565e', '4de164057', '87ba924b1', '4d05e2995', '2c0babb55', 'e9375ad86', '8988e8da5', '8a1b76aaf', '724b993fd', '654dd8a3b', 'f423cf205', '3b54cc2cf', 'e04141e42', 'cacc1edae', '314396b31', '2c339d4f2', '3f8614071', '16d1d6204', '80b6e9a8b', 'a84cbdab5', '1a6d13c4a'],
['a9819bda9', 'ea26c7fe6', '3a89d003b', '1029d9146', '759c9e85d', '1f71b76c1', '854e37761', '56cb93fd8', '946d16369', '33e4f9a0e', '5a6a1ec1a', '4c835bd02', 'b3abb64d2', 'fe0dd1a15', 'de63b3487', 'c059f2574', 'e36687647', 'd58172aef', 'd746efbfe', 'ccf6632e6', 'f1c272f04', 'da7f4b066', '3a7771f56', '5807de036', 'b22eb2036', 'b77c707ef', 'e4e9c8cc6', 'ff3b49c1d', '800f38b6b', '9a1d8054b', '0c9b00a91', 'fe28836c3', '1f8415d03', '6a542a40a', 'd53d64307', 'e700276a2', 'bb6f50464', '988518e2d', 'f0eb7b98f', 'd7447b2c5'],
['87ffda550', '63c094ba4', '2e103d632', '1c71183bb', 'd5fa73ead', 'e078302ef', 'a6b6bc34a', 'f6eba969e', '0d51722ca', 'ce3d7595b', '6c5c8869c', 'dfd179071', '122c135ed', 'b4cfe861f', 'b7c931383', '44d5b820f', '4bcf15776', '51d4053c7', '1fe5d56b9', 'ea772e115', 'ad009c8b9', '68a945b18', '62fb56487', 'c10f31664', 'cbb673163', 'c8d582dd2', '8781e4b91', 'bd6da0cca', 'ca2b906e8', '11e12dbe8', 'bb0ce54e9', 'c0d2348b7', '77deffdf0', 'f97d9431e', 'a09a238d0', '935ca66a9', '9de83dc23', '861076e21', 'f02ecb19c', '166008929'],
['f3cf9341c', 'fa11da6df', 'd47c58fe2', '0d5215715', '555f18bd3', '134ac90df', '716e7d74d', 'c00611668', '1bf8c2597', '1f6b2bafa', '174edf08a', 'f1851d155', '5bc7ab64f', 'a61aa00b0', 'b2e82c050', '26417dec4', '53a550111', '51707c671', 'e8d9394a0', 'cbbc9c431', '6b119d8ce', 'f296082ec', 'be2e15279', '698d05d29', '38e6f8d32', '93ca30057', '7af000ac2', '1fd0a1f2a', '41bc25fef', '0df1d7b9a', '88d29cfaf', '2b2b5187e', 'bf59c51c3', 'cfe749e26', 'ad207f7bb', '11114a47a', '341daa7d1', 'a8dd5cea5', '7b672b310', 'b88e5de84'],
]<load_from_csv>
def get_nbest(prelim_predictions, crops, example, n_best_size):
seen, nbest = set() , []
for pred in prelim_predictions:
if len(nbest)>= n_best_size:
break
crop = crops[pred.crop_index]
orig_doc_start, orig_doc_end = -1, -1
if pred.start_index > 0:
if pred.end_index == -1:
tok_tokens = crop.tokens[pred.start_index: pred.start_index + 11]
else:
tok_tokens = crop.tokens[pred.start_index: pred.end_index + 1]
tok_text = " ".join(tok_tokens)
 tok_text = clean_text(tok_text)
 orig_doc_start = int(crop.token_to_orig_map[pred.start_index])
 if pred.end_index == -1:
orig_doc_end = orig_doc_start + 10
else:
orig_doc_end = int(crop.token_to_orig_map[pred.end_index])
 final_text = tok_text
if final_text in seen:
continue
else:
final_text = ""
seen.add(final_text)
 nbest.append(NbestPrediction(
 text=final_text,
start_logit=pred.start_logit, end_logit=pred.end_logit,
start_index=pred.start_index, end_index=pred.end_index,
orig_doc_start=orig_doc_start, orig_doc_end=orig_doc_end,
crop_index=pred.crop_index))
if len(nbest)in(0, 1):
nbest.insert(0, NbestPrediction(text="empty",
start_logit=0.0, end_logit=0.0,
start_index=-1, end_index=-1,
orig_doc_start=-1, orig_doc_end=-1,
crop_index=UNMAPPED))
assert len(nbest)>= 1
return nbest
pattern_1964666 = pd.read_csv('.. /input/pattern-found/pattern_1964666.66.csv')
 pattern_1166666 = pd.read_csv('.. /input/pattern-found/pattern_1166666.66.csv')
 pattern_812666 = pd.read_csv('.. /input/pattern-found/pattern_812666.66.csv')
 pattern_2002166 = pd.read_csv('.. /input/pattern-found/pattern_2002166.66.csv')
 pattern_3160000 = pd.read_csv('.. /input/pattern-found/pattern_3160000.csv')
 pattern_3255483 = pd.read_csv('.. /input/pattern-found/pattern_3255483.88.csv' )<drop_column>
def write_predictions(examples_gen, all_crops, all_results, n_best_size,
max_answer_length, output_prediction_file,
output_nbest_file, output_null_log_odds_file, verbose_logging,
short_null_score_diff, long_null_score_diff):

 logger.info("Writing predictions to: %s" % output_prediction_file)
 logger.info("Writing nbest to: %s" % output_nbest_file)
 example_index_to_crops = collections.defaultdict(list)
 for crop in all_crops:
example_index_to_crops[crop.example_index].append(crop)
 unique_id_to_result = {result.unique_id: result for result in all_results}
all_predictions = collections.OrderedDict()
all_nbest_json = collections.OrderedDict()
scores_diff_json = collections.OrderedDict()
short_num_empty, long_num_empty = 0, 0
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info(f'[{example_index}]: {short_num_empty} short and {long_num_empty} long empty')
 crops = example_index_to_crops[example_index]
short_prelim_predictions, long_prelim_predictions = [], []
for crop_index, crop in enumerate(crops):
assert crop.unique_id in unique_id_to_result, f"{crop.unique_id}"
result = unique_id_to_result[crop.unique_id]
start_indexes = np.argpartition(result.start_logits, -n_best_size)[-n_best_size:]
start_indexes = [int(x)for x in start_indexes]
end_indexes = np.argpartition(result.end_logits, -n_best_size)[-n_best_size:]
end_indexes = [int(x)for x in end_indexes]
for start_index in start_indexes:
if start_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[start_index] == UNMAPPED:
continue
if not crop.token_is_max_context[start_index]:
continue
for end_index in end_indexes:
if end_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[end_index] == UNMAPPED:
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
short_prelim_predictions.append(PrelimPrediction(
 crop_index=crop_index,
start_index=start_index,
end_index=end_index,
start_logit=result.start_logits[start_index],
end_logit=result.end_logits[end_index]))
long_indexes = np.argpartition(result.long_logits, -n_best_size)[-n_best_size:].tolist()
for long_index in long_indexes:
if long_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[long_index] == UNMAPPED:
continue
if not crop.token_is_max_context[long_index]:
continue
long_prelim_predictions.append(PrelimPrediction(
 crop_index=crop_index,
start_index=long_index, end_index=-1,
start_logit=result.long_logits[long_index],
end_logit=result.long_logits[long_index]))
short_prelim_predictions = sorted(short_prelim_predictions,
key=lambda x: x.start_logit + x.end_logit, reverse=True)
 short_nbest = get_nbest(short_prelim_predictions, crops,
example, n_best_size)
 short_best_non_null = None
for entry in short_nbest:
if short_best_non_null is None:
if entry.text != "":
short_best_non_null = entry
long_prelim_predictions = sorted(long_prelim_predictions,
key=lambda x: x.start_logit, reverse=True)
 long_nbest = get_nbest(long_prelim_predictions, crops,
example, n_best_size)
 long_best_non_null = None
for entry in long_nbest:
if long_best_non_null is None:
if entry.text != "":
long_best_non_null = entry
nbest_json = {'short': [], 'long': []}
for kk, entries in [('short', short_nbest),('long', long_nbest)]:
for i, entry in enumerate(entries):
output = {}
output["text"] = entry.text
output["start_logit"] = entry.start_logit
output["end_logit"] = entry.end_logit
output["start_index"] = entry.start_index
output["end_index"] = entry.end_index
output["orig_doc_start"] = entry.orig_doc_start
output["orig_doc_end"] = entry.orig_doc_end
nbest_json[kk].append(output)
 assert len(nbest_json['short'])>= 1
assert len(nbest_json['long'])>= 1
try:
crop_unique_id = crops[short_best_non_null.crop_index].unique_id
start_score_null = unique_id_to_result[crop_unique_id].start_logits[CLS_INDEX]
end_score_null = unique_id_to_result[crop_unique_id].end_logits[CLS_INDEX]
short_score_null = start_score_null + end_score_null
short_score_diff = short_score_null -(short_best_non_null.start_logit +
short_best_non_null.end_logit)
 if short_score_diff > short_null_score_diff:
final_pred =("", -1, -1)
 short_num_empty += 1
else:
final_pred =(short_best_non_null.text, short_best_non_null.orig_doc_start,
short_best_non_null.orig_doc_end)
 except Exception as e:
print(e)
 final_pred =("", -1, -1)
 short_num_empty += 1
try:
long_score_null = unique_id_to_result[crops[
long_best_non_null.crop_index].unique_id].long_logits[CLS_INDEX]
long_score_diff = long_score_null - long_best_non_null.start_logit
scores_diff_json[example.qas_id] = {'short_score_diff': short_score_diff,
'long_score_diff': long_score_diff}
if long_score_diff > long_null_score_diff:
final_pred +=("", -1)
 long_num_empty += 1
else:
final_pred +=(long_best_non_null.text, long_best_non_null.orig_doc_start)
 except Exception as e:
print(e)
 final_pred +=("", -1)
 long_num_empty += 1
all_predictions[example.qas_id] = final_pred
all_nbest_json[example.qas_id] = nbest_json
if output_prediction_file is not None:
with open(output_prediction_file, "w")as writer:
writer.write(json.dumps(all_predictions, indent=2))
if output_nbest_file is not None:
with open(output_nbest_file, "w")as writer:
writer.write(json.dumps(all_nbest_json, indent=2))
if output_null_log_odds_file is not None:
with open(output_null_log_odds_file, "w")as writer:
writer.write(json.dumps(scores_diff_json, indent=2))
logger.info(f'{short_num_empty} short and {long_num_empty} long empty of'
f' {example_index}')
 return all_predictions
pattern_1964666.drop(['Unnamed: 0','value_count'],axis=1,inplace=True)
 pattern_1166666.drop(['Unnamed: 0','value_count'],axis=1,inplace=True)
 pattern_812666.drop(['Unnamed: 0','value_count'],axis=1,inplace=True)
 pattern_2002166.drop(['Unnamed: 0','value_count'],axis=1,inplace=True)
 pattern_3160000.drop(['Unnamed: 0','value_count'],axis=1,inplace=True)
 pattern_3255483.drop(['Unnamed: 0','value_count'],axis=1,inplace=True )<rename_columns>
def convert_preds_to_df(preds, candidates):
num_found_long, num_searched_long = 0, 0
df = {'example_id': [], 'PredictionString': []}
for example_id, pred in preds.items() :
short_text, start_token, end_token, long_text, long_token = pred
df['example_id'].append(example_id + '_short')
 short_answer = ''
if start_token != -1:
short_answer = f'{start_token}:{end_token + 1}'
df['PredictionString'].append(short_answer)
 long_answer = ''
found_long = False
min_dist = 1_000_000
if long_token != -1:
num_searched_long += 1
for candidate in candidates[example_id]:
cstart, cend = candidate.start_token, candidate.end_token
dist = abs(cstart - long_token)
 if dist < min_dist:
min_dist = dist
if long_token == cstart:
long_answer = f'{cstart}:{cend}'
found_long = True
break
if found_long:
num_found_long += 1
else:
logger.info(f"Not found: {min_dist}")
 df['example_id'].append(example_id + '_long')
 df['PredictionString'].append(long_answer)
 df = pd.DataFrame(df)
 print(f'Found {num_found_long} of {num_searched_long}(total {len(preds)})')
 return df
pattern_1166666.rename(columns={'8.50E+43': '850027e38'},inplace=True )<data_type_conversions>
class TFBertForNaturalQuestionAnswering(TFBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.num_labels = config.num_labels
self.bert = TFBertMainLayer(config, name='bert')
 self.initializer = get_initializer(config.initializer_range)
 self.qa_outputs = L.Dense(config.num_labels,
kernel_initializer=self.initializer, name='qa_outputs')
 self.long_outputs = L.Dense(1, kernel_initializer=self.initializer,
name='long_outputs')
 def call(self, inputs, **kwargs):
outputs = self.bert(inputs, **kwargs)
 sequence_output = outputs[0]
logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, -1)
 end_logits = tf.squeeze(end_logits, -1)
 long_logits = tf.squeeze(self.long_outputs(sequence_output), -1)
 return start_logits, end_logits, long_logits
class TFRobertaForNaturalQuestionAnswering(TFRobertaPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.num_labels = config.num_labels
self.roberta = TFRobertaMainLayer(config, name='roberta')
 self.initializer = get_initializer(config.initializer_range)
 self.qa_outputs = L.Dense(config.num_labels,
kernel_initializer=self.initializer, name='qa_outputs')
 self.long_outputs = L.Dense(1, kernel_initializer=self.initializer,
name='long_outputs')
 def call(self, inputs, **kwargs):
outputs = self.roberta(inputs, **kwargs)
 sequence_output = outputs[0]
logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, -1)
 end_logits = tf.squeeze(end_logits, -1)
 long_logits = tf.squeeze(self.long_outputs(sequence_output), -1)
 return start_logits, end_logits, long_logits
l=[]
l.append(pattern_1964666.columns.values.tolist())
 l.append(pattern_1166666.columns.values.tolist())
 l.append(pattern_812666.columns.values.tolist())
 l.append(pattern_2002166.columns.values.tolist())
 l.append(pattern_3160000.columns.values.tolist())
 l.append(pattern_3255483.columns.values.tolist() )<define_variables>
def enumerate_tags(text_split):

 special_tokens = ['<P>', '<Table>']
special_token_counts = [0 for _ in range(len(special_tokens)) ]
for index, token in enumerate(text_split):
for special_token_index, special_token in enumerate(special_tokens):
if token == special_token:
cnt = special_token_counts[special_token_index]
if cnt <= 10:
text_split[index] = f'<{special_token[1: -1]}{cnt}>'
special_token_counts[special_token_index] = cnt + 1
return text_split
ss = l + colgroups<save_to_csv>
ALL_MODELS = sum(( tuple(conf.pretrained_config_archive_map.keys())
 for conf in(BertConfig,)) ,())
 MODEL_CLASSES = {
'bert':(BertConfig, TFBertForNaturalQuestionAnswering, BertTokenizer),
'roberta':(RobertaConfig, TFRobertaForNaturalQuestionAnswering, RobertaTokenizer),
}
RawResult = namedtuple("RawResult", ["unique_id", "start_logits", "end_logits",
"long_logits"])
 def set_seed(args):
random.seed(args.seed)
 np.random.seed(args.seed)
 tf.random.set_seed(args.seed )
def _get_leak(df, cols,extra_feats, lag=0):
f1 = cols[:(( lag+2)* -1)]
f2 = cols[(lag+2):]
for ef in extra_feats:
f1 += ef[:(( lag+2)* -1)]
f2 += ef[(lag+2):]
d1 = df[f1].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d1.to_csv('extra_d1.csv')
 d2 = df[f2].apply(tuple, axis=1 ).to_frame().rename(columns={0: 'key'})
 d2['pred'] = df[cols[lag]]
d3 = d2[~d2.duplicated(['key'], keep=False)]
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 d6 = d1.merge(d5, how='left', on='key')
 d6.to_csv('extra_d6.csv')
 return d1.merge(d5, how='left', on='key' ).pred.fillna(0 )<feature_engineering>
def submit(args, model, tokenizer):
csv_fn = 'submission.csv'
eval_dataset, crops, entries = load_and_cache_crops(args, tokenizer, evaluate=True)
 args.eval_batch_size = args.per_tpu_eval_batch_size
eval_dataset_length = len(eval_dataset[0])
 padded_length = math.ceil(eval_dataset_length / args.eval_batch_size)* args.eval_batch_size
num_pad = padded_length - eval_dataset[0].shape[0]
for ti, t in enumerate(eval_dataset):
pad_tensor = tf.expand_dims(tf.zeros_like(t[0]), 0)
 pad_tensor = tf.repeat(pad_tensor, num_pad, 0)
 eval_dataset[ti] = tf.concat([t, pad_tensor], 0)
 eval_ds = tf.data.Dataset.from_tensor_slices({
'input_ids': tf.constant(eval_dataset[0]),
'attention_mask': tf.constant(eval_dataset[1]),
'token_type_ids': tf.constant(eval_dataset[2]),
'example_index': tf.range(padded_length, dtype=tf.int32)
 })
 eval_ds = eval_ds.batch(batch_size=args.eval_batch_size, drop_remainder=True)
 print("***** Running evaluation *****")
 print(" Num examples = %d", eval_dataset_length)
 print(" Batch size = %d", args.eval_batch_size)
 @tf.function
def predict_step(batch):
outputs = model(batch, training=False)
 return outputs
all_results = []
tic = time.time()
for batch_ind, batch in tqdm(enumerate(eval_ds), total=padded_length // args.per_tpu_eval_batch_size):
example_indexes = batch['example_index']
outputs = predict_step(batch)
 batched_start_logits = outputs[0].numpy()
batched_end_logits = outputs[1].numpy()
batched_long_logits = outputs[2].numpy()
for i, example_index in enumerate(example_indexes):
if example_index >= eval_dataset_length:
continue
eval_crop = crops[example_index]
unique_id = int(eval_crop.unique_id)
 start_logits = batched_start_logits[i].tolist()
end_logits = batched_end_logits[i].tolist()
long_logits = batched_long_logits[i].tolist()
result = RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits,
long_logits=long_logits)
 all_results.append(result)
 eval_time = time.time() - tic
print(" Evaluation done in total %f secs(%f sec per example)",
eval_time, eval_time / padded_length)
 examples_gen = read_nq_examples(entries, is_training=False)
 preds = write_predictions(examples_gen, crops, all_results, args.n_best_size,
args.max_answer_length,
None, None, None,
args.verbose_logging,
args.short_null_score_diff_threshold, args.long_null_score_diff_threshold)
 del crops, all_results
gc.collect()
candidates = read_candidates(['.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'], do_cache=False)
 sub = convert_preds_to_df(preds, candidates ).sort_values('example_id')
 sub.to_csv(csv_fn, index=False, columns=['example_id', 'PredictionString'])
 print(f'***** Wrote submission to {csv_fn} *****')
 result = {}
return result
def compiled_leak_result() :
max_nlags = len(cols)-2
train_leak = train[["ID", "target"] + cols]
train_leak["compiled_leak"] = 0
train_leak["nonzero_mean"] = train[transact_cols].apply(
 lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 
)
 scores = []
leaky_value_counts = []
leaky_value_corrects = []
leaky_cols = []
for i in range(max_nlags):
c = "leaked_target_"+str(i)
 print('Processing lag', i)
 train_leak[c] = _get_leak(train, cols,ss, i)
 leaky_cols.append(c)
 train_leak = train.join(
 train_leak.set_index("ID")[leaky_cols+["compiled_leak", "nonzero_mean"]],
on="ID", how="left" 
)[["ID", "target"] + cols + leaky_cols+["compiled_leak", "nonzero_mean"]]
zeroleak = train_leak["compiled_leak"]==0
train_leak.loc[zeroleak, "compiled_leak"] = train_leak.loc[zeroleak, c]
leaky_value_counts.append(sum(train_leak["compiled_leak"] > 0))
_correct_counts = sum(train_leak["compiled_leak"]==train_leak["target"])
 leaky_value_corrects.append(_correct_counts*1.0/leaky_value_counts[-1])
 print("Leak values found in train", leaky_value_counts[-1])
 print(
 "% of correct leaks values in train ",
leaky_value_corrects[-1] 
)
 tmp = train_leak.copy()
tmp.loc[zeroleak, "compiled_leak"] = tmp.loc[zeroleak, "nonzero_mean"]
print('Na count',tmp.compiled_leak.isna().sum())
 scores.append(np.sqrt(mean_squared_error(y, np.log1p(tmp["compiled_leak"] ).fillna(14.49))))
print(
 'Score(filled with nonzero mean)',
scores[-1] 
)
 result = dict(
 score=scores,
leaky_count=leaky_value_counts,
leaky_correct=leaky_value_corrects, 
)
 return train_leak, result<create_dataframe>
def get_convert_args() :
convert_args = argparse.Namespace()
convert_args.fn = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
convert_args.version = 'v0.0.1'
convert_args.prefix = 'nq'
convert_args.num_samples = 1_000_000
convert_args.val_ids = None
convert_args.do_enumerate = False
convert_args.do_not_dump = True
convert_args.num_max_tokens = 400_000
return convert_args
<save_to_csv><EOS> 
def load_and_cache_crops(args, tokenizer, evaluate=False):
do_cache = False
cached_crops_fn = 'cached_{}_{}.pkl'.format('test', str(args.max_seq_length))
if os.path.exists(cached_crops_fn)and do_cache:
print("Loading crops from cached file %s", cached_crops_fn)
 with open(cached_crops_fn, "rb")as f:
crops = pickle.load(f)
 else:
entries = convert_nq_to_squad(get_convert_args())
 examples_gen = read_nq_examples(entries, is_training=not evaluate)
 crops = convert_examples_to_crops(examples_gen=examples_gen,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=not evaluate,
cls_token_segment_id=0,
pad_token_segment_id=0,
p_keep_impossible=args.p_keep_impossible if not evaluate else 1.0)
 if do_cache:
with open(cached_crops_fn, "wb")as f:
pickle.dump(crops, f)
 all_input_ids = tf.stack([c.input_ids for c in crops], 0)
 all_attention_mask = tf.stack([c.attention_mask for c in crops], 0)
 all_token_type_ids = tf.stack([c.token_type_ids for c in crops], 0)
 all_attention_mask = tf.cast(all_attention_mask, tf.int32)
 if evaluate:
dataset = [all_input_ids, all_attention_mask, all_token_type_ids]
else:
all_start_positions = tf.convert_to_tensor([f.start_position for f in crops], dtype=tf.int32)
 all_end_positions = tf.convert_to_tensor([f.end_position for f in crops], dtype=tf.int32)
 all_long_positions = tf.convert_to_tensor([f.long_position for f in crops], dtype=tf.int32)
 dataset = [all_input_ids, all_attention_mask, all_token_type_ids,
all_start_positions, all_end_positions, all_long_positions]
return dataset, crops, entries