Kaggle competition: TensorFlow 2.0 Question Answering
<SOS> metric: NQMicroF1 Kaggle data source: tensorflow-2.0-question-answering<set_options>
!pip install --upgrade.. /input/tokenizers-repo/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl
%matplotlib inline
!pip install pycountry-convert
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.log_device_placement = True
sess = tf.Session(config=config)
 set_session(sess )<define_variables>
batch_size = 340*3
valid_percent = 1
data_files = 100
size = 64
num_classes = 340
class_paths = os.listdir(".. /input/quickdraw-doodle-recognition/train_simplified/")
 cat_names = [item[:-4] for item in class_paths]
cat_names.sort(key=lambda x: str.lower(x))
UNKNOWN_COUNTRY = 'YY'
class ContextMode(Enum):
no_context = 1
area_context = 2<define_variables>
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
valid_countries_dict = pycountry_convert.map_countries(cn_name_format="default")
 valid_country_codes = list(set([v['alpha_2'] for k, v in valid_countries_dict.items() ]))
all_country_codes= set(valid_country_codes)| set([UNKNOWN_COUNTRY] )<concatenate>
UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',
def ValidateCountry(country_code):
return country_code if country_code in all_country_codes else UNKNOWN_COUNTRY<load_from_csv>
def find_closing_tag(tokens, opening_tag):
closing_tag = f'</{opening_tag[1: -1]}>'
index, stack = -1, []
for token_index, token in enumerate(tokens):
if token == opening_tag:
stack.insert(0, opening_tag)
 elif token == closing_tag:
stack.pop()
if len(stack)== 0:
index = token_index
break
return index
our_mapping = pd.read_csv('.. /input/area-context-country-clustering/area_mapping.csv')
 print(our_mapping.head())
 our_dict = pd.Series(our_mapping.groups.values, index = our_mapping.alpha3 ).to_dict()
def get_area(iso_code):
try:
return our_dict[iso_code]
except KeyError:
return 0<define_variables>
def read_candidates(candidate_files, do_cache=True):
assert isinstance(candidate_files,(tuple, list)) , candidate_files
for fn in candidate_files:
assert os.path.exists(fn), f'Missing file {fn}'
cache_fn = 'candidates.pkl'
candidates = {}
if not os.path.exists(cache_fn):
for fn in candidate_files:
with open(fn)as f:
for line in tqdm(f):
entry = json.loads(line)
 example_id = str(entry['example_id'])
 cnds = entry.pop('long_answer_candidates')
 cnds = [LongAnswerCandidate(c['start_token'], c['end_token'],
c['top_level'])for c in cnds]
candidates[example_id] = cnds
if do_cache:
with open(cache_fn, 'wb')as f:
pickle.dump(candidates, f)
 else:
print(f'Loading from cache: {cache_fn}')
 with open(cache_fn, 'rb')as f:
candidates = pickle.load(f)
 return candidates
num_groups = max(our_mapping.groups)+1
print(num_groups )<define_variables>
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" or ord(c)== 0x202F:
return True
return False
def image_generator(batch_size, isTraining):
while True:
if isTraining:
index_table = np.random.permutation(data_files-valid_percent)
 else:
index_table = np.random.permutation(valid_percent)+(data_files-valid_percent)
 _path = '.. /input/shuffle-and-filter/'
for k in index_table:
filename = os.path.join(_path,'train_k{}.csv.gz'.format(k))
for chunk in pd.read_csv(filename, chunksize=batch_size,
usecols=['countrycode','y','drawing','word'],
keep_default_na=False):
ch_size = len(chunk)
 if(ch_size<batch_size):
continue
else:
yield chunk<load_from_csv>
def read_nq_examples(input_file_or_data, is_training):

 if isinstance(input_file_or_data, str):
with open(input_file_or_data, "r", encoding='utf-8')as f:
input_data = json.load(f)["data"]
else:
input_data = input_file_or_data
for entry_index, entry in enumerate(tqdm(input_data, total=len(input_data))):
assert len(entry["paragraphs"])== 1
paragraph = entry["paragraphs"][0]
paragraph_text = paragraph["context"]
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in paragraph_text:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 assert len(paragraph["qas"])== 1
qa = paragraph["qas"][0]
start_position = None
end_position = None
long_position = None
orig_answer_text = None
short_is_impossible = False
long_is_impossible = False
if is_training:
short_is_impossible = qa["short_is_impossible"]
short_answers = qa["short_answers"]
if len(short_answers)>= 2:
short_answers = sorted(short_answers, key=lambda sa: sa["answer_start"])
 short_answers = short_answers[0: 1]
if not short_is_impossible:
answer = short_answers[0]
orig_answer_text = answer["text"]
answer_offset = answer["answer_start"]
answer_length = len(orig_answer_text)
 start_position = char_to_word_offset[answer_offset]
end_position = char_to_word_offset[
answer_offset + answer_length - 1]
actual_text = " ".join(doc_tokens[start_position:
end_position + 1])
 cleaned_answer_text = " ".join(
 whitespace_tokenize(orig_answer_text))
if actual_text.find(cleaned_answer_text)== -1:
logger.warning(
 "Could not find answer: '%s' vs.'%s'",
actual_text, cleaned_answer_text)
 continue
else:
start_position = -1
end_position = -1
orig_answer_text = ""
long_is_impossible = qa["long_is_impossible"]
long_answers = qa["long_answers"]
if(len(long_answers)!= 1)and not long_is_impossible:
raise ValueError(f"For training, each question"
f" should have exactly 1 long answer.")
 if not long_is_impossible:
long_answer = long_answers[0]
long_answer_offset = long_answer["answer_start"]
long_position = char_to_word_offset[long_answer_offset]
else:
long_position = -1
if not short_is_impossible and not long_is_impossible:
assert long_position <= start_position
if not short_is_impossible and long_is_impossible:
assert False, f'Invalid pair short, long pair'
example = NQExample(
 qas_id=qa["id"],
question_text=qa["question"],
doc_tokens=doc_tokens,
orig_answer_text=orig_answer_text,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible,
crop_start=qa["crop_start"])
 yield example
DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
def test_gen() :
_path = '.. /input/quickdraw-doodle-recognition/test_simplified.csv'
while True:
for _chunk in pd.read_csv(_path, chunksize=batch_size,
usecols=['drawing','countrycode'],
keep_default_na=False):
if(len(_chunk)<batch_size):
_chunk = pd.concat([_chunk, _chunk[0:batch_size-len(_chunk)]])
 yield _chunk
gen_TEST = test_gen()<categorify>
def get_spans(doc_stride, max_tokens_for_doc, max_len):
doc_spans = []
start_offset = 0
while start_offset < max_len:
length = max_len - start_offset
if length > max_tokens_for_doc:
length = max_tokens_for_doc
doc_spans.append(DocSpan(start=start_offset, length=length))
if start_offset + length == max_len:
break
start_offset += min(length, doc_stride)
 return doc_spans
areas_oneh = OneHotEncoder().fit(np.arange(num_groups ).reshape(-1,1))<choose_model_class>
def convert_examples_to_crops(examples_gen, tokenizer, max_seq_length,
doc_stride, max_query_length, is_training,
cls_token='[CLS]', sep_token='[SEP]', pad_id=0,
sequence_a_segment_id=0,
sequence_b_segment_id=1,
cls_token_segment_id=0,
pad_token_segment_id=0,
mask_padding_with_zero=True,
p_keep_impossible=None,
sep_token_extra=False):

 assert p_keep_impossible is not None, '`p_keep_impossible` is required'
unique_id = 1000000000
num_short_pos, num_short_neg = 0, 0
num_long_pos, num_long_neg = 0, 0
sub_token_cache = {}
crops = []
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info('Converting %s: short_pos %s short_neg %s'
' long_pos %s long_neg %s',
example_index, num_short_pos, num_short_neg,
num_long_pos, num_long_neg)
 query_tokens = tokenizer.tokenize(example.question_text)
 if len(query_tokens)> max_query_length:
query_tokens = query_tokens[0:max_query_length]
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for i, token in enumerate(example.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = sub_token_cache.get(token)
 if sub_tokens is None:
sub_tokens = tokenizer.tokenize(token)
 sub_token_cache[token] = sub_tokens
tok_to_orig_index.extend([i for _ in range(len(sub_tokens)) ])
 all_doc_tokens.extend(sub_tokens)
 tok_start_position = None
tok_end_position = None
if is_training and example.short_is_impossible:
tok_start_position = -1
tok_end_position = -1
if is_training and not example.short_is_impossible:
tok_start_position = orig_to_tok_index[example.start_position]
if example.end_position < len(example.doc_tokens)- 1:
tok_end_position = orig_to_tok_index[
example.end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
tok_long_position = None
if is_training and example.long_is_impossible:
tok_long_position = -1
if is_training and not example.long_is_impossible:
tok_long_position = orig_to_tok_index[example.long_position]
special_tokens_count = 3
if sep_token_extra:
special_tokens_count += 1
max_tokens_for_doc = max_seq_length - len(query_tokens)- special_tokens_count
assert max_tokens_for_doc > 0
doc_spans = get_spans(doc_stride, max_tokens_for_doc, len(all_doc_tokens))
for doc_span_index, doc_span in enumerate(doc_spans):
tokens = []
token_to_orig_map = UNMAPPED * np.ones(( max_seq_length,), dtype=np.int32)
 token_is_max_context = np.zeros(( max_seq_length,), dtype=np.bool)
 token_type_ids = []
short_is_impossible = example.short_is_impossible
start_position = None
end_position = None
special_tokens_offset = special_tokens_count - 1
doc_offset = len(query_tokens)+ special_tokens_offset
if is_training and not short_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_start_position >= doc_start and tok_end_position <= doc_end):
start_position = 0
end_position = 0
short_is_impossible = True
else:
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
long_is_impossible = example.long_is_impossible
long_position = None
if is_training and not long_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_long_position >= doc_start and tok_long_position <= doc_end):
long_position = 0
long_is_impossible = True
else:
long_position = tok_long_position - doc_start + doc_offset
if long_is_impossible:
if np.random.rand() > p_keep_impossible:
continue
tokens.append(cls_token)
 token_type_ids.append(cls_token_segment_id)
 tokens += query_tokens
token_type_ids += [sequence_a_segment_id] * len(query_tokens)
 tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 if sep_token_extra:
tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[
split_token_index] + example.crop_start
token_is_max_context[len(tokens)] = check_is_max_context(doc_spans,
doc_span_index, split_token_index)
 tokens.append(all_doc_tokens[split_token_index])
 token_type_ids.append(sequence_b_segment_id)
 paragraph_len = doc_span.length
tokens.append(sep_token)
 token_type_ids.append(sequence_b_segment_id)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
 while len(input_ids)< max_seq_length:
input_ids.append(pad_id)
 attention_mask.append(0 if mask_padding_with_zero else 1)
 token_type_ids.append(pad_token_segment_id)
 input_ids = np.array(input_ids, dtype=np.int32)
 attention_mask = np.array(attention_mask, dtype=np.bool)
 token_type_ids = np.array(token_type_ids, dtype=np.uint8)
 if is_training and short_is_impossible:
start_position = CLS_INDEX
end_position = CLS_INDEX
if is_training and long_is_impossible:
long_position = CLS_INDEX
if example_index in(0, 10):
if doc_span_index in(0, 5):
logger.info("*** Example ***")
 logger.info("unique_id: %s" %(unique_id))
logger.info("example_index: %s" %(example_index))
logger.info("doc_span_index: %s" %(doc_span_index))
logger.info("tokens: %s" % " ".join(tokens))
logger.info("input_ids: %s" % input_ids)
 logger.info("attention_mask: %s" % np.uint8(attention_mask))
logger.info("token_type_ids: %s" % token_type_ids)
 if is_training and short_is_impossible:
logger.info("short impossible example")
 if is_training and long_is_impossible:
logger.info("long impossible example")
 if is_training and not short_is_impossible:
answer_text = " ".join(tokens[start_position: end_position + 1])
 logger.info("start_position: %d" %(start_position))
logger.info("end_position: %d" %(end_position))
logger.info("answer: %s" %(answer_text))
if short_is_impossible:
num_short_neg += 1
else:
num_short_pos += 1
if long_is_impossible:
num_long_neg += 1
else:
num_long_pos += 1
crop = Crop(
 unique_id=unique_id,
example_index=example_index,
doc_span_index=doc_span_index,
tokens=tokens,
token_to_orig_map=token_to_orig_map,
token_is_max_context=token_is_max_context,
input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
paragraph_len=paragraph_len,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible)
 crops.append(crop)
 unique_id += 1
return crops
class GeneratorMode(Enum):
training = 1
validation = 2
testing = 3
def main_generator(whatGeneratorMode, whatContextMode=ContextMode.no_context):
while True:
if whatGeneratorMode==GeneratorMode.training:
results = next(train_generator)
 elif whatGeneratorMode==GeneratorMode.validation:
results = next(valid_generator)
 elif whatGeneratorMode==GeneratorMode.testing:
results = next(gen_TEST)
 results['drawing'] = results['drawing'].apply(json.loads)
 x = np.zeros(( batch_size, size, size))
for i, raw_strokes in enumerate(results.drawing.values):
x[i] = draw_cv2(raw_strokes, size=size)
 x = x / 255.
 x = x.reshape(( batch_size, size, size, 1)).astype(np.float32)
 if whatGeneratorMode!=GeneratorMode.testing:
y = keras.utils.to_categorical(results.y, num_classes=num_classes)
 if whatContextMode==ContextMode.no_context:
if whatGeneratorMode==GeneratorMode.testing:
yield x
else:
yield x, y
elif whatContextMode==ContextMode.area_context:
_countries = results['countrycode'].apply(ValidateCountry ).values
_areas = np.array(list(map(get_area, _countries)))
 xac = areas_oneh.transform(_areas.reshape(-1,1)).toarray()
if whatGeneratorMode==GeneratorMode.testing:
yield [x, xac]
else:
yield [x, xac], y<compute_test_metric>
PrelimPrediction = collections.namedtuple("PrelimPrediction",
["crop_index", "start_index", "end_index", "start_logit", "end_logit"])
 NbestPrediction = collections.namedtuple("NbestPrediction", [
"text", "start_logit", "end_logit",
"start_index", "end_index",
"orig_doc_start", "orig_doc_end", "crop_index"])
def in_top_3(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<choose_model_class>
def clean_text(tok_text):
tok_text = tok_text.replace("
tok_text = tok_text.replace("
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
 return tok_text
def create_model(whatContextMode=ContextMode.no_context, alpha=1.0, show=False):
K.clear_session()
if whatContextMode.value==ContextMode.no_context.value:
my_model = MobileNet(input_shape=(size, size, 1), weights=None, alpha=alpha, classes=num_classes)
 my_model.compile(optimizer=Adam() , loss='categorical_crossentropy',
metrics=['accuracy',in_top_3])
 elif whatContextMode.value==ContextMode.area_context.value:
dropout=0.001
shape0 =(1, 1, int(1024*alpha))
shape1 =(1, 1, int(1024*alpha+num_groups))
Input_area = Input(shape=(num_groups,))
my_model = MobileNet(input_shape=(size, size, 1), weights=None, alpha=alpha, include_top=False)
 x = my_model.get_layer('conv_pw_13_relu' ).output
x = GlobalAveragePooling2D()(x)
 x = Reshape(shape0, name='reshape_0' )(x)
 x = Flatten()(x)
 x = concatenate([x, Input_area])
 x = Reshape(shape1, name='reshape_1' )(x)
 x = Dropout(dropout, name='dropout' )(x)
 x = Conv2D(num_classes,(1, 1), padding='same', name='conv_preds' )(x)
 x = Activation('softmax', name='act_softmax' )(x)
 preds = Reshape(( num_classes,), name='reshape_2' )(x)
 my_model = Model(inputs=[my_model.input, Input_area], outputs=preds)
 my_model.compile(optimizer=Adam() , loss='categorical_crossentropy',
metrics=['accuracy',in_top_3])
 else:
print('Context not recognized')
 return
if show:
my_model.summary()
return my_model<choose_model_class>
def get_nbest(prelim_predictions, crops, example, n_best_size):
seen, nbest = set() , []
for pred in prelim_predictions:
if len(nbest)>= n_best_size:
break
crop = crops[pred.crop_index]
orig_doc_start, orig_doc_end = -1, -1
orig_doc_start, orig_doc_end = -1, -1
if pred.start_index > 0:
if pred.end_index == -1:
tok_tokens = crop.tokens[pred.start_index: pred.start_index + 11]
else:
tok_tokens = crop.tokens[pred.start_index: pred.end_index + 1]
tok_text = " ".join(tok_tokens)
 tok_text = clean_text(tok_text)
 orig_doc_start = int(crop.token_to_orig_map[pred.start_index])
 if pred.end_index == -1:
orig_doc_end = orig_doc_start + 10
else:
orig_doc_end = int(crop.token_to_orig_map[pred.end_index])
 final_text = tok_text
if final_text in seen:
continue
else:
final_text = ""
seen.add(final_text)
 nbest.append(NbestPrediction(
 text=final_text,
start_logit=pred.start_logit, end_logit=pred.end_logit,
start_index=pred.start_index, end_index=pred.end_index,
orig_doc_start=orig_doc_start, orig_doc_end=orig_doc_end,
crop_index=pred.crop_index))
if len(nbest)in(0, 1):
nbest.insert(0, NbestPrediction(text="empty",
start_logit=0.0, end_logit=0.0,
start_index=-1, end_index=-1,
orig_doc_start=-1, orig_doc_end=-1,
crop_index=UNMAPPED))
assert len(nbest)>= 1
return nbest
ourContextMode=ContextMode.area_context
create_model(whatContextMode=ourContextMode, alpha=0.5, show=True )<define_variables>
def write_predictions(examples_gen, all_crops, all_results, n_best_size,
max_answer_length, output_prediction_file,
output_nbest_file, output_null_log_odds_file, verbose_logging,
short_null_score_diff, long_null_score_diff):

 logger.info("Writing predictions to: %s" % output_prediction_file)
 logger.info("Writing nbest to: %s" % output_nbest_file)
 example_index_to_crops = collections.defaultdict(list)
 for crop in all_crops:
example_index_to_crops[crop.example_index].append(crop)
 unique_id_to_result = {result.unique_id: result for result in all_results}
all_predictions = collections.OrderedDict()
all_nbest_json = collections.OrderedDict()
scores_diff_json = collections.OrderedDict()
short_num_empty, long_num_empty = 0, 0
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info(f'[{example_index}]: {short_num_empty} short and {long_num_empty} long empty')
 crops = example_index_to_crops[example_index]
short_prelim_predictions, long_prelim_predictions = [], []
for crop_index, crop in enumerate(crops):
assert crop.unique_id in unique_id_to_result, f"{crop.unique_id}"
result = unique_id_to_result[crop.unique_id]
start_indexes = np.argpartition(result.start_logits, -n_best_size)[-n_best_size:]
start_indexes = [int(x)for x in start_indexes]
end_indexes = np.argpartition(result.end_logits, -n_best_size)[-n_best_size:]
end_indexes = [int(x)for x in end_indexes]
for start_index in start_indexes:
if start_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[start_index] == UNMAPPED:
continue
if not crop.token_is_max_context[start_index]:
continue
for end_index in end_indexes:
if end_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[end_index] == UNMAPPED:
continue
if end_index < start_index:
continue
length = end_index - start_index + 1
if length > max_answer_length:
continue
short_prelim_predictions.append(PrelimPrediction(
 crop_index=crop_index,
start_index=start_index,
end_index=end_index,
start_logit=result.start_logits[start_index],
end_logit=result.end_logits[end_index]))
long_indexes = np.argpartition(result.long_logits, -n_best_size)[-n_best_size:].tolist()
for long_index in long_indexes:
if long_index >= len(crop.tokens):
continue
if crop.token_to_orig_map[long_index] == UNMAPPED:
continue
if not crop.token_is_max_context[long_index]:
continue
long_prelim_predictions.append(PrelimPrediction(
 crop_index=crop_index,
start_index=long_index, end_index=-1,
start_logit=result.long_logits[long_index],
end_logit=result.long_logits[long_index]))
short_prelim_predictions = sorted(short_prelim_predictions,
key=lambda x: x.start_logit + x.end_logit, reverse=True)
 short_nbest = get_nbest(short_prelim_predictions, crops,
example, n_best_size)
 short_best_non_null = None
for entry in short_nbest:
if short_best_non_null is None:
if entry.text != "":
short_best_non_null = entry
long_prelim_predictions = sorted(long_prelim_predictions,
key=lambda x: x.start_logit, reverse=True)
 long_nbest = get_nbest(long_prelim_predictions, crops,
example, n_best_size)
 long_best_non_null = None
for entry in long_nbest:
if long_best_non_null is None:
if entry.text != "":
long_best_non_null = entry
nbest_json = {'short': [], 'long': []}
for kk, entries in [('short', short_nbest),('long', long_nbest)]:
for i, entry in enumerate(entries):
output = {}
output["text"] = entry.text
output["start_logit"] = entry.start_logit
output["end_logit"] = entry.end_logit
output["start_index"] = entry.start_index
output["end_index"] = entry.end_index
output["orig_doc_start"] = entry.orig_doc_start
output["orig_doc_end"] = entry.orig_doc_end
nbest_json[kk].append(output)
 assert len(nbest_json['short'])>= 1
assert len(nbest_json['long'])>= 1
try:
crop_unique_id = crops[short_best_non_null.crop_index].unique_id
start_score_null = unique_id_to_result[crop_unique_id].start_logits[CLS_INDEX]
end_score_null = unique_id_to_result[crop_unique_id].end_logits[CLS_INDEX]
short_score_null = start_score_null + end_score_null
short_score_diff = short_score_null -(short_best_non_null.start_logit +
short_best_non_null.end_logit)
 if short_score_diff > short_null_score_diff:
final_pred =("", -1, -1)
 short_num_empty += 1
else:
final_pred =(short_best_non_null.text, short_best_non_null.orig_doc_start,
short_best_non_null.orig_doc_end)
 except Exception as e:
print(e)
 final_pred =("", -1, -1)
 short_num_empty += 1
try:
long_score_null = unique_id_to_result[crops[
long_best_non_null.crop_index].unique_id].long_logits[CLS_INDEX]
long_score_diff = long_score_null - long_best_non_null.start_logit
scores_diff_json[example.qas_id] = {'short_score_diff': short_score_diff,
'long_score_diff': long_score_diff}
if long_score_diff > long_null_score_diff:
final_pred +=("", -1)
 long_num_empty += 1
else:
final_pred +=(long_best_non_null.text, long_best_non_null.orig_doc_start)
 except Exception as e:
print(e)
 final_pred +=("", -1)
 long_num_empty += 1
all_predictions[example.qas_id] = final_pred
all_nbest_json[example.qas_id] = nbest_json
if output_prediction_file is not None:
with open(output_prediction_file, "w")as writer:
writer.write(json.dumps(all_predictions, indent=2))
if output_nbest_file is not None:
with open(output_nbest_file, "w")as writer:
writer.write(json.dumps(all_nbest_json, indent=2))
if output_null_log_odds_file is not None:
with open(output_null_log_odds_file, "w")as writer:
writer.write(json.dumps(scores_diff_json, indent=2))
logger.info(f'{short_num_empty} short and {long_num_empty} long empty of'
f' {example_index}')
 return all_predictions
nseed(2018)
 set_random_seed(2018 )<load_pretrained>
def convert_preds_to_df(preds, candidates):
num_found_long, num_searched_long = 0, 0
df = {'example_id': [], 'PredictionString': []}
for example_id, pred in preds.items() :
short_text, start_token, end_token, long_text, long_token = pred
df['example_id'].append(example_id + '_short')
 short_answer = ''
if start_token != -1:
short_answer = f'{start_token}:{end_token + 1}'
df['PredictionString'].append(short_answer)
 long_answer = ''
found_long = False
min_dist = 1_000_000
if long_token != -1:
num_searched_long += 1
for candidate in candidates[example_id]:
cstart, cend = candidate.start_token, candidate.end_token
dist = abs(cstart - long_token)
 if dist < min_dist:
min_dist = dist
if long_token == cstart:
long_answer = f'{cstart}:{cend}'
found_long = True
break
if found_long:
num_found_long += 1
else:
logger.info(f"Not found: {min_dist}")
 df['example_id'].append(example_id + '_long')
 df['PredictionString'].append(long_answer)
 df = pd.DataFrame(df)
 print(f'Found {num_found_long} of {num_searched_long}(total {len(preds)})')
 return df
ourContextMode = ContextMode.area_context
train_generator = image_generator(batch_size=batch_size, isTraining=True)
 valid_generator = image_generator(batch_size=batch_size, isTraining=False)
 train_gen=main_generator(whatGeneratorMode=GeneratorMode.training,
whatContextMode=ourContextMode)
 valid_gen=main_generator(whatGeneratorMode=GeneratorMode.validation,
whatContextMode=ourContextMode)
 model = create_model(whatContextMode=ourContextMode )<train_model>
import tensorflow as tf
from tensorflow.keras import layers as L
from transformers import TFBertMainLayer, TFBertPreTrainedModel, TFRobertaMainLayer, TFRobertaPreTrainedModel
from transformers.modeling_tf_utils import get_initializer
train_epoch=44
train_steps=1000
validation_steps=34
cb = [ReduceLROnPlateau(monitor='val_acc', factor=0.5,
patience=5, mode='max', cooldown=3, verbose=0)]
history = model.fit_generator(
 generator=train_gen, steps_per_epoch=train_steps,
validation_data=valid_gen, validation_steps=validation_steps,
callbacks=cb,
epochs=train_epoch, verbose=1 )<predict_on_test>
class TFBertForNaturalQuestionAnswering(TFBertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.num_labels = config.num_labels
self.bert = TFBertMainLayer(config, name='bert')
 self.initializer = get_initializer(config.initializer_range)
 self.qa_outputs = L.Dense(config.num_labels,
kernel_initializer=self.initializer, name='qa_outputs')
 self.long_outputs = L.Dense(1, kernel_initializer=self.initializer,
name='long_outputs')
 def call(self, inputs, **kwargs):
outputs = self.bert(inputs, **kwargs)
 sequence_output = outputs[0]
logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, -1)
 end_logits = tf.squeeze(end_logits, -1)
 long_logits = tf.squeeze(self.long_outputs(sequence_output), -1)
 return start_logits, end_logits, long_logits
_test_gen = main_generator(whatGeneratorMode=GeneratorMode.testing,
whatContextMode=ourContextMode)
 predictions = model.predict_generator(_test_gen, steps=np.ceil(112199/batch_size), verbose=1 )<load_from_csv>
class TFRobertaForNaturalQuestionAnswering(TFRobertaPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.num_labels = config.num_labels
self.roberta = TFRobertaMainLayer(config, name='roberta')
 self.initializer = get_initializer(config.initializer_range)
 self.qa_outputs = L.Dense(config.num_labels,
kernel_initializer=self.initializer, name='qa_outputs')
 self.long_outputs = L.Dense(1, kernel_initializer=self.initializer,
name='long_outputs')
 def call(self, inputs, **kwargs):
outputs = self.roberta(inputs, **kwargs)
 sequence_output = outputs[0]
logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, -1)
 end_logits = tf.squeeze(end_logits, -1)
 long_logits = tf.squeeze(self.long_outputs(sequence_output), -1)
 return start_logits, end_logits, long_logits
def top3cats(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3],
columns=['word','word2','word3'])
 test_simp = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv',
nrows=np.shape(predictions)[0])
 categories_dict = {_id: cat_name.replace(' ', '_')for _id, cat_name in enumerate(cat_names)}
output = top3cats(predictions ).replace(categories_dict)
 output =(output.word + ' ' ).str.cat([output.word2 + ' ', output.word3])
 output.head()<concatenate>
import json
import numpy as np
import argparse
import pandas as pd
from tqdm.notebook import tqdm
output = pd.concat([test_simp['key_id'], output.loc[0:112198]], axis=1)
 output.tail()<save_to_csv>
def enumerate_tags(text_split):

 special_tokens = ['<P>', '<Table>']
special_token_counts = [0 for _ in range(len(special_tokens)) ]
for index, token in enumerate(text_split):
for special_token_index, special_token in enumerate(special_tokens):
if token == special_token:
cnt = special_token_counts[special_token_index]
if cnt <= 10:
text_split[index] = f'<{special_token[1: -1]}{cnt}>'
special_token_counts[special_token_index] = cnt + 1
return text_split
output.to_csv(ourContextMode.name + '_myResults.csv',index=False )<define_variables>
ALL_MODELS = sum(( tuple(conf.pretrained_config_archive_map.keys())
 for conf in(BertConfig,)) ,() )
LABELS = ["HasDetections"] 
<load_from_csv>
MODEL_CLASSES = {
'bert':(BertConfig, TFBertForNaturalQuestionAnswering, BertTokenizer),
'roberta':(RobertaConfig, TFRobertaForNaturalQuestionAnswering, RobertaTokenizer),
}
predict_list = []
predict_list.append(pd.read_csv(".. /input/ensmbl2/submission.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/blending-0-697/ens_sub_v12.csv")[LABELS].values)
<save_to_csv>
RawResult = namedtuple("RawResult", ["unique_id", "start_logits", "end_logits",
"long_logits"] )
print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('super_blend.csv', index=False )<set_options>
def set_seed(args):
random.seed(args.seed)
 np.random.seed(args.seed)
 tf.random.set_seed(args.seed )
%matplotlib inline
plt.style.use('ggplot')
 py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore")
 logging.basicConfig(filename='log.txt',level=logging.DEBUG, format='%(asctime)s %(message)s')
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 print(os.listdir(".. /input/")) 
<define_variables>
def submit(args, model, tokenizer):
csv_fn = 'submission.csv'
eval_dataset, crops, entries = load_and_cache_crops(args, tokenizer, evaluate=True)
 args.eval_batch_size = args.per_tpu_eval_batch_size
eval_dataset_length = len(eval_dataset[0])
 padded_length = math.ceil(eval_dataset_length / args.eval_batch_size)* args.eval_batch_size
num_pad = padded_length - eval_dataset[0].shape[0]
for ti, t in enumerate(eval_dataset):
pad_tensor = tf.expand_dims(tf.zeros_like(t[0]), 0)
 pad_tensor = tf.repeat(pad_tensor, num_pad, 0)
 eval_dataset[ti] = tf.concat([t, pad_tensor], 0)
 eval_ds = tf.data.Dataset.from_tensor_slices({
'input_ids': tf.constant(eval_dataset[0]),
'attention_mask': tf.constant(eval_dataset[1]),
'token_type_ids': tf.constant(eval_dataset[2]),
'example_index': tf.range(padded_length, dtype=tf.int32)
 })
 eval_ds = eval_ds.batch(batch_size=args.eval_batch_size, drop_remainder=True)
 print("***** Running evaluation *****")
 print(" Num examples = %d", eval_dataset_length)
 print(" Batch size = %d", args.eval_batch_size)
 @tf.function
def predict_step(batch):
outputs = model(batch, training=False)
 return outputs
all_results = []
tic = time.time()
for batch_ind, batch in tqdm(enumerate(eval_ds), total=padded_length // args.per_tpu_eval_batch_size):
example_indexes = batch['example_index']
outputs = predict_step(batch)
 batched_start_logits = outputs[0].numpy()
batched_end_logits = outputs[1].numpy()
batched_long_logits = outputs[2].numpy()
for i, example_index in enumerate(example_indexes):
if example_index >= eval_dataset_length:
continue
eval_crop = crops[example_index]
unique_id = int(eval_crop.unique_id)
 start_logits = batched_start_logits[i].tolist()
end_logits = batched_end_logits[i].tolist()
long_logits = batched_long_logits[i].tolist()
result = RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits,
long_logits=long_logits)
 all_results.append(result)
 eval_time = time.time() - tic
print(" Evaluation done in total %f secs(%f sec per example)",
eval_time, eval_time / padded_length)
 examples_gen = read_nq_examples(entries, is_training=False)
 preds = write_predictions(examples_gen, crops, all_results, args.n_best_size,
args.max_answer_length,
None, None, None,
args.verbose_logging,
args.short_null_score_diff_threshold, args.long_null_score_diff_threshold)
 del crops, all_results
gc.collect()
candidates = read_candidates(['.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'], do_cache=False)
 sub = convert_preds_to_df(preds, candidates ).sort_values('example_id')
 sub.to_csv(csv_fn, index=False, columns=['example_id', 'PredictionString'])
 print(f'***** Wrote submission to {csv_fn} *****')
 result = {}
return result
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<define_variables>
def get_convert_args() :
convert_args = argparse.Namespace()
convert_args.fn = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
convert_args.version = 'v0.0.1'
convert_args.prefix = 'nq'
convert_args.num_samples = 1_000_000
convert_args.val_ids = None
convert_args.do_enumerate = False
convert_args.do_not_dump = True
convert_args.num_max_tokens = 400_000
return convert_args
<load_from_csv><EOS> 
def load_and_cache_crops(args, tokenizer, evaluate=False):
do_cache = False
cached_crops_fn = 'cached_{}_{}.pkl'.format('test', str(args.max_seq_length))
if os.path.exists(cached_crops_fn)and do_cache:
print("Loading crops from cached file %s", cached_crops_fn)
 with open(cached_crops_fn, "rb")as f:
crops = pickle.load(f)
 else:
entries = convert_nq_to_squad(get_convert_args())
 examples_gen = read_nq_examples(entries, is_training=not evaluate)
 crops = convert_examples_to_crops(examples_gen=examples_gen,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=not evaluate,
cls_token_segment_id=0,
pad_token_segment_id=0,
p_keep_impossible=args.p_keep_impossible if not evaluate else 1.0)
 if do_cache:
with open(cached_crops_fn, "wb")as f:
pickle.dump(crops, f)
 all_input_ids = tf.stack([c.input_ids for c in crops], 0)
 all_attention_mask = tf.stack([c.attention_mask for c in crops], 0)
 all_token_type_ids = tf.stack([c.token_type_ids for c in crops], 0)
 all_attention_mask = tf.cast(all_attention_mask, tf.int32)
 if evaluate:
dataset = [all_input_ids, all_attention_mask, all_token_type_ids]
else:
all_start_positions = tf.convert_to_tensor([f.start_position for f in crops], dtype=tf.int32)
 all_end_positions = tf.convert_to_tensor([f.end_position for f in crops], dtype=tf.int32)
 all_long_positions = tf.convert_to_tensor([f.long_position for f in crops], dtype=tf.int32)
 dataset = [all_input_ids, all_attention_mask, all_token_type_ids,
all_start_positions, all_end_positions, all_long_positions]
return dataset, crops, entries