Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<feature_engineering>
!pip install bert-tensorflow
print(tf.version.VERSION )
answers_df['long_indexes_and_scores'] = df_long_index_score(answers_df)
 answers_df['short_indexes_and_scores'] = df_short_index_score(answers_df)
 answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(
 create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(
 create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df = answers_df.drop(
 ['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 sample_submission = pd.read_csv(
 ".. /.. /.. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains(
 "_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains(
 "_short"), "PredictionString"] = short_prediction_strings 
<set_options>
print(transformers.__version__ )
torch.cuda.empty_cache()<drop_column>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
del BertTokenizer, BertModel, BertPreTrainedModel, BertConfig, AlbertTokenizer, AlbertModel, AlbertPreTrainedModel, AlbertConfig<install_modules>
SEQUENCE_LENGTH = 128
DATA_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 BERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')
 BERT_PATH_SAVEDMODEL = BERT_PATH + "/bert_multi_from_tfhub"
WEIGHTS_PATH = '.. /input/jigsaw-weights'
OUTPUT_PATH = "/kaggle/working"
<import_modules>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub2 = pd.read_csv('.. /input/ensemble/submission.csv' )
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',<save_to_csv>
def get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):
bert_layer = tf.saved_model.load(bert_path)
 bert_layer = hub.KerasLayer(bert_layer, trainable=False)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
cased = bert_layer.resolved_object.do_lower_case.numpy()
tf.gfile = tf.io.gfile
tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)
 return tokenizer
tokenizer = get_tokenizer()
path = '/kaggle/working'
os.chdir(path)
 sample_submission.to_csv('submission2.csv', index=False )<load_from_csv>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
submission_no2 = pd.read_csv("submission1.csv")
 count = 0
for i in range(len(submission_no2)) :
if ':' not in str(submission_no2["PredictionString"][i])and ':' in str(sample_submission["PredictionString"][i]):
count += 1
submission_no2["PredictionString"][i] = sample_submission["PredictionString"][i]
print('count:', count )<drop_column>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 5
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
path = 'submission1.csv'
if os.path.exists(path):
os.remove(path)
 path = 'submission2.csv'
if os.path.exists(path):
os.remove(path )<save_to_csv>
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)
 fast_tokenizer
submission_no2.to_csv('submission.csv', index=False)
<install_modules>
x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN )
!pip install --upgrade.. /input/tokenizers-repo/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl<set_options>
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values
<define_variables>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
<define_variables>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = tf.keras.layers.Dense(300, activation='relu' )(cls_token)
 out = tf.keras.layers.Dense(128, activation='relu' )(out)
 out = tf.keras.layers.Dense(128, activation='relu' )(out)
 out = Dense(1, activation='sigmoid' )(out)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',<drop_column>
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFDistilBertModel 
.from_pretrained('distilbert-base-multilingual-cased')
)
 model = build_model(transformer_layer, max_len=MAX_LEN )
def find_closing_tag(tokens, opening_tag):
closing_tag = f'</{opening_tag[1: -1]}>'
index, stack = -1, []
for token_index, token in enumerate(tokens):
if token == opening_tag:
stack.insert(0, opening_tag)
 elif token == closing_tag:
stack.pop()
if len(stack)== 0:
index = token_index
break
return index 
<load_pretrained>
model.load_weights(WEIGHTS_PATH+"/weights.h5" )
def read_candidates(candidate_files, do_cache=True):
assert isinstance(candidate_files,(tuple, list)) , candidate_files
for fn in candidate_files:
assert os.path.exists(fn), f'Missing file {fn}'
cache_fn = 'candidates.pkl'
candidates = {}
if not os.path.exists(cache_fn):
for fn in candidate_files:
with open(fn)as f:
for line in tqdm(f):
entry = json.loads(line)
 example_id = str(entry['example_id'])
 cnds = entry.pop('long_answer_candidates')
 cnds = [LongAnswerCandidate(c['start_token'], c['end_token'],
c['top_level'])for c in cnds]
candidates[example_id] = cnds
if do_cache:
with open(cache_fn, 'wb')as f:
pickle.dump(candidates, f)
 else:
print(f'Loading from cache: {cache_fn}')
 with open(cache_fn, 'rb')as f:
candidates = pickle.load(f)
 return candidates 
<string_transform>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
)
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" or ord(c)== 0x202F:
return True
return False 
<load_pretrained>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2,
)
def read_nq_examples(input_file_or_data, is_training):

 if isinstance(input_file_or_data, str):
with open(input_file_or_data, "r", encoding='utf-8')as f:
input_data = json.load(f)["data"]
else:
input_data = input_file_or_data
for entry_index, entry in enumerate(tqdm(input_data, total=len(input_data))):
assert len(entry["paragraphs"])== 1
paragraph = entry["paragraphs"][0]
paragraph_text = paragraph["context"]
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in paragraph_text:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 assert len(paragraph["qas"])== 1
qa = paragraph["qas"][0]
start_position = None
end_position = None
long_position = None
orig_answer_text = None
short_is_impossible = False
long_is_impossible = False
if is_training:
short_is_impossible = qa["short_is_impossible"]
short_answers = qa["short_answers"]
if len(short_answers)>= 2:
short_answers = sorted(short_answers, key=lambda sa: sa["answer_start"])
 short_answers = short_answers[0: 1]
if not short_is_impossible:
answer = short_answers[0]
orig_answer_text = answer["text"]
answer_offset = answer["answer_start"]
answer_length = len(orig_answer_text)
 start_position = char_to_word_offset[answer_offset]
end_position = char_to_word_offset[
answer_offset + answer_length - 1]
actual_text = " ".join(doc_tokens[start_position:
end_position + 1])
 cleaned_answer_text = " ".join(
 whitespace_tokenize(orig_answer_text))
if actual_text.find(cleaned_answer_text)== -1:
logger.warning(
 "Could not find answer: '%s' vs.'%s'",
actual_text, cleaned_answer_text)
 continue
else:
start_position = -1
end_position = -1
orig_answer_text = ""
long_is_impossible = qa["long_is_impossible"]
long_answers = qa["long_answers"]
if(len(long_answers)!= 1)and not long_is_impossible:
raise ValueError(f"For training, each question"
f" should have exactly 1 long answer.")
 if not long_is_impossible:
long_answer = long_answers[0]
long_answer_offset = long_answer["answer_start"]
long_position = char_to_word_offset[long_answer_offset]
else:
long_position = -1
if not short_is_impossible and not long_is_impossible:
assert long_position <= start_position
if not short_is_impossible and long_is_impossible:
assert False, f'Invalid pair short, long pair'
example = NQExample(
 qas_id=qa["id"],
question_text=qa["question"],
doc_tokens=doc_tokens,
orig_answer_text=orig_answer_text,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible,
crop_start=qa["crop_start"])
 yield example
DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
<define_variables>
model.save_weights("weights.h5" )
def get_spans(doc_stride, max_tokens_for_doc, max_len):
doc_spans = []
start_offset = 0
while start_offset < max_len:
length = max_len - start_offset
if length > max_tokens_for_doc:
length = max_tokens_for_doc
doc_spans.append(DocSpan(start=start_offset, length=length))
if start_offset + length == max_len:
break
start_offset += min(length, doc_stride)
 return doc_spans 
<categorify>
print(model )
def convert_examples_to_crops(examples_gen, tokenizer, max_seq_length,
doc_stride, max_query_length, is_training,
cls_token='[CLS]', sep_token='[SEP]', pad_id=0,
sequence_a_segment_id=0,
sequence_b_segment_id=1,
cls_token_segment_id=0,
pad_token_segment_id=0,
mask_padding_with_zero=True,
p_keep_impossible=None,
sep_token_extra=False):

 assert p_keep_impossible is not None, '`p_keep_impossible` is required'
unique_id = 1000000000
num_short_pos, num_short_neg = 0, 0
num_long_pos, num_long_neg = 0, 0
sub_token_cache = {}
crops = []
for example_index, example in enumerate(examples_gen):
if example_index % 1000 == 0 and example_index > 0:
logger.info('Converting %s: short_pos %s short_neg %s'
' long_pos %s long_neg %s',
example_index, num_short_pos, num_short_neg,
num_long_pos, num_long_neg)
 query_tokens = tokenizer.tokenize(example.question_text)
 if len(query_tokens)> max_query_length:
query_tokens = query_tokens[0:max_query_length]
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for i, token in enumerate(example.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = sub_token_cache.get(token)
 if sub_tokens is None:
sub_tokens = tokenizer.tokenize(token)
 sub_token_cache[token] = sub_tokens
tok_to_orig_index.extend([i for _ in range(len(sub_tokens)) ])
 all_doc_tokens.extend(sub_tokens)
 tok_start_position = None
tok_end_position = None
if is_training and example.short_is_impossible:
tok_start_position = -1
tok_end_position = -1
if is_training and not example.short_is_impossible:
tok_start_position = orig_to_tok_index[example.start_position]
if example.end_position < len(example.doc_tokens)- 1:
tok_end_position = orig_to_tok_index[
example.end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
tok_long_position = None
if is_training and example.long_is_impossible:
tok_long_position = -1
if is_training and not example.long_is_impossible:
tok_long_position = orig_to_tok_index[example.long_position]
special_tokens_count = 3
if sep_token_extra:
special_tokens_count += 1
max_tokens_for_doc = max_seq_length - len(query_tokens)- special_tokens_count
assert max_tokens_for_doc > 0
doc_spans = get_spans(doc_stride, max_tokens_for_doc, len(all_doc_tokens))
for doc_span_index, doc_span in enumerate(doc_spans):
tokens = []
token_to_orig_map = UNMAPPED * np.ones(( max_seq_length,), dtype=np.int32)
 token_is_max_context = np.zeros(( max_seq_length,), dtype=np.bool)
 token_type_ids = []
short_is_impossible = example.short_is_impossible
start_position = None
end_position = None
special_tokens_offset = special_tokens_count - 1
doc_offset = len(query_tokens)+ special_tokens_offset
if is_training and not short_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_start_position >= doc_start and tok_end_position <= doc_end):
start_position = 0
end_position = 0
short_is_impossible = True
else:
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
long_is_impossible = example.long_is_impossible
long_position = None
if is_training and not long_is_impossible:
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
if not(tok_long_position >= doc_start and tok_long_position <= doc_end):
long_position = 0
long_is_impossible = True
else:
long_position = tok_long_position - doc_start + doc_offset
if long_is_impossible:
if np.random.rand() > p_keep_impossible:
continue
tokens.append(cls_token)
 token_type_ids.append(cls_token_segment_id)
 tokens += query_tokens
token_type_ids += [sequence_a_segment_id] * len(query_tokens)
 tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 if sep_token_extra:
tokens.append(sep_token)
 token_type_ids.append(sequence_a_segment_id)
 for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[
split_token_index] + example.crop_start
token_is_max_context[len(tokens)] = check_is_max_context(doc_spans,
doc_span_index, split_token_index)
 tokens.append(all_doc_tokens[split_token_index])
 token_type_ids.append(sequence_b_segment_id)
 paragraph_len = doc_span.length
tokens.append(sep_token)
 token_type_ids.append(sequence_b_segment_id)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
 while len(input_ids)< max_seq_length:
input_ids.append(pad_id)
 attention_mask.append(0 if mask_padding_with_zero else 1)
 token_type_ids.append(pad_token_segment_id)
 input_ids = np.array(input_ids, dtype=np.int32)
 attention_mask = np.array(attention_mask, dtype=np.bool)
 token_type_ids = np.array(token_type_ids, dtype=np.uint8)
 if is_training and short_is_impossible:
start_position = CLS_INDEX
end_position = CLS_INDEX
if is_training and long_is_impossible:
long_position = CLS_INDEX
if example_index in(0, 10):
if doc_span_index in(0, 5):
logger.info("*** Example ***")
 logger.info("unique_id: %s" %(unique_id))
logger.info("example_index: %s" %(example_index))
logger.info("doc_span_index: %s" %(doc_span_index))
logger.info("tokens: %s" % " ".join(tokens))
logger.info("input_ids: %s" % input_ids)
 logger.info("attention_mask: %s" % np.uint8(attention_mask))
logger.info("token_type_ids: %s" % token_type_ids)
 if is_training and short_is_impossible:
logger.info("short impossible example")
 if is_training and long_is_impossible:
logger.info("long impossible example")
 if is_training and not short_is_impossible:
answer_text = " ".join(tokens[start_position: end_position + 1])
 logger.info("start_position: %d" %(start_position))
logger.info("end_position: %d" %(end_position))
logger.info("answer: %s" %(answer_text))
if short_is_impossible:
num_short_neg += 1
else:
num_short_pos += 1
if long_is_impossible:
num_long_neg += 1
else:
num_long_pos += 1
crop = Crop(
 unique_id=unique_id,
example_index=example_index,
doc_span_index=doc_span_index,
tokens=tokens,
token_to_orig_map=token_to_orig_map,
token_is_max_context=token_is_max_context,
input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
paragraph_len=paragraph_len,
start_position=start_position,
end_position=end_position,
long_position=long_position,
short_is_impossible=short_is_impossible,
long_is_impossible=long_is_impossible)
 crops.append(crop)
 unique_id += 1
return crops 
<define_variables>
model.load_weights("weights.h5" )
PrelimPrediction = collections.namedtuple("PrelimPrediction",
["crop_index", "start_index", "end_index", "start_logit", "end_logit"])
 NbestPrediction = collections.namedtuple("NbestPrediction", [
"text", "start_logit", "end_logit",
"start_index", "end_index",
"orig_doc_start", "orig_doc_end", "crop_index"])
<string_transform>
def clean_text(tok_text):
tok_text = tok_text.replace("
tok_text = tok_text.replace("
tok_text = tok_text.strip()
tok_text = " ".join(tok_text.split())
 return tok_text 
<define_variables>
def get_nbest(prelim_predictions, crops, example, n_best_size):
seen, nbest = set() , []
for pred in prelim_predictions:
if len(nbest)>= n_best_size:
break
crop = crops[pred.crop_index]
orig_doc_start, orig_doc_end = -1, -1
orig_doc_start, orig_doc_end = -1, -1
if pred.start_index > 0:
if pred.end_index == -1:
tok_tokens = crop.tokens[pred.start_index: pred.start_index + 11]
else:
tok_tokens = crop.tokens[pred.start_index: pred.end_index + 1]
tok_text = " ".join(tok_tokens)
 tok_text = clean_text(tok_text)
 orig_doc_start = int(crop.token_to_orig_map[pred.start_index])
 if pred.end_index == -1:
orig_doc_end = orig_doc_start + 10
else:
orig_doc_end = int(crop.token_to_orig_map[pred.end_index])
 final_text = tok_text
if final_text in seen:
continue
else:
final_text = ""
seen.add(final_text)
 nbest.append(NbestPrediction(
 text=final_text,
start_logit=pred.start_logit, end_logit=pred.end_logit,
start_index=pred.start_index, end_index=pred.end_index,
orig_doc_start=orig_doc_start, orig_doc_end=orig_doc_end,
crop_index=pred.crop_index))
if len(nbest)in(0, 1):
nbest.insert(0, NbestPrediction(text="empty",
start_logit=0.0, end_logit=0.0,
start_index=-1, end_index=-1,
orig_doc_start=-1, orig_doc_end=-1,
crop_index=UNMAPPED))
assert len(nbest)>= 1
return nbest 
<save_to_csv>
checkpoint = ModelCheckpoint(OUTPUT_PATH, monitor='history = model.fit() ', verbose=1, save_best_only=True, mode='max')
 print(checkpoint )
<categorify><EOS> 
sub1.rename(columns={'toxic':'toxic1'}, inplace=True)
 sub2.rename(columns={'toxic':'toxic2'}, inplace=True)
 sub3 = pd.merge(sub1, sub2, how='left', on='id')
 sub3['toxic'] =(sub3['toxic1'] * 0.1)+(sub3['toxic2'] * 0.9)
 sub3['toxic'] =(sub3['toxic2'] * 0.39)+(sub3['toxic'] * 0.61)
 sub3[['id', 'toxic']].to_csv('submission.csv', index=False )