Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<import_modules>
import codecs
import copy
import csv
import gc
import os
import random
import time
from typing import Dict, List, Tuple
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',<set_options>
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import roc_auc_score, roc_curve
import tensorflow as tf
import tensorflow_addons as tfa
from transformers import TFXLMRobertaModel, XLMRobertaConfig
from transformers import AutoTokenizer, XLMRobertaTokenizer
tf.logging.set_verbosity(tf.logging.INFO)
 tf.disable_eager_execution()
def del_all_flags(FLAGS):
flags_dict = FLAGS._flags()
keys_list = [keys for keys in flags_dict]
for keys in keys_list:
FLAGS.__delattr__(keys)
 del_all_flags(tf.app.flags.FLAGS)
 flags = tf.app.flags
flags.DEFINE_string("bert_config_file", "/kaggle/input/bert-disjoint/bert_config.json", " ")
 flags.DEFINE_string("vocab_file_bert", "/kaggle/input/bert-disjoint/vocab-nq.txt", " ")
 flags.DEFINE_string("output_dir_bert", "/kaggle/input/bert-disjoint/", " ")
 flags.DEFINE_string("init_checkpoint_bert", "/kaggle/input/bert-disjoint/model.ckpt-15000", " ")
 flags.DEFINE_string("albert_config_file", "/kaggle/input/albert-yes-no/albert_config.json", " ")
 flags.DEFINE_string("vocab_file_albert", "/kaggle/input/albert-yes-no/30k-clean.vocab", " ")
 flags.DEFINE_string("spm_model_file", "/kaggle/input/albert-yes-no/30k-clean.model", " ")
 flags.DEFINE_string("output_dir_albert", "/kaggle/input/albert-yes-no/", " ")
 flags.DEFINE_string("init_checkpoint_albert", "/kaggle/input/albert-yes-no/model.ckpt-1206", " ")
 flags.DEFINE_string("eval_record_file_bert", "eval_bert.tf_record", " ")
 flags.DEFINE_integer("max_seq_length", 512, " ")
 flags.DEFINE_integer("predict_batch_size", 8, " ")
 flags.DEFINE_string(
 "predict_file", "/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl",
"NQ json for predictions.E.g., dev-v1.1.jsonl.gz or test-v1.1.jsonl.gz")
 flags.DEFINE_boolean("logtostderr", True, "Logs to stderr")
 flags.DEFINE_boolean("undefok", True, "it's okay to be undefined")
 flags.DEFINE_string('f', '', 'kernel')
 flags.DEFINE_string('HistoryManager.hist_file', '', 'kernel')
 FLAGS = flags.FLAGS<features_selection>
def generate_random_seed() -> int:
return random.randint(0, 2147483648 )
bert_tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file_bert, do_lower_case=True)
 eval_examples = bert_disjoint_utils.read_nq_examples(input_file=FLAGS.predict_file)
 eval_writer = bert_disjoint_utils.FeatureWriter(filename=FLAGS.eval_record_file_bert, is_training=False)
 eval_features = []
def append_feature(feature):
eval_features.append(feature)
 eval_writer.process_feature(feature)
 num_spans_to_ids = bert_disjoint_utils.convert_examples_to_features(examples=eval_examples,
tokenizer=bert_tokenizer,
output_fn=append_feature)
 eval_writer.close()
eval_filename = eval_writer.filename<choose_model_class>
def regular_encode(texts: List[str], tokenizer: XLMRobertaTokenizer,
maxlen: int)-> Tuple[np.ndarray, np.ndarray]:
err_msg = '"{0}" is wrong type for the text list!'.format(type(texts))
assert isinstance(texts, list)or isinstance(texts, tuple), err_msg
enc_di = tokenizer.batch_encode_plus(
 texts,
return_token_type_ids=False,
padding='max_length',
max_length=maxlen 
)
 err_msg = '{0} != {1}'.format(len(texts), len(enc_di['input_ids']))
assert len(texts)== len(enc_di['input_ids']), err_msg
err_msg = '{0} != {1}'.format(len(texts), len(enc_di['attention_mask']))
assert len(texts)== len(enc_di['attention_mask']), err_msg
encoded_tokens = np.zeros(( len(texts), maxlen), dtype=np.int32)
 encoded_masks = np.zeros(( len(texts), maxlen), dtype=np.int32)
 for sample_idx,(encoded_cur_text, encoded_cur_mask)in enumerate(
 zip(enc_di['input_ids'], enc_di['attention_mask'])
):
n_text = len(encoded_cur_text)
 n_mask = len(encoded_cur_mask)
 err_msg = 'Tokens and masks of texts "{0}" are different! '\
'{1} != {2}'.format(texts[sample_idx], n_text, n_mask)
 assert n_text == n_mask, err_msg
if n_text >= maxlen:
encoded_tokens[sample_idx] = np.array(encoded_cur_text[0:maxlen],
dtype=np.int32)
 encoded_masks[sample_idx] = np.array(encoded_cur_mask[0:maxlen],
dtype=np.int32)
 else:
padding = [0 for _ in range(maxlen - n_text)]
encoded_tokens[sample_idx] = np.array(encoded_cur_text + padding,
dtype=np.int32)
 encoded_masks[sample_idx] = np.array(encoded_cur_mask + padding,
dtype=np.int32)
 return encoded_tokens, encoded_masks
bert_config = bert_disjoint_modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
 run_config = tf.estimator.RunConfig(model_dir=FLAGS.output_dir_bert)
 model_fn = bert_disjoint_fn_builder.model_fn_builder(bert_config=bert_config,
init_checkpoint=FLAGS.init_checkpoint_bert)
 estimator = tf.estimator.Estimator(model_fn=model_fn,
model_dir=FLAGS.output_dir_bert,
config=run_config,
params={"batch_size": FLAGS.predict_batch_size})
 predict_input_fn = bert_disjoint_fn_builder.input_fn_builder(input_file=eval_filename,
seq_length=FLAGS.max_seq_length,
drop_remainder=False)
 tf.logging.info("***** Running predictions *****")
 tf.logging.info(" Num orig examples = %d", len(eval_examples))
tf.logging.info(" Num split examples = %d", len(eval_features))
tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size)
 for spans, ids in num_spans_to_ids.items() :
tf.logging.info(" Num split into %d = %d", spans, len(ids))
all_results = []
for result in estimator.predict(predict_input_fn, yield_single_examples=True):
if len(all_results)% 1000 == 0:
tf.logging.info("Processing example: %d" %(len(all_results)))
 unique_id = int(result["unique_ids"])
 start_logits = [float(x)for x in result["start_logits"].flat]
end_logits = [float(x)for x in result["end_logits"].flat]
all_results.append(bert_disjoint_utils.RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits))
candidates_dict = bert_disjoint_utils.read_candidates(FLAGS.predict_file)
 eval_features = [tf.train.Example.FromString(r)for r in tf.python_io.tf_record_iterator(eval_filename)]
nq_pred_dict = bert_disjoint_utils.compute_pred_dict(candidates_dict, eval_features, [r._asdict() for r in all_results])
 predictions = list(nq_pred_dict.values())
 del bert_config, run_config, model_fn, estimator, predict_input_fn
del all_results, candidates_dict, eval_features, nq_pred_dict<randomize_order>
def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
assert len(sentiment_fields)> 0, 'List of sentiment fields is empty!'
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
for cur_field in sentiment_fields:
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(
 cur_field)
 assert cur_field in header, err_msg2
text_field_index = header.index(text_field)
 try:
lang_field_index = header.index(lang_field)
 except:
lang_field_index = -1
indices_of_sentiment_fields = []
for cur_field in sentiment_fields:
indices_of_sentiment_fields.append(header.index(cur_field))
else:
if len(row)== len(header):
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
max_proba = 0.0
for cur_field_idx in indices_of_sentiment_fields:
try:
cur_proba = float(row[cur_field_idx])
 except:
cur_proba = -1.0
err_msg2 = err_msg + ' Value {0} is wrong!'.format(
 row[cur_field_idx] 
)
 assert(cur_proba >= 0.0)and(cur_proba <= 1.0), err_msg2
if cur_proba > max_proba:
max_proba = cur_proba
new_label = 1 if max_proba >= 0.5 else 0
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, new_label))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang
albert_tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file_albert,
do_lower_case=True,
spm_model_file=FLAGS.spm_model_file)
 eval_examples = albert_yes_no_utils.read_nq_examples(input_file="yes_no_questions.jsonl")
 eval_writer = albert_yes_no_utils.FeatureWriter(filename="yes_no_questions.eval.tf_record",
albert_tokenizer=albert_tokenizer)
 n_features, example_to_feature_map, feature_to_example_map = \
eval_writer.file_based_convert_examples_to_features_span(eval_examples )<choose_model_class>
def load_test_set(file_name: str, id_field: str, text_field: str,
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(id_field)
 assert id_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(lang_field)
 assert lang_field in header, err_msg2
id_field_index = header.index(id_field)
 text_field_index = header.index(text_field)
 lang_field_index = header.index(lang_field)
 else:
if len(row)== len(header):
try:
id_value = int(row[id_field_index])
 except:
id_value = -1
err_msg2 = err_msg + ' {0} is wrong ID!'.format(
 row[id_field_index])
 assert id_value >= 0, err_msg2
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, id_value))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang
albert_config = albert_yes_no_modeling.AlbertConfig.from_json_file(FLAGS.albert_config_file)
 run_config = tf.estimator.RunConfig(model_dir=FLAGS.output_dir_albert)
 model_fn = albert_yes_no_fn_builder.model_fn_builder(albert_config=albert_config,
init_checkpoint=FLAGS.init_checkpoint_albert)
 estimator = tf.estimator.Estimator(model_fn=model_fn,
model_dir=FLAGS.output_dir_albert,
config=run_config,
params={"batch_size": FLAGS.predict_batch_size})
 predict_input_fn = albert_yes_no_fn_builder.input_fn_builder(input_file="yes_no_questions.eval.tf_record",
seq_length=FLAGS.max_seq_length,
drop_remainder=False)
 tf.logging.info(" Num features: %d" % n_features)
 all_results = []
for result in estimator.predict(predict_input_fn, yield_single_examples=True):
if len(all_results)% 100 == 0:
tf.logging.info("Processing example: %d" %(len(all_results)))
 answer_type_logits = [float(x)for x in result["answer_type_logits"].flat]
all_results.append(answer_type_logits)
 yes_no_pred_dict = albert_yes_no_utils.compute_pred_dicts_cls("yes_no_questions.jsonl",
all_results,
example_to_feature_map,
feature_to_example_map)
 del albert_config, run_config, model_fn, estimator, predict_input_fn
del all_results<merge>
def build_dataset(texts: Dict[str, List[Tuple[str, int]]],
dataset_size: int, tokenizer: XLMRobertaTokenizer,
maxlen: int, batch_size: int,
shuffle: bool)-> Tuple[tf.data.Dataset, int]:
texts_and_labels = []
dataset_size_by_lang = int(round(dataset_size / float(len(texts))))
for lang in texts:
print('{0}:'.format(lang))
n_lang = 0
if shuffle:
if len(texts[lang])> dataset_size_by_lang:
texts_and_labels += random.sample(texts[lang], k=dataset_size_by_lang)
 n_lang += dataset_size_by_lang
elif len(texts[lang])< dataset_size_by_lang:
texts_and_labels += texts[lang]
n_lang += len(texts[lang])
 n = dataset_size_by_lang - len(texts[lang])
 while n >= len(texts[lang]):
texts_and_labels += texts[lang]
n -= len(texts[lang])
 n_lang += len(texts[lang])
 if n > 0:
texts_and_labels += random.sample(texts[lang], k=n)
 n_lang += n
else:
texts_and_labels += texts[lang]
n_lang += len(texts[lang])
 else:
texts_and_labels += texts[lang]
n_lang += len(texts[lang])
 print(' number of samples is {0};'.format(n_lang))
random.shuffle(texts_and_labels)
 n_steps = len(texts_and_labels)// batch_size
print('Samples number of the data set is {0}.'.format(len(texts_and_labels)))
 tokens_of_texts, mask_of_texts = regular_encode(
 texts=[cur[0] for cur in texts_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 toxicity_labels = np.array([cur[1] for cur in texts_and_labels], dtype=np.int32)
 print('Number of positive siamese samples is {0} from {1}.'.format(
 int(sum(toxicity_labels)) , toxicity_labels.shape[0]))
if shuffle:
err_msg = '{0} is too small number of samples for the data set!'.format(
 len(texts_and_labels))
assert n_steps >= 50, err_msg
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_texts, mask_of_texts 
),
toxicity_labels 
)
 ).repeat().batch(batch_size)
 else:
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_texts, mask_of_texts 
),
toxicity_labels 
)
 ).batch(batch_size)
 del texts_and_labels
return dataset, n_steps
create_submission.merge_predictions(predictions, yes_no_pred_dict)
 create_submission.create_answers(predictions, "submission.csv" )<define_variables>
def build_classifier(transformer_name: str, hidden_state_size: int,
max_len: int, lr: float)-> tf.keras.Model:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_word_ids")
 attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_attention_mask")
 transformer_layer = TFXLMRobertaModel.from_pretrained(
 pretrained_model_name_or_path=transformer_name,
name='Transformer' 
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
pooled_output = tf.keras.layers.GlobalAvgPool1D(
 name='AvePool'
)(sequence_output, mask=attention_mask)
 kernel_init = tf.keras.initializers.GlorotNormal(seed=generate_random_seed())
 bias_init = tf.keras.initializers.Constant(value=0.0)
 cls_layer = tf.keras.layers.Dense(
 units=1, activation='sigmoid',
kernel_initializer=kernel_init,
bias_initializer=bias_init,
name='OutputLayer'
)(pooled_output)
 cls_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=cls_layer,
name='ToxicityClassifier' 
)
 cls_model.compile(
 optimizer=tfa.optimizers.AdamW(learning_rate=lr, weight_decay=1e-5),
loss='binary_crossentropy' 
)
 return cls_model
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
nq_train_file = '.. /input/tensorflow2-question-answering/simplified-nq-train.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
def train_classifier(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,
steps_per_epoch: int, validset: tf.data.Dataset, max_duration: int,
classifier_file_name: str):
assert steps_per_trainset >= steps_per_epoch
n_epochs = int(round(10.0 * steps_per_trainset / float(steps_per_epoch)))
 print(f'Maximal duration of the XLMR-based classifier training is {max_duration} seconds.')
 callbacks = [
tf.keras.callbacks.EarlyStopping(patience=7, monitor='val_loss', mode='min',
restore_best_weights=False, verbose=True),
tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', mode="min",
save_weights_only=True, save_best_only=True,
filepath=classifier_file_name),
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)
 ]
history = nn.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs,
callbacks=callbacks 
)
 show_training_process(history, 'loss')
 nn.load_weights(classifier_file_name )
if True:
importlib.reload(bert_utils )<init_hyperparams>
def predict_with_classifier(texts: Dict[str, List[Tuple[str, int]]],
tokenizer: XLMRobertaTokenizer, maxlen: int,
classifier: tf.keras.Model, batch_size: int,
max_dataset_size: int = 0)-> \
Dict[str, Tuple[np.ndarray, np.ndarray]]:
languages = sorted(list(texts.keys()))
predictions_by_languages = dict()
if max_dataset_size > 0:
max_size_per_lang = max_dataset_size // len(languages)
 err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)
 assert max_size_per_lang > 0, err_msg
else:
max_size_per_lang = 0
for cur_lang in languages:
selected_indices = list(range(len(texts[cur_lang])))
 if max_size_per_lang > 0:
if len(selected_indices)> max_size_per_lang:
selected_indices = random.sample(
 population=selected_indices,
k=max_size_per_lang 
)
 tokens_of_texts, mask_of_texts = regular_encode(
 texts=[texts[cur_lang][idx][0] for idx in selected_indices],
tokenizer=tokenizer, maxlen=maxlen 
)
 predictions = []
n_batches = int(np.ceil(len(selected_indices)/ float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(len(selected_indices), batch_start + batch_size)
 res = classifier.predict_on_batch(
 [
tokens_of_texts[batch_start:batch_end],
mask_of_texts[batch_start:batch_end]
] 
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
predictions.append(res.reshape(( res.shape[0],)))
 del res
predictions = np.concatenate(predictions)
 identifiers = np.array(
 [texts[cur_lang][idx][1] for idx in selected_indices],
dtype=np.int32 
)
 predictions_by_languages[cur_lang] =(predictions, identifiers)
 del predictions, identifiers, selected_indices
return predictions_by_languages
config = {'attention_probs_dropout_prob':0.5,
'hidden_act':'gelu',
'hidden_dropout_prob':0.5,
'hidden_size':1024,
'initializer_range':0.02,
'intermediate_size':4096,
'max_position_embeddings':512,
'num_attention_heads':16,
'num_hidden_layers':24,
'type_vocab_size':2,
'vocab_size':30522}<choose_model_class>
experiment_start_time = time.time()
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=3, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<categorify>
random.seed(42)
 np.random.seed(42)
 tf.random.set_seed(42 )
model= mk_model(config )<categorify>
tfa.register_all()
class DummyObject:
def __init__(self,**kwargs):
self.__dict__.update(kwargs)
 FLAGS=DummyObject(skip_nested_contexts=True,
max_position=50,
max_contexts=48,
max_query_length=64,
max_seq_length=512,
doc_stride=128,
include_unknowns=0.02,
n_best_size=20,
max_answer_length=30,
warmup_proportion=0.1,
learning_rate=1e-3,
num_train_epochs=3.0,
train_batch_size=8,
num_train_steps=100000,
num_warmup_steps=10000,
max_eval_steps=100,
use_tpu=False,
eval_batch_size=4,
max_predictions_per_seq=20 )<define_variables>
xlmr_learning_rate = 1e-5
dataset_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'
final_classifier_name = '/kaggle/working/xlmr_for_toxicity.h5'
eval_records = ".. /input/bert-joint-baseline/nq-test.tfrecords"
if on_kaggle_server and private_dataset:
eval_records='nq-test.tfrecords'
if not os.path.exists(eval_records):
eval_writer = bert_utils.FeatureWriter(
 filename=os.path.join(eval_records),
is_training=False)
 tokenizer = tokenization.FullTokenizer(vocab_file='.. /input/bert-joint-baseline/vocab-nq.txt',
do_lower_case=True)
 features = []
convert = bert_utils.ConvertExamples2Features(tokenizer=tokenizer,
is_training=False,
output_fn=eval_writer.process_feature,
collect_stat=False)
 n_examples = 0
tqdm_notebook= tqdm.tqdm_notebook if not on_kaggle_server else None
for examples in bert_utils.nq_examples_iter(input_file=nq_test_file,
is_training=False,
tqdm=tqdm_notebook):
for example in examples:
n_examples += convert(example)
 eval_writer.close()
print('number of test examples: %d, written to file: %d' %(n_examples,eval_writer.num_features))<categorify>
xlmroberta_tokenizer = AutoTokenizer.from_pretrained(model_name)
 xlmroberta_config = XLMRobertaConfig.from_pretrained(model_name)
 print(xlmroberta_config )
seq_length = FLAGS.max_seq_length
name_to_features = {
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"input_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
"input_mask": tf.io.FixedLenFeature([seq_length], tf.int64),
"segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64),
}
def _decode_record(record, name_to_features=name_to_features):

 example = tf.io.parse_single_example(serialized=record, features=name_to_features)
 for name in list(example.keys()):
t = example[name]
if name != 'unique_id':
t = tf.cast(t, dtype=tf.int64)
 example[name] = t
return example
def _decode_tokens(record):
return tf.io.parse_single_example(serialized=record,
features={
"unique_id": tf.io.FixedLenFeature([], tf.int64),
"token_map" : tf.io.FixedLenFeature([seq_length], tf.int64)
 })
<categorify>
sentence_embedding_size = xlmroberta_config.hidden_size
print('Sentence embedding size is {0}'.format(sentence_embedding_size))
assert max_seq_len <= xlmroberta_config.max_position_embeddings
raw_ds = tf.data.TFRecordDataset(eval_records)
 token_map_ds = raw_ds.map(_decode_tokens)
 decoded_ds = raw_ds.map(_decode_record)
 ds = decoded_ds.batch(batch_size=16,drop_remainder=False )<predict_on_test>
corpus_for_training = load_train_set(
 os.path.join(dataset_dir, "jigsaw-toxic-comment-train.csv"),
text_field="comment_text", lang_field="lang",
sentiment_fields=["toxic", "severe_toxic", "obscene", "threat", "insult",
"identity_hate"] 
)
 assert 'en' in corpus_for_training
result=model.predict_generator(ds,verbose=1 )<load_pretrained>
multilingual_corpus = load_train_set(
 os.path.join(dataset_dir, "validation.csv"),
text_field="comment_text", lang_field="lang", sentiment_fields=["toxic", ] 
)
 assert 'en' not in multilingual_corpus
max_size = 0
print('Multilingual data:')
 for language in sorted(list(multilingual_corpus.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(multilingual_corpus[language])))
 assert set(map(lambda cur: cur[1], multilingual_corpus[language])) == {0, 1}
if len(multilingual_corpus[language])> max_size:
max_size = len(multilingual_corpus[language] )
np.savez_compressed('bert-joint-baseline-output.npz',
**dict(zip(['uniqe_id','start_logits','end_logits','answer_type_logits'],
result)) )<create_dataframe>
nonenglish_languages = sorted(list(multilingual_corpus.keys()))
corpus_for_validation = dict()
for lang in nonenglish_languages:
random.shuffle(multilingual_corpus[lang])
 n = len(multilingual_corpus[lang])// 2
corpus_for_validation[lang] = multilingual_corpus[lang][0:n]
corpus_for_training[lang] = multilingual_corpus[lang][n:]
del multilingual_corpus[lang]
Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx", "score"] )<import_modules>
texts_for_submission = load_test_set(
 os.path.join(dataset_dir, "test.csv"),
text_field="content", lang_field="lang", id_field="id" 
)
 print('Data for submission:')
 for language in sorted(list(texts_for_submission.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(texts_for_submission[language])) )
class ScoreSummary(object):
def __init__(self):
self.predicted_label = None
self.short_span_score = None
self.cls_token_score = None
self.answer_type_logits = None<prepare_output>
dataset_for_training, n_batches_per_data = build_dataset(
 texts=corpus_for_training, dataset_size=150000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_xlmr, shuffle=True
)
class EvalExample(object):

 def __init__(self, example_id, candidates):
self.example_id = example_id
self.candidates = candidates
self.results = {}
self.features = {}<find_best_params>
dataset_for_validation, n_batches_per_epoch = build_dataset(
 texts=corpus_for_validation, dataset_size=6000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_xlmr, shuffle=False
)
def get_best_indexes(logits, n_best_size):

 index_and_score = sorted(
 enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
 best_indexes = []
for i in range(len(index_and_score)) :
if i >= n_best_size:
break
best_indexes.append(index_and_score[i][0])
 return best_indexes
def top_k_indices(logits,n_best_size,token_map):
indices = np.argsort(logits[1:])+1
indices = indices[token_map[indices]!=-1]
return indices[-n_best_size:]<count_duplicates>
del corpus_for_training
gc.collect()
def remove_duplicates(span):
start_end = []
for s in span:
cont = 0
if not start_end:
start_end.append(Span(s[0], s[1], s[2]))
cont += 1
else:
for i in range(len(start_end)) :
if start_end[i][0] == s[0] and start_end[i][1] == s[1]:
cont += 1
if cont == 0:
start_end.append(Span(s[0], s[1], s[2]))
return start_end<define_variables>
preparing_duration = int(round(time.time() - experiment_start_time))
print("Duration of data loading and preparing to the Siamese NN training is "
"{0} seconds.".format(preparing_duration))
def get_short_long_span(predictions, example):
sorted_predictions = sorted(predictions, reverse=True)
 short_span = []
long_span = []
for prediction in sorted_predictions:
score, _, summary, start_span, end_span = prediction
if score > 0:
short_span.append(Span(int(start_span), int(end_span), float(score)))
 short_span = remove_duplicates(short_span)
 for s in range(len(short_span)) :
for c in example.candidates:
start = short_span[s].start_token_idx
end = short_span[s].end_token_idx
if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
long_span.append(Span(int(c["start_token"]), int(c["end_token"]), float(short_span[s].score)))
 break
long_span = remove_duplicates(long_span)
 if not long_span:
long_span = [Span(-1, -1, -10000.0)]
if not short_span:
short_span = [Span(-1, -1, -10000.0)]
return short_span, long_span<find_best_params>
with strategy.scope() :
xlmr_based_classifier = build_classifier(
 transformer_name=model_name,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
lr=xlmr_learning_rate
)
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for _,(start_index,end_index)in enumerate(indexes):
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
short_span = [Span(-1, -1, -10000.0)]
long_span = [Span(-1, -1, -10000.0)]
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 summary.predicted_label = {
"example_id": int(example.example_id),
"long_answers": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"short_answers": {
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1,
"yes_no_answer": "NONE"
}
}
return summary<load_pretrained>
train_classifier(nn=xlmr_based_classifier, trainset=dataset_for_training,
steps_per_trainset=n_batches_per_data,
steps_per_epoch=min(5 * n_batches_per_epoch, n_batches_per_data),
validset=dataset_for_validation,
max_duration=int(round(2.0 * 3600.0 - preparing_duration)) ,
classifier_file_name=final_classifier_name )
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict<string_transform>
val_predictions = predict_with_classifier(
 texts=corpus_for_validation,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
classifier=xlmr_based_classifier,
batch_size=batch_size_for_xlmr
)
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict<load_pretrained>
del corpus_for_validation
gc.collect()
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4)
 print('done!' )<load_from_disk>
calculated_probas = []
true_labels = []
for lang in val_predictions:
probabilities_, true_labels_ = val_predictions[lang]
calculated_probas.append(probabilities_)
 true_labels.append(true_labels_)
 calculated_probas = np.concatenate(calculated_probas)
 true_labels = np.concatenate(true_labels )
answers_df = pd.read_json(".. /working/predictions.json")
 answers_df.head()<feature_engineering>
del val_predictions
gc.collect()
def df_long_index_score(df):
answers = []
cont = 0
for e in df['long_answers']['tokens_and_score']:
if e[2] > 3:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_short_index_score(df):
answers = []
cont = 0
for e in df['short_answers']['tokens_and_score']:
if e[2] > 7:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_example_id(df):
return df['example_id']<feature_engineering>
del true_labels, calculated_probas
answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)
 answers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)
 answers_df.head()<drop_column>
del dataset_for_training
del dataset_for_validation
gc.collect()
answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df.head()<string_transform>
experiment_duration = int(round(time.time() - experiment_start_time))
print('Duration of XLM-RoBERTa training is {0} seconds.'.format(
 experiment_duration))
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':'+ str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
<feature_engineering>
final_predictions = predict_with_classifier(
 texts=texts_for_submission,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
classifier=xlmr_based_classifier,
batch_size=batch_size_for_xlmr
)
answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df.head()<drop_column>
with codecs.open('submission.csv', mode='w', encoding='utf-8', errors='ignore')as fp:
fp.write('id,toxic
')
 for lang in final_predictions:
probabilities, IDs = final_predictions[lang]
assert probabilities.shape == IDs.shape
assert len(probabilities.shape)== 1
for sample_idx in range(probabilities.shape[0]):
id_val = IDs[sample_idx]
proba_val = probabilities[sample_idx]
fp.write('{0},{1:.9f}
'.format(id_val, proba_val))
<feature_engineering><EOS> 
print('Experiment duration is {0:.3f}.'.format(time.time() - experiment_start_time))