Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
import json
import numpy as np
import argparse
import pandas as pd
from tqdm.notebook import tqdm<string_transform>
!pip install keras
def enumerate_tags(text_split):

 special_tokens = ['<P>', '<Table>']
special_token_counts = [0 for _ in range(len(special_tokens)) ]
for index, token in enumerate(text_split):
for special_token_index, special_token in enumerate(special_tokens):
if token == special_token:
cnt = special_token_counts[special_token_index]
if cnt <= 10:
text_split[index] = f'<{special_token[1: -1]}{cnt}>'
special_token_counts[special_token_index] = cnt + 1
return text_split<define_variables>
warnings.filterwarnings("ignore")
ALL_MODELS = sum(( tuple(conf.pretrained_config_archive_map.keys())
 for conf in(BertConfig,)) ,() )<define_variables>
MODEL_CLASSES = {
'bert':(BertConfig, TFBertForNaturalQuestionAnswering, BertTokenizer),
'roberta':(RobertaConfig, TFRobertaForNaturalQuestionAnswering, RobertaTokenizer),
}<define_variables>
RawResult = namedtuple("RawResult", ["unique_id", "start_logits", "end_logits",
"long_logits"] )<define_variables>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
def set_seed(args):
random.seed(args.seed)
 np.random.seed(args.seed)
 tf.random.set_seed(args.seed )<define_variables>
def regular_encode(texts, tokenizer, maxlen = 512):
enc_di = tokenizer.batch_encode_plus(texts,return_attention_mask = False, return_token_type_ids = False,
pad_to_max_length = True, max_length = maxlen, truncation = True)
 return np.array(enc_di["input_ids"])
def submit(args, model, tokenizer):
csv_fn = 'submission.csv'
eval_dataset, crops, entries = load_and_cache_crops(args, tokenizer, evaluate=True)
 args.eval_batch_size = args.per_tpu_eval_batch_size
eval_dataset_length = len(eval_dataset[0])
 padded_length = math.ceil(eval_dataset_length / args.eval_batch_size)* args.eval_batch_size
num_pad = padded_length - eval_dataset[0].shape[0]
for ti, t in enumerate(eval_dataset):
pad_tensor = tf.expand_dims(tf.zeros_like(t[0]), 0)
 pad_tensor = tf.repeat(pad_tensor, num_pad, 0)
 eval_dataset[ti] = tf.concat([t, pad_tensor], 0)
 eval_ds = tf.data.Dataset.from_tensor_slices({
'input_ids': tf.constant(eval_dataset[0]),
'attention_mask': tf.constant(eval_dataset[1]),
'token_type_ids': tf.constant(eval_dataset[2]),
'example_index': tf.range(padded_length, dtype=tf.int32)
 })
 eval_ds = eval_ds.batch(batch_size=args.eval_batch_size, drop_remainder=True)
 print("***** Running evaluation *****")
 print(" Num examples = %d", eval_dataset_length)
 print(" Batch size = %d", args.eval_batch_size)
 @tf.function
def predict_step(batch):
outputs = model(batch, training=False)
 return outputs
all_results = []
tic = time.time()
for batch_ind, batch in tqdm(enumerate(eval_ds), total=padded_length // args.per_tpu_eval_batch_size):
example_indexes = batch['example_index']
outputs = predict_step(batch)
 batched_start_logits = outputs[0].numpy()
batched_end_logits = outputs[1].numpy()
batched_long_logits = outputs[2].numpy()
for i, example_index in enumerate(example_indexes):
if example_index >= eval_dataset_length:
continue
eval_crop = crops[example_index]
unique_id = int(eval_crop.unique_id)
 start_logits = batched_start_logits[i].tolist()
end_logits = batched_end_logits[i].tolist()
long_logits = batched_long_logits[i].tolist()
result = RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits,
long_logits=long_logits)
 all_results.append(result)
 eval_time = time.time() - tic
print(" Evaluation done in total %f secs(%f sec per example)",
eval_time, eval_time / padded_length)
 examples_gen = read_nq_examples(entries, is_training=False)
 preds = write_predictions(examples_gen, crops, all_results, args.n_best_size,
args.max_answer_length,
None, None, None,
args.verbose_logging,
args.short_null_score_diff_threshold, args.long_null_score_diff_threshold)
 del crops, all_results
gc.collect()
candidates = read_candidates(['.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'], do_cache=False)
 sub = convert_preds_to_df(preds, candidates ).sort_values('example_id')
 sub.to_csv(csv_fn, index=False, columns=['example_id', 'PredictionString'])
 print(f'***** Wrote submission to {csv_fn} *****')
 result = {}
return result 
<load_pretrained>
import transformers
def get_convert_args() :
convert_args = argparse.Namespace()
convert_args.fn = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
convert_args.version = 'v0.0.1'
convert_args.prefix = 'nq'
convert_args.num_samples = 1_000_000
convert_args.val_ids = None
convert_args.do_enumerate = False
convert_args.do_not_dump = True
convert_args.num_max_tokens = 400_000
return convert_args 
<load_pretrained>
def build_model(transformer,maxlen = 512):
input_word_ids = Input(shape=(maxlen,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def load_and_cache_crops(args, tokenizer, evaluate=False):
do_cache = False
cached_crops_fn = 'cached_{}_{}.pkl'.format('test', str(args.max_seq_length))
if os.path.exists(cached_crops_fn)and do_cache:
print("Loading crops from cached file %s", cached_crops_fn)
 with open(cached_crops_fn, "rb")as f:
crops = pickle.load(f)
 else:
entries = convert_nq_to_squad(get_convert_args())
 examples_gen = read_nq_examples(entries, is_training=not evaluate)
 crops = convert_examples_to_crops(examples_gen=examples_gen,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=not evaluate,
cls_token_segment_id=0,
pad_token_segment_id=0,
p_keep_impossible=args.p_keep_impossible if not evaluate else 1.0)
 if do_cache:
with open(cached_crops_fn, "wb")as f:
pickle.dump(crops, f)
 all_input_ids = tf.stack([c.input_ids for c in crops], 0)
 all_attention_mask = tf.stack([c.attention_mask for c in crops], 0)
 all_token_type_ids = tf.stack([c.token_type_ids for c in crops], 0)
 all_attention_mask = tf.cast(all_attention_mask, tf.int32)
 if evaluate:
dataset = [all_input_ids, all_attention_mask, all_token_type_ids]
else:
all_start_positions = tf.convert_to_tensor([f.start_position for f in crops], dtype=tf.int32)
 all_end_positions = tf.convert_to_tensor([f.end_position for f in crops], dtype=tf.int32)
 all_long_positions = tf.convert_to_tensor([f.long_position for f in crops], dtype=tf.int32)
 dataset = [all_input_ids, all_attention_mask, all_token_type_ids,
all_start_positions, all_end_positions, all_long_positions]
return dataset, crops, entries 
<install_modules>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
!pip install.. /input/apex01cp37cp37mlinux-x86-64whl/apex-0.1-cp37-cp37m-linux_x86_64.whl<install_modules>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 152
MODEL = 'jplu/tf-xlm-roberta-large'
<predict_on_test>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
def bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len):
class TFQADataset(Dataset):
def __init__(self, id_list):
self.id_list=id_list
def __len__(self):
return len(self.id_list)
 def __getitem__(self, index):
return self.id_list[index]
class Collator(object):
def __init__(self, data_dict, new_token_dict, tokenizer, max_seq_len=384, max_question_len=64):
self.data_dict = data_dict
self.new_token_dict = new_token_dict
self.tokenizer = tokenizer
self.max_seq_len = max_seq_len
self.max_question_len = max_question_len
def _get_input_ids(self, doc_id, candidate_index):
data = self.data_dict[doc_id]
question_tokens = self.tokenizer.tokenize(data['question_text'])[:self.max_question_len]
doc_words = data['document_text'].split()
max_answer_tokens = self.max_seq_len-len(question_tokens)-3
candidate = data['long_answer_candidates'][candidate_index]
candidate_start = candidate['start_token']
candidate_end = candidate['end_token']
candidate_words = doc_words[candidate_start:candidate_end]
for i, word in enumerate(candidate_words):
if re.match(r'<.+>', word):
if word in self.new_token_dict:
candidate_words[i] = self.new_token_dict[word]
else:
candidate_words[i] = '<'
words_to_tokens_index = []
candidate_tokens = []
for i, word in enumerate(candidate_words):
words_to_tokens_index.append(len(candidate_tokens))
tokens = self.tokenizer.tokenize(word)
 if len(candidate_tokens)+len(tokens)> max_answer_tokens:
break
for token in tokens:
candidate_tokens.append(token)
 input_tokens = ['[CLS]'] + question_tokens + ['[SEP]'] + candidate_tokens + ['[SEP]']
input_ids = self.tokenizer.convert_tokens_to_ids(input_tokens)
 return input_ids, words_to_tokens_index, len(input_ids), len(question_tokens)+2
def __call__(self, batch_ids):
batch_size = len(batch_ids)
 batch_input_ids_temp = []
batch_seq_len = []
batch_offset = []
batch_words_to_tokens_index = []
for i,(doc_id, candidate_index)in enumerate(batch_ids):
input_ids, words_to_tokens_index, seq_len, offset = self._get_input_ids(doc_id, candidate_index)
 batch_input_ids_temp.append(input_ids)
 batch_seq_len.append(seq_len)
 batch_offset.append(offset)
 batch_words_to_tokens_index.append(words_to_tokens_index)
 batch_max_seq_len = max(batch_seq_len)
 batch_input_ids = np.zeros(( batch_size, batch_max_seq_len), dtype=np.int64)
 batch_token_type_ids = np.ones(( batch_size, batch_max_seq_len), dtype=np.int64)
 for i in range(batch_size):
input_ids = batch_input_ids_temp[i]
batch_input_ids[i, :len(input_ids)] = input_ids
batch_token_type_ids[i, :len(input_ids)] = [0 if k<=input_ids.index(102)else 1 for k in range(len(input_ids)) ]
batch_attention_mask = batch_input_ids > 0
return torch.from_numpy(batch_input_ids), torch.from_numpy(batch_attention_mask), torch.from_numpy(batch_token_type_ids), batch_words_to_tokens_index, batch_offset, batch_max_seq_len
class BertForQuestionAnswering(BertPreTrainedModel):
def __init__(self, config):
super(BertForQuestionAnswering, self ).__init__(config)
 self.bert = BertModel(config)
 self.qa_outputs = nn.Linear(config.hidden_size, 2)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.init_weights()
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 sequence_output = outputs[0]
pooled_output = outputs[1]
qa_logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = qa_logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 pooled_output = self.dropout(pooled_output)
 classifier_logits = self.classifier(pooled_output)
 return start_logits, end_logits, classifier_logits
max_seq_len = 360
max_question_len = 64
batch_size = int(384/4)
 model_path = '.. /models/models/bert/'
config = BertConfig.from_pretrained(model_path)
 config.num_labels = 5
config.vocab_size = 30531
tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = BertForQuestionAnswering.from_pretrained(model_dir, config=config)
 new_token_dict = {
'<P>':'qw1',
'<Table>':'qw2',
'<Tr>':'qw3',
'<Ul>':'qw4',
'<Ol>':'qw5',
'<Fl>':'qw6',
'<Li>':'qw7',
'<Dd>':'qw8',
'<Dt>':'qw9',
}
new_token_list = [
'qw1',
'qw2',
'qw3',
'qw4',
'qw5',
'qw6',
'qw7',
'qw8',
'qw9',
]
num_added_toks = tokenizer.add_tokens(new_token_list)
 model.resize_token_embeddings(len(tokenizer))
model.cuda()
optimizer = optim.Adam(model.parameters() , lr=1e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)
 if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
 test_datagen = TFQADataset(id_list=id_candidate_list_sorted)
 test_collate = Collator(data_dict=data_dict,
new_token_dict=new_token_dict,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
max_question_len=max_question_len)
 test_generator = DataLoader(dataset=test_datagen,
collate_fn=test_collate,
batch_size=batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
 model.eval()
list_offset = []
list_words_to_tokens_index = []
test_prob1 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob2 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob3 = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 for j,(batch_input_ids, batch_attention_mask, batch_token_type_ids, batch_words_to_tokens_index, batch_offset, batch_max_seq_len)in tqdm(enumerate(test_generator)) :
with torch.no_grad() :
start = j*batch_size
end = start+batch_size
if j == len(test_generator)-1:
end = len(test_generator.dataset)
 batch_input_ids = batch_input_ids.cuda()
batch_attention_mask = batch_attention_mask.cuda()
batch_token_type_ids = batch_token_type_ids.cuda()
logits1, logits2, logits3 = model(batch_input_ids, batch_attention_mask, batch_token_type_ids)
 test_prob1[start:end, :batch_max_seq_len] += F.softmax(logits1,dim=1 ).cpu().data.numpy()
test_prob2[start:end, :batch_max_seq_len] += F.softmax(logits2,dim=1 ).cpu().data.numpy()
test_prob3[start:end] += F.softmax(logits3,dim=1 ).cpu().data.numpy()
list_words_to_tokens_index += batch_words_to_tokens_index
list_offset += batch_offset
test_word_prob1 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 test_word_prob2 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 for i in range(len(id_candidate_list_sorted)) :
for j in range(len(list_words_to_tokens_index[i])) :
test_word_prob1[i,j] = test_prob1[i, list_words_to_tokens_index[i][j]+list_offset[i]]
test_word_prob2[i,j] = test_prob2[i, list_words_to_tokens_index[i][j]+list_offset[i]]
return test_word_prob1, test_word_prob2, test_prob3 
<categorify>
start_time = time.time()
data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted = reduce1(n_candidate=10, th_candidate=0.2)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
id_candidate_list_sorted = reduce2(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, n_candidate=4, th_candidate=0.35)
 print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
word_len = 360
start_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 end_prob = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 start_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 end_label = np.zeros(( len(id_candidate_list_sorted),),dtype=int)
 class_prob = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 print("before bert predicting")
 weight_albert = 0.7
model_dir = '.. /models/models/bert/'
test_prob1, test_prob2, test_prob3 = bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob +=(1-weight_albert)*test_prob1
end_prob +=(1-weight_albert)*test_prob2
class_prob +=(1-weight_albert)*test_prob3
print("before albert predicting")
 model_dir = '.. /models/models/albert/'
test_prob1, test_prob2, test_prob3 = albert_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len)
 start_prob += weight_albert*test_prob1
end_prob += weight_albert*test_prob2
class_prob += weight_albert*test_prob3
start_label = np.argmax(start_prob, axis=1)
 end_label = np.argmax(end_prob, axis=1)
 temp_dict = {}
for doc_id in id_list:
temp_dict[doc_id] = {
'long_answer': {'start_token': -1, 'end_token': -1},
'long_answer_score': -1.0,
'short_answers': [{'start_token': -1, 'end_token': -1}],
'short_answers_score': -1.0,
'yes_no_answer': 'NONE'
}
for i,(doc_id, candidate_index)in tqdm(enumerate(id_candidate_list_sorted)) :
long_answer_score = 1.0 - class_prob[i,0]
if long_answer_score > temp_dict[doc_id]['long_answer_score']:
temp_dict[doc_id]['long_answer_score'] = long_answer_score
temp_dict[doc_id]['long_answer']['start_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
temp_dict[doc_id]['long_answer']['end_token'] = data_dict[doc_id]['long_answer_candidates'][candidate_index]['end_token']
short_answer_score = 1.0 - class_prob[i,0] - class_prob[i,1]
temp_dict[doc_id]['short_answers_score'] = short_answer_score
temp_dict[doc_id]['short_answers'][0]['start_token'] = -1
temp_dict[doc_id]['short_answers'][0]['end_token'] = -1
temp_dict[doc_id]['yes_no_answer'] = 'NONE'
if max([class_prob[i,3], class_prob[i,4]])> class_prob[i,2]:
if class_prob[i,3] > class_prob[i,4]:
temp_dict[doc_id]['yes_no_answer'] = 'NO'
else:
temp_dict[doc_id]['yes_no_answer'] = 'YES'
else:
short_start_word = int(start_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
short_end_word = int(end_label[i])+ data_dict[doc_id]['long_answer_candidates'][candidate_index]['start_token']
if short_end_word > short_start_word:
temp_dict[doc_id]['short_answers'][0]['start_token'] = short_start_word
temp_dict[doc_id]['short_answers'][0]['end_token'] = short_end_word
final_dict = {}
final_dict['predictions'] = []
for doc_id in id_list:
prediction_dict = {
'example_id': doc_id,
'long_answer': {'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['long_answer']['start_token'], 'end_token': temp_dict[doc_id]['long_answer']['end_token']},
'long_answer_score': temp_dict[doc_id]['long_answer_score'],
'short_answers': [{'start_byte': -1, 'end_byte': -1, 'start_token': temp_dict[doc_id]['short_answers'][0]['start_token'], 'end_token': temp_dict[doc_id]['short_answers'][0]['end_token']}],
'short_answers_score': temp_dict[doc_id]['short_answers_score'],
'yes_no_answer': temp_dict[doc_id]['yes_no_answer']
}
final_dict['predictions'].append(prediction_dict)
 print("--- %s seconds ---" %(time.time() - start_time))<define_variables>
import pandas as pd
answers_df = pd.DataFrame.from_dict(final_dict)
 long_best_threshold = 0.7
short_best_threshold = 0.7
def df_long_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['long_answer_score'] > long_best_threshold:
index['start'] = df["predictions"][i]['long_answer']['start_token']
index['end'] = df["predictions"][i]['long_answer']['end_token']
index['score'] = df["predictions"][i]['long_answer_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_short_index_score(df):
answers = []
cont = 0
for i in range(len(df)) :
index = {}
if df["predictions"][i]['short_answers_score'] > short_best_threshold and df["predictions"][i]['short_answers'][0]['start_token'] != -1:
index['start'] = df["predictions"][i]['short_answers'][0]['start_token']
index['end'] = df["predictions"][i]['short_answers'][0]['end_token']
index['score'] = df["predictions"][i]['short_answers_score']
index = [index]
answers.append(index)
 else:
answers.append([])
 return answers
def df_example_id(df):
return df['example_id']
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':' + str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
<feature_engineering>
import pandas as pd
answers_df['long_indexes_and_scores'] = df_long_index_score(answers_df)
 answers_df['short_indexes_and_scores'] = df_short_index_score(answers_df)
 answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(
 create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(
 create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df = answers_df.drop(
 ['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 sample_submission = pd.read_csv(
 ".. /.. /.. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains(
 "_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains(
 "_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains(
 "_short"), "PredictionString"] = short_prediction_strings 
<set_options>
train_data = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 val_data = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv")
 test_data = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv")
 unintended_bias_train = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
torch.cuda.empty_cache()<drop_column>
del BertTokenizer, BertModel, BertPreTrainedModel, BertConfig, AlbertTokenizer, AlbertModel, AlbertPreTrainedModel, AlbertConfig<install_modules>
train = pd.concat([
train_data[['comment_text', 'toxic']],
unintended_bias_train[['comment_text', 'toxic']].query('toxic==1'),
unintended_bias_train[['comment_text', 'toxic']].query('toxic==0' ).sample(n=50000, random_state=0)
 ] )
!pip install --upgrade.. /input/tokenizers0011cp37/tokenizers-0.0.11-cp37-cp37m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl<import_modules>
import numpy as np
logger = logging.getLogger(__name__)
 NQExample = collections.namedtuple("NQExample", [
"qas_id", "question_text", "doc_tokens", "orig_answer_text",
"start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible", "crop_start"])
 Crop = collections.namedtuple("Crop", ["unique_id", "example_index", "doc_span_index",
"tokens", "token_to_orig_map", "token_is_max_context",
"input_ids", "attention_mask", "token_type_ids",
"paragraph_len", "start_position", "end_position", "long_position",
"short_is_impossible", "long_is_impossible"])
 LongAnswerCandidate = collections.namedtuple('LongAnswerCandidate', [
'start_token', 'end_token', 'top_level'])
 UNMAPPED = -123
CLS_INDEX = 0
def get_add_tokens(do_enumerate):
tags = ['Dd', 'Dl', 'Dt', 'H1', 'H2', 'H3', 'Li', 'Ol', 'P', 'Table', 'Td', 'Th', 'Tr', 'Ul']
opening_tags = [f'<{tag}>' for tag in tags]
closing_tags = [f'</{tag}>' for tag in tags]
added_tags = opening_tags + closing_tags
special_tokens = ['<P>', '<Table>']
if do_enumerate:
for special_token in special_tokens:
for j in range(11):
added_tags.append(f'<{special_token[1: -1]}{j}>')
 add_tokens = ['Td_colspan', 'Th_colspan', '``',<load_from_csv>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
submission_no2 = pd.read_csv("submission1.csv")
 for i in range(len(submission_no2)) :
if submission_no2["PredictionString"][i] is np.nan and sample_submission["PredictionString"][i] is not np.nan:
submission_no2["PredictionString"][i] = sample_submission["PredictionString"][i] 
<drop_column>
%%time
x_train = regular_encode(list(train.comment_text.values), tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(list(val_data.comment_text.values), tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(list(test_data.content.values), tokenizer, maxlen=MAX_LEN)
 y_valid = val_data.toxic.values
path = 'submission1.csv'
if os.path.exists(path):
os.remove(path )<save_to_csv>
y_train = train.toxic.values
submission_no2.to_csv('submission.csv', index=False)
<import_modules>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
sys.path.extend([
'.. /input/bert-joint-baseline/'])
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<define_variables>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, maxlen=MAX_LEN)
 model.summary()
on_kaggle_server = os.path.exists('/kaggle')
 nq_test_file = '.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl'
public_dataset = os.path.getsize(nq_test_file)<20_000_000
private_dataset = os.path.getsize(nq_test_file)>=20_000_000<load_pretrained>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
if True:
importlib.reload(bert_utils )<load_from_disk>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
with open('.. /input/bert-joint-baseline/bert_config.json','r')as f:
config = json.load(f)
 print(json.dumps(config,indent=4)) 
<choose_model_class>
y_t = model.predict(test_dataset, verbose=1 )
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=2, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
def mk_model(config):
seq_len = config['max_position_embeddings']
unique_id = tf.keras.Input(shape=(1,),dtype=tf.int64,name='unique_id')
 input_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_ids')
 input_mask = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='input_mask')
 segment_ids = tf.keras.Input(shape=(seq_len,),dtype=tf.int32,name='segment_ids')
 BERT = modeling.BertModel(config=config,name='bert')
 pooled_output, sequence_output = BERT(input_word_ids=input_ids,
input_mask=input_mask,
input_type_ids=segment_ids)
 logits = TDense(2,name='logits' )(sequence_output)
 start_logits,end_logits = tf.split(logits,axis=-1,num_or_size_splits= 2,name='split')
 start_logits = tf.squeeze(start_logits,axis=-1,name='start_squeeze')
 end_logits = tf.squeeze(end_logits, axis=-1,name='end_squeeze')
 ans_type = TDense(5,name='ans_type' )(pooled_output)
 return tf.keras.Model([input_ for input_ in [unique_id,input_ids,input_mask,segment_ids]
if input_ is not None],
[unique_id,start_logits,end_logits,ans_type],
name='bert-baseline' )<categorify>
s = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv" )
model= mk_model(config )<load_pretrained>
sub = pd.DataFrame({"id": s.id , "toxic":y_t.squeeze() }, index = None )
model.load_weights('.. /input/unk0201128w/weights' )<save_model>
sub.to_csv('submission.csv', index=False )
<install_modules><EOS> 
sub.to_csv('submission.csv', index=False )