Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
train_loader, val_loader, test_loader = get_loader(train_df, val_df, test_df )<define_variables>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
steps_per_epoch = len(train_loader)
 steps_per_epoch<compute_train_metric>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
scores = []
criterion = torch.nn.CrossEntropyLoss()
val_pr_avg = [np.zeros(val_y.shape)for _ in range(num_train_epochs)]
test_pr_avg = np.zeros(( test_df.shape[0], 3))<find_best_model_class>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
for model_id in range(n_models):
model, optimizer = get_gap_model(bert_model, n_bertlayers, dropout,
steps_per_epoch, device)
 total_psize, trainalbe_psize = get_param_size(model)
 print(f"Total params: {total_psize}
Trainable params: {trainalbe_psize}")
 for e in range(num_train_epochs):
t1 = time.time()
tr_loss = run_epoch(model, train_loader, optimizer, criterion, device)
 val_pr = predict(model, val_loader, device)
 val_pr_avg[e] += val_pr
val_loss = log_loss(val_y, val_pr)
 val_avg_loss = log_loss(val_y, val_pr_avg[e] /(model_id + 1))
elapsed = time.time() - t1
print(f"Epoch:{e + 1} tr_loss:{tr_loss:.4f} val_loss:{val_loss:.4f}"
f" val_avg_loss: {val_avg_loss:.4f} time:{elapsed:.1f}s")
 scores.append({"model_id": model_id, "epoch": e + 1, "time": elapsed,
"tr_loss": tr_loss, "val_loss": val_loss, "val_avg_loss": val_avg_loss})
 test_pr = predict(model, test_loader, device)
 test_pr_avg += test_pr
del model, optimizer
torch.cuda.empty_cache()<create_dataframe>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
df = pd.DataFrame(scores)
 pd.set_option("precision", 5)
 print("
Single model")
 print(df.groupby("epoch")[['tr_loss', 'val_loss']].mean())
 if n_models > 1:
print(f"
Avg of {n_models} models")
 print(df[df.model_id == n_models - 1][['epoch', 'val_avg_loss']].set_index('epoch'))<save_to_csv>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
test_pr_avg /= n_models
df_sub = pd.DataFrame(test_pr_avg, columns=["A", "B", "NEITHER"])
 df_sub["ID"] = test_df.ID
df_sub.to_csv("submission.csv", index=False)
<import_modules>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
import pandas as pd
import numpy as np
import os<load_from_csv>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
train = pd.read_csv('.. /input/test_stage_2.tsv', delimiter = '\t' ).rename(columns={'A': 'A_Noun', 'B': 'B_Noun'})
 train = train.rename(columns={'A_Noun':'A', 'B_Noun':'B'})
 train['A'] = pd.to_numeric(train['A'], errors = 'ignore')
 train['A'] =(( - np.random.randint(1, 12359, train.shape[0])+ 12359)/ 2)/ 10000
train['B'] = pd.to_numeric(train['B'], errors = 'ignore')
 train['B'] =(( - np.random.randint(1, 12359, train.shape[0])+ 12359)/ 2)/ 10000
sub = pd.read_csv('.. /input/sample_submission_stage_2.csv')
 sub['A'] = sub['A'] - pd.to_numeric(train['A'], errors = 'ignore')
 sub['A'] =(( - np.random.randint(1, 12359, train.shape[0])+ 12359)/ 2)/ 10000
sub['B'] = sub['B'] - pd.to_numeric(train['B'], errors = 'ignore')
 sub['B'] =(( - np.random.randint(1, 12359, train.shape[0])+ 12359)/ 2)/ 10000
sub['NEITHER'] =(( 10.0 -(train['A'] + train['B'])) / 6)/ 10
sub[['ID', 'A', 'B', 'NEITHER']].to_csv("submission.csv", index=False )<install_modules>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
!conda remove -y greenlet
!pip install pytorch-pretrained-bert
!pip install allennlp<load_pretrained>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )
nlp = English()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )<string_transform>
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
def candidate_length(candidate):
count = 0
for i in range(len(candidate)) :
if candidate[i] != " ": count += 1
return count
def count_char(text, offset):
count = 0
for pos in range(offset):
if text[pos] != " ": count +=1
return count
def count_token_length_special(token):
count = 0
special_token = ["
for i in range(len(token)) :
if token[i] not in special_token:
count+=1
return count
def find_word_index(tokenized_text, char_start, target):
tar_len = candidate_length(target)
 char_count = 0
word_index = []
special_token = ["[CLS]", "[SEP]"]
for i in range(len(tokenized_text)) :
token = tokenized_text[i]
if char_count in range(char_start, char_start+tar_len):
if token in special_token:
continue
word_index.append(i)
 if token not in special_token:
token_length = count_token_length_special(token)
 char_count += token_length
if len(word_index)== 1:
return [word_index[0], word_index[0]]
else:
return [word_index[0], word_index[-1]]
def create_tokenizer_input(sents):
tokenizer_input = str()
for i, sent in enumerate(sents):
if i == 0:
tokenizer_input += "[CLS] "+sent.text+" [SEP] "
elif i == len(sents)- 1:
tokenizer_input += sent.text+" [SEP]"
else:
tokenizer_input += sent.text+" [SEP] "
return tokenizer_input
def create_inputs(dataframe):
idxs = dataframe.index
columns = ['indexed_token', 'offset']
features_df = pd.DataFrame(index=idxs, columns=columns)
 max_len = 0
for i in tqdm(range(len(dataframe))):
text = dataframe.loc[i, 'Text']
Pronoun_offset = dataframe.loc[i, 'Pronoun-offset']
A_offset = dataframe.loc[i, "A-offset"]
B_offset = dataframe.loc[i, "B-offset"]
Pronoun = dataframe.loc[i, "Pronoun"]
A = dataframe.loc[i, "A"]
B = dataframe.loc[i, "B"]
doc = nlp(text)
 sents = []
for sent in doc.sents: sents.append(sent)
 token_input = create_tokenizer_input(sents)
 token_input = token_input.replace("
tokenized_text = tokenizer.tokenize(token_input)
 if len(tokenized_text)> max_len:
max_len = len(tokenized_text)
 indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
 A_char_start, B_char_start = count_char(text, A_offset), count_char(text, B_offset)
 Pronoun_char_start = count_char(text, Pronoun_offset)
 word_indexes = []
for char_start, target in zip([A_char_start, B_char_start, Pronoun_char_start], [A, B, Pronoun]):
word_indexes.append(find_word_index(tokenized_text, char_start, target))
features_df.iloc[i] = [indexed_tokens, word_indexes]
print('max length of sentence:', max_len)
 return features_df<load_from_csv>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
train_df = pd.read_table('.. /input/gap-coreference/gap-test.tsv')
 test_df = pd.read_table('.. /input/gap-coreference/gap-development.tsv')
 val_df = pd.read_table('.. /input/gap-coreference/gap-validation.tsv')
 new_train_df = create_inputs(train_df)
 new_test_df = create_inputs(test_df)
 new_val_df = create_inputs(val_df )<categorify>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
def get_label(dataframe):
labels = []
for i in range(len(dataframe)) :
if dataframe.loc[i, 'A-coref']:
labels.append(0)
 elif dataframe.loc[i, 'B-coref']:
labels.append(1)
 else:
labels.append(2)
 return labels
new_train_df['label'] = get_label(train_df)
 new_val_df['label'] = get_label(val_df)
 new_df = pd.concat([new_train_df, new_val_df])
 new_df = new_df.reset_index(drop=True)
 new_df.to_csv('train.csv', index=False)
 new_test_df['label'] = get_label(test_df)
 new_test_df.to_csv('test.csv', index=False )<drop_column>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
<set_options><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )