Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<load_from_url>
MAX_LEN = 192
LR = 1e-5
BATCH_SIZE = 16
TOTAL_STEPS_STAGE1 = 300
VALIDATE_EVERY_STAGE1 = 100
TOTAL_STEPS_STAGE2 = 200
VALIDATE_EVERY_STAGE2 = 100
PRETRAINED_MODEL = 'jplu/tf-xlm-roberta-large'
D = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'
print(tf.__version__)
 logging.getLogger().setLevel(logging.NOTSET)
 AUTO = tf.data.experimental.AUTOTUNE
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-development.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-test.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-validation.tsv -q<install_modules>
def connect_to_TPU() :

 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
global_batch_size = BATCH_SIZE * strategy.num_replicas_in_sync
return tpu, strategy, global_batch_size
tpu, strategy, global_batch_size = connect_to_TPU()
print("REPLICAS: ", strategy.num_replicas_in_sync )
!pip install pytorch-pretrained-bert<import_modules>
train_df = pd.read_csv(D+'jigsaw-toxic-comment-train.csv')
 val_df = pd.read_csv(D+'validation.csv')
 test_df = pd.read_csv(D+'test.csv')
 sub_df = pd.read_csv(D+'sample_submission.csv')
 train_df = pd.concat([
train_df.query('toxic==1'),
train_df.query('toxic==0' ).sample(sum(train_df.toxic),random_state=42)
 ])
 train_df = train_df.sample(frac=1, random_state = 42 )
import time
import os
import random
import numpy as np
import pandas as pd
import torch
from torch.optim import Adam
from torch.utils.data import Dataset
from torch.nn import Module, Linear, Dropout
import torch.nn.functional as F
from pytorch_pretrained_bert.modeling import BertModel, BertLayer
from pytorch_pretrained_bert import BertTokenizer
from pytorch_pretrained_bert.optimization import BertAdam
from pytorch_pretrained_bert.optimization import warmup_linear
from torch.utils.data import DataLoader
from torch.utils.data import RandomSampler
from sklearn.metrics import log_loss
import matplotlib.pyplot as plt<set_options>
%%time
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL)
 X_train = regular_encode(train_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_val = regular_encode(val_df.comment_text.values, tokenizer, maxlen=MAX_LEN)
 X_test = regular_encode(test_df.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train_df.toxic.values.reshape(-1,1)
 y_val = val_df.toxic.values.reshape(-1,1 )
seed = 9876
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<init_hyperparams>
def create_dist_dataset(X, y=None, training=False):
dataset = tf.data.Dataset.from_tensor_slices(X)
 if y is not None:
dataset_y = tf.data.Dataset.from_tensor_slices(y)
 dataset = tf.data.Dataset.zip(( dataset, dataset_y))
if training:
dataset = dataset.shuffle(len(X)).repeat()
dataset = dataset.batch(global_batch_size ).prefetch(AUTO)
 dist_dataset = strategy.experimental_distribute_dataset(dataset)
 return dist_dataset
train_dist_dataset = create_dist_dataset(X_train, y_train, True)
 val_dist_dataset = create_dist_dataset(X_val)
 test_dist_dataset = create_dist_dataset(X_test )
bert_model = "bert-large-cased"
n_bertlayers = 22
dropout = 0.1
do_lower_case = False
train_batch_size = 4
gradient_accumulation_steps = 5
lr = 1e-5
num_train_epochs = 2
warmup_proportion = 0.1
optim = "bertadam"
weight_decay = False
n_models = 10
eval_batch_size = 32
device = torch.device("cuda")
 data_dir = ""<string_transform>
%%time
def create_model_and_optimizer() :
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(PRETRAINED_MODEL)
 model = build_model(transformer_layer)
 optimizer = tf.keras.optimizers.Adam(learning_rate=LR, epsilon=1e-08)
 return model, optimizer
def build_model(transformer):
inp = Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 x = transformer(inp)[0][:, 0, :]
out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[inp], outputs=[out])
 return model
model, optimizer = create_model_and_optimizer()
model.summary()
def insert_tag(row):

 to_be_inserted = sorted([ 
(row["A-offset"], " [A] "), 
(row["B-offset"], " [B] "), 
(row["Pronoun-offset"], " [P] ")
 ], key=lambda x: x[0], reverse=True)
 text = row["Text"]
for offset, tag in to_be_inserted:
text = text[:offset] + tag + text[offset:]
return text
def tokenize(text, tokenizer):

 entries = {}
final_tokens = []
for token in tokenizer.tokenize(text):
if token in("[A]", "[B]", "[P]"):
entries[token] = len(final_tokens)
 continue
final_tokens.append(token)
 return final_tokens,(entries["[A]"], entries["[B]"], entries["[P]"])
 class GAPDataset(Dataset):

 def __init__(self, df, tokenizer, labeled=True):
self.labeled = labeled
if labeled:
tmp = df[["A-coref", "B-coref"]].copy()
tmp["Neither"] = ~(df["A-coref"] | df["B-coref"])
 self.y = tmp.values.astype("bool")
 self.offsets, self.tokens = [], []
self.seq_len = []
for _, row in df.iterrows() :
text = insert_tag(row)
 tokens, offsets = tokenize(text, tokenizer)
 self.offsets.append(offsets)
 self.tokens.append(tokenizer.convert_tokens_to_ids(
 ["[CLS]"] + tokens + ["[SEP]"]))
self.seq_len.append(len(self.tokens[-1]))
def __len__(self):
return len(self.tokens)
 def __getitem__(self, idx):
if self.labeled:
return self.tokens[idx], self.offsets[idx], self.y[idx]
return self.tokens[idx], self.offsets[idx]
def get_seq_len(self):
return self.seq_len
def collate_examples(batch, truncate_len=500):

 transposed = list(zip(*batch))
max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels<categorify>
def define_losses_and_metrics() :
with strategy.scope() :
loss_object = tf.keras.losses.BinaryCrossentropy(
 reduction=tf.keras.losses.Reduction.NONE, from_logits=False)
 def compute_loss(labels, predictions):
per_example_loss = loss_object(labels, predictions)
 loss = tf.nn.compute_average_loss(
 per_example_loss, global_batch_size = global_batch_size)
 return loss
train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')
 return compute_loss, train_accuracy_metric
def train(train_dist_dataset, val_dist_dataset=None, y_val=None,
total_steps=5000, validate_every=500):
step = 0
for tensor in train_dist_dataset:
distributed_train_step(tensor)
 step+=1
if(step % validate_every == 0):
train_metric = train_accuracy_metric.result().numpy()
print("Step %d, train AUC: %.5f" %(step, train_metric))
if val_dist_dataset:
val_metric = roc_auc_score(y_val, predict(val_dist_dataset))
print(" validation AUC: %.5f" % val_metric)
 train_accuracy_metric.reset_states()
if step == total_steps:
break
@tf.function
def distributed_train_step(data):
strategy.experimental_run_v2(train_step, args=(data,))
def train_step(inputs):
features, labels = inputs
with tf.GradientTape() as tape:
predictions = model(features, training=True)
 loss = compute_loss(labels, predictions)
 gradients = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_accuracy_metric.update_state(labels, predictions)
 def predict(dataset):
predictions = []
for tensor in dataset:
predictions.append(distributed_prediction_step(tensor))
predictions = np.vstack(list(map(np.vstack,predictions)))
 return predictions
@tf.function
def distributed_prediction_step(data):
predictions = strategy.experimental_run_v2(prediction_step, args=(data,))
return strategy.experimental_local_results(predictions)
 def prediction_step(inputs):
features = inputs
predictions = model(features, training=False)
 return predictions
compute_loss, train_accuracy_metric = define_losses_and_metrics()
def get_pretrained_bert(modelname, num_hidden_layers=None):
bert = BertModel.from_pretrained(modelname)
 if num_hidden_layers is None:
return bert
old_num_hidden_layers = bert.config.num_hidden_layers
if num_hidden_layers < old_num_hidden_layers:
del bert.encoder.layer[num_hidden_layers:]
elif num_hidden_layers > old_num_hidden_layers:
for i in range(old_num_hidden_layers, num_hidden_layers):
bert.encoder.layer.add_module(str(i), BertLayer(bert.config))
if num_hidden_layers != old_num_hidden_layers:
bert.config.num_hidden_layers = num_hidden_layers
bert.init_bert_weights(bert.pooler.dense)
 return bert
class BertCl_GAP(Module):

 def __init__(self, bert, dropout, n_offsets=3):
super().__init__()
self.bert = bert
self.bert_hidden_size = self.bert.config.hidden_size
self.dropout = Dropout(dropout)
 self.classifier = Linear(self.bert.config.hidden_size * n_offsets, n_offsets)
 def forward(self, token_tensor, offsets, label_id=None):
bert_outputs, _ = self.bert(
 token_tensor, attention_mask=(token_tensor > 0 ).long() ,
token_type_ids=None, output_all_encoded_layers=False)
 extracted_outputs = bert_outputs.gather(
 1, offsets.unsqueeze(2 ).expand(-1, -1, bert_outputs.size(2))
).view(bert_outputs.size(0), -1)
 outputs = self.classifier(self.dropout(extracted_outputs))
return outputs
def get_param_size(model):
trainable_psize = np.sum([np.prod(p.size())for p in model.parameters() if p.requires_grad])
 total_psize = np.sum([np.prod(p.size())for p in model.parameters() ])
 return total_psize, trainable_psize
def run_epoch(model, dataloader, optimizer, criterion, device,
verbose_step=10000):
model.train()
t1 = time.time()
tr_loss = 0
for step, batch in enumerate(dataloader):
batch = tuple(t.to(device)for t in batch)
 label_ids = batch[-1]
outputs = model(*batch[:-1])
 if criterion._get_name() == "BCEWithLogitsLoss":
outputs = outputs[:, 0]
label_ids = label_ids.float()
loss = criterion(outputs, label_ids)
 if gradient_accumulation_steps > 1:
loss = loss / gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
if(step + 1)% verbose_step == 0:
loss_now = gradient_accumulation_steps * tr_loss /(step + 1)
 print(f'step:{step+1} loss:{loss_now:.7f} time:{time.time() - t1:.1f}s')
 if(step + 1)% gradient_accumulation_steps == 0:
optimizer.step()
model.zero_grad()
return gradient_accumulation_steps * tr_loss /(step + 1)
 def predict(model, data_loader, device, proba=True, to_numpy=True):
model.eval()
preds = []
for step, batch in enumerate(data_loader):
batch = batch[:2]
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
logits = model(*batch)
 preds.append(logits.detach().cpu())
 preds = torch.cat(preds)if len(preds)> 1 else preds[0]
if proba:
if preds.size(-1)> 1:
preds = F.softmax(preds, dim=1)
 else:
preds = torch.sigmoid(preds)
 if to_numpy:
preds = preds.numpy()
return preds<choose_model_class>
%%time
train(train_dist_dataset, val_dist_dataset, y_val,
TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1 )
def get_gap_model(bert_model, n_bertlayers, dropout,
steps_per_epoch, device):
bert = get_pretrained_bert(bert_model, n_bertlayers)
 model = BertCl_GAP(bert, dropout)
 model.to(device)
 param_optimizer = list(model.named_parameters())
 if weight_decay:
no_decay = ["bias", "gamma", "beta", "head"]
optimizer_grouped_parameters = [
{"params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
"weight_decay": 0.01},
{"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
"weight_decay": 0.0}
]
else:
optimizer_grouped_parameters = [
{"params": [p for n, p in param_optimizer], "weight_decay": 0.0}
]
t_total = int(
 steps_per_epoch / gradient_accumulation_steps * num_train_epochs)
 if optim == 'bertadam':
optimizer = BertAdam(optimizer_grouped_parameters,
lr=lr,
warmup=warmup_proportion,
t_total=t_total)
 elif optim == 'adam':
optimizer = Adam(optimizer_grouped_parameters, lr=lr)
 return model, optimizer
def get_loader(train_df, val_df, test_df):
tokenizer = BertTokenizer.from_pretrained(
 bert_model,
do_lower_case=do_lower_case,
never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "[A]", "[B]", "[P]")
)
 tokenizer.vocab['[A]'] = -1
tokenizer.vocab['[B]'] = -1
tokenizer.vocab['[P]'] = -1
train_ds = GAPDataset(train_df, tokenizer)
 val_ds = GAPDataset(val_df, tokenizer)
 test_ds = GAPDataset(test_df, tokenizer, labeled=False)
 train_loader = DataLoader(
 train_ds,
collate_fn=collate_examples,
batch_size=train_batch_size,
shuffle=True,
drop_last=True 
)
 val_loader = DataLoader(
 val_ds,
collate_fn=collate_examples,
batch_size=eval_batch_size,
shuffle=False 
)
 test_loader = DataLoader(
 test_ds,
collate_fn=collate_examples,
batch_size=eval_batch_size,
shuffle=False 
)
 return train_loader, val_loader, test_loader<load_from_csv>
%%time
val_dist_dataset_4_training = create_dist_dataset(X_val, y_val, training=True)
 train(val_dist_dataset_4_training,
total_steps = TOTAL_STEPS_STAGE2,
validate_every = VALIDATE_EVERY_STAGE2 )
train_df = pd.concat([pd.read_csv(data_dir + "gap-test.tsv", delimiter="\t"),
pd.read_csv(data_dir + "gap-development.tsv", delimiter="\t")])
 val_df = pd.read_csv(data_dir + "gap-validation.tsv", delimiter="\t")
 val_y = val_df[['A-coref', 'B-coref']].astype(int)
 val_y['None'] = 1 - val_y.sum(1)
 test_df = pd.read_csv(".. /input/test_stage_2.tsv", delimiter="\t")
 print(f"Train:{train_df.shape[0]}, Valid:{val_df.shape[0]}, Test:{test_df.shape[0]}" )<load_pretrained>
%%time
sub_df['toxic'] = predict(test_dist_dataset)[:,0]
sub_df.to_csv('submission.csv', index=False )