Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<create_dataframe>
! pip install transformers
def get_dataloaders(num_workers, batch_size, x_train, y_train, x_valid, y_valid):

 train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, num_workers=num_workers)
 loader = {
"train": train_loader,
"valid": valid_loader
}
return loader<load_pretrained>
import os
from typing import Tuple, List
from functools import partial
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, RandomSampler
from torch.nn.utils.rnn import pad_sequence
from transformers import BertTokenizer, BertModel, AdamW, get_linear_schedule_with_warmup, BertPreTrainedModel
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from tqdm import tqdm
def get_testloaders(num_workers, batch_size, x_test):

 test_dataset = TestDataset(x_test)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers)
 loader = {
'test': test_loader
}
return loader<train_model>
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip;
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip;
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip;
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip;
def train_model(n_epochs, loaders, model, optimizer, criterion, device, save_path):

 scheduler = StepLR(optimizer, step_size=2, gamma=0.96)
 valid_loss_min = np.Inf
print(valid_loss_min)
 for epoch in range(1, n_epochs + 1):
scheduler.step()
train_loss = 0.0
valid_loss = 0.0
model.train()
for data in loaders['train']:
data_input, data_target = data['input'].to(device), data['target'].to(device)
 optimizer.zero_grad()
output = model(data_input)
 loss = criterion(output, data_target)
 loss.backward()
optimizer.step()
train_loss +=(loss.item() / len(loaders['train']))
model.eval()
for data in loaders['valid']:
data_input, data_target = data['input'].to(device), data['target'].to(device)
 output = model(data_input)
 loss = criterion(output, data_target)
 valid_loss +=(loss.item() / len(loaders['valid']))
if valid_loss < valid_loss_min:
print(f'Epoch: \t{epoch}\tValidation loss decreased({valid_loss_min} -> {valid_loss} ).Saving the model...')
 torch.save(model.state_dict() , save_path)
 valid_loss_min = valid_loss
return model<split>
path = "./"
bert_model_name = 'bert-base-cased'
device = torch.device('cpu')
 if torch.cuda.is_available() :
device = torch.device('cuda:0')
 tokenizer = BertTokenizer.from_pretrained(bert_model_name)
 assert tokenizer.pad_token_id == 0, "Padding value used in masks is set to zero, please change it everywhere"
train_df = pd.read_csv(os.path.join(path, 'train.csv'))
train_df, val_df = train_test_split(train_df, test_size=0.05 )
def run_training(seed, kfold, batch_size, epochs, learning_rate, weight_decay):
set_seed(seed)
 for fold,(train_idx, valid_idx)in enumerate(kfold.split(X=train, y=train_targets)) :
x_train, x_valid = train[train_idx], train[valid_idx]
y_train, y_valid = train_targets[train_idx], train_targets[valid_idx]
dataloaders = get_dataloaders(0, batch_size, x_train, y_train, x_valid, y_valid)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Model(875, 206, 1024 ).to(device)
 criterion_moa = nn.BCEWithLogitsLoss()
optimizer_moa = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=weight_decay)
 train_model(epochs, dataloaders, model, optimizer_moa, criterion_moa, device, f'models/model_seed_{seed}_fold_{fold}.pt' )<define_variables>
class ToxicDataset(Dataset):
def __init__(self, tokenizer: BertTokenizer, dataframe: pd.DataFrame, lazy: bool = False):
self.tokenizer = tokenizer
self.pad_idx = tokenizer.pad_token_id
self.lazy = lazy
if not self.lazy:
self.X = []
self.Y = []
for i,(row)in tqdm(dataframe.iterrows()):
x, y = self.row_to_tensor(self.tokenizer, row)
 self.X.append(x)
 self.Y.append(y)
 else:
self.df = dataframe
@staticmethod
def row_to_tensor(tokenizer: BertTokenizer, row: pd.Series)-> Tuple[torch.LongTensor, torch.LongTensor]:
tokens = tokenizer.encode(row["comment_text"], add_special_tokens=True)
 if len(tokens)> 120:
tokens = tokens[:119] + [tokens[-1]]
x = torch.LongTensor(tokens)
 y = torch.FloatTensor(row[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]])
 return x, y
def __len__(self):
if self.lazy:
return len(self.df)
 else:
return len(self.X)
 def __getitem__(self, index: int)-> Tuple[torch.LongTensor, torch.LongTensor]:
if not self.lazy:
return self.X[index], self.Y[index]
else:
return self.row_to_tensor(self.tokenizer, self.df.iloc[index])
 def collate_fn(batch: List[Tuple[torch.LongTensor, torch.LongTensor]], device: torch.device)\
-> Tuple[torch.LongTensor, torch.LongTensor]:
x, y = list(zip(*batch))
x = pad_sequence(x, batch_first=True, padding_value=0)
 y = torch.stack(y)
 return x.to(device), y.to(device)
 train_dataset = ToxicDataset(tokenizer, train_df, lazy=True)
 dev_dataset = ToxicDataset(tokenizer, val_df, lazy=True)
 collate_fn = partial(collate_fn, device=device)
 BATCH_SIZE = 32
train_sampler = RandomSampler(train_dataset)
 dev_sampler = RandomSampler(dev_dataset)
 train_iterator = DataLoader(train_dataset, batch_size=BATCH_SIZE, sampler=train_sampler, collate_fn=collate_fn)
 dev_iterator = DataLoader(dev_dataset, batch_size=BATCH_SIZE, sampler=dev_sampler, collate_fn=collate_fn )
FOLDS = 5
WORKERS = 0
BATCH_SIZE = 128
EPOCHS = 50
LEARNING_RATE = 0.0002
WEIGHT_DECAY = 0.00001
SEED = 42<set_options>
class BertClassifier(nn.Module):
def __init__(self, bert: BertModel, num_classes: int):
super().__init__()
self.bert = bert
self.classifier = nn.Linear(bert.config.hidden_size, num_classes)
 def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 cls_output = outputs[1]
cls_output = self.classifier(cls_output)
 cls_output = torch.sigmoid(cls_output)
 criterion = nn.BCELoss()
loss = 0
if labels is not None:
loss = criterion(cls_output, labels)
 return loss, cls_output
model = BertClassifier(BertModel.from_pretrained(bert_model_name), 6 ).to(device )
%mkdir models<train_model>
def train(model, iterator, optimizer, scheduler):
model.train()
total_loss = 0
for x, y in tqdm(iterator):
optimizer.zero_grad()
mask =(x != 0 ).float()
loss, outputs = model(x, attention_mask=mask, labels=y)
 total_loss += loss.item()
loss.backward()
optimizer.step()
scheduler.step()
print(f"Train loss {total_loss / len(iterator)}")
 def evaluate(model, iterator):
model.eval()
pred = []
true = []
with torch.no_grad() :
total_loss = 0
for x, y in tqdm(iterator):
mask =(x != 0 ).float()
loss, outputs = model(x, attention_mask=mask, labels=y)
 total_loss += loss
true += y.cpu().numpy().tolist()
pred += outputs.cpu().numpy().tolist()
true = np.array(true)
 pred = np.array(pred)
 for i, name in enumerate(['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']):
print(f"{name} roc_auc {roc_auc_score(true[:, i], pred[:, i])}")
 print(f"Evaluate loss {total_loss / len(iterator)}" )
mskf = MultilabelStratifiedKFold(n_splits=FOLDS)
 for seed in range(40, 45):
run_training(seed, mskf, BATCH_SIZE, EPOCHS, LEARNING_RATE, WEIGHT_DECAY )<predict_on_test>
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
EPOCH_NUM = 2
warmup_steps = 10 ** 3
total_steps = len(train_iterator)* EPOCH_NUM - warmup_steps
optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5, eps=1e-8)
 scheduler = get_linear_schedule_with_warmup(optimizer, warmup_steps, total_steps)
def inference(loaders, model, device):

 model.eval()
preds = []
for data in loaders['test']:
data_input = data['input'].to(device)
 with torch.no_grad() :
output = model(data_input)
 pred = output.sigmoid().detach().cpu().numpy()
preds.append(pred)
 return np.concatenate(preds )<define_search_space>
for i in range(EPOCH_NUM):
print('=' * 50, f"EPOCH {i}", '=' * 50)
 train(model, train_iterator, optimizer, scheduler)
 evaluate(model, dev_iterator )
def run_inferencing(seed):
set_seed(seed)
 preds = np.zeros(( len(test), 206))
for i in range(0, FOLDS):
dataloaders = get_testloaders(0, 128, test)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Model(875, 206, 1024)
 model.load_state_dict(torch.load(f'models/model_seed_{seed}_fold_{i}.pt'))
model.to(device)
 pred = inference(dataloaders, model, device)
 preds += pred
preds = preds / FOLDS
return preds<define_variables>
model.eval()
test_df = pd.read_csv(os.path.join(path, 'test.csv'))
submission = pd.read_csv(os.path.join(path, 'sample_submission.csv'))
columns = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
for i in tqdm(range(len(test_df)// BATCH_SIZE + 1)) :
batch_df = test_df.iloc[i * BATCH_SIZE:(i + 1)* BATCH_SIZE]
assert(batch_df["id"] == submission["id"][i * BATCH_SIZE:(i + 1)* BATCH_SIZE] ).all() , f"Id mismatch"
texts = []
for text in batch_df["comment_text"].tolist() :
text = tokenizer.encode(text, add_special_tokens=True)
 if len(text)> 120:
text = text[:119] + [tokenizer.sep_token_id]
texts.append(torch.LongTensor(text))
x = pad_sequence(texts, batch_first=True, padding_value=tokenizer.pad_token_id ).to(device)
 mask =(x != tokenizer.pad_token_id ).float().to(device)
 with torch.no_grad() :
_, outputs = model(x, attention_mask=mask)
 outputs = outputs.cpu().numpy()
submission.iloc[i * BATCH_SIZE:(i + 1)* BATCH_SIZE][columns] = outputs
preds = np.zeros(( len(test), 206))
for seed in range(40, 45):
preds += run_inferencing(seed)
 preds = preds / 5<save_to_csv>
submission.to_csv("submission.csv", index=False )
sample_submission[targets] = preds
sample_submission.loc[test_features['cp_type']=='ctl_vehicle', targets] = 0
sample_submission.to_csv('submission.csv', index=False )<set_options>
submission.to_csv("submission.csv", index=False )