Kaggle competition: Toxic Comment Classification Challenge
<SOS> metric: MCAUC Kaggle data source: toxic-comment-classification-challenge<import_modules>
! pip install pytorch_pretrained_bert pytorch-nlp -q
from sklearn.cluster import KMeans<groupby>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device
train[['Target', 'clusters', 'v2a1']].groupby(['Target', 'clusters'] ).agg(['count', 'mean'] )<filter>
!unzip -o '/kaggle/input/jigsaw-toxic-comment-classification-challenge/*.zip' -d /kaggle/working > /dev/null
cl = '4_3'
train['idhogar'][train['clusters'] == cl]<drop_column>
df_train = pd.read_csv("train.csv")
 df_test = pd.read_csv("test.csv")
 sample_submission = pd.read_csv("sample_submission.csv")
 print(df_train.shape, df_test.shape, sample_submission.shape)
 df_train.head()
train = train[~(train['idhogar'] == '563cc81b7')].drop(columns='clusters' )<concatenate>
def strip(text):
soup = BeautifulSoup(text, 'html.parser')
 text = re.sub('\[[^]]*\]', '', soup.get_text())
 pattern=r"[^a-zA-z0-9\s,']"
text=re.sub(pattern,'',text)
 return text
df_train["comment_text"] = df_train["comment_text"].apply(strip)
 df_test["comment_text"] = df_test["comment_text"].apply(strip )
cluster(kmeans_groups )<groupby>
train_sentences = df_train["comment_text"]
test_sentences = df_test["comment_text"]
train_sentences = ["[CLS] "+ i + " [SEP]"for i in train_sentences]
test_sentences = ["[CLS] "+ i + " [SEP]"for i in test_sentences]
train_sentences[0], test_sentences[0]
train[['Target', 'clusters', 'v2a1']].groupby(['Target', 'clusters'] ).agg(['count', 'mean'] )<randomize_order>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case = True)
 train_tokenizer_texts = list(map(lambda t: tokenizer.tokenize(t)[:510], tqdm(train_sentences)))
 test_tokenizer_texts = list(map(lambda t: tokenizer.tokenize(t)[:510], tqdm(test_sentences)))
 np.array(train_tokenizer_texts[0]), np.array(test_tokenizer_texts[0] )
split = 0.2
val = []
for c in train['clusters'].unique() : val += list(train['Id'][train['clusters'] == c].sample(frac=split, random_state=random_state))
idxs = train['Id'].isin(val )<groupby>
labels = df_train[["toxic","severe_toxic","obscene","threat","insult","identity_hate"]].to_numpy()
def oversample(df):
d = df.copy()
max_group = d[['clusters', 'v2a1']].groupby(['clusters'] ).count().max().values[0]
for c in d['clusters'].unique() :
ratio = 1 if c[0] != '4' else 3
to_sample = max(int(max_group/ratio)- d['clusters'][d['clusters'] == c].count() , 0)
 d = d.append(df[df['clusters'] == c].sample(
 n=to_sample, replace=True, random_state=random_state), ignore_index=True)
 return d
t_set = oversample(t_set )<prepare_x_and_y>
MAX_LEN = 128
input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tqdm(train_tokenizer_texts)]
input_ids = pad_sequences(sequences = input_ids, maxlen = MAX_LEN, dtype = 'long', padding='post', truncating='post')
 test_input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tqdm(test_tokenizer_texts)]
test_input_ids = pad_sequences(sequences = test_input_ids, maxlen = MAX_LEN, dtype = 'long', padding='post', truncating='post')
 input_ids[0], test_input_ids[0]
to_drop = ['Id', 'idhogar', 'parentesco1_mean', 'parentesco1_sum']
X_train = t_set.copy().drop(columns=(to_drop + ['Target', 'clusters']))
X_val = v_set.copy().drop(columns=(to_drop + ['Target', 'clusters']))<data_type_conversions>
def create_attention_masks(input_ids):
attention_masks = []
for seq in tqdm(input_ids):
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask)
 return np.array(attention_masks)
 attention_masks = create_attention_masks(input_ids)
 test_attention_masks = create_attention_masks(test_input_ids)
 attention_masks[0], test_attention_masks[0]
sc = StandardScaler()
def to_float(df):
for c in df.columns: df[c] = df[c].astype('float64')
 return df
X_train = sc.fit_transform(to_float(X_train))
X_val = sc.transform(to_float(X_val))<data_type_conversions>
X_train, X_val, y_train, y_val = train_test_split(input_ids, labels, random_state = 123, test_size = 0.20)
 attention_masks_train, attention_masks_val = train_test_split(attention_masks, random_state = 123, test_size = 0.20 )
y_train = t_set['Target'].copy().astype('category' ).cat.as_ordered().astype('float64')- 1
y_val = v_set['Target'].copy().astype('category' ).cat.as_ordered().astype('float64')- 1<import_modules>
X_train = torch.tensor(X_train)
 X_val = torch.tensor(X_val)
 y_train = torch.tensor(y_train)
 y_val = torch.tensor(y_val)
 attention_masks_train = torch.tensor(attention_masks_train)
 attention_masks_val = torch.tensor(attention_masks_val)
 test_input_ids = torch.tensor(test_input_ids)
 test_attention_masks = torch.tensor(test_attention_masks )
from sklearn.metrics import f1_score<import_modules>
BATCH_SIZE = 32
train_data = TensorDataset(X_train, attention_masks_train, y_train)
 val_data = TensorDataset(X_val, attention_masks_val, y_val)
 test_data = TensorDataset(test_input_ids, test_attention_masks)
 train_sampler = RandomSampler(train_data)
 val_sampler = SequentialSampler(val_data)
 test_sampler = SequentialSampler(test_data)
 train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size = BATCH_SIZE)
 val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size = BATCH_SIZE)
 test_dataloader = DataLoader(test_data, sampler = test_sampler, batch_size = BATCH_SIZE )
import torch as t
import torch.nn as nn
from torch import autograd, optim
import torch.nn.functional as F<train_on_grid>
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels = 6)
 model.cuda()
class Net(nn.Module):
def __init__(self, X, y, nodes=[200], p=0, act='lrelu'):
super().__init__()
self.input_size = X.shape[1]
self.output_size = int(y.nunique())
 self.nodes = nodes
self.p = p
self.act = act
if self.p > 1 or self.p < 0: raise Exception(f"Expected p to be <0,1> but got {self.p}")
 if nodes == []: raise Exception("No node sizes input")
 self.h1 = nn.Linear(self.input_size, self.nodes[0])
 self.hid = nn.ModuleList([nn.Linear(self.nodes[i], self.nodes[i + 1])for i in range(len(self.nodes)- 1)])
 self.out = nn.Linear(self.nodes[-1], self.output_size)
 self.bn = nn.ModuleList([nn.BatchNorm1d(self.nodes[i])for i in range(1, len(self.nodes)) ])
 if self.act == 'lrelu': self.act1 = F.leaky_relu
elif self.act == 'relu': self.act1 = F.relu
elif self.act == 'tanh': self.act1 = F.tanh
else: raise Exception("Invalid activation function - choose between lrelu, relu or tanh")
 self.act2 = F.softmax
def forward(self, X):
self.inp = autograd.Variable(t.cuda.FloatTensor(X))
o = self.act1(self.h1(self.inp))
for h, b in zip(self.hid, self.bn):
o = F.dropout(b(self.act1(h(o))), p=self.p)
 o = self.act2(self.out(o), dim=1)
 return o<choose_model_class>
param_optimizer = list(model.named_parameters())
 no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{
'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.01
},
{
'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay_rate': 0.0
}
]
optimizer = BertAdam(optimizer_grouped_parameters, lr = 2e-5, warmup =.1 )
def train_nn(model, n_epochs=10000, lr=1e-6, bs=900, wd=0, update_cycle=200, use_SGD=True, t_cycles=1, updates=True, cycle_double=True):
loss_plt = []
f1s = []
vf1s = []
opt = optim.Adam(params=model.parameters() , lr=lr, weight_decay=wd)
 targ = autograd.Variable(t.cuda.FloatTensor(y_train.values ).long())
 for i in range(1, t_cycles + 1):
if t_cycles > 1 and updates: print('='*100, f'
Cycle: {i}, Current Learning Rate: {lr}, Epochs at Cycle: {n_epochs} - updates every {update_cycle} epoch')
 for epoch in range(1, n_epochs + 1):
targ.require_grad = False
if use_SGD:
idxs = np.arange(len(X_train))
np.random.shuffle(idxs)
 for b in range(0, len(X_train), bs):
loss = 0
targb = autograd.Variable(t.cuda.FloatTensor(y_train.values[idxs[b:b+bs]] ).long())
 targb.require_grad = False
out = model(X_train[idxs[b:b+bs]])
 loss = F.nll_loss(out, targb)
 else:
loss = 0
out = model(X_train)
 loss = F.nll_loss(out, targ)
 model.zero_grad()
loss.backward()
opt.step()
if epoch % update_cycle == 0:
out = model(X_train)
 loss = F.nll_loss(out, targ)
 loss_plt.append(loss.cpu().data.numpy())
 vout = model(X_val)
 f1 = f1_score(y_train.values, out.max(dim=1)[1].cpu().data.numpy() , average='macro')
 vf1 = f1_score(y_val.values, vout.max(dim=1)[1].cpu().data.numpy() , average='macro')
 f1s.append(f1)
 vf1s.append(vf1)
 if updates: print("_"*50,'
', f'Epoch: {epoch}, Training Loss: {loss.cpu().data.numpy() }, Training F1: {f1}, Validation F1: {vf1}')
 lr /= 10
if cycle_double:
n_epochs *= 2
update_cycle *= 2
print(f'Final validation F1: {vf1s[-1:][0]}')
 return model, vf1s[-1:][0]<train_model>
torch.cuda.empty_cache()
gc.collect()
def train_ensamble(nodes = [89, 89], p = 0.7, wd = 1e-7, lr = 1e-1, bs =(int(X_train.shape[0] / 4 + 1)) ,
t_cycles = 6, epochs_per_cycle = 1, cycle_double = True, cycles = 1, n_models = 31, stop_threshold = 0.46):
global models
models = []
for n in range(n_models): models.append(Net(X_train, y_train, nodes=nodes, p=p ).cuda())
 final_res = []
mod = 1
for model in models:
print('='*50, f'
MODEL {mod}')
 model.train(mode=True)
 for i in range(1, cycles + 1):
print(f'Cycle {i}')
 _, res = train_nn(model, n_epochs = epochs_per_cycle,
lr = lr, bs = bs, wd=wd, update_cycle=epochs_per_cycle, t_cycles = t_cycles,
updates=False, cycle_double=cycle_double)
 if res >= stop_threshold: break
final_res.append(res)
 mod +=1
plt.scatter(np.arange(len(final_res)) , final_res)
 plt.plot([np.mean(final_res)for a in range(len(final_res)) ], color='r', linestyle=':')
 plt.show()<train_model>
torch.cuda.empty_cache()
epochs = 2
for _ in trange(epochs, desc = "Epoch"):
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
val_loss = 0
nb_val_examples, nb_val_steps = 0, 0
for batch in tqdm(train_dataloader):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
optimizer.zero_grad()
logits = model(b_input_ids, token_type_ids = None, attention_mask = b_input_mask)
 loss_func = BCEWithLogitsLoss()
loss = loss_func(logits, b_labels.type_as(logits))
loss.backward()
optimizer.step()
tr_loss += loss.item()
nb_tr_examples += b_input_ids.size(0)
 nb_tr_steps += 1
print("Train Loss: {}".format(tr_loss/nb_tr_examples))
model.eval()
for step, batch in enumerate(val_dataloader):
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
b_input_ids, b_input_mask, b_labels = batch
logits = model(b_input_ids, token_type_ids = None, attention_mask = b_input_mask)
 loss_func = BCEWithLogitsLoss()
loss = loss_func(logits, b_labels.type_as(logits))
val_loss += loss.item()
nb_val_examples += b_input_ids.size(0)
 nb_val_steps += 1
print("Valid Loss: {}".format(val_loss/nb_val_examples))
%%time
train_ensamble()<predict_on_test>
outputs = []
for step, batch in enumerate(test_dataloader):
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
b_input_ids, b_input_mask = batch
logits = model(b_input_ids, token_type_ids = None, attention_mask = b_input_mask)
 s = Sigmoid()
out = s(logits ).to('cpu' ).numpy()
outputs.extend(out)
 df_test = pd.merge(df_test, sample_submission, on = "id")
 df_test[["toxic","severe_toxic","obscene","threat","insult","identity_hate"]] = outputs
df_test.drop(["comment_text"], axis = 1, inplace = True)
 df_test.to_csv("sample_submission.csv", index = False )