Kaggle competition: Toxic Comment Classification Challenge
names = [f"F{i+1}" for i in range(28)]<create_dataframe>
import shutil
from allennlp.common.params import Params
from allennlp.common.util import prepare_environment, dump_metrics
from allennlp.data.iterators import BasicIterator, BucketIterator
from allennlp.data.token_indexers import SingleIdTokenIndexer
from allennlp.data.token_indexers import PretrainedBertIndexer
from allennlp.data.tokenizers import WordTokenizer
from allennlp.data.tokenizers.word_splitter import BertBasicWordSplitter
from allennlp.data.vocabulary import Vocabulary
from allennlp.modules.seq2seq_encoders import PytorchSeq2SeqWrapper
from allennlp.modules.seq2vec_encoders import BagOfEmbeddingsEncoder
from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.training import Trainer
from pytorch_pretrained_bert import convert_tf_checkpoint_to_pytorch
from pytorch_pretrained_bert.optimization import BertAdam
from sklearn.model_selection import train_test_split
import torch
import torch.optim as optim
from swemencoder import SWEMEncoder
from toxiccommentclassificationreader import ToxicCommentClassificationReader
from toxiccommentpredictor import ToxicCommentPredictor
from toxicbaseclassifier import ToxicBaseClassifier
from toxicbertclassifier import ToxicBertClassifier
piv = pd.DataFrame(ids[:, 0], columns=["id"] )<feature_engineering>
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
<create_dataframe>
class Lookahead(Optimizer):
def __init__(self, base_optimizer,alpha=0.5, k=6):
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
 if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
 self.optimizer = base_optimizer
self.param_groups = self.optimizer.param_groups
self.alpha = alpha
self.k = k
for group in self.param_groups:
group["step_counter"] = 0
self.slow_weights = [[p.clone().detach() for p in group['params']]
for group in self.param_groups]
for w in it.chain(*self.slow_weights):
w.requires_grad = False
self.state = base_optimizer.state
def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
loss = self.optimizer.step()
for group,slow_weights in zip(self.param_groups,self.slow_weights):
group['step_counter'] += 1
if group['step_counter'] % self.k != 0:
continue
for p,q in zip(group['params'],slow_weights):
if p.grad is None:
continue
q.data.add_(self.alpha,p.data - q.data)
 p.data.copy_(q.data)
 self.state = self.optimizer.state
return loss
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2<save_to_csv>
reader = ToxicCommentClassificationReader(token_indexers={
"tokens1": SingleIdTokenIndexer() ,
"tokens2": SingleIdTokenIndexer() ,
})
 all_dataset = reader.read('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 train_dataset, validation_dataset = train_test_split(all_dataset, test_size=0.2, random_state=11)
 vocab = Vocabulary.from_instances(train_dataset, min_count={'tokens': 3})
 iterator = BucketIterator(
 batch_size=512,
sorting_keys=[("tokens", "num_tokens")], 
)
 iterator.index_with(vocab )
sub.to_csv("submission.csv", index=False )<import_modules>
glove_params = Params({
'pretrained_file': '.. /input/glove-stanford/glove.twitter.27B.200d.txt',
'embedding_dim': 200,
'trainable': False
})
 fasttext_params = Params({
'pretrained_file': '.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec',
'embedding_dim': 300,
'trainable': False
})
 glove_embedding = Embedding.from_params(vocab, glove_params)
 fasttext_embedding = Embedding.from_params(vocab, fasttext_params)
 word_embeddings = BasicTextFieldEmbedder({"tokens1": glove_embedding, "tokens2": fasttext_embedding})
 seq2vec_encoder = BagOfEmbeddingsEncoder(embedding_dim=word_embeddings.get_output_dim() )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc<categorify>
model = ToxicBaseClassifier(
 text_field_embedder=word_embeddings,
seq2seq_encoder=None,
seq2vec_encoder=seq2vec_encoder,
dropout=0.5,
num_labels=6,
vocab=vocab 
)
 model.cuda()
trainer = Trainer(
 model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=10 
)
 metrics = trainer.train()
def preprocess_sales(sales, start=1400, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale1",
"sales1","start","sales2","scale2"],
var_name='d', value_name='demand')
 return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar["nb"] = calendar.index + 1
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1400, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales_aug.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 calendar = calendar.loc[calendar.nb>=start]
gc.collect()
return sales, calendar 
<compute_test_metric>
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))
def qloss_np(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = np.array([qs])
 e = y_true - y_pred
v = np.maximum(q*e,(q-1)*e)
 return np.mean(v )<create_dataframe>
seq2vec_encoder = SWEMEncoder(embedding_dim=word_embeddings.get_output_dim() )
%%time
CATEGORIZE = True;
START = 1400; UPPER = 1970;
maps = {}
mods = {}
sales, calendar = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )<feature_engineering>
model = ToxicBaseClassifier(
 text_field_embedder=word_embeddings,
seq2seq_encoder=None,
seq2vec_encoder=seq2vec_encoder,
dropout=0.5,
num_labels=6,
vocab=vocab 
)
 model.cuda()
trainer = Trainer(
 model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=10 
)
 metrics = trainer.train()
sales["x"] = sales["demand"] / sales["scale1"] 
<prepare_x_and_y>
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))
NITEMS = 42840
scale = sales['scale1'].values.reshape(( NITEMS, -1))
ids = sales['id'].values.reshape(( NITEMS, -1))
ys = sales[['x','sales1']].values.reshape(( NITEMS, -1, 2))
Z = sales[FEATS].values.reshape(( NITEMS, -1, len(FEATS)) )<define_variables>
lstm = torch.nn.LSTM(
 bidirectional=True,
input_size=word_embeddings.get_output_dim() ,
hidden_size=40,
num_layers=2,
batch_first=True 
)
 seq2seq_encoder = PytorchSeq2SeqWrapper(lstm)
 seq2vec_encoder = SWEMEncoder(embedding_dim=seq2seq_encoder.get_output_dim() )
LEN = 1969 - START + 1
MAX_LAG = max(LAGS)
 print(LEN, MAX_LAG )<define_variables>
model = ToxicBaseClassifier(
 text_field_embedder=word_embeddings,
seq2seq_encoder=seq2seq_encoder,
seq2vec_encoder=seq2vec_encoder,
dropout=0.5,
num_labels=6,
vocab=vocab 
)
 model.cuda()
trainer = Trainer(
 model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=10 
)
 metrics = trainer.train()
CATCOLS = ['snap_CA', 'snap_TX', 'snap_WI', 'wday', 'month', 'year', 'event_name', 'nday',
'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']<import_modules>
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf
import itertools as it<categorify>
BERT_MODEL_PATH = '.. /input/bertpretrained/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
WORK_DIR = ".. /working/"
convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(
 BERT_MODEL_PATH + 'bert_model.ckpt',
BERT_MODEL_PATH + 'bert_config.json',
WORK_DIR + 'pytorch_model.bin' 
)
 shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json' )
def make_data(c, z, y):
x = {"snap_CA":c[:,:,0], "snap_TX":c[:,:,1], "snap_WI":c[:,:,2], "wday":c[:,:,3],
"month":c[:,:,4], "year":c[:,:,5], "event":c[:,:,6], "nday":c[:,:,7],
"item":c[:,:,8], "dept":c[:,:,9], "cat":c[:,:,10], "store":c[:,:,11],
"state":c[:,:,12], "num":z}
t = y
return x, t<prepare_x_and_y>
token_indexer = PretrainedBertIndexer(
 pretrained_model=BERT_MODEL_PATH,
max_pieces=128,
do_lowercase=True, 
)
 tokenizer = WordTokenizer(word_splitter=BertBasicWordSplitter())
 reader = ToxicCommentClassificationReader(
 tokenizer=tokenizer,
token_indexers={"bert": token_indexer} 
)
 all_dataset = reader.read('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 train_dataset, validation_dataset = train_test_split(all_dataset, test_size=0.2, random_state=11)
 iterator = BucketIterator(
 batch_size=64,
sorting_keys=[("tokens", "num_tokens")], 
)
 vocab = Vocabulary()
iterator.index_with(vocab)
 model = ToxicBertClassifier(
 vocab=vocab,
bert_model=WORK_DIR,
num_labels=6
)
class DataGenerator(tf.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, x, brks, batch_size=32, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.c = x[0]
self.z = x[1]
self.y = x[2]
self.brks = brks.copy()
self.list_IDs = np.array(range(42840))
self.shuffle = shuffle
self.nb_batch = int(np.ceil(len(self.list_IDs)/ self.batch_size))
self.n_windows = brks.shape[0]
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.list_IDs)/ self.batch_size)) *(self.n_windows)
 def __getitem__(self, index):
'Generate one batch of data'
idx, kx = self.ids[index]
batch_ids = self.list_IDs[idx*self.batch_size:(idx+1)*self.batch_size]
X, y = self.__data_generation(batch_ids, kx)
 return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.ids = list(it.product(np.arange(0, self.nb_batch), self.brks))
if self.shuffle == True:
np.random.shuffle(self.ids)
 def __data_generation(self, batch_ids, kx):
'Generates data containing batch_size samples'
x_batch, y_batch = make_data(self.c[batch_ids, kx-28:kx], self.z[batch_ids,kx-28:kx],
self.y[batch_ids, kx-28:kx])
 return x_batch, y_batch 
<prepare_x_and_y>
model.cuda()
trainer = Trainer(model=model,
optimizer=Lookahead(RAdam(model.parameters())) ,
iterator=iterator,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
cuda_device=0,
num_epochs=1000,
grad_norm=5.0,
grad_clipping=1.0,
patience=3)
 metrics = trainer.train()
xv, yv = make_data(C[:,LEN-56:LEN-28,:], Z[:,LEN-56:LEN-28,:], ys[:,LEN-56:LEN-28])
 xe, ye = make_data(C[:,LEN-28:LEN,:], Z[:,LEN-28:LEN,:], ys[:,LEN-28:LEN] )<compute_test_metric>
print('metrics: {}'.format(metrics))
print('best_validation_loss: {}'.format(metrics['best_validation_loss']))
def wqloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true[:,:,:1] - y_pred
v = tf.maximum(q*e,(q-1)*e)
 psl = tf.reduce_mean(v, axis=[1,2])
 weights = y_true[:,0,1] / K.sum(y_true[:,0,1])
 return tf.reduce_sum(psl * weights)
 def qloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([[qs]]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def make_model(n_in):
seq_len = 28
num = L.Input(( seq_len, n_in,), name="num")
 ca = L.Input(( seq_len,), name="snap_CA")
 tx = L.Input(( seq_len,), name="snap_TX")
 wi = L.Input(( seq_len,), name="snap_WI")
 wday = L.Input(( seq_len,), name="wday")
 month = L.Input(( seq_len,), name="month")
 year = L.Input(( seq_len,), name="year")
 event = L.Input(( seq_len,), name="event")
 nday = L.Input(( seq_len,), name="nday")
 item = L.Input(( seq_len,), name="item")
 dept = L.Input(( seq_len,), name="dept")
 cat = L.Input(( seq_len,), name="cat")
 store = L.Input(( seq_len,), name="store")
 state = L.Input(( seq_len,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
context = L.Concatenate(name="context" )(p)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.GRU(100, return_sequences=True, name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.GRU(100, return_sequences=True, name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.GRU(100, return_sequences=True, name="d3" )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=wqloss, optimizer="adam")
 return model<choose_model_class>
test_dataset = reader.read('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 seq_iterator = BasicIterator(batch_size=64)
 seq_iterator.index_with(vocab )
<train_model><EOS> 
predictor = ToxicCommentPredictor(model, seq_iterator, cuda_device=0)
 test_preds = predictor.predict(test_dataset)
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = test_preds
submission.to_csv('submission.csv', index=False )