Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<categorify>
!pip install wandb
!pip install bert-tensorflow
print(tf.version.VERSION )
class MyDataset(Dataset):
def __init__(self, dataframe, transform=None):
self.df = dataframe
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
index_token = self.df.loc[idx, 'indexed_token']
index_token = literal_eval(index_token)
 index_token = pad_sequences([index_token], maxlen=360, padding='post')[0]
offset = self.df.loc[idx, 'offset']
offset = literal_eval(offset)
 offset = np.asarray(offset, dtype='int32')
 label = int(self.df.loc[idx, 'label'])
 distP_A = self.df.loc[idx, 'D_PA']
distP_B = self.df.loc[idx, 'D_PB']
if self.transform:
index_token = self.transform(index_token)
 offset = self.transform(offset)
 label = self.transform(label)
 return(index_token, offset, distP_A, distP_B), label<compute_train_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
class score(torch.nn.Module):
def __init__(self, embed_dim, hidden_dim):
super(score, self ).__init__()
self.score = torch.nn.Sequential(
 torch.nn.Linear(embed_dim, hidden_dim),
torch.nn.ReLU(inplace=True),
torch.nn.Dropout(0.6),
torch.nn.Linear(hidden_dim, 1))
def forward(self, x):
return self.score(x)
 class mentionpair_score(torch.nn.Module):
def __init__(self, input_dim, hidden_dim):
super(mentionpair_score, self ).__init__()
self.score = score(input_dim, hidden_dim)
 def forward(self, g1, g2, dist_embed):
element_wise = g1 * g2
pair_score = self.score(torch.cat(( g1, g2, element_wise, dist_embed), dim=-1))
return pair_score
class score_model(torch.nn.Module):
def __init__(self):
super(score_model, self ).__init__()
self.buckets = [1, 2, 3, 4, 5, 8, 16, 32, 64]
self.bert = BertModel.from_pretrained('bert-base-uncased')
 self.embedding = torch.nn.Embedding(len(self.buckets)+1, 20)
 self.span_extractor = EndpointSpanExtractor(768, "x,y,x*y")
 self.pair_score = mentionpair_score(2304*3+20, 150)
 def forward(self, sent, offsets, distP_A, distP_B):
bert_output, _ = self.bert(sent, output_all_encoded_layers=False)
 distPA_embed = self.embedding(distP_A)
 distPB_embed = self.embedding(distP_B)
 span_repres = self.span_extractor(bert_output, offsets)
 span_repres = torch.unbind(span_repres, dim=1)
 span_norm = []
for i in range(len(span_repres)) :
span_norm.append(F.normalize(span_repres[i], p=2, dim=1))
ap_score = self.pair_score(span_norm[2], span_norm[0], distPA_embed)
 bp_score = self.pair_score(span_norm[2], span_norm[1], distPB_embed)
 nan_score = torch.zeros_like(ap_score)
 output = torch.cat(( ap_score, bp_score, nan_score), dim=1)
 return output<categorify>
SEQUENCE_LENGTH = 128
DATA_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 BERT_PATH = KaggleDatasets().get_gcs_path('bert-multi')
 BERT_PATH_SAVEDMODEL = BERT_PATH + "/bert_multi_from_tfhub"
OUTPUT_PATH = "/kaggle/working"
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, torch.nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))<load_from_csv>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub2 = pd.read_csv('.. /input/ensemble/submission.csv' )
train_dist = pd.read_csv('.. /input/bert-score-layer-lb-0-475/train_dist_df.csv')
 val_dist = pd.read_csv('.. /input/bert-score-layer-lb-0-475/val_dist_df.csv')
 test_dist = pd.read_csv('.. /input/bert-score-layer-lb-0-475/test_dist_df.csv')
 train_dist = pd.concat([train_dist, val_dist])
 train_dist = train_dist.reset_index(drop=True)
 train_dist.head()<load_from_csv>
def get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):

 bert_layer = tf.saved_model.load(bert_path)
 bert_layer = hub.KerasLayer(bert_layer, trainable=False)
 vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
cased = bert_layer.resolved_object.do_lower_case.numpy()
tf.gfile = tf.io.gfile
tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)
 return tokenizer
tokenizer = get_tokenizer()
n_split = 5
train = pd.read_csv('.. /working/train.csv')
 test = pd.read_csv('.. /working/test.csv')
 train = pd.concat([train, train_dist], axis=1)
 test = pd.concat([test, test_dist], axis=1)
 train.head()
Kfold = StratifiedKFold(n_splits=n_split, random_state=2019 ).split(train, train['label'] )<define_variables>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
def softmax(x):
exp_x = np.exp(x)
 y = exp_x / np.sum(exp_x, axis=1, keepdims=True)
 return y
output = np.zeros(( len(test_df), 3))
testset = MyDataset(test)
 test_loader = torch.utils.data.DataLoader(testset, batch_size=20)
 n_epochs = 30
for n_fold,(train_index, val_index)in enumerate(Kfold):
min_val_loss = 100.0
PATH = "./best_model_{}.hdf5".format(n_fold+1)
 train_df = train.loc[train_index]
train_df = train_df.reset_index(drop=True)
 val_df = train.loc[val_index]
val_df = val_df.reset_index(drop=True)
 trainset = MyDataset(train_df)
 train_loader = torch.utils.data.DataLoader(trainset, batch_size=20, shuffle=True)
 valset = MyDataset(val_df)
 val_loader = torch.utils.data.DataLoader(valset, batch_size=20, shuffle=True)
 model = score_model()
set_trainable(model.bert, False)
 set_trainable(model.embedding, True)
 set_trainable(model.pair_score, True)
 model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=0.001, weight_decay=0.00001)
 criterion = torch.nn.CrossEntropyLoss().cuda()
print('fold:', n_fold+1)
 for i in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for idx,(inputs, label)in enumerate(train_loader):
index_token, offset, distP_A, distP_B = inputs
index_token = index_token.type(torch.LongTensor ).cuda()
offset = offset.type(torch.LongTensor ).cuda()
label = label.type(torch.LongTensor ).cuda()
distP_A = distP_A.type(torch.LongTensor ).cuda()
distP_B = distP_B.type(torch.LongTensor ).cuda()
optimizer.zero_grad()
output_train = model(index_token, offset, distP_A, distP_B)
 loss = criterion(output_train, label)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 avg_val_loss = 0.
 model.eval()
with torch.no_grad() :
for idx,(inputs, label)in enumerate(val_loader):
index_token, offset, distP_A, distP_B = inputs
index_token = index_token.type(torch.LongTensor ).cuda()
offset = offset.type(torch.LongTensor ).cuda()
label = label.type(torch.LongTensor ).cuda()
distP_A = distP_A.type(torch.LongTensor ).cuda()
distP_B = distP_B.type(torch.LongTensor ).cuda()
output_test = model(index_token, offset, distP_A, distP_B)
 avg_val_loss += criterion(output_test, label ).item() / len(val_loader)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 i + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
if min_val_loss > avg_val_loss:
min_val_loss = avg_val_loss
torch.save(model.state_dict() , PATH)
 del model
model = score_model()
model.load_state_dict(torch.load(PATH))
model.cuda()
model.eval()
with torch.no_grad() :
for idx,(inputs, label)in enumerate(test_loader):
index_token, offset, distP_A, distP_B = inputs
index_token = index_token.type(torch.LongTensor ).cuda()
offset = offset.type(torch.LongTensor ).cuda()
label = label.type(torch.LongTensor ).cuda()
distP_A = distP_A.type(torch.LongTensor ).cuda()
distP_B = distP_B.type(torch.LongTensor ).cuda()
y_pred = model(index_token, offset, distP_A, distP_B)
 y_pred = softmax(y_pred.cpu().numpy())
 start = idx * 20
end = start + 20
output[start:end, :] += y_pred<feature_engineering>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 5
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
output /= 5
sub_df_path = os.path.join('.. /input/gendered-pronoun-resolution/', 'sample_submission_stage_1.csv')
 sub_df = pd.read_csv(sub_df_path)
 sub_df.loc[:, 'A'] = pd.Series(output[:, 0])
 sub_df.loc[:, 'B'] = pd.Series(output[:, 1])
 sub_df.loc[:, 'NEITHER'] = pd.Series(output[:, 2])
 sub_df.head(20 )<save_to_csv>
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)
 fast_tokenizer
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values
y_test = pd.read_csv('.. /working/test.csv')['label']
y_one_hot = np.zeros(( 2000, 3))
for i in range(len(y_test)) :
y_one_hot[i, y_test[i]] = 1
log_loss(y_one_hot, output)
<compute_test_metric>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
_output = np.argmax(output, axis=1)
 print('acc:', np.asarray(np.where(_output == y_test)).shape[1]/ 2000 )<load_from_csv>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = tf.keras.layers.Dense(192, activation='relu' )(cls_token)
 out = tf.keras.layers.Dense(64, activation='relu' )(out)
 out = tf.keras.layers.Dense(64, activation='relu' )(out)
 out = Dense(1, activation='sigmoid' )(out)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
test_stage_1 = pd.read_csv(".. /input/test_stage_1.tsv", sep="\t")
 test_stage_2 = pd.read_csv(".. /input/test_stage_2.tsv", sep="\t" )<load_from_csv>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
gap_test = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-test.tsv", delimiter='\t')
 gap_valid = pd.read_csv("https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-validation.tsv", delimiter='\t' )<feature_engineering>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)
def get_prior(df):
Neither_count = len(df)- sum(df["A-coref"] | df["B-coref"])
 A_count = sum(df["A-coref"])
 B_count = sum(df["B-coref"])
 test_total = len(df)
 Neither_prior = Neither_count/test_total
A_prior = A_count/test_total
B_prior = B_count/test_total
print("Prior probabilities:")
 print("Neither: "+str(Neither_prior),"A: "+str(A_prior),"B: "+str(B_prior))
assert Neither_count + A_count + B_count == test_total
return A_prior, B_prior, Neither_prior
A_prior,B_prior,Neither_prior = get_prior(gap_test)
<load_from_csv>
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub1 = sub[['id', 'toxic']]
<feature_engineering><EOS> 
sub1.rename(columns={'toxic':'toxic1'}, inplace=True)
 sub2.rename(columns={'toxic':'toxic2'}, inplace=True)
 sub3 = pd.merge(sub1, sub2, how='left', on='id')
 sub3['toxic'] =(sub3['toxic1'] * 0.1)+(sub3['toxic2'] * 0.9)
 sub3['toxic'] =(sub3['toxic2'] * 0.39)+(sub3['toxic'] * 0.61)
 sub3[['id', 'toxic']].to_csv('submission.csv', index=False )