Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<concatenate>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import tensorflow as tf
from tensorflow.keras.layers import Dense, Dropout, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer
train = assign_prior(gap_test)
 valid = assign_prior(gap_valid )<something_strange>
def clean_text(text):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 return text.strip()
def get_gold(df):
gold = []
for index, row in df.iterrows() :
if(row["A-coref"]):
gold.append("A")
 else:
if(row["B-coref"]):
gold.append("B")
 else:
gold.append("NEITHER")
 return gold<feature_engineering>
def text_process(text):
ws = text.split(' ')
 if(len(ws)>160):
text = ' '.join(ws[:160])+ ' ' + ' '.join(ws[-32:])
 return text
train_gold = get_gold(gap_test)
 valid_gold = get_gold(gap_valid )<compute_test_metric>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
train_pred = train[["A","B","NEITHER"]]
log_loss(train_gold,train_pred )<compute_test_metric>
tokenizer = transformers.BertTokenizer.from_pretrained('bert-large-uncased')
 save_path = '/kaggle/working/bert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('bert_base_uncased/vocab.txt', lowercase=True)
 fast_tokenizer
valid_pred = valid[["A","B","NEITHER"]]
log_loss(valid_gold,valid_pred )<feature_engineering>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
sub1 = assign_prior(test_stage_1 )<save_to_csv>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS_1 = 20
EPOCHS_2 = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
SHUFFLE = 2048
VERBOSE = 1
sub1.to_csv("submission_1.csv", index=False )<merge>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/val-en-df/validation_en.csv')
 test1 = pd.read_csv('/kaggle/input/test-en-df/test_en.csv')
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
set(gap_test["Pronoun"] ).union(set(gap_valid["Pronoun"])).union(set(test_stage_1["Pronoun"]))<define_variables>
train['comment_text'] = train.apply(lambda x: clean_text(x['comment_text']), axis=1)
 valid['comment_text_en'] = valid.apply(lambda x: clean_text(x['comment_text_en']), axis=1)
 test1['content'] = test1.apply(lambda x: clean_text(x['content_en']), axis=1)
 test2['content'] = test2.apply(lambda x: clean_text(x['translated']), axis=1)
 train['comment_text'] = train['comment_text'].apply(lambda x: text_process(x))
valid['comment_text_en'] = valid['comment_text_en'].apply(lambda x: text_process(x))
test1['content'] = test1['content_en'].apply(lambda x: text_process(x))
test2['content'] = test2['translated'].apply(lambda x: text_process(x))
female_pronouns = ['she','her','hers']
male_pronouns = ['he','him','his']<data_type_conversions>
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text_en.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test1 = fast_encode(test1.content_en.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test2 = fast_encode(test2.translated.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values
female_gap_test = gap_test[gap_test["Pronoun"].str.lower().isin(female_pronouns)]
male_gap_test = gap_test[gap_test["Pronoun"].str.lower().isin(male_pronouns)]
female_gap_valid = gap_valid[gap_valid["Pronoun"].str.lower().isin(female_pronouns)]
male_gap_valid = gap_valid[gap_valid["Pronoun"].str.lower().isin(male_pronouns)] 
<categorify>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(SHUFFLE)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test1_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(BATCH_SIZE)
)
 test2_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(BATCH_SIZE)
 )
train_female = assign_prior(female_gap_test)
 train_male = assign_prior(male_gap_test)
 valid_female = assign_prior(female_gap_valid)
 valid_male = assign_prior(male_gap_valid )<categorify>
lrs = ReduceLROnPlateau(monitor='val_accuracy', mode ='max', factor = 0.7, min_lr= 1e-7, verbose = 1, patience = 2)
 es1 = EarlyStopping(monitor='val_accuracy', mode='max', verbose = 1, patience = 5, restore_best_weights=True)
 es2 = EarlyStopping(monitor='accuracy', mode='max', verbose = 1, patience = 1, restore_best_weights=True)
 callbacks_list1 = [lrs,es1]
callbacks_list2 = [lrs,es2]
train_gold_female = get_gold(female_gap_test)
 train_gold_male = get_gold(male_gap_test )<compute_test_metric>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
x = tf.keras.layers.Dropout(0.4 )(cls_token)
 out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(name='auc'), 'accuracy'])
 return model
train_pred_female = train_female[["A","B","NEITHER"]]
log_loss(train_gold_female,train_pred_female )<compute_test_metric>
n_train_steps = train.shape[0] //(BATCH_SIZE*16)
 model_history_1 = model.fit(
 train_dataset,
steps_per_epoch=n_train_steps,
validation_data=valid_dataset,
epochs=EPOCHS_1,
callbacks=callbacks_list1,
verbose=VERBOSE
)
train_pred_male = train_male[["A","B","NEITHER"]]
log_loss(train_gold_male,train_pred_male )<categorify>
n_valid_steps = valid.shape[0] //(BATCH_SIZE)
 model_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_valid_steps,
epochs=EPOCHS_2,
callbacks=callbacks_list2,
verbose=VERBOSE
)
valid_gold_female = get_gold(female_gap_valid)
 valid_gold_male = get_gold(male_gap_valid )<compute_test_metric>
eng1 = model.predict(test1_dataset, verbose=1)
 eng2 = model.predict(test2_dataset, verbose=1 )
valid_pred_female = valid_female[["A","B","NEITHER"]]
log_loss(valid_gold_female,valid_pred_female )<compute_test_metric>
sub.to_csv('submission.csv', index=False )
<data_type_conversions><EOS> 
sub.to_csv('submission.csv', index=False )