Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<prepare_output>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
print(preds)
 preds = np.argmax(preds, axis = 1)+1
preds<load_from_csv>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
sample_submission = pd.read_csv(".. /input/sample_submission.csv")
 sample_submission.head()<save_to_csv>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
sample_submission['Target'] = preds
sample_submission.to_csv('submission_{:.6f}.csv'.format(err_mean), index=False)
 sample_submission.head()<prepare_output>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
np.mean(preds )<load_from_csv>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
print(os.listdir(".. /input"))
sample_submission = pd.read_csv(".. /input/gendered-pronoun-resolution/sample_submission_stage_1.csv")
 final_test = pd.read_csv(".. /input/gendered-pronoun-resolution/test_stage_2.tsv", sep = "\t")
 nlp = spacy.load('en_core_web_sm')
 dep = ["ACL", "ACOMP", "ADVCL", "ADVMOD", "AGENT", "AMOD", "APPOS", "ATTR", "AUX", "AUXPASS",
"CASE", "CC", "CCOMP", "COMPOUND", "CONJ", "CSUBJ", "CSUBJPASS", "DATIVE", "DEP", "DET", "DOBJ"
, "EXPL", "INTJ", "MARK", "META", "NEG", "NOUNMOD", "NPMOD", "NSUBJ", "NSUBJPASS", "NUMMOD"
, "OPRD", "PARATAXIS", "PCOMP", "POBJ", "POSS", "PRECONJ", "PREDET", "PREP", "PRT", "PUNCT", "QUANTMOD",
"RELCL", "ROOT", "XCOMP", "COMPLM","INFMOD","PARTMOD","HMOD","HYPH","IOBJ","NUM",
"NUMBER","NMOD","NN","NPADVMOD","POSSESSIVE","RCMOD","SUBTOK"] 
<import_modules>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 1
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 275
MODEL = 'jplu/tf-xlm-roberta-large'
import tensorflow as tf<load_from_url>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
!wget https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-development.tsv
!wget https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-validation.tsv
!wget https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-test.tsv
!ls 
<load_from_csv>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 submission = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
train_data = pd.read_csv("gap-development.tsv", sep = "\t")
 validation_data = pd.read_csv("gap-validation.tsv", sep = "\t")
 test_data = pd.read_csv("gap-test.tsv", sep = "\t" )<concatenate>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )
merge_data = pd.concat([train_data,validation_data] ).reset_index(drop = True)
 merge_data = pd.concat([merge_data,train_data] ).reset_index(drop = True)
 count = 0<feature_engineering>
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
feature = get_features(merge_data)
<data_type_conversions>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
Y = merge_data[["A-coref", "B-coref"]]
Y.columns = ["A","B"]
Y["A"] = Y["A"].astype(int)
 Y["B"] = Y["B"].astype(int)
 Y["NEITHER"] = 1-(Y["A"] + Y["B"] )<split>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
x1, x2, y1, y2 = model_selection.train_test_split(feature.fillna(-1), Y, test_size=0.2, random_state=1)
 x1.head()
x2.head()
y2<train_on_grid>
EPOCHS = 1
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
scaler = StandardScaler()
x1 = scaler.fit_transform(x1)
 x2 = scaler.transform(x2)
 model = multiclass.OneVsRestClassifier(ensemble.RandomForestClassifier(max_depth = 7, n_estimators=1000, random_state=33))
model.fit(x1, y1)
 print('log_loss', metrics.log_loss(y2, model.predict_proba(x2)) )<load_from_csv>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
final_test = pd.read_csv(".. /input/gendered-pronoun-resolution/test_stage_2.tsv", sep = "\t")
 feature = get_features(final_test)
 print(feature)
<correct_missing_values>
sub['toxic'] = model.predict(test_dataset, verbose=1)
feature = feature.fillna(-1)
 print(feature)
<predict_on_test>
submission=sub[['id','toxic']]
<save_to_csv><EOS> 
submission.to_csv('submission.csv',index=False )