Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<train_model>
import os,re,gc,pickle,random,sys
import numpy as np
import pandas as pd
import tensorflow as tf
import transformers
from transformers import TFAutoModel
from tensorflow.data.experimental import sample_from_datasets
from tensorflow.data import Dataset
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm.notebook import tqdm
learn.fit_one_cycle(20, 1e-4 )<save_model>
np.random.seed(1234)
 random.seed(1234)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
with strategy.scope() :
tf.random.set_seed(1234 )
learn.save('stage-07' )<train_model>
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
HEAD = "mean"
input1 = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"
inpath = ".. /input/jwtc-xlmroberta-encoding-192-pickle/datain/"
form = "training/encode_{}.pkl"
langs = ["en","en2","es","fr","it","pt","ru","tr"]
used_data = langs
learn.validate(learn.data.train_dl )<train_model>
def pick_load_format(path):
with open(inpath+path,"rb")as f:
return pickle.load(f)
 def load(path):
return pick_load_format(form.format(path))
learn.validate(learn.data.valid_dl )<find_best_params>
def get_cong(n,verb=True):
tot = round(1+(n*2)/10_000)*10_000
if verb: print("Pos: {}, Sample neg: {}, Total: {}".format(n,tot-n,tot))
return tot,tot-n
def load_data(seed=1214):
train = []
for i in used_data:
df1 = load(i+"_l1")
 size, sample_size = get_cong(df1.shape[0])
 df0 = load(i+"_l0" ).sample(n=sample_size, random_state=seed)
 train += [df1,df0]
train = pd.concat(train)
 train = np.stack(train.comment_text.values, axis=0 ).astype("int32"),train.toxic.values
valid = pick_load_format("valid.pkl")
 x_valid = np.stack(valid.comment_text.values, axis=0 ).astype("int32")
 y_valid = valid.toxic.values
test = pick_load_format("test.pkl")
 x_test = np.stack(test.content.values, axis=0 ).astype("int32")
 return train,(x_valid,y_valid),x_test
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()<choose_model_class>
%%time
train,valid,test = load_data()
gc.collect()
valid_size = len(valid[1])
 train_size = len(train[1])
 print(train_size,valid_size )
valid_tta_preds, y, losses = learn.TTA(scale=1.10, with_loss=True )<compute_test_metric>
def make_dataset_pipeline(dataset, cache=False,repeat_and_shuffle=False,shuffle_size=128_000,seed=386491):
if cache: dataset = dataset.cache()
if repeat_and_shuffle:
dataset = dataset.repeat().shuffle(shuffle_size,seed)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
accuracy(valid_tta_preds, y )<compute_test_metric>
train_dataset = Dataset.from_tensor_slices(train)
 train_dataset = make_dataset_pipeline(train_dataset,True, repeat_and_shuffle=True)
 valid_dataset = make_dataset_pipeline(Dataset.from_tensor_slices(valid))
valid_dataset2 = make_dataset_pipeline(Dataset.from_tensor_slices(( valid)) , True,
shuffle_size=8000, repeat_and_shuffle=True)
 test_dataset = make_dataset_pipeline(Dataset.from_tensor_slices(test))
confusion_matrix(y.cpu().numpy() , valid_tta_preds.argmax(1 ).cpu().numpy() )<import_modules>
def get_cls(x):
return x[:, 0, :]
dic = {"mean":GlobalAveragePooling1D() ,
"max":GlobalMaxPool1D() ,
"cls":get_cls}
def build_model(transformer,head="cls" , loss='binary_crossentropy',
max_len=512, drop_rate=None, lr=1e-5,seed=940208):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
x = dic[head](sequence_output)
 if drop_rate is not None:
x = Dropout(drop_rate )(x)
 out = Dense(1, activation='sigmoid',kernel_initializer=GlorotUniform(seed))(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=lr), loss=loss, metrics=[AUC() ])
 return model
learn.export('model.pkl' )<predict_on_test>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN)
 model.summary()
preds, y = learn.get_preds(DatasetType.Test )<save_to_csv>
model_path = "xlm-roberta.h5"
checkpoint = ModelCheckpoint(model_path, monitor='val_auc', mode='max', save_best_only=True, save_weights_only=True, verbose=1)
 es = EarlyStopping(monitor='val_auc', mode='max', patience=6, restore_best_weights=False, verbose=1)
 rp = ReduceLROnPlateau(monitor='val_auc', factor=0.8, patience=3, verbose=1, mode='max')
 callback_list = [checkpoint,es,rp]
sample_df.iloc[:,1:] = preds.numpy()
sample_df.to_csv('submission.csv', index=False )<import_modules>
%%time
N_STEPS = train_size //(BATCH_SIZE*4)
 EPOCHS = 10
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=valid_dataset,
callbacks=callback_list,
epochs=EPOCHS
)
<install_modules>
del model
gc.collect()
tf.tpu.experimental.initialize_tpu_system(tpu )
!pip install -q efficientnet<import_modules>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=8e-6)
 model.load_weights(model_path )
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
from sklearn import metrics
from sklearn.model_selection import train_test_split
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
from tensorflow.keras.applications.nasnet import NASNetLarge
from efficientnet.tfkeras import EfficientNetB7, EfficientNetL2
from tensorflow.keras.layers import Flatten,Dense,Dropout,BatchNormalization
from tensorflow.keras.models import Model,Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint<choose_model_class>
%%time
n_steps = valid_size //(BATCH_SIZE)
 EPOCHS = 1
train_history_2 =model.fit(
 valid_dataset2,
steps_per_epoch=n_steps,
epochs= EPOCHS
)
<define_variables><EOS> 
sub = pd.read_csv(input1 + "sample_submission.csv")
 sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )