Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<merge>
!pip install -U transformers
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )<feature_engineering>
import os
import gc
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )<drop_column>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_mask=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
truncation=True 
)
 return np.array(enc_di['input_ids'] )
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )<drop_column>
def dict_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_mask=True,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
truncation=True 
)
 return {
"input_ids": np.array(enc_di['input_ids']),
"attention_mask": np.array(enc_di['attention_mask'])
 }
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )<filter>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 attention_mask = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 sequence_output = transformer({"input_ids": input_word_ids, "attention_mask": attention_mask})[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs={
"input_ids": input_word_ids,
"attention_mask": attention_mask
}, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
data_test = data[data.eval_set=='test']
data_test.head()<drop_column>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()<drop_column>
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
tokenizer = AutoTokenizer.from_pretrained(MODEL )
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()<train_model>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv' ).sample(frac=1.0)
 n_valid_steps = valid.shape[0] // BATCH_SIZE
test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
<train_model>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] ).sample(frac=1.0)
 n_train_steps = train.shape[0] // BATCH_SIZE
del train1, train2
gc.collect()
X_train, X_val, y_train, y_val = train_test_split(data_train.drop('reordered', axis=1), data_train.reordered, test_size=0.8, random_state=42)
 rfc = RandomForestClassifier(n_estimators=12, n_jobs=-1 ,random_state=42)
 model = rfc.fit(X_train, y_train)
 y_val_pred =(model.predict_proba(X_val)[:,1] >= 0.22 ).astype(int)
 print("RESULTS ON VALIDATION SET
====================")
 print("F1 Score: ",f1_score(y_val, y_val_pred, average='binary'), "
====================")
 print("Classification Report
", classification_report(y_val, y_val_pred), "
====================")
 print("Confusion Matrix
", confusion_matrix(y_val, y_val_pred))
del [X_val, y_val]<predict_on_test>
!free -h
test_pred = model.predict(data_test ).astype(int)
 test_pred =(model.predict_proba(data_test)[:,1] >= 0.22 ).astype(int)
 test_pred[0:20]<predict_on_test>
%%time
x_train = dict_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
del train
gc.collect()
x_valid = dict_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 y_valid = valid.toxic.values
del valid
gc.collect()
x_test = dict_encode(test.content.values, tokenizer, maxlen=MAX_LEN )
data_test['prediction'] = test_pred
data_test.head(10 )<filter>
!free -h
orders_test = orders.loc[orders.eval_set=='test',("user_id", "order_id")]
orders_test.head()<merge>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(4096)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
)
 del x_train, x_valid, y_train, y_valid
gc.collect()
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
final = final.drop('user_id', axis=1)
<data_type_conversions>
EPOCHS = 2
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_train_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_valid_steps,
epochs=EPOCHS
)
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d<prepare_output>
model.save_weights("final_weights.h5" )
<save_to_csv><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )