Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<count_missing_values>
import os
import gc
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
display(poverty_train.loc[(poverty_train["tipovivi4" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi4" ] == 1, "v2a1")]))<count_missing_values>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
display(poverty_train.loc[(poverty_train["tipovivi5" ] == 1, "v2a1")].isna().sum() ,
len(poverty_train.loc[(poverty_train["tipovivi5" ] == 1, "v2a1")]))<categorify>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
poverty_train["v2a1"].fillna(0, inplace = True)
 print("Top Columns having missing values(%)")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()<count_missing_values>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
display(poverty_train.loc[(poverty_train["v18q" ] == 0, "v18q1")].isna().sum() ,
len(poverty_train[poverty_train["v18q"] == 0]))<feature_engineering>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
poverty_train.loc[(poverty_train["v18q" ] == 0, "v18q1")] = 0
print("Top Columns having missing values(%)")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head()<feature_engineering>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
poverty_train['rez_esc'] = poverty_train.apply(
 lambda row: 0 if(row['instlevel1'] == 1)else row['rez_esc'],
axis=1 
)
 poverty_train['rez_esc'] = poverty_train.apply(
 lambda row: poverty_train["rez_esc"].median() if np.isnan(row["rez_esc"])else row['rez_esc'],
axis=1 
)
 print("Top Columns having missing values(%)")
 missmap = poverty_train.isnull().sum().to_frame().sort_values(0, ascending = False)/ len(poverty_train)
 missmap.head() 
<count_values>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 train2.toxic = train2.toxic.round().astype(int)
 train = pd.concat([train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000)
 ])
 del train1,train2
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
poverty_train["dependency"].replace('yes', 1, inplace = True)
 poverty_train["dependency"].replace('no', 0, inplace = True)
 poverty_train["edjefa"].replace('yes', 1, inplace = True)
 poverty_train["edjefa"].replace('no', 0, inplace = True)
 poverty_train["edjefe"].replace('yes', 1, inplace = True)
 poverty_train["edjefe"].replace('no', 0, inplace = True)
 display(poverty_train["dependency"].value_counts() ,
poverty_train["edjefa"].value_counts() ,
poverty_train["edjefe"].value_counts() )<data_type_conversions>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS 
)
 EPOCHS_V=10
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS_V
)
<define_variables><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )