Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
%matplotlib inline
plt.style.use('ggplot')
 seed =420
pd.options.display.max_rows = 100<load_from_csv>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tokenizers import Tokenizer, models
nrows = 100000
path = '.. /input/'
app_train = pd.read_csv(path+'application_train.csv', nrows= nrows)
 app_test = pd.read_csv(path+'application_test.csv', nrows= None)
 bureau_balance = pd.read_csv(path+'bureau_balance.csv', nrows=nrows)
 bureau = pd.read_csv(path+'bureau.csv', nrows=nrows)
 installments_payments = pd.read_csv(path+ 'installments_payments.csv', nrows= nrows)
 POS_CASH_balance = pd.read_csv(path+ 'POS_CASH_balance.csv', nrows= nrows)
 credit_card_balance = pd.read_csv(path+'credit_card_balance.csv',nrows=nrows)
 previous_application = pd.read_csv(path+ 'previous_application.csv', nrows= nrows)
 gc.collect()
test_index = app_test['SK_ID_CURR']<feature_engineering>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
 def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def basic_details(df):

 k = pd.DataFrame()
k['missing_value'] = df.isnull().sum()
k['%missing_value'] = round(df.isnull().sum() *100/df.shape[0],2)
 k['dtypes'] = df.dtypes
k['N unique'] = df.nunique()
return k<categorify>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def missing_value_fill(df ,columns, mean_or_mode='mode'):

 for i in columns:
if(df[i].isnull().sum() >0)and(mean_or_mode =='mode'):
df[i].fillna(df[i].mode() [0], inplace=True)
 elif(df[i].isnull().sum() >0)and(mean_or_mode =='mean'):
df[i].fillna(df[i].mean() , inplace=True )<categorify>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
def replace_XNA_XAP(df):
"Replace XNA,XAP"
df.replace(to_replace = {'XNA':np.nan,'XAP':np.nan},inplace=True,value= None)
 return df<categorify>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
def one_hot_encoding(df,columns,nan_as_category = True):

 print('*'*5,'One hot encoding of categorical variable','*'*5)
 print('Original shape',df.shape)
 original_columns = df.columns
df = pd.get_dummies(df, columns= columns,drop_first=True,dummy_na=nan_as_category)
 new_columns = [i for i in df.columns if i not in original_columns]
print('After OHE',df.shape)
 return df,new_columns<data_type_conversions>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ])
 x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
def descriptive_stat_feat(df,columns):

 print('*'*5,'Descriptive statistics feature','*'*5)
 print('Before',df.shape)
 mean = df[columns].mean()
median = df[columns].median()
Q1 = np.percentile(df[columns], 25, axis=0)
 Q3 = np.percentile(df[columns], 75, axis=0)
 for i,j in enumerate(columns):
df['mean_'+j] =(df[j] < mean[i] ).astype('int8')
 df['median_'+j] =(df[j] > median[i] ).astype('int8')
 df['Q1'+j] =(df[j] < Q1[i] ).astype('int8')
 df['Q3'+j] =(df[j] > Q3[i] ).astype('int8')
 print('After ',df.shape )<categorify>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
def binary_encoding(df,columns):

 print('*'*5,'Binary encoding','*'*5)
 lb = LabelBinarizer()
print('Original shape:',df.shape)
 original_col = df.columns
for i in columns:
if df[i].nunique() >2:
result = lb.fit_transform(df[i].fillna(df[i].mode() [0],axis=0))
col = ['BIN_'+ str(i)+'_'+str(c)for c in lb.classes_]
result1 = pd.DataFrame(result, columns=col)
 df = df.join(result1)
 print('After:',df.shape)
 new_col = [c for c in df.columns if c not in original_col]
return df, new_col<define_variables>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
test_index = app_test['SK_ID_CURR']
app_train_col_drop = []<count_values>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS 
)
 n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
<data_type_conversions><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )