Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<feature_engineering>
import os
import gc
import pandas as pd
import numpy as np
import transformers
from tqdm.notebook import tqdm
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, Conv1D, GlobalMaxPooling1D, Dropout, BatchNormalization
from tensorflow.keras.activations import sigmoid
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import AUC
from tensorflow.keras.regularizers import l1, l2
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from sklearn.ensemble import AdaBoostClassifier
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
kaggle = '/kaggle/input/undersampling-xlmtokenized/'
x_train = np.load(f'{kaggle}x_train.npy')
 y_train = np.load(f'{kaggle}y_train.npy')
 x_valid = np.load(f'{kaggle}x_valid.npy')
 y_valid = np.load(f'{kaggle}y_valid.npy')
 x_test = np.load(f'{kaggle}x_test.npy' )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
BATCHSIZE = 128
EPOCHS = 10
AUTO = tf.data.experimental.AUTOTUNE
NUMCORES = strategy.num_replicas_in_sync
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
def build(transformer, maxlen=512):

 input_ids = Input(shape=(maxlen,), dtype=tf.int32, name="input_word_ids")
 cls_token = transformer(input_ids)[0][:,0,:]
out = tf.reshape(cls_token, [-1, cls_token.shape[1], 1])
 out = Dropout(0.25 )(out)
 out = Conv1D(100, 3, padding='valid', activation='relu', strides=1 )(out)
 out = BatchNormalization(axis=2 )(out)
 out = Conv1D(100, 4, padding='valid', activation='relu', strides=1 )(out)
 out = BatchNormalization(axis=2 )(out)
 out = Conv1D(100, 5, padding='valid', activation='relu', strides=1 )(out)
 out = BatchNormalization(axis=2 )(out)
 out = GlobalMaxPooling1D()(out)
 out = Dropout(0.5 )(out)
 out = Dense(1, activation='sigmoid' )(out)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(Adam(lr=1e-5),
loss=BinaryCrossentropy(label_smoothing=0.1), metrics=['accuracy', AUC() ])
 return model
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(x_train.shape[0])
.batch(BATCHSIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCHSIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCHSIZE)
 )
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.26*df1['Confidence'] + 0.74*df2['Confidence']
df.head()<save_to_csv>
%%time
with strategy.scope() :
transformer_layer = transformers.TFAutoModel.from_pretrained('jplu/tf-xlm-roberta-large')
 model = build(transformer_layer, maxlen=192)
 model.summary()
df.to_csv('submission.csv', index=False )<install_modules>
n_steps = 680
training_history = model.fit(train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS )
!pip install -q https://github.com/pfnet-research/xfeat/archive/master.zip<load_from_csv>
n_steps = x_valid.shape[0] // 128
CLASS_WEIGHT = [1., 7.]
valid_history = model.fit(valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=4,
class_weight=CLASS_WEIGHT )
<normalization><EOS> 
sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub['toxic'] = model.predict(test_dataset, verbose=1)[0:len(sub['toxic'])]
sub.to_csv('submission.csv', index=False )