Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<find_best_params>
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math
import os
import re
import time
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.metrics import AUC
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
best_grid_hyp = grid_hyp.iloc[grid_hyp['score'].idxmax() ].copy()
best_random_hyp = random_hyp.iloc[random_hyp['score'].idxmax() ].copy()<compute_test_metric>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
print('Average validation score of grid search = {:.5f}.'.format(np.mean(grid_hyp['score'])))
 print('Average validation score of random search = {:.5f}.'.format(np.mean(random_hyp['score'])) )<data_type_conversions>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
random_hyp['score'] = random_hyp['score'].astype(float)
 best_random_hyp = random_hyp.loc[0, :].copy()<load_from_csv>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-3), loss='binary_crossentropy', metrics=['accuracy'])
 return model
train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv')
 test_ids = test['SK_ID_CURR']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )<train_on_grid>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
train_set = lgb.Dataset(train, label = train_labels)
 hyperparameters = dict(**random_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS )<compute_test_metric>
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 20
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 200
MODEL = 'jplu/tf-xlm-roberta-large'
print('The cross validation score on the full dataset = {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )<train_model>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]<save_to_csv>
train = pd.read_csv('/kaggle/input/toxic-comment-classification/train.csv')
 valid = pd.read_csv('/kaggle/input/toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/toxic-comment-classification/test.csv' )
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_simple_features_random.csv', index = False )<train_model>
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.comment_text.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
clf_xgBoost = xgb.XGBClassifier(
 learning_rate =0.01, n_estimators=1000, max_depth=4, min_child_weight=4, subsample=0.8, colsample_bytree=0.8, objective= 'binary:logistic',
nthread=4, scale_pos_weight=2, seed=27)
 clf_xgBoost.fit(train,train_labels )<predict_on_test>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
pred = clf_xgBoost.predict_proba(test)[:, 1]<save_to_csv>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': pred})
 submission.to_csv('submission_xgboost.csv', index = False)
 submission.head()<import_modules>
filepath = "roberta_model.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True, mode='max')
 early = EarlyStopping(monitor='val_accuracy', mode='max', patience=5)
 reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=2, min_lr=1e-5 )
import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold<load_from_csv>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
callbacks=[early, checkpoint, reduce_lr]
)
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_id = test['id']
train.drop(['id'], axis=1, inplace=True)
 test.drop(['id'], axis=1, inplace=True )<groupby>
y_true = test['toxic'].values
y_pred = model.predict(test_dataset ).ravel()
<feature_engineering><EOS> 
roc_auc_score(y_true, y_pred )