Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<import_modules>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR
from sklearn.kernel_ridge import KernelRidge
from sklearn.metrics import mean_absolute_error, make_scorer
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold<load_from_csv>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64} )<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
pd.options.display.precision = 15<prepare_x_and_y>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128
rows = 150_000
shift_step = int(np.floor(rows / 2))
segments = int(np.floor(train.shape[0] / rows))
segments_augmented = 2*segments - 1
X_train = pd.DataFrame(index=range(segments_augmented), dtype=np.float64,
columns=['ave', 'std', 'max', 'min','q95','q99', 'q05','q01'])
 y_train = pd.DataFrame(index=range(segments_augmented), dtype=np.float64,
columns=['time_to_failure'])
 for segment in tqdm(range(segments)) :
for do_shift in [False,True]:
if(do_shift):
shift = shift_step
idx = segments + segment
if(segment==segments-1):
continue
else:
shift = 0
idx = segment
seg = train.iloc[segment*rows+shift:segment*rows+shift+rows]
x = seg['acoustic_data'].values
y = seg['time_to_failure'].values[-1]
y_train.loc[idx, 'time_to_failure'] = y
X_train.loc[idx, 'ave'] = x.mean()
X_train.loc[idx, 'std'] = x.std()
X_train.loc[idx, 'max'] = x.max()
X_train.loc[idx, 'min'] = x.min()
X_train.loc[idx, 'q95'] = np.quantile(x,0.95)
 X_train.loc[idx, 'q99'] = np.quantile(x,0.99)
 X_train.loc[idx, 'q05'] = np.quantile(x,0.05)
 X_train.loc[idx, 'q01'] = np.quantile(x,0.01)
<define_search_space>
tokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')
 tokenizer.save_pretrained('.')
X_train.loc[[0,1,2,3,4,0+segments,1+segments,2+segments,3+segments]]<normalization>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
scaler = StandardScaler()
scaler.fit(X_train)
 X_train_scaled = scaler.transform(X_train )<string_transform>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ] )
nFolds = 3
customKFoldAvoidLeakToValidation = []
for train_id, valid_id in KFold(nFolds,shuffle=True ).split(X_train):
must_remove = []
for v in valid_id:
if(v>=segments):
must_remove.append(v-segments)
 must_remove.append(v-segments+1)
 else:
must_remove.append(v+segments-1)
 must_remove.append(v+segments)
 train_id = [t for t in train_id if t not in(must_remove)]
customKFoldAvoidLeakToValidation.append(( train_id, valid_id))<compute_test_metric>
train.toxic.value_counts()
scorer = make_scorer(mean_absolute_error, greater_is_better=False )<train_on_grid>
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )
parameters = [{ 'gamma': [0.6, 0.7, 0.8],
'C': [2.35, 2.4, 2.45, 2.5],
'nu': [0.85, 0.9, 0.95]}]
reg1 = GridSearchCV(NuSVR(kernel='rbf', tol=0.01), parameters, cv = customKFoldAvoidLeakToValidation, scoring=scorer)
 reg1.fit(X_train_scaled, y_train.values.flatten())
 y_pred1 = reg1.predict(X_train_scaled)
 print(reg1.best_params_)
 print(reg1.best_score_ )<train_on_grid>
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
parameters = [{'gamma': [0.06, 0.1, 0.08, 0.09],
'alpha': [0.005, 0.01, 0.05]}]
reg2 = GridSearchCV(KernelRidge(kernel='rbf'), parameters, cv = customKFoldAvoidLeakToValidation, scoring=scorer)
 reg2.fit(X_train_scaled, y_train.values.flatten())
 y_pred2 = reg2.predict(X_train_scaled)
 print(reg2.best_params_)
 print(reg2.best_score_ )<compute_test_metric>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
score1 = mean_absolute_error(y_train.values.flatten() , y_pred1)
 print(f'Score1: {score1:0.3f}')
 score2 = mean_absolute_error(y_train.values.flatten() , y_pred2)
 print(f'Score2: {score2:0.3f}')
 score3 = mean_absolute_error(y_train.values.flatten() ,(0.5*y_pred1 + 0.5*y_pred2))
print(f'Score3: {score3:0.3f}' )<load_from_csv>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id' )<create_dataframe>
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead 
.from_pretrained('jplu/tf-xlm-roberta-large')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index )<feature_engineering>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
for seg_id in X_test.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
X_test.loc[seg_id, 'ave'] = x.mean()
X_test.loc[seg_id, 'std'] = x.std()
X_test.loc[seg_id, 'max'] = x.max()
X_test.loc[seg_id, 'min'] = x.min()
X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)
 X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)
 X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)
 X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01 )<save_to_csv>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
<set_options><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False)