Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<drop_column>
import os
import numpy as np
import pandas as pd
import sklearn
from sklearn.utils.class_weight import compute_class_weight
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.losses import BinaryCrossentropy
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True )<feature_engineering>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()<save_to_csv>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
submission.to_csv('result-with-best.csv', index=False )<set_options>
loss = BinaryCrossentropy()
%config InlineBackend.figure_format = 'retina'
%matplotlib inline<load_from_csv>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss=loss, metrics=['accuracy'])
 return model
train_set = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', header=0)
 test_set =pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', header=0 )<concatenate>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
full_data = pd.concat(( train_set.loc[:,'MSSubClass':'YrSold'],test_set.loc[:,'MSSubClass':'YrSold']))<feature_engineering>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
train_set["SalePrice"] = np.log1p(train_set["SalePrice"])
 numeric_feats = full_data.dtypes[full_data.dtypes != "object"].index
skewed_feats = train_set[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
full_data[skewed_feats] = np.log1p(full_data[skewed_feats] )<categorify>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
full_data = pd.get_dummies(full_data )<data_type_conversions>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
full_data = full_data.fillna(full_data.mean() )<prepare_x_and_y>
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
print('x_train.shape:', x_train.shape )
x_train = full_data[:train_set.shape[0]]
x_test = full_data[train_set.shape[0]:]
y = train_set.SalePrice<compute_train_metric>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.shuffle(2048)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, x_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )<train_on_grid>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(x_train, y )<compute_train_metric>
n_steps = x_train.shape[0] // BATCH_SIZE
class_weights = compute_class_weight(class_weight = 'balanced', classes = np.unique(train['toxic'].values), y = train['toxic'].values)
 train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
class_weight = class_weights
)
r = rmse_cv(model_lasso ).mean()
print(r )<choose_model_class>
n_steps = x_valid.shape[0] // BATCH_SIZE
class_weights = compute_class_weight(class_weight = 'balanced', classes = np.unique(valid['toxic'].values), y = valid['toxic'].values)
 train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS,
class_weight = class_weights
)
model_ridge = Ridge()<define_search_space>
predictions = model.predict(test_dataset, verbose=1 )
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]<drop_column>
predictions = predictions.round().astype(int)
 pseudo_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test, predictions)) 
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
train_set = train_set[train_set.GrLivArea < 8.25]
train_set = train_set[train_set.SalePrice < 13]
train_set = train_set[train_set.SalePrice > 10.75]
train_set.drop("Id", axis=1, inplace=True )<train_on_grid>
n_steps = x_test.shape[0] // BATCH_SIZE
model.fit(
 pseudo_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
<train_model><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )