Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<init_hyperparams>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
import pickle
optimized_param = None
lowest_cv = 1000
total_iteration = 100
for i in range(total_iteration):
print('-'*20, 'For {} of {} iterations'.format(i+1, total_iteration), '-'*20)
 learning_rate = np.random.rand() * 0.02
n_folds = 3
num_class = len(np.unique(y))
params = {}
params['application'] = 'multiclass'
params['metric'] = 'multi_logloss'
params['num_class'] = num_class
params['class_weight'] = 'balanced'
params['num_leaves'] = np.random.randint(24, 48)
 params['max_depth'] = np.random.randint(5, 8)
 params['min_child_weight'] = np.random.randint(5, 50)
 params['min_split_gain'] = np.random.rand() * 0.09
params['colsample_bytree'] = np.random.rand() *(0.9 - 0.1)+ 0.1
params['subsample'] = np.random.rand() *(1 - 0.8)+ 0.8
params['bagging_freq'] = np.random.randint(1, 5)
 params['bagging_seed'] = np.random.randint(1, 5)
 params['reg_alpha'] = np.random.rand() * 2
params['reg_lambda'] = np.random.rand() * 2
params['learning_rate'] = np.random.rand() * 0.02
params['seed'] =1989
d_train = lgb.Dataset(data=new_train, label=y.values-1, categorical_feature=new_categorical_feats, free_raw_data=False)
 cv_results = lgb.cv(params=params, train_set=d_train, num_boost_round=10000, categorical_feature=new_categorical_feats,
nfold=n_folds, stratified=True, shuffle=True, early_stopping_rounds=1, verbose_eval=1000)
 min_cv_results = min(cv_results['multi_logloss-mean'])
 if min_cv_results < lowest_cv:
lowest_cv = min_cv_results
optimized_param = params<compute_train_metric>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=0.000009), loss='binary_crossentropy', metrics=['accuracy'])
 return model
N_Folds = 20
SEED = 1989
predicts_result, feat_importance_df = LGB_OOF(optimized_param, new_categorical_feats, N_Folds, SEED=1989 )<save_to_csv>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
submission['Target'] = np.array(predicts_result ).mean(axis=0 ).round().astype(int)
 submission.to_csv('submission_shap_randomized_search.csv', index = False )<set_options>
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
%matplotlib inline 
<load_from_csv>
tokenizer = AutoTokenizer.from_pretrained(MODEL )
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()<load_from_csv>
%%time
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_train.pkl"
x_train = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_trainOA.pkl"
x_trainOA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_trainA.pkl"
x_trainA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_valid.pkl"
x_valid = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_x_test.pkl"
x_test = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_train.pkl"
y_train = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_trainOA.pkl"
y_trainOA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_trainA.pkl"
y_trainA = pickle.load(open(filename, 'rb'))
filename = "/kaggle/input/jigsawtpuxlmrobertacopypickledata/jigsaw_multilingual_y_valid.pkl"
y_valid = pickle.load(open(filename, 'rb'))
test_df = pd.read_csv('.. /input/test.csv' )<count_missing_values>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_datasetOA =(
 tf.data.Dataset 
.from_tensor_slices(( x_trainOA, y_trainOA)) 
.repeat() 
.shuffle(x_trainOA.shape[0])
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 train_datasetA =(
 tf.data.Dataset 
.from_tensor_slices(( x_trainA, y_trainA)) 
.repeat() 
.shuffle(x_trainOA.shape[0])
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
train_df.isnull().values.any()<count_missing_values>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
test_df.isnull().values.any()<count_missing_values>
n_steps = x_train.shape[0] // BATCH_SIZE
model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1
)
train_df.isnull().values.sum(axis=0 )<count_missing_values>
checkpointPredictions1 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions1[:10] )
test_df.isnull().values.sum(axis=0 )<prepare_x_and_y>
n_steps = x_trainA.shape[0] // BATCH_SIZE
model.fit(
 train_datasetA,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1
)
y = train_df['Target'].values-1<concatenate>
checkpointPredictions2 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions2[:10] )
train_test_df = pd.concat([train_df[columns_to_use], test_df[columns_to_use]], axis=0)
 object_cols = [f_ for f_ in train_test_df.columns if train_test_df[f_].dtype == 'object']<categorify>
n_steps = x_valid.shape[0] // BATCH_SIZE
model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=2
)
for col in object_cols:
le = LabelEncoder()
print(col)
 le.fit(train_test_df[col].astype(str))
train_df[col] = le.transform(train_df[col].astype(str))
test_df[col] = le.transform(test_df[col].astype(str))
del le<categorify>
checkpointPredictions3 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions3[:10] )
def dprint(*args, **kwargs):
print("[{}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) + \
" ".join(map(str,args)) , **kwargs)
 id_name = 'Id'
target_name = 'Target'
df_all = pd.concat([train_df, test_df], axis=0)
 cols = [f_ for f_ in df_all.columns if df_all[f_].dtype == 'object' and f_ != id_name]
print(cols)
 for c in tqdm(cols):
le = preprocessing.LabelEncoder()
le.fit(df_all[c].astype(str))
train_df[c] = le.transform(train_df[c].astype(str))
test_df[c] = le.transform(test[c].astype(str))
del le
gc.collect()
def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
extract_features(train_df)
 extract_features(test_df )<prepare_x_and_y>
n_steps = x_trainOA.shape[0] // BATCH_SIZE
model.fit(
 train_datasetOA,
steps_per_epoch=n_steps,
epochs=1
)
cnt = 0
p_buf = []
n_splits = 20
n_repeats = 1
kf = RepeatedKFold(
 n_splits=n_splits,
n_repeats=n_repeats,
random_state=None)
 err_buf = []
cols_to_drop = [
id_name,
target_name,
]
X = train_df.drop(cols_to_drop, axis=1, errors='ignore')
 feature_names = list(X.columns)
 X = X.fillna(0)
 X = X.values
y = train_df[target_name].values
classes = np.unique(y)
 dprint('Number of classes: {}'.format(len(classes)))
 c2i = {}
i2c = {}
for i, c in enumerate(classes):
c2i[c] = i
i2c[i] = c
y_le = np.array([c2i[c] for c in y])
 X_test = test_df.drop(cols_to_drop, axis=1, errors='ignore')
 X_test = X_test.fillna(0)
 X_test = X_test.values
id_test = test_df[id_name].values
dprint(X.shape, y.shape)
 dprint(X_test.shape)
 n_features = X.shape[1]
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': -1,
'num_leaves': 14,
'learning_rate': 0.1,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': -1,
'num_threads': 8,
'colsample_bytree': 0.89,
'min_child_samples': 90,
'subsample': 0.96,
'lambda_l2': 1.0,
'min_gain_to_split': 0,
'num_class': len(np.unique(y)) ,
}<train_model>
checkpointPredictions4 = model.predict(test_dataset, verbose=1)
 print(checkpointPredictions4[:10] )
<compute_test_metric><EOS> 
sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
 sub['toxic'] =(checkpointPredictions1*0.05)+(checkpointPredictions2*0.10)+(checkpointPredictions3*0.76)+(checkpointPredictions4*0.09)
 sub.to_csv('submission.csv', index=False )