Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
param_grid = {
'boosting_type': ['gbdt', 'goss', 'dart'],
'num_leaves': list(range(20, 150)) ,
'learning_rate': list(np.logspace(np.log10(0.005), np.log10(0.5), base = 10, num = 1000)) ,
'subsample_for_bin': list(range(20000, 300000, 20000)) ,
'min_child_samples': list(range(20, 500, 5)) ,
'reg_alpha': list(np.linspace(0, 1)) ,
'reg_lambda': list(np.linspace(0, 1)) ,
'colsample_bytree': list(np.linspace(0.6, 1, 10)) ,
'subsample': list(np.linspace(0.5, 1, 100)) ,
'is_unbalance': [True, False]
}<define_variables>
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
import time
import os
import re
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input,GRU
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras import optimizers
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
a = 0
b = 0
for x in param_grid['learning_rate']:
if x >= 0.005 and x < 0.05:
a += 1
elif x >= 0.05 and x < 0.5:
b += 1
print('There are {} values between 0.005 and 0.05'.format(a))
print('There are {} values between 0.05 and 0.5'.format(b))<create_dataframe>
def align_data() :
en=pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')
 f1=pd.read_csv('.. /input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')
 result=[]
last_cell=[]
for i in tqdm(range(len(en['id']))):
index=np.where(f1['id']==en['id'][i])[0]
l=len(index)
 if l==1:
cell=[f1['comment_text'][index[0]],'es',f1['toxic'][index[0]]]
result.append(cell)
 last_cell=cell
else:
cell=last_cell
result.append(cell)
 print(len(en['toxic']))
print(len(result))
column=['comment_text','lang','toxic']
toxic=pd.DataFrame(columns=column,data=result)
 toxic.to_csv('/kaggle/working/train1_es_align.csv'%(LANG,LANG),index=False )
random_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 grid_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)) )<count_unique_values>
def mix_data() :
ES_FRAC=0.132232182034727
TR_FRAC=0.21939447125932426
RU_FRAC=0.17156647652479157
IT_FRAC=0.13310975991976431
FR_FRAC=0.17112768758227292
PT_FRAC=0.17256942267911993
train_tr=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_tr.csv.csv')
 train_es=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_es.csv.csv')
 train_it=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_it.csv.csv')
 train_fr=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_fr.csv.csv')
 train_ru=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_ru.csv.csv')
 train_pt=pd.read_csv('.. /input/jigsaw_toxic_comment_train_align/train1_pt.csv.csv')
 result=[]
label=[]
for i in tqdm(range(0, len(train_es['comment_text']))):
rf = random.random()
if rf < ES_FRAC:
cell = [train_es['comment_text'][i], train_es['lang'][i], train_es['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC):
cell = [train_fr['comment_text'][i], train_fr['lang'][i], train_fr['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC + TR_FRAC):
cell = [train_tr['comment_text'][i], train_tr['lang'][i], train_tr['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC + TR_FRAC + IT_FRAC):
cell = [train_it['comment_text'][i], train_it['lang'][i], train_it['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 elif rf <(ES_FRAC + FR_FRAC + TR_FRAC + IT_FRAC + PT_FRAC):
cell = [train_pt['comment_text'][i], train_pt['lang'][i], train_pt['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 else:
cell = [train_ru['comment_text'][i], train_ru['lang'][i], train_ru['toxic'][i]]
label.append(cell[2])
 result.append(cell)
 print(len(result))
print(len(label))
label=np.array(label)
 np.save('/kaggle/working/train1_label.npy')
 column=['comment_text','lang','toxic']
toxic=pd.DataFrame(columns=column,data=result)
 toxic.to_csv('/kaggle/working/train1_mix.csv')
 x_train=regular_encode(toxic['comment_text'], tokenizer, maxlen=MAX_LEN)
 np.save('/kaggle/working/train1_mix.npy' )
com = 1
for x in param_grid.values() :
com *= len(x)
 print('There are {} combinations'.format(com))<train_model>
def build_model(transformer, max_len=512,learnig_rate=1e-5):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=learnig_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
print('This would take {:.0f} years to finish.'.format(( 100 * com)/(60 * 60 * 24 * 365)) )<train_on_grid>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
 EPOCHS = 3
BATCH_SIZE = 24 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
grid_results = grid_search(param_grid)
 print('The best validation score was {:.5f}'.format(grid_results.loc[0, 'score']))
print('
The best hyperparameters were:')
 pprint.pprint(grid_results.loc[0, 'params'] )<train_on_grid>
valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
grid_search_params = grid_results.loc[0, 'params']
model = lgb.LGBMClassifier(**grid_search_params, random_state=42)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('The best model from grid search scores {:.5f} ROC AUC on the test set.'.format(roc_auc_score(test_labels, preds)) )<set_options>
x_valid=np.load('/kaggle/input/toxic-npy/x_valid_encode192.npy')
 print(x_valid)
 print(x_valid.shape)
 y_valid = valid.toxic.values
print(y_valid)
 print(y_valid.shape )
pd.options.display.max_colwidth = 1000
grid_results['params'].values<init_hyperparams>
x_test=np.load('/kaggle/input/toxic-npy/x_test_encode192.npy')
 print(x_test)
 print(x_test.shape)
 print("finish x_test_encode" )
random.seed(50)
 random_params = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
random_params['subsample'] = 1.0 if random_params['boosting_type'] == 'goss' else random_params['subsample']
random_params<create_dataframe>
valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
)
def random_search(param_grid, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 for i in range(MAX_EVALS):
hyperparameters = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
hyperparameters['subsample'] = 1.0 if hyperparameters['boosting_type'] == 'goss' else hyperparameters['subsample']
eval_results = objective(hyperparameters, i)
 results.loc[i, :] = eval_results
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results<train_model>
def train_train2max() :
print('train train2max')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 x_train=np.load('/kaggle/input/mix-data2/train2max21.npy')
 y_train=np.load('/kaggle/input/mix-data2/train2max_label21.npy')
 y_train=y_train.round().astype(int)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.batch(BATCH_SIZE)
.shuffle(2048)
.prefetch(AUTO)
)
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN,learnig_rate=5e-6)
 model.summary()
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1 
)
 model.save_weights('/kaggle/working/model_mix_1.h5' )
random_results = random_search(param_grid)
 print('The best validation score was {:.5f}'.format(random_results.loc[0, 'score']))
print('
The best hyperparameters were:')
 pprint.pprint(random_results.loc[0, 'params'] )<find_best_model_class>
def train_train1() :
print('train train1')
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 x_train=np.load('/kaggle/input/mix-data2/train1_mix_data21.npy')
 y_train=np.load('/kaggle/input/mix-data2/train1_label21.npy')
 print(y_train)
 print(y_train.shape)
 train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.batch(BATCH_SIZE)
.shuffle(2048)
.prefetch(AUTO)
)
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN,learnig_rate=5e-6)
 model.load_weights('/kaggle/working/model_mix_1.h5')
 print("learning rate:5e-6")
 n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=1 
)
 model.save_weights('/kaggle/working/model_mix_1.h5' )
random_search_params = random_results.loc[0, 'params']
model = lgb.LGBMClassifier(**random_search_params, random_state = 42)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('The best model from random search scores {:.5f} ROC AUC on the test set.'.format(roc_auc_score(test_labels, preds)) )<save_to_csv>
def train_valid() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN,learnig_rate=4e-6)
 model.load_weights('/kaggle/working/model_mix_1.h5')
 n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=2
)
out_file = 'random_search_trials.csv'
of_connection = open(out_file, 'w')
 writer = csv.writer(of_connection)
 headers = ['score', 'hyperparameters', 'iteration']
writer.writerow(headers)
 of_connection.close()<save_to_csv>
def predict() :
score = model.predict(test_dataset,verbose=1)
 sub['toxic']=score[:sub['toxic'].shape[0]]
sub.to_csv('submission.csv', index=False )
def random_search(param_grid, out_file, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 for i in range(MAX_EVALS):
random_params = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
random_params['subsample'] = 1.0 if random_params['boosting_type'] == 'goss' else random_params['subsample']
eval_results = objective(random_params, i)
 results.loc[i, :] = eval_results
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow(eval_results)
 of_connection.close()
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results<save_to_csv>
submission_train2mix1 = pd.read_csv('/kaggle/input/haveatry/submission9450.csv')
 submission_train2mix2 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_2.csv')
 submission_train2mix3 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_3.csv')
 submission_train2mix4 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_4.csv')
 submission_train2mix5 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_5.csv')
 submission_train2mix6 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_6.csv')
 submission_train2mix7 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_7.csv')
 submission_train2mix8 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_8.csv')
 submission_train2mix9 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_9.csv')
 submission_train2mix10 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_10.csv')
 submission_train2mix11 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_11.csv')
 submission_train2mix12 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_12.csv')
 submission_train2mix13 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_13.csv')
 submission_train2mix14 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_14.csv')
 submission_train2mix15 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_15.csv')
 submission_train2mix16 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_16.csv')
 submission_train2mix17 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_17.csv')
 submission_train2mix18 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_18.csv')
 submission_train2mix19 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_19.csv')
 submission_train2mix20 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_20.csv')
 submission_train2mix21 = pd.read_csv('/kaggle/input/haveatry/submission_train2mix_21.csv' )
def grid_search(param_grid, out_file, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 keys, values = zip(*param_grid.items())
 i = 0
for v in itertools.product(*values):
parameters = dict(zip(keys, v))
parameters['subsample'] = 1.0 if parameters['boosting_type'] == 'goss' else parameters['subsample']
eval_results = objective(parameters, i)
 results.loc[i, :] = eval_results
i += 1
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow(eval_results)
 of_connection.close()
if i > MAX_EVALS:
break
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results<load_from_csv>
for i in range(len(submission_train2mix1['toxic'])) :
sub['toxic'][i]=(submission_train2mix1['toxic'][i]+submission_train2mix2['toxic'][i]
+submission_train2mix3['toxic'][i]+submission_train2mix4['toxic'][i]
+submission_train2mix5['toxic'][i]+submission_train2mix6['toxic'][i]
+submission_train2mix7['toxic'][i]+submission_train2mix8['toxic'][i]
+submission_train2mix9['toxic'][i]+submission_train2mix10['toxic'][i]
+submission_train2mix11['toxic'][i]+submission_train2mix12['toxic'][i]
+submission_train2mix13['toxic'][i]+submission_train2mix14['toxic'][i]
+submission_train2mix15['toxic'][i]+submission_train2mix16['toxic'][i]
+submission_train2mix17['toxic'][i]+submission_train2mix18['toxic'][i]
+submission_train2mix19['toxic'][i]+submission_train2mix20['toxic'][i]
+submission_train2mix21['toxic'][i])/21
random_results = pd.read_csv('.. /input/home-credit-model-tuning/random_search_trials_1000.csv')
 grid_results = pd.read_csv('.. /input/home-credit-model-tuning/grid_search_trials_1000.csv' )<categorify>
sub.to_csv('submission.csv', index=False )