Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<feature_engineering>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=3e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC() ])
 return model
sub2 = assign_prior(test_stage_2 )<save_to_csv>
def roc_auc(predictions,target):

 fpr, tpr, thresholds = metrics.roc_curve(target, predictions)
 roc_auc = metrics.auc(fpr, tpr)
 return roc_auc
sub2.to_csv("submission.csv", index=False )<import_modules>
import numpy as np
import pandas as pd
from tqdm import tqdm
import os
from sklearn import metrics
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tokenizers import BertWordPieceTokenizer
import pandas as pd<import_modules>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
import pandas as pd<save_to_csv>
DATA_PATH = ".. /input/jigsaw-multilingual-toxic-comment-classification"
answers = pd.read_csv('.. /input/gapanswers/answers.csv', dtype={'A-coref': int, 'B-coref': int})
 answers.rename(columns={'A-coref': 'A', 'B-coref': 'B'}, inplace=True)
 answers['NEITHER'] = answers.eval('1 - A - B')
 answers[['ID', 'A', 'B', 'NEITHER']].to_csv("submission.csv", index=False )<load_from_csv>
wiki_toxic_comment_data = "jigsaw-toxic-comment-train.csv"
toxic_unintended_data = "jigsaw-unintended-bias-train.csv"
valid_data = "validation.csv"
test_data = "test.csv"
sub_sample = "sample_submission.csv"
wiki_toxic_comment = pd.read_csv(os.path.join(DATA_PATH, wiki_toxic_comment_data))
toxic_unintended = pd.read_csv(os.path.join(DATA_PATH, toxic_unintended_data))
valid = pd.read_csv(os.path.join(DATA_PATH, valid_data))
test = pd.read_csv(os.path.join(DATA_PATH, test_data))
sub = pd.read_csv(os.path.join(DATA_PATH, sub_sample))
train = pd.read_csv('.. /input/allstate-claims-severity/train.csv')
 test = pd.read_csv('.. /input/allstate-claims-severity/test.csv')
 sample = pd.read_csv('.. /input/allstate-claims-severity/sample_submission.csv' )<count_values>
EPOCHS = 2
BATCH_SIZE_PER_CORE = 64
BATCH_SIZE = BATCH_SIZE_PER_CORE * strategy.num_replicas_in_sync
MAX_LEN = 200
train.dtypes.value_counts()<count_missing_values>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
train.isna().any().sum()<drop_column>
tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-multilingual-cased')
 tokenizer.save_pretrained('.')
 fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)
 fast_tokenizer
train = train.drop(['cont1', 'cont11', 'cont10'], axis=1)
 test = test.drop(['cont1', 'cont11', 'cont10'], axis=1 )<categorify>
%%time
x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values
cat_cols = train.select_dtypes(include='object' ).columns
le = LabelEncoder()
for i in cat_cols:
test_unique = test[i].unique()
train_unique = train[i].unique()
labels = list(set(test_unique)| set(train_unique))
le.fit(labels)
 train[i] = le.transform(train[i])
 test[i] = le.transform(test[i] )<prepare_x_and_y>
AUTO = tf.data.experimental.AUTOTUNE
X = train.drop(['loss'], axis=1)
 y = np.log(train['loss']+1 )<split>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(len(x_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
X_train,X_val,y_train, y_val = train_test_split(X,y,test_size=0.1)
 model = LGBMRegressor(n_estimators=300, learning_rate=0.1, random_state=123)
 model.fit(X_train,y_train)
 preds = model.predict(X_val)
 print(mean_absolute_error(preds,y_val))<save_to_csv>
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFBertModel 
.from_pretrained('bert-base-multilingual-cased')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
test_predictions = model.predict(test)
 sample['loss'] = np.expm1(test_predictions)
 sample.to_csv('submission.csv', index=False )<set_options>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
warnings.simplefilter("ignore")
 print(os.listdir(".. /input"))<load_from_csv>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)
df=pd.read_csv(".. /input/train.csv")
 del df['id']
print(df.shape)
 display(df.head() )<load_from_csv>
scores = model.predict(valid_dataset)
 print("Auc: %.2f%%" %(roc_auc(scores,y_valid)*100))
test=pd.read_csv(".. /input/test.csv")
 idx=test['id']
del test['id']
print(test.shape)
 display(test.head() )<define_variables>
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )
cat_int=['cat12|cat80', 'cat79|cat80', 'cat1|cat12|cat80',
'cat12|cat80|cat81', 'cat12|cat79|cat80', 'cat1|cat80',
'cat103|cat12|cat80', 'cat57|cat80', 'cat12|cat72|cat80',
'cat1|cat103|cat80', 'cat57|cat79|cat80', 'cat1|cat80|cat81',
'cat12|cat57|cat80', 'cat2|cat57|cat80', 'cat10|cat80',
'cat12|cat81', 'cat103|cat12', 'cat1|cat12', 'cat1|cat114|cat80',
'cat10|cat103|cat80', 'cat1|cat111|cat80', 'cat10|cat111|cat80',
'cat12|cat53', 'cat1|cat103', 'cat12|cat53|cat80',
'cat10|cat101|cat80', 'cat12|cat53|cat81', 'cat12|cat79',
'cat1|cat81', 'cat2|cat80|cat81', 'cat81|cat87', 'cat12|cat72',
'cat103|cat111', 'cat1|cat72', 'cat10|cat114|cat81', 'cat4|cat5',
'cat103|cat111|cat80', 'cat10|cat72|cat80', 'cat80|cat81',
'cat103|cat111|cat12', 'cat1|cat2|cat72', 'cat10|cat81|cat87',
'cat2|cat57', 'cat1|cat72|cat79', 'cat79|cat81|cat87',
'cat2|cat44', 'cat114|cat2', 'cat13|cat53', 'cat111|cat13|cat53',
'cat10|cat81']
for each in cat_int:
l=each.split("|")
 if(len(l)==3):
a=l[0]
b=l[1]
c=l[2]
df[a+"_"+b+"_"+c]=df[a]+df[b]+df[c]
test[a+"_"+b+"_"+c]=test[a]+test[b]+test[c]
else:
a=l[0]
b=l[1]
df[a+"_"+b]=df[a]+df[b]
test[a+"_"+b]=test[a]+test[b]<categorify>
EPOCHS = 2
BATCH_SIZE_PER_CORE = 32
BATCH_SIZE = BATCH_SIZE_PER_CORE * strategy.num_replicas_in_sync
MAX_LEN = 200
c=list(df.columns)
 for each in c:
if each[0:3]=='cat':
res=pd.factorize(pd.concat(( df[each],test[each])).values,sort=True)[0]
df[each]=res[0:188318]
test[each]=res[188318:]<prepare_x_and_y>
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )
c=list(df.columns)
 c.remove('loss')
 X=df.loc[:,c]
Y=df.loss
X.head()<feature_engineering>
tokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )
X['mul_cont2_cont7']=X['cont2']*X['cont7']
X['add_cont2_cont7']=X['cont2']+X['cont7']
X['add_cont2_cont12']=X['cont2']+X['cont12']
X['add_cont2_cont11']=X['cont2']+X['cont11']
X['sub_cont1_cont7']=X['cont1']-X['cont7']
X['add_cont7_cont14']=X['cont7']+X['cont14']
X['div_cont2_cont4']=X['cont2']/X['cont4']
X['add_cont3_cont7']=X['cont3']+X['cont7']
test['mul_cont2_cont7']=test['cont2']*test['cont7']
test['add_cont2_cont7']=test['cont2']+test['cont7']
test['add_cont2_cont12']=test['cont2']+test['cont12']
test['add_cont2_cont11']=test['cont2']+test['cont11']
test['sub_cont1_cont7']=test['cont1']-test['cont7']
test['add_cont7_cont14']=test['cont7']+test['cont14']
test['div_cont2_cont4']=test['cont2']/test['cont4']
test['add_cont3_cont7']=test['cont3']+test['cont7']<feature_engineering>
%%time
x_train = regular_encode(train1.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train1.toxic.values
y_valid = valid.toxic.values
<init_hyperparams>
AUTO = tf.data.experimental.AUTOTUNE
params={'bagging_fraction': 0.5021393514653055,
'feature_fraction': 0.6190345305094423,
'lambda_l1': 0.4932225704923126,
'max_depth': 39,
'min_data_in_leaf': 22,
'min_gain_to_split': 0.2,
'num_leaves': 45,
'silent':True,
'objective':'fair',
'random_state':60,
'n_estimators':10000,
'learning_rate':0.03} 
<count_unique_values>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(len(x_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
r=X.nunique()
r.sort_values(ascending=False )<split>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
P=np.zeros(( test.shape[0]))
trainALL=[]
valALL=[]
kf = KFold(n_splits=5,random_state=60)
 k=1
for train_index, test_index in kf.split(X):
print("FOLD :",k)
 X_train, X_val = X.loc[train_index,:], X.loc[test_index,:]
Y_train, Y_val = Y[train_index], Y[test_index]
reg=lgb.LGBMRegressor(**params)
 reg.fit(X_train, Y_train,eval_metric='mae',eval_set=[(X_val,Y_val)],verbose=False,early_stopping_rounds=50)
 trainR=mean_absolute_error(np.expm1(Y_train),np.expm1(reg.predict(X_train)))
 valR=mean_absolute_error(np.expm1(Y_val),np.expm1(reg.predict(X_val)))
 print("MAE Train : ",trainR)
 print("MAE Test : ",valR)
 print("****************")
 trainALL.append(trainR)
 valALL.append(valR)
 k+=1
P+=np.expm1(reg.predict(test))<compute_test_metric>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS*2
)
print("Train Score: ",np.mean(trainALL)," Std dev:",np.std(trainALL))
print("Val Score: ",np.mean(valALL)," Std dev:",np.std(valALL))<save_to_csv>
scores = model.predict(valid_dataset)
 print("Auc: %.2f%%" %(roc_auc(scores,y_valid)*100))
<load_from_csv><EOS> 
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )