Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<merge>
import os,gc,pickle,random,sys
import numpy as np
import pandas as pd
import tensorflow as tf
import transformers
from transformers import TFAutoModel
from tqdm.notebook import tqdm
from tensorflow.data import Dataset
from tensorflow.data.experimental import sample_from_datasets
user = user.merge(user1, on='user_id', how='inner')
 del user1
gc.collect()
user.head()<merge>
MAX_LEN = 192
HEAD = "max"
PATH = ".. /input/jigsaw-multilingual-toxic-comment-classification/"
INPATH = ".. /input/jmtc-monolinguish-encoding-pickle/"
TRANSFER = ".. /input/buffer/submission-9462.csv"
LANG = "pt"
lang_valid = ["it","es","tr"]
MODEL = "neuralmind/bert-large-portuguese-cased"
user = user.merge(results, on='user_id', how='inner')
 del results
gc.collect()
user.head()<prepare_output>
np.random.seed(1234)
 random.seed(1234)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
with strategy.scope() :
tf.random.set_seed(1234 )
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd.head()<groupby>
models = {"pt":["neuralmind/bert-large-portuguese-cased"],
"it":["dbmdz/bert-base-italian-xxl-uncased",
"m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0"],
"es":["dccuchile/bert-base-spanish-wwm-cased"],
"tr":["dbmdz/bert-base-turkish-128k-cased",
"dbmdz/electra-base-turkish-cased-discriminator"],
"ru":["DeepPavlov/rubert-base-cased"],
"fr":["flaubert/flaubert_large_cased",
"camembert/camembert-large"]}
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()<groupby>
trans = pd.read_csv(TRANSFER ).toxic.values.astype('float32' )
avg_pos = op.groupby('product_id')[['add_to_cart_order']].mean()
avg_pos.columns=['mean_add_to_cart_order']
avg_pos.head()<merge>
fetch_title = lambda x: x.split("/")[-1]
def pick_load_format(path):
with open(INPATH+path,"rb")as f:
return pickle.load(f)
 def get_cong(n,verb=True):
tot = round(1+(n*2)/10_000)*10_000
if verb: print("Pos: {}, Sample neg: {}, Total: {}".format(n,tot-n,tot))
return tot-n
def balance_shuffle(list_arr,pos_thred=0.6,neg_thred=0.4,balance=True):
target = list_arr[-1]
idx = np.arange(len(target))
pos_idx = idx[target>=pos_thred]
neg_idx = idx[target<=neg_thred]
psize,nsize = len(pos_idx),len(neg_idx)
 cong = get_cong(psize,balance)
 if balance and nsize>cong:
neg_idx = np.random.choice(neg_idx,cong,replace=False)
 idx = np.concatenate(( pos_idx, neg_idx))
np.random.shuffle(idx)
 return tuple(arr[idx] for arr in list_arr)
 def load_data(model,lang,frac=.05):
title = fetch_title(model)
 x_train,y_train = balance_shuffle(pick_load_format("train/train_%s"%title))
train = x_train,y_train.astype("float32")
 if lang in lang_valid:
valid = balance_shuffle(pick_load_format("valid/valid_%s"%title),balance=False)
 else:
thres = round(len(train[-1])*frac)
 valid = tuple(arr[:thres] for arr in train)
 train = tuple(arr[thres:] for arr in train)
 x_test,id_test = pick_load_format("test/test_%s"%title)
 quesdo = balance_shuffle(( x_test,trans[id_test]),balance=False)
 return train,valid,quesdo,x_test,id_test
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()<feature_engineering>
train,valid,quesdo,test,ids = load_data(MODEL,LANG)
 train_size,valid_size,quesdo_size = len(train[1]),len(valid[1]),len(quesdo[1])
 print(train_size,valid_size,quesdo_size )
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()<merge>
def make_dataset_pipeline(dataset, cache=False,repeat_and_shuffle=False,shuffle_size=128_000,seed=386491):
if cache: dataset = dataset.cache()
if repeat_and_shuffle:
dataset = dataset.repeat().shuffle(shuffle_size,seed)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def build_datasets(train,valid,quesdo,test):
dtrain = Dataset.from_tensor_slices(train)
 dvalid = Dataset.from_tensor_slices(valid)
 dquesdo = Dataset.from_tensor_slices(quesdo)
 dtest = Dataset.from_tensor_slices(test)
 train_dataset = make_dataset_pipeline(dtrain,True, repeat_and_shuffle=True)
 valid_dataset = make_dataset_pipeline(dvalid, True,repeat_and_shuffle=True)
 quesdo_dataset = make_dataset_pipeline(dquesdo, True,repeat_and_shuffle=True)
 validset = make_dataset_pipeline(dvalid)
 testset = make_dataset_pipeline(dtest)
 return train_dataset,valid_dataset,quesdo_dataset,validset,testset
def mix_dataset(dss,szs,weight=None,seed=1214):
if weight is None: weight = np.ones(len(szs))
prop = np.array(szs)*weight
return sample_from_datasets(dss,prop/np.sum(prop),seed),sum(szs )
prd = prd.merge(avg_pos, on='product_id', how='left')
 del avg_pos
gc.collect()
prd.head()<prepare_output>
train_dataset,valid_dataset,quesdo_dataset,validset,testset = build_datasets(train,valid,quesdo,test )
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()<groupby>
def get_cls(x):
return x[:, 0, :]
dic = {"mean":GlobalAveragePooling1D() ,
"max":GlobalMaxPool1D() ,
"cls":get_cls}
def build_model(transformer,head="cls" , loss='binary_crossentropy',
max_len=512, drop_rate=None, lr=1e-5,seed=940208):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
x = dic[head](sequence_output)
 if drop_rate is not None:
x = Dropout(drop_rate )(x)
 out = Dense(1, activation='sigmoid',kernel_initializer=GlorotUniform(seed))(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=lr), loss=loss, metrics=[AUC() ])
 return model
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()<prepare_output>
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=1e-5)
 model.summary()
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()<merge>
train_dataset,train_size = mix_dataset([train_dataset,quesdo_dataset],[train_size,quesdo_size],weight=None )
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head(20 )<feature_engineering>
%%time
N_STEPS = train_size //(BATCH_SIZE*2)
 EPOCHS = 8
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=validset,
callbacks=callback_list,
epochs=EPOCHS
)
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head(30 )<merge>
del model
gc.collect()
tf.tpu.experimental.initialize_tpu_system(tpu )
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=5e-6)
 model.load_weights(model_path )
uxp_ratio['uxp_order_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
%%time
n_steps = valid_size //(BATCH_SIZE)
 EPOCHS = 1
train_history_2 =model.fit(
 valid_dataset,
steps_per_epoch=n_steps,
epochs= EPOCHS
)
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()<drop_column>
!rm model1.h5
<merge><EOS> 
sub = pd.read_csv(TRANSFER)
 sub.loc[ids,'toxic'] = model.predict(testset, verbose=1 ).reshape(-1)
 sub.to_csv('submission.csv', index=False )