Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<data_type_conversions>
warnings.filterwarnings("ignore")
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )<merge>
def clean_text(text):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 return text.strip()
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()<create_dataframe>
def text_process(text):
ws = text.split(' ')
 if(len(ws)>160):
text = ' '.join(ws[:160])+ ' ' + ' '.join(ws[-32:])
 return text
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 add1 = op.groupby(['user_id','product_id'])['product_id'].count().to_frame('times_p_order')
 add1 = add1.reset_index()
add1_new = add1.groupby('user_id')['times_p_order'].sum().to_frame('total_products')
 add1= pd.merge(add1,add1_new,on=["user_id"],how="left")
 add1["percentage_of_bought_products_in_total"] = add1.times_p_order/add1.total_products
del add1_new
add1 = add1.drop(["times_p_order","total_products"],axis=1)
 add1.head() 
<create_dataframe>
def dict_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_mask=True,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
truncation=True 
)
 return {
"input_ids": np.array(enc_di['input_ids']),
"attention_mask": np.array(enc_di['attention_mask'])
 }
summ_priordays_per_product = op.groupby(['product_id' ])['days_since_prior_order'].sum().to_frame('sum_prior_days_per_product')
 summ_priordays_per_product.head()<create_dataframe>
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )
total_orders_per_prod=op.groupby(['product_id' ])['order_id'].count().to_frame('TotalOrdersPerProd')
 total_orders_per_prod.head()<data_type_conversions>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
test1=summ_priordays_per_product['sum_prior_days_per_product'].astype(str ).astype(float)
 test1.dtypes 
<data_type_conversions>
MODEL = 'jplu/tf-xlm-roberta-large'
AUTO = tf.data.experimental.AUTOTUNE
SEED = 2020
EPOCHS_1 = 20
EPOCHS_2 = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
SHUFFLE = 2048
VERBOSE = 1
test2=total_orders_per_prod['TotalOrdersPerProd'].astype(str ).astype(float)
 test2.dtypes<data_type_conversions>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2 = pd.concat([train2[['comment_text', 'toxic']].query('toxic<=0.4'),
train2[['comment_text', 'toxic']].query('toxic>=0.6')])
 train2.toxic = train2.toxic.round().astype(int)
 train2 = train2.sample(frac = 1, random_state=SEED)
 train3 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv")
 train4 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv")
 train5 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv")
 train6 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv")
 train7 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv")
 train8 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 valid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 valid2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')
 valid2.rename(columns={'comment_text': 'old_comment_text', 'translated': 'comment_text'}, inplace=True)
 test = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )
our_ratio=our_ratio['our_ratio'].astype(str ).astype(float)
 our_ratio.dtypes<merge>
%%time
x_train = dict_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = dict_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test1 = dict_encode(test1.content.values, tokenizer, maxlen=MAX_LEN)
 x_test2 = dict_encode(test2.translated.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
del train, valid
gc.collect()
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()<merge>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(SHUFFLE)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test1_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(BATCH_SIZE)
)
 test2_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(BATCH_SIZE)
)
 del x_train, x_valid, y_train, y_valid, x_test1, x_test2
gc.collect()
items = pd.merge(left =pd.merge(left=products, right=departments, how='left'), right=aisles, how='left')
 items.head()<create_dataframe>
lrs = ReduceLROnPlateau(monitor='val_accuracy', mode ='max', factor = 0.7, min_lr= 1e-7, verbose = 1, patience = 2)
 es1 = EarlyStopping(monitor='val_accuracy', mode='max', verbose = 1, patience = 5, restore_best_weights=True)
 es2 = EarlyStopping(monitor='accuracy', mode='max', verbose = 1, patience = 1, restore_best_weights=True)
 callbacks_list1 = [lrs,es1]
callbacks_list2 = [lrs,es2]
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd= prd.reset_index() 
<merge>
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 attention_mask = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 sequence_output = transformer({"input_ids": input_word_ids, "attention_mask": attention_mask})[0]
avg_pool = GlobalAveragePooling1D()(sequence_output)
 max_pool = GlobalMaxPooling1D()(sequence_output)
 cls_token = Concatenate()([avg_pool, max_pool])
 samples = []
for n in range(2):
sample = Dropout(0.4 )(cls_token)
 sample = Dense(1, activation='sigmoid', name=f'sample_{n}' )(sample)
 samples.append(sample)
 out = Average(name='output' )(samples)
 model = Model(inputs={
"input_ids": input_word_ids,
"attention_mask": attention_mask
},
outputs=out,
name='XLM-R')
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(name='auc'), 'accuracy'])
 return model
items = pd.merge(left =pd.merge(left=products, right=departments, how='left'), right=aisles, how='left')
 items.head()<concatenate>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
order_products_all = pd.concat([order_products_train, order_products_prior], axis=0)
 order_products_all.head()<groupby>
model_history_1 = model.fit(
 train_dataset,
steps_per_epoch=n_train_steps,
validation_data=valid_dataset,
epochs=EPOCHS_1,
callbacks=callbacks_list1,
verbose=VERBOSE
)
grouped = order_products_all.groupby("reordered")["product_id"].aggregate({'Total_products': 'count'} ).reset_index()
grouped['Ratios'] = grouped["Total_products"].apply(lambda x: x /grouped['Total_products'].sum())
 grouped.head()<groupby>
model_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_valid_steps,
epochs=EPOCHS_2,
callbacks=callbacks_list2,
verbose=VERBOSE
)
grouped1 = items.groupby("department")["product_id"].aggregate({'Total_products': 'count'} ).reset_index()
grouped1['RatioDepartments'] = grouped1["Total_products"].apply(lambda x: x /grouped['Total_products'].sum())
 grouped1.head()<groupby>
sub.to_csv('submission.csv', index=False )
grouped2 = items.groupby("aisle")["product_id"].aggregate({'Total_products': 'count'} ).reset_index()
grouped2['RatioAisles'] = grouped2["Total_products"].apply(lambda x: x /grouped['Total_products'].sum())
 grouped2.head()<groupby>
sub.to_csv('submission.csv', index=False )