Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
reorder_u = prior_orders.groupby(by='user_id')['reordered'].aggregate('mean' ).to_frame('u_reorder_ratio' ).reset_index()
reorder_u['u_reorder_ratio'] = reorder_u['u_reorder_ratio'].astype(np.float16)
 reorder_u.head()<merge>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print(tpu_strategy.num_replicas_in_sync )
users = users.merge(avg_products, on='user_id', how='left')
 users = users.merge(dow, on='user_id', how='left')
 users = users.merge(hod, on='user_id', how='left')
 users = users.merge(reorder_u, on='user_id', how='left')
 users.head()<drop_column>
MY_GCS_PATH=KaggleDatasets().get_gcs_path('dataencoding' )
del [reorder_u, dow, hod, avg_products]
gc.collect()<data_type_conversions>
data_path = ".. /input/dataencoding/"
train_ids = np.load(os.path.join(data_path, "ids.npy"))
train_labels = np.load(os.path.join(data_path, "labels.npy")).astype(int)
 val_ids = np.load(os.path.join(data_path, "val_ids.npy"))
val_labels = np.load(os.path.join(data_path, "val_labels.npy"))
prd = prior_orders.groupby(by='product_id')['order_id'].aggregate('count' ).to_frame('p_num_of_times' ).reset_index()
prd.head()<data_type_conversions>
MODEL = "jplu/tf-xlm-roberta-large"
SEQUENCE_LENGTH = 192
BATCH_SIZE = 16 * tpu_strategy.num_replicas_in_sync
reorder_p = prior_orders.groupby(by='product_id')['reordered'].aggregate('mean' ).to_frame('p_reorder_ratio' ).reset_index()
reorder_p.head()<data_type_conversions>
pos = train_ids[np.where(train_labels == 1)[0]]
neg = train_ids[np.where(train_labels == 0)[0]]
pos_labels = train_labels[np.where(train_labels==1)[0]]
neg_labels = train_labels[np.where(train_labels==0)[0]]
add_to_cart = prior_orders.groupby(by='product_id')['add_to_cart_order'].aggregate('mean' ).to_frame('p_avg_cart_position' ).reset_index()
add_to_cart.head()<merge>
def make_ds(features, labels):
ds = tf.data.Dataset.from_tensor_slices(( features, labels))
ds = ds.shuffle(1500000 ).repeat()
return ds
prd = prd.merge(reorder_p, on='product_id', how='left')
 prd = prd.merge(add_to_cart, on='product_id', how='left')
 del [reorder_p, add_to_cart]
gc.collect()<data_type_conversions>
del train_ids
del train_labels
uxp = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('uxp_times_bought')
 uxp = uxp.reset_index()
uxp.head()<data_type_conversions>
pos_ds = make_ds(pos, pos_labels)
 neg_ds = make_ds(neg, neg_labels)
 resampled_ds = tf.data.experimental.sample_from_datasets([pos_ds, neg_ds], weights=[0.5, 0.5])
 resampled_ds = resampled_ds.batch(BATCH_SIZE ).prefetch(AUTO)
times = prior_orders.groupby(by=['user_id', 'product_id'])['order_id'].aggregate('count' ).to_frame('times_bought')
 times = times.reset_index()
times.head()<prepare_output>
del pos
del neg
total_orders = prior_orders.groupby('user_id')['order_number'].max().to_frame('total_orders' ).reset_index()
total_orders.head()<merge>
val_dataset =(tf.data.Dataset.from_tensor_slices(( val_ids, val_labels)) 
.shuffle(len(val_ids)) 
.repeat() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
span = pd.merge(total_orders, first_order_num, on='user_id', how='right')
 span.head()<feature_engineering>
def make_model(embed_model):
input_ids = tf.keras.layers.Input(shape=(SEQUENCE_LENGTH,), name='input_token', dtype='int32')
 embed_layer = embed_model([input_ids])[0]
avg_pool = tf.reduce_mean(embed_layer, axis=1)
 max_pool = tf.reduce_max(embed_layer, axis=1)
 X = tf.concat([avg_pool, max_pool], axis=1)
 X = tf.keras.layers.Dropout(0.3 )(X)
 X = tf.keras.layers.Dense(1, activation="sigmoid" )(X)
 model = tf.keras.Model(inputs=input_ids, outputs = X)
 return model
span['Order_Range_D'] = span.total_orders - span.first_order_num + 1
span.head()<merge>
with tpu_strategy.scope() :
xlm_roberta = TFXLMRobertaModel.from_pretrained(MODEL)
 xr_model = make_model(xlm_roberta)
 xr_model.summary()
xr_model.compile(optimizer=tf.keras.optimizers.Adam(lr=3e-5), loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1), metrics=[tf.keras.metrics.AUC() ] )
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
N_STEPS = 1200000//BATCH_SIZE
N_STEPS
VAL_STEPS = val_ids.shape[0]//BATCH_SIZE
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.times_bought / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
def build_lrfn(lr_start=0.000001, lr_max=0.000002,
lr_min=0.0000001, lr_rampup_epochs=7,
lr_sustain_epochs=0, lr_exp_decay=.87):
lr_max = lr_max * tpu_strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
lrfn = build_lrfn()
es = EarlyStopping(monitor='val_accuracy', mode='max', patience=2,
restore_best_weights=True, verbose=1)
 lr_callback = LearningRateScheduler(lrfn, verbose=1)
 callbacks = [es, lr_callback]
uxp_ratio.drop(['times_bought', 'total_orders', 'first_order_num', 'Order_Range_D'], axis=1, inplace=True)
 uxp_ratio.head()<drop_column>
%%time
history = xr_model.fit(resampled_ds,
validation_data=val_dataset,
epochs=2,
steps_per_epoch=N_STEPS,
validation_steps = VAL_STEPS,
callbacks=callbacks
)
del [times, span, first_order_num, total_orders]
gc.collect()<merge>
val_history = xr_model.fit(val_dataset,
epochs=2,
steps_per_epoch=VAL_STEPS,
callbacks=callbacks
)
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
gc.collect()<data_type_conversions>
xr_model.save_weights("weights.h5" )
<groupby><EOS> 
sub = pd.read_csv(os.path.join('.. /input/jigsaw-multilingual-toxic-comment-classification/','sample_submission.csv'))
test_ids = np.load(".. /input/dataencoding/test_ids.npy", allow_pickle=True)
 sub['toxic'] = xr_model.predict(test_ids, verbose=1)
 sub.to_csv('submission.csv', index=False )