Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<merge>
model_name = 'jplu/tf-xlm-r-ner-40-lang'
tokenizer_name = "novinsh/xlm-roberta-large-toxicomments-12k"
data = uxp.merge(user, on='user_id', how='left')
 data.head()<merge>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer
data = data.merge(prd, on='product_id', how='left')
 data.head()<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
del op, user, prd, uxp
gc.collect()<filter>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 if model_name == 't5-large':
EPOCHs = 1
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
else:
EPOCHS = 3
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
MAX_LEN = 128
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
<merge>
tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer_name)
 if model_name == 't5-large':
tokenizer.pad_token = tokenizer.eos_token
tokenizer.save_pretrained('.')
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )<filter>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
data_train = data[data.eval_set=='train']
data_train.head()<merge>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ])
 print(train.toxic.value_counts())
 print(valid.toxic.value_counts() )
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )<feature_engineering>
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid.head()
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )<drop_column>
olid_test = pd.read_csv('/kaggle/input/olid2019/testset-levela.tsv', sep='\t')
 olid_test_label = pd.read_csv('/kaggle/input/olid2019/labels-levela.csv', names = ['id', 'label'] )
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )<drop_column>
olid_test_label.label.value_counts()
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )<filter>
olid_test = olid_test.set_index('id' ).join(olid_test_label.set_index('id'))
data_test = data[data.eval_set=='test']
data_test.head()<drop_column>
olid_test['toxic'] =(olid_test.label == 'OFF' ).astype(int ).values
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()<drop_column>
olid_train, olid_valid = train_test_split(olid, test_size=3240, random_state = 2020)
 olid_train_1k = olid_train[0:1000]
olid_train_2k = olid_train[0:2000]
olid_train_5k = olid_train[0:5000]
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()<train_model>
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )
X_train, y_train = data_train.drop('reordered', axis=1), data_train.reordered
rfc = RandomForestClassifier(n_estimators=10, n_jobs=-1 ,random_state=42)
 model = rfc.fit(X_train, y_train )<predict_on_test>
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
prediction = model.predict(data_test)
 prediction =(rfc.predict_proba(data_test)[:,1] >= 0.3 ).astype(int )<prepare_output>
olid_train_encode = fast_encode_xlm(olid_train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_1k_encode = fast_encode_xlm(olid_train_1k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_2k_encode = fast_encode_xlm(olid_train_2k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_5k_encode = fast_encode_xlm(olid_train_5k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_valid_encode = fast_encode_xlm(olid_valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_test_encode = fast_encode_xlm(olid_test.tweet.astype(str), tokenizer, maxlen = MAX_LEN)
data_test['prediction'] = prediction
data_test.tail(50 )<filter>
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_valid = olid_valid.toxic.values
y_olid_test =(olid_test_label.label == 'OFF' ).astype(int ).values
orders_test = orders.loc[orders.eval_set=='test',("user_id", "order_id")]
orders_test.head()<merge>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
final = final.merge(orders_test, on='user_id', how='left')
 final.head()<drop_column>
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_valid_encode, y_olid_valid)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
final = final.drop('user_id', axis=1)
<data_type_conversions>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()<feature_engineering>
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead.from_pretrained(model_name)
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d<prepare_output>
n_steps = x_train.shape[0] // BATCH_SIZE
model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()<save_to_csv>
n_steps = x_valid.shape[0] // BATCH_SIZE
model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
sub.to_csv('sub.csv', index=False )<load_from_csv>
sub['toxic'] = model.predict(test_dataset, verbose=0)
 sub.to_csv('submission.csv', index=False )
print('loading prior')
 priors = pd.read_csv(IDIR + 'order_products__prior.csv', dtype={
'order_id': np.int32,
'product_id': np.uint16,
'add_to_cart_order': np.int16,
'reordered': np.int8})
 print('loading train')
 train = pd.read_csv(IDIR + 'order_products__train.csv', dtype={
'order_id': np.int32,
'product_id': np.uint16,
'add_to_cart_order': np.int16,
'reordered': np.int8})
 print('loading orders')
 orders = pd.read_csv(IDIR + 'orders.csv', dtype={
'order_id': np.int32,
'user_id': np.int32,
'eval_set': 'category',
'order_number': np.int16,
'order_dow': np.int8,
'order_hour_of_day': np.int8,
'days_since_prior_order': np.float32})
 print('loading products')
 products = pd.read_csv(IDIR + 'products.csv', dtype={
'product_id': np.uint16,
'order_id': np.int32,
'aisle_id': np.uint8,
'department_id': np.uint8},
usecols=['product_id', 'aisle_id', 'department_id'])
 print('priors {}: {}'.format(priors.shape, ', '.join(priors.columns)))
 print('orders {}: {}'.format(orders.shape, ', '.join(orders.columns)))
 print('train {}: {}'.format(train.shape, ', '.join(train.columns)) )<data_type_conversions>
from sklearn.metrics import roc_auc_score
print('computing product f')
 prods = pd.DataFrame()
prods['orders'] = priors.groupby(priors.product_id ).size().astype(np.int32)
 prods['reorders'] = priors['reordered'].groupby(priors.product_id ).sum().astype(np.float32)
 prods['reorder_rate'] =(prods.reorders / prods.orders ).astype(np.float32)
 products = products.join(prods, on='product_id')
 products.set_index('product_id', drop=False, inplace=True)
 del prods
print('add order info to priors')
 orders.set_index('order_id', inplace=True, drop=False)
 priors = priors.join(orders, on='order_id', rsuffix='_')
 priors.drop('order_id_', inplace=True, axis=1)
<data_type_conversions>
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )
print('computing user f')
 usr = pd.DataFrame()
usr['average_days_between_orders'] = orders.groupby('user_id')['days_since_prior_order'].mean().astype(np.float32)
 usr['nb_orders'] = orders.groupby('user_id' ).size().astype(np.int16)
 users = pd.DataFrame()
users['total_items'] = priors.groupby('user_id' ).size().astype(np.int16)
 users['all_products'] = priors.groupby('user_id')['product_id'].apply(set)
 users['total_distinct_items'] =(users.all_products.map(len)).astype(np.int16)
 users = users.join(usr)
 del usr
users['average_basket'] =(users.total_items / users.nb_orders ).astype(np.float32)
 print('user f', users.shape)
<feature_engineering>
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]
print('compute userXproduct f - this is long...')
 priors['user_product'] = priors.product_id + priors.user_id * 100000 
<create_dataframe>
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
d= dict()
for row in priors.itertuples() :
z = row.user_product
if z not in d:
d[z] =(1, 
(row.order_number, row.order_id),
row.add_to_cart_order)
 else:
d[z] =(d[z][0] + 1,
max(d[z][1],(row.order_number, row.order_id)) ,
d[z][2] + row.add_to_cart_order)
 print('to dataframe(less memory)')
 userXproduct = pd.DataFrame.from_dict(d, orient='index')
 del d
userXproduct.columns = ['nb_orders', 'last_order_id', 'sum_pos_in_cart']
userXproduct.nb_orders = userXproduct.nb_orders.astype(np.int16)
 userXproduct.last_order_id = userXproduct.last_order_id.map(lambda x: x[1] ).astype(np.int32)
 userXproduct.sum_pos_in_cart = userXproduct.sum_pos_in_cart.astype(np.int16)
 print('user X product f', len(userXproduct))
del priors<split>
model.save_weights("/kaggle/working/ckpt.h5")
 model.predict(olid_test_dataset, verbose=0)[0:5]
print('split orders : train, test')
 test_orders = orders[orders.eval_set == 'test']
train_orders = orders[orders.eval_set == 'train']
train.set_index(['order_id', 'product_id'], inplace=True, drop=False )<define_variables>
with strategy.scope() :
model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
def features(selected_orders, labels_given=False):
print('build candidate list')
 order_list = []
product_list = []
labels = []
i=0
for row in selected_orders.itertuples() :
i+=1
if i%10000 == 0: print('order row',i)
 order_id = row.order_id
user_id = row.user_id
user_products = users.all_products[user_id]
product_list += user_products
order_list += [order_id] * len(user_products)
 if labels_given:
labels += [(order_id, product)in train.index for product in user_products]
df = pd.DataFrame({'order_id':order_list, 'product_id':product_list}, dtype=np.int32)
 labels = np.array(labels, dtype=np.int8)
 del order_list
del product_list
print('user related features')
 df['user_id'] = df.order_id.map(orders.user_id)
 df['user_total_orders'] = df.user_id.map(users.nb_orders)
 df['user_total_items'] = df.user_id.map(users.total_items)
 df['total_distinct_items'] = df.user_id.map(users.total_distinct_items)
 df['user_average_days_between_orders'] = df.user_id.map(users.average_days_between_orders)
 df['user_average_basket'] = df.user_id.map(users.average_basket)
 print('order related features')
 df['order_hour_of_day'] = df.order_id.map(orders.order_hour_of_day)
 df['days_since_prior_order'] = df.order_id.map(orders.days_since_prior_order)
 df['days_since_ratio'] = df.days_since_prior_order / df.user_average_days_between_orders
print('product related features')
 df['aisle_id'] = df.product_id.map(products.aisle_id)
 df['department_id'] = df.product_id.map(products.department_id)
 df['product_orders'] = df.product_id.map(products.orders ).astype(np.int32)
 df['product_reorders'] = df.product_id.map(products.reorders)
 df['product_reorder_rate'] = df.product_id.map(products.reorder_rate)
 print('user_X_product related features')
 df['z'] = df.user_id * 100000 + df.product_id
df.drop(['user_id'], axis=1, inplace=True)
 df['UP_orders'] = df.z.map(userXproduct.nb_orders)
 df['UP_orders_ratio'] =(df.UP_orders / df.user_total_orders ).astype(np.float32)
 df['UP_last_order_id'] = df.z.map(userXproduct.last_order_id)
 df['UP_average_pos_in_cart'] =(df.z.map(userXproduct.sum_pos_in_cart)/ df.UP_orders ).astype(np.float32)
 df['UP_reorder_rate'] =(df.UP_orders / df.user_total_orders ).astype(np.float32)
 df['UP_orders_since_last'] = df.user_total_orders - df.UP_last_order_id.map(orders.order_number)
 df['UP_delta_hour_vs_last'] = abs(df.order_hour_of_day - df.UP_last_order_id.map(orders.order_hour_of_day)).map(lambda x: min(x, 24-x)).astype(np.int8)
 df.drop(['UP_last_order_id', 'z'], axis=1, inplace=True)
 print(df.dtypes)
 print(df.memory_usage())
 return(df, labels )<define_variables>
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]
df_train, labels = features(train_orders, labels_given=True)
 f_to_use = ['user_total_orders', 'user_total_items', 'total_distinct_items',
'user_average_days_between_orders', 'user_average_basket',
'order_hour_of_day', 'days_since_prior_order', 'days_since_ratio',
'aisle_id', 'department_id', 'product_orders', 'product_reorders',
'product_reorder_rate', 'UP_orders', 'UP_orders_ratio',
'UP_average_pos_in_cart', 'UP_reorder_rate', 'UP_orders_since_last',
'UP_delta_hour_vs_last']
print('formating for lgb')
 d_train = lgb.Dataset(df_train[f_to_use],
label=labels,
categorical_feature=['aisle_id', 'department_id'])
 del df_train<predict_on_test>
n_steps = olid_train_2k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': {'binary_logloss'},
'num_leaves': 96,
'max_depth': 10,
'feature_fraction': 0.9,
'bagging_fraction': 0.95,
'bagging_freq': 5
}
ROUNDS = 100
print('light GBM train :-)')
 bst = lgb.train(params, d_train, ROUNDS)
 del d_train
df_test, _ = features(test_orders)
 print('light GBM predict')
 preds = bst.predict(df_test[f_to_use])
 df_test['pred'] = preds
TRESHOLD = 0.22
d = dict()
for row in df_test.itertuples() :
if row.pred > TRESHOLD:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in test_orders.order_id:
if order not in d:
d[order] = 'None'<save_to_csv>
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.to_csv('sub.csv', index=False )<load_pretrained>
n_steps = olid_train_5k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)) )
uxp = pd.read_pickle('.. /input/ml-instacart-f1-0-38-part-one-features/uxp.pkl')
 uxp.head()<load_from_csv>
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]
<filter><EOS> 
n_steps = olid_train.shape[0] // BATCH_SIZE
model.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))