Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<save_to_csv>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer
test_res = test_leak[leaky_cols+['compiled_leak']].replace(0.0, np.nan)
 test_res.to_csv('test_leak.csv', index=False )<feature_engineering>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
test_leak.loc[test_leak["compiled_leak"]==0, "compiled_leak"] = test_leak.loc[test_leak["compiled_leak"]==0, "nonzero_mean"]<save_to_csv>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128
sub = pd.read_csv(DATA_DIR+"test.csv", usecols=["ID"])
 sub["target"] = 0
sub.iloc[non_ugly_indexes, 1] = test_leak["compiled_leak"].values
sub.to_csv(f"non_fake_sub_lag_{best_lag}.csv", index=False)
 print(f"non_fake_sub_lag_{best_lag}.csv saved" )<set_options>
tokenizer = transformers.AutoTokenizer.from_pretrained("bert-base-cased")
 tokenizer.save_pretrained('.')
%reset -f
warnings.filterwarnings('ignore' )<import_modules>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
import sys
import random
import gc<import_modules>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ])
 print(train.toxic.value_counts())
 print(valid.toxic.value_counts() )
import numpy as np
import pandas as pd<set_options>
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid.head()
%matplotlib inline
matplotlib.rc('figure', figsize=(14, 11))
set_matplotlib_formats('retina' )<set_options>
olid_test = pd.read_csv('/kaggle/input/olid2019/testset-levela.tsv', sep='\t')
 olid_test_label = pd.read_csv('/kaggle/input/olid2019/labels-levela.csv', names = ['id', 'label'] )
init_notebook_mode(connected=True )<import_modules>
olid_test_label.label.value_counts()
import scipy
from scipy import stats
from scipy.special import boxcox1p
from scipy.special import inv_boxcox1p<import_modules>
olid_test = olid_test.set_index('id' ).join(olid_test_label.set_index('id'))
import sklearn
import sklearn.decomposition
import sklearn.model_selection<import_modules>
olid_test['toxic'] =(olid_test.label == 'OFF' ).astype(int ).values
import lightgbm as lgb
from catboost import CatBoostRegressor, Pool<set_options>
olid_train, olid_valid = train_test_split(olid, test_size=3240, random_state = 2020)
 olid_train_1k = olid_train[0:1000]
olid_train_2k = olid_train[0:2000]
olid_train_5k = olid_train[0:5000]
random.seed(321)
 np.random.seed(321)
 np.set_printoptions(suppress=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 9999<load_from_csv>
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )
def getDatasetInformation(csv_filepath):

 dataset_tmp = pd.read_csv(csv_filepath)
 dataset_tmp_shape = pd.DataFrame(list(dataset_tmp.shape), index=['No of Rows', 'No of Columns'], columns=['Total'])
 dataset_tmp_shape = dataset_tmp_shape.reset_index()
dataset_tmp_dtypes = dataset_tmp.dtypes.reset_index()
dataset_tmp_dtypes.columns = ['Column Names', 'Column Data Types']
dataset_tmp_desc = pd.DataFrame(dataset_tmp.describe())
 dataset_tmp_desc = dataset_tmp_desc.transpose()
dataset_tmp_unique = dataset_tmp.nunique().reset_index()
dataset_tmp_unique.columns = ["Column Name", "Unique Value(s)Count"]
dataset_tmp_missing = dataset_tmp.isnull().sum(axis=0 ).reset_index()
dataset_tmp_missing.columns = ['Column Names', 'NULL value count per Column']
dataset_tmp_missing = dataset_tmp_missing.sort_values(by='NULL value count per Column', ascending=False)
 dataset_tmp_corr = pd.DataFrame()
return [dataset_tmp, dataset_tmp_shape, dataset_tmp_dtypes, dataset_tmp_desc, dataset_tmp_unique, dataset_tmp_missing, dataset_tmp_corr]<load_from_csv>
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
dataset_santandervp_train, df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr = getDatasetInformation('.. /input/train.csv' )<drop_column>
olid_train_encode = fast_encode_xlm(olid_train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_1k_encode = fast_encode_xlm(olid_train_1k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_2k_encode = fast_encode_xlm(olid_train_2k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_5k_encode = fast_encode_xlm(olid_train_5k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_valid_encode = fast_encode_xlm(olid_valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_test_encode = fast_encode_xlm(olid_test.tweet.astype(str), tokenizer, maxlen = MAX_LEN)
del(df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr )<define_variables>
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_valid = olid_valid.toxic.values
y_olid_test =(olid_test_label.label == 'OFF' ).astype(int ).values
dataset_santandervp_train_target = dataset_santandervp_train['target']<create_dataframe>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
getContinuousVariableDistributionGraph(tuple(range(dataset_santandervp_train.shape[0])) , dataset_santandervp_train_target, 'Target - Distribution' )<create_dataframe>
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_valid_encode, y_olid_valid)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
getContinuousVariableDistributionGraph(tuple(range(dataset_santandervp_train.shape[0])) , np.log1p(dataset_santandervp_train_target), 'Target - log1p - Distribution' )<feature_engineering>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
dataset_santandervp_train_target = \
np.log1p(dataset_santandervp_train_target )<drop_column>
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead.from_pretrained('bert-base-cased')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
dataset_santandervp_train.drop(['ID', 'target'], axis=1, inplace=True )<count_unique_values>
for layer in model.layers[:-1]:
layer.trainable = False
santandervp_columns_one_unique_value = \
dataset_santandervp_train.columns[dataset_santandervp_train.nunique()
== 1].tolist()
dataset_santandervp_train.drop(santandervp_columns_one_unique_value,
axis=1, inplace=True)
 dataset_santandervp_train = dataset_santandervp_train.replace([0, 1563411.76],
np.nan )<feature_engineering>
n_steps = x_train.shape[0] // BATCH_SIZE
model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
dataset_santandervp_train = np.log1p(dataset_santandervp_train)
 dataset_santandervp_train.head()<feature_engineering>
n_steps = x_valid.shape[0] // BATCH_SIZE
model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
dataset_santandervp_train_tmp = pd.DataFrame()
dataset_santandervp_train_tmp['svp_quantile'] = \
dataset_santandervp_train[dataset_santandervp_train
> 0].quantile(0.5, axis=1)
 dataset_santandervp_train_tmp['svp_max'] = \
dataset_santandervp_train.max(axis=1)
 dataset_santandervp_train_tmp['svp_sum_nonzeroval'] = \ 
(dataset_santandervp_train != 0 ).sum(axis=1)
 dataset_santandervp_train_tmp['svp_skew'] = \
dataset_santandervp_train.skew(axis=1)
 dataset_santandervp_train_tmp['svp_kurtosis'] = \
dataset_santandervp_train.kurtosis(axis=1)
 dataset_santandervp_train_tmp['svp_sum_all'] = \
dataset_santandervp_train.sum(axis=1)
 dataset_santandervp_train_tmp['svp_variance'] = \
dataset_santandervp_train.var(axis=1)
 dataset_santandervp_train_tmp.head()<concatenate>
sub['toxic'] = model.predict(test_dataset, verbose=0)
 sub.to_csv('submission.csv', index=False )
dataset_santandervp_train = pd.concat([dataset_santandervp_train,
dataset_santandervp_train_tmp], axis=1)
 dataset_santandervp_train.head(15 )<split>
from sklearn.metrics import roc_auc_score
( X_train, X_test, y_train, y_test)= \
sklearn.model_selection.train_test_split(dataset_santandervp_train,
dataset_santandervp_train_target, test_size=0.20 )<train_model>
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )
catboost_params = {
'iterations': 10000,
'learning_rate': 0.05,
'depth': 11,
'l2_leaf_reg': 40,
'bootstrap_type': 'Bernoulli',
'eval_metric': 'RMSE',
'metric_period': 100,
'od_type': 'Iter',
'od_wait': 80,
'allow_writing_files': False,
'task_type': 'CPU',
'thread_count': 4,
}
catboost_model = CatBoostRegressor(**catboost_params)
 catboost_model.fit(X_train, y_train, eval_set=(X_test, y_test), cat_features=[], use_best_model=True )<create_dataframe>
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]
dataset_transformed_train_lgb = lgb.Dataset(data=X_train, label=y_train)
 dataset_transformed_test_lgb = lgb.Dataset(data=X_test, label=y_test)
 watchlist = [dataset_transformed_train_lgb,
dataset_transformed_test_lgb]
evaluation_results = {}
lightgbm_params = {
'objective': 'regression_l2',
'metric': 'rmse',
'boosting': 'gbdt',
'num_leaves': 144,
'learning_rate': 0.004,
'bagging_fraction': 0.4,
'feature_fraction': 0.6,
'bagging_freq': 4,
'max_depth': 4,
'reg_alpha': 1,
'reg_lambda': 0.1,
'min_child_weight': 10,
'zero_as_missing': True,
'is_training_metric': True,
'verbosity': -1,
'device': 'cpu',
'nthread': 4,
}
lightgbm_model = lgb.train(
 lightgbm_params,
train_set=dataset_transformed_train_lgb,
num_boost_round=10000,
valid_sets=watchlist,
early_stopping_rounds=100,
evals_result=evaluation_results,
verbose_eval=100,
)<features_selection>
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
max_num_features=37
importance = lightgbm_model.feature_importance()
feature_name = lightgbm_model.feature_name()
tuples = sorted(zip(feature_name, importance), key=lambda x: x[1])
 tuples = tuples[-max_num_features:]
labels, values = zip(*tuples)
(X_train, X_test)=(X_train.loc[ : , labels], X_test.loc[ : , labels] )<create_dataframe>
model.save_weights("/kaggle/working/ckpt.h5")
 model.predict(olid_test_dataset, verbose=0)[0:5]
dataset_transformed_train_lgb = lgb.Dataset(data=X_train, label=y_train)
 dataset_transformed_test_lgb = lgb.Dataset(data=X_test, label=y_test)
 watchlist = [dataset_transformed_train_lgb,
dataset_transformed_test_lgb]
evaluation_results = {}
lightgbm_params = {
'objective': 'regression_l2',
'metric': 'rmse',
'boosting': 'gbdt',
'num_leaves': 144,
'learning_rate': 0.004,
'bagging_fraction': 0.4,
'feature_fraction': 0.6,
'bagging_freq': 4,
'max_depth': 4,
'reg_alpha': 1,
'reg_lambda': 0.1,
'min_child_weight': 10,
'zero_as_missing': True,
'is_training_metric': True,
'verbosity': -1,
'device': 'cpu',
'nthread': 4,
}
lightgbm_model = lgb.train(
 lightgbm_params,
train_set=dataset_transformed_train_lgb,
num_boost_round=10000,
valid_sets=watchlist,
early_stopping_rounds=100,
evals_result=evaluation_results,
verbose_eval=100,
)<drop_column>
with strategy.scope() :
model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
del(dataset_santandervp_train, dataset_santandervp_train_target, dataset_santandervp_train_tmp, dataset_transformed_train_lgb, dataset_transformed_test_lgb, watchlist, X_train, X_test, y_train, y_test)
 gc.collect()<load_from_csv>
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]
dataset_santandervp_test, df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr = getDatasetInformation('.. /input/test.csv')
 dataset_santandervp_test.head()<drop_column>
n_steps = olid_train_2k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
del(df_shape, df_dtypes, df_describe, df_unique, df_missing, df_corr )<drop_column>
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]
dataset_santandervp_test_ID = dataset_santandervp_test['ID']
dataset_santandervp_test.drop(['ID'], axis=1, inplace=True)
 dataset_santandervp_test.drop(santandervp_columns_one_unique_value,
axis=1, inplace=True)
 dataset_santandervp_test = dataset_santandervp_test.replace([0, 1563411.76],
np.nan )<feature_engineering>
n_steps = olid_train_5k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)) )
dataset_santandervp_test = np.log1p(dataset_santandervp_test)
 dataset_santandervp_test.head()<feature_engineering>
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]
<concatenate><EOS> 
n_steps = olid_train.shape[0] // BATCH_SIZE
model.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))