Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<load_from_csv>
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer
su=pd.read_csv("submission.csv")
 su<import_modules>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
print(tf.__version__)
 print(tf.keras.__version__ )<install_modules>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
!pip install efficientnet 
<set_options>
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 128
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<train_model>
tokenizer = transformers.AutoTokenizer.from_pretrained("jplu/tf-xlm-roberta-large")
 tokenizer.save_pretrained('.')
img = plt.imread('.. /input/plant-pathology-2020-fgvc7/images/Train_0.jpg')
 print(img.shape)
 plt.imshow(img )<load_from_csv>
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_paths = train.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
train_labels = train.loc[:, 'healthy':].values<define_variables>
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid['comment_text'] = olid['comment_text'].str.replace('@USER', '')
 olid.head()
nb_classes = 4
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 224
EPOCHS = 10<normalization>
olid_train, olid_test = train_test_split(olid, test_size=3240, random_state = 2020)
 olid_train_1k = olid_train[0:1000]
olid_train_2k = olid_train[0:2000]
olid_train_5k = olid_train[0:5000]
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=2020):
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 image = tf.image.random_crop(image, size=[img_size, img_size, 3])
 if label is None:
return image
else:
return image, label<categorify>
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ] )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<categorify>
train.toxic.value_counts()
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )<init_hyperparams>
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )
LR_START = 0.001
LR_MAX = 0.001 * strategy.num_replicas_in_sync
LR_MIN = 0.0001
LR_RAMPUP_EPOCHS = 10
LR_SUSTAIN_EPOCHS = 2
LR_EXP_DECAY =.6
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto',
baseline=None, restore_best_weights=False 
)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
def get_model() :
base_model = efn.EfficientNetB7(weights='imagenet', include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
x = base_model.output
out = Dense(2048, activation="softmax" )(x)
 predictions = Dense(nb_classes, activation="softmax" )(out)
 return Model(inputs=base_model.input, outputs=predictions )<choose_model_class>
olid_train_encode = fast_encode_xlm(olid_train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_1k_encode = fast_encode_xlm(olid_train_1k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_2k_encode = fast_encode_xlm(olid_train_2k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_5k_encode = fast_encode_xlm(olid_train_5k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_test_encode = fast_encode_xlm(olid_test.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
with strategy.scope() :
model = get_model()
model.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy'] )<train_model>
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_test = olid_test.toxic.values
%%time
model.fit(
 train_dataset,
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=[lr_callback, early_stopping],
epochs=40
)<predict_on_test>
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )
%%time
probs = model.predict(test_dataset )<save_to_csv>
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_test_dataset_wlabel =(
 tf.data.Dataset 
.from_tensor_slices(( olid_test_encode, y_olid_test)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()<define_variables>
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
Path.ls = lambda x: list(x.iterdir())
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 effnet_path = '.. /input/efficientnet-pytorch/'
sys.path.append(effnet_path)
 test_path = Path(".. /input/cassava-leaf-disease-classification/test_images")
 test_fnames = test_path.ls()<load_from_csv>
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead 
.from_pretrained('jplu/tf-xlm-roberta-large')
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()
test_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 train_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 num_classes = train_df['label'].nunique()<set_options>
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)
mean = [0.4589, 0.5314, 0.3236]
std = [0.2272, 0.2297, 0.2200]
test_tfms = albumentations.Compose([
albumentations.RandomResizedCrop(256, 256),
albumentations.HorizontalFlip(p=0.5),
albumentations.HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
albumentations.RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
albumentations.Normalize(
 mean=mean,
std=std,
max_pixel_value=255.0,
p=1.0 
),
ToTensorV2() ], p=1.)
class LeafDataTest(Dataset):
def __init__(self, df, split="test"):
self.transforms = test_tfms
self.paths = [test_path/id_ for id_ in df['image_id'].values]
def __getitem__(self, idx):
img = cv2.imread(str(self.paths[idx])) [..., ::-1]
img = self.transforms(image=img)['image']
return img
def __len__(self):
return len(self.paths)
 def make_dataloaders(batch_size=32, num_workers=4, pin_memory=True, **kwargs):
dataset = LeafDataTest(**kwargs)
 dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers,
pin_memory=pin_memory, shuffle=True if kwargs['split'] == "train" else False)
 return dataloader<load_pretrained>
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)
test_dl = make_dataloaders(df=test_df, split="test" )<choose_model_class>
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False)
class EfficientNetModel(nn.Module):
def __init__(self, arch="b4", dropout=0.2, n_out=5,
pretrained=True, freeze=True):
super().__init__()
if pretrained:
self.model = EfficientNet.from_pretrained(f"efficientnet-{arch}")
 if freeze:
for p in self.model.parameters() :
p.requires_grad = False
else:
self.model = EfficientNet.from_name(f"efficientnet-{arch}")
 self.lin1 = nn.Linear(1792 * 2, 512)
 self.lin2 = nn.Linear(512, n_out)
 self.bn1 = nn.BatchNorm1d(1792 * 2)
 self.bn2 = nn.BatchNorm1d(512)
 self.dropout = dropout
def forward(self, x):
x = self.model.extract_features(x)
 avg = F.adaptive_avg_pool2d(x, 1)
 max_ = F.adaptive_max_pool2d(x, 1)
 cat = torch.cat(( avg.view(x.size(0), -1), max_.view(x.size(0), -1)) , dim=1)
 x = self.bn1(cat)
 x = F.dropout(x, self.dropout)
 x = F.relu(self.bn2(self.lin1(x)))
 x = self.lin2(x)
 return x<load_pretrained>
from sklearn.metrics import roc_auc_score
model = model = EfficientNetModel(pretrained=False, freeze=False ).to(device)
 model.load_state_dict(torch.load(".. /input/pytorch-better-normalization-onecycle-lr-train/effnet.pt", map_location=device))
model.eval() ;<predict_on_test>
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=1)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )
def inference_one_pass(model, test_dl):
model.eval()
all_preds = []
with torch.no_grad() :
for batch in tqdm(test_dl):
preds = model(batch.to(device))
all_preds.append(preds)
 return torch.cat(all_preds, dim=0 )<define_variables>
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]
num_passes = 5
tta = None
for _ in range(num_passes):
all_preds = inference_one_pass(model, test_dl)
 if tta is None:
tta = all_preds
else:
tta += all_preds
tta /= float(num_passes)
 label_preds = tta.argmax(dim=1 )<feature_engineering>
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
test_df['label'] = label_preds.cpu().numpy()<save_to_csv>
model_1k = model
model_1k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
train_history_1k = model_1k.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model_1k.predict(olid_test_dataset, verbose=1)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
 del model_1k
test_df.to_csv("submission.csv", index=False )<define_variables>
model_2k = model
model_2k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_2k.shape[0] // BATCH_SIZE
train_history_2k = model_2k.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model_2k.predict(olid_test_dataset, verbose=1)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
 del model_2k
tez_path = '.. /input/tez-lib/'
effnet_path = '.. /input/efficientnet-pytorch/'
sys.path.append(tez_path)
 sys.path.append(effnet_path )<import_modules>
model_5k = model
model_5k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_5k.shape[0] // BATCH_SIZE
train_history_5 = model_5k.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model_5k.predict(olid_test_dataset, verbose=1)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)))
 del model_5k
<feature_engineering><EOS> 
model_10k = model
model_10k.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train.shape[0] // BATCH_SIZE
train_history_10 = model_10k.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_test_dataset_wlabel,
epochs=10 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model_10k.predict(olid_test_dataset, verbose=1)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))
 del model_10k