Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<set_options>
import codecs
import copy
import csv
import gc
import os
import pickle
import random
import time
from typing import Dict, List, Sequence, Set, Tuple
test_aug = albumentations.Compose([
albumentations.RandomResizedCrop(256, 256),
albumentations.Transpose(p=0.5),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HueSaturationValue(
 hue_shift_limit=0.2,
sat_shift_limit=0.2,
val_shift_limit=0.2,
p=0.5 
),
albumentations.RandomBrightnessContrast(
 brightness_limit=(-0.1,0.1),
contrast_limit=(-0.1, 0.1),
p=0.5 
),
albumentations.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
max_pixel_value=255.0,
p=1.0 
)
 ], p=1.)<load_from_csv>
import matplotlib.pyplot as plt
import numpy as np
from sklearn.manifold import TSNE
from sklearn.metrics import roc_auc_score, roc_curve
import tensorflow as tf
from tensorflow.python.framework import ops, tensor_util
from tensorflow.python.keras.utils import losses_utils, tf_utils
from tensorflow.python.ops import math_ops
from tensorflow.python.ops.losses import util as tf_losses_util
import tensorflow_addons as tfa
from transformers import AutoTokenizer, XLMRobertaTokenizer
from transformers import TFXLMRobertaModel, XLMRobertaConfig
dfx = pd.read_csv(".. /input/cassava-leaf-disease-classification/sample_submission.csv")
 image_path = ".. /input/cassava-leaf-disease-classification/test_images/"
test_image_paths = [os.path.join(image_path, x)for x in dfx.image_id.values]
test_targets = dfx.label.values
test_dataset = ImageDataset(
 image_paths=test_image_paths,
targets=test_targets,
resize=None,
augmentations=test_aug,
)<load_from_csv>
class LossFunctionWrapper(tf.keras.losses.Loss):
def __init__(self,
fn,
reduction=losses_utils.ReductionV2.AUTO,
name=None,
**kwargs):
super(LossFunctionWrapper, self ).__init__(reduction=reduction, name=name)
 self.fn = fn
self._fn_kwargs = kwargs
def call(self, y_true, y_pred):
if tensor_util.is_tensor(y_pred)and tensor_util.is_tensor(y_true):
y_pred, y_true = tf_losses_util.squeeze_or_expand_dimensions(y_pred, y_true)
 return self.fn(y_true, y_pred, **self._fn_kwargs)
 def get_config(self):
config = {}
for k, v in six.iteritems(self._fn_kwargs):
config[k] = tf.keras.backend.eval(v)if tf_utils.is_tensor_or_variable(v)\
else v
base_config = super(LossFunctionWrapper, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
train_dfx = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 model = LeafModel(num_classes=train_dfx.label.nunique())
 model.load(".. /input/leafmodel/model.bin" )<predict_on_test>
def distance_based_log_loss(y_true, y_pred):
y_pred = ops.convert_to_tensor(y_pred)
 y_true = math_ops.cast(y_true, y_pred.dtype)
 margin = 1.0
p =(1.0 + tf.math.exp(-margin)) /(1.0 + tf.math.exp(y_pred - margin))
return tf.keras.backend.binary_crossentropy(target=y_true, output=p )
final_preds = None
for j in range(5):
preds = model.predict(test_dataset, batch_size=32, n_jobs=-1, device="cuda")
 temp_preds = None
for p in preds:
if temp_preds is None:
temp_preds = p
else:
temp_preds = np.vstack(( temp_preds, p))
if final_preds is None:
final_preds = temp_preds
else:
final_preds += temp_preds
final_preds /= 5<prepare_output>
class DBLLogLoss(LossFunctionWrapper):
def __init__(self, reduction=losses_utils.ReductionV2.AUTO,
name='distance_based_log_loss'):
super(DBLLogLoss, self ).__init__(distance_based_log_loss, name=name,
reduction=reduction )
final_preds = final_preds.argmax(axis=1 )<prepare_output>
def dice_loss(y_true, y_pred):
y_pred = ops.convert_to_tensor(y_pred)
 y_true = math_ops.cast(y_true, y_pred.dtype)
 dice_loss_1 = tf.keras.backend.mean(y_true * y_pred, axis=-1)
 dice_loss_2 = tf.keras.backend.mean(y_true * y_true, axis=-1)
 dice_loss_3 = tf.keras.backend.mean(y_pred * y_pred, axis=-1)
 return 1.0 -(2 * dice_loss_1 + 1.0)/(dice_loss_2 + dice_loss_3 + 1.0 )
dfx.label = final_preds<save_to_csv>
class DiceLoss(LossFunctionWrapper):
def __init__(self, reduction=losses_utils.ReductionV2.AUTO, name='dice_loss'):
super(DiceLoss, self ).__init__(dice_loss, name=name, reduction=reduction )
dfx.to_csv("submission.csv", index=False )<set_options>
class MaskCalculator(tf.keras.layers.Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MaskCalculator, self ).__init__(**kwargs)
 def build(self, input_shape):
super(MaskCalculator, self ).build(input_shape)
 def call(self, inputs, **kwargs):
return tf.keras.backend.permute_dimensions(
 x=tf.keras.backend.repeat(
 x=tf.keras.backend.cast(
 x=tf.keras.backend.greater(
 x=inputs,
y=0 
),
dtype='float32' 
),
n=self.output_dim 
),
pattern=(0, 2, 1)
)
 def compute_output_shape(self, input_shape):
assert len(input_shape)== 1
shape = list(input_shape)
 shape.append(self.output_dim)
 return tuple(shape )
%matplotlib inline
pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None )<load_from_csv>
def regular_encode(texts: List[str], tokenizer: XLMRobertaTokenizer,
maxlen: int)-> Tuple[np.ndarray, np.ndarray]:
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=True,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids']), np.array(enc_di['attention_mask'] )
data_dir = '/kaggle/input/cassava-leaf-disease-classification'
train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))<set_options>
def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
assert len(sentiment_fields)> 0, 'List of sentiment fields is empty!'
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
for cur_field in sentiment_fields:
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(
 cur_field)
 assert cur_field in header, err_msg2
text_field_index = header.index(text_field)
 try:
lang_field_index = header.index(lang_field)
 except:
lang_field_index = -1
indices_of_sentiment_fields = []
for cur_field in sentiment_fields:
indices_of_sentiment_fields.append(header.index(cur_field))
else:
if len(row)== len(header):
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
max_proba = 0.0
for cur_field_idx in indices_of_sentiment_fields:
try:
cur_proba = float(row[cur_field_idx])
 except:
cur_proba = -1.0
err_msg2 = err_msg + ' Value {0} is wrong!'.format(
 row[cur_field_idx] 
)
 assert(cur_proba >= 0.0)and(cur_proba <= 1.0), err_msg2
if cur_proba > max_proba:
max_proba = cur_proba
new_label = 1 if max_proba >= 0.5 else 0
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, new_label))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True<count_values>
def load_test_set(file_name: str, id_field: str, text_field: str,
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(id_field)
 assert id_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(lang_field)
 assert lang_field in header, err_msg2
id_field_index = header.index(id_field)
 text_field_index = header.index(text_field)
 lang_field_index = header.index(lang_field)
 else:
if len(row)== len(header):
try:
id_value = int(row[id_field_index])
 except:
id_value = -1
err_msg2 = err_msg + ' {0} is wrong ID!'.format(
 row[id_field_index])
 assert id_value >= 0, err_msg2
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, id_value))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang
train['label'].value_counts().sort_index()<categorify>
def build_siamese_dataset(texts: Dict[str, List[Tuple[str, int]]],
dataset_size: int, tokenizer: XLMRobertaTokenizer,
maxlen: int, batch_size: int,
for_training: bool)-> Tuple[tf.data.Dataset, int]:
language_pairs = set()
for language in texts.keys() :
for other_language in texts:
if other_language == language:
language_pairs.add(( language, other_language))
else:
new_pair =(language, other_language)
 new_pair_2 =(other_language, language)
 if(new_pair not in language_pairs)and(new_pair_2 not in language_pairs):
language_pairs.add(new_pair)
 language_pairs = sorted(list(language_pairs))
print('Possible language pairs are: {0}.'.format(language_pairs))
err_msg = '{0} is too small size of the data set!'.format(dataset_size)
 assert dataset_size >=(len(language_pairs)* 10), err_msg
n_samples_for_lang_pair = int(np.ceil(dataset_size / float(len(language_pairs))))
text_pairs_and_labels = []
for left_lang, right_lang in language_pairs:
print('{0}-{1}:'.format(left_lang, right_lang))
left_positive_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] > 0, range(len(texts[left_lang])) 
))
left_negative_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] == 0, range(len(texts[left_lang])) 
))
right_positive_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] > 0, range(len(texts[right_lang])) 
))
right_negative_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] == 0, range(len(texts[right_lang])) 
))
used_pairs = set()
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4):
left_idx = random.choice(left_positive_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1,
1,
1 
)
)
 number_of_samples += 1
print(' number of "1-1" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4,(2 * n_samples_for_lang_pair)// 4):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_negative_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_negative_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1,
0,
0 
)
)
 number_of_samples += 1
print(' number of "0-0" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(( 2 * n_samples_for_lang_pair)// 4, n_samples_for_lang_pair):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
if random.random() >= 0.5:
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
0,
0,
1 
)
)
 else:
text_pairs_and_labels.append(
(
 texts[right_lang][right_idx][0],
texts[left_lang][left_idx][0],
0,
1,
0 
)
)
 number_of_samples += 1
print(' number of "0-1" or "1-0" pairs is {0}.'.format(number_of_samples))
random.shuffle(text_pairs_and_labels)
 n_steps = len(text_pairs_and_labels)// batch_size
print('Samples number of the data set is {0}.'.format(len(text_pairs_and_labels)))
 print('Samples number per each language pair is {0}.'.format(n_samples_for_lang_pair))
tokens_of_left_texts, mask_of_left_texts = regular_encode(
 texts=[cur[0] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 tokens_of_right_texts, mask_of_right_texts = regular_encode(
 texts=[cur[1] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 siamese_labels = np.array([cur[2] for cur in text_pairs_and_labels], dtype=np.int32)
 left_labels = np.array([cur[3] for cur in text_pairs_and_labels], dtype=np.int32)
 right_labels = np.array([cur[4] for cur in text_pairs_and_labels], dtype=np.int32)
 print('Number of positive siamese samples is {0} from {1}.'.format(
 int(sum(siamese_labels)) , siamese_labels.shape[0]))
print('Number of positive left samples is {0} from {1}.'.format(
 int(sum(left_labels)) , left_labels.shape[0]))
print('Number of positive right samples is {0} from {1}.'.format(
 int(sum(right_labels)) , right_labels.shape[0]))
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_left_texts, mask_of_left_texts,
tokens_of_right_texts, mask_of_right_texts 
), 
(
 siamese_labels,
left_labels,
right_labels 
)
)
 ).batch(batch_size)
 if for_training:
dataset = dataset.repeat()
del text_pairs_and_labels
return dataset, n_steps
class CassavaDataset(Dataset):
def __init__(self, data_dir, transform=None, phase='train', df=None):
self.df = df
self.data_dir = data_dir
self.transform = transform
self.phase = phase
if self.phase == 'test':
img_dir = 'test_images'
else:
img_dir = 'train_images'
self.img_path = glob.glob(os.path.join(self.data_dir, img_dir, '*.jpg'))
def __len__(self):
if self.df is None:
return len(self.img_path)
 else:
return len(self.df)
 def __getitem__(self, idx):
if self.phase == 'test':
target_img_path = self.img_path[idx]
else:
row = self.df.iloc[idx]
target_img_id = row['image_id']
target_img_path = os.path.join(self.data_dir, 'train_images', f'{target_img_id}')
 img = cv2.imread(target_img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img_id = os.path.basename(target_img_path)
 if self.transform is not None:
img = self.transform(img, self.phase)
 else:
img = torch.from_numpy(img.transpose(( 2, 0, 1)))
 img = img / 255.
 if self.phase == 'test':
return img, img_id
else:
label = self.df[self.df['image_id'] == img_id]['label'].values
label = torch.tensor(label, dtype=torch.long)
 return img, label<set_options>
def build_feature_extractor(transformer_name: str, hidden_state_size: int,
max_len: int)-> Tuple[tf.keras.Model, tf.keras.Model]:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_word_ids")
 attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_attention_mask")
 transformer_layer = TFXLMRobertaModel.from_pretrained(
 pretrained_model_name_or_path=transformer_name,
name='Transformer' 
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
output_mask = MaskCalculator(
 output_dim=hidden_state_size, trainable=False,
name='OutMaskCalculator'
)(attention_mask)
 masked_sequence_output = tf.keras.layers.Multiply(
 name='OutMaskMultiplicator'
)([output_mask, sequence_output])
 masked_sequence_output = tf.keras.layers.Masking(
 name='OutMasking'
)(masked_sequence_output)
 pooled_output = tf.keras.layers.GlobalAvgPool1D(name='AvePool' )(masked_sequence_output)
 text_embedding = tf.keras.layers.Lambda(
 lambda x: tf.math.l2_normalize(x, axis=1),
name='Emdedding'
)(pooled_output)
 cls_layer = tf.keras.layers.Dropout(rate=0.3, name='ClsDropout' )(pooled_output)
 cls_layer = tf.keras.layers.Dense(
 units=1, activation='sigmoid', use_bias=True,
kernel_initializer=tf.keras.initializers.GlorotNormal(seed=42),
kernel_regularizer=tf.keras.regularizers.l2(l=1e-4),
name='ClsOutput'
)(cls_layer)
 cls_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=cls_layer,
name='Classifier' 
)
 cls_model.build(input_shape=[(None, max_len),(None, max_len)])
 fe_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=text_embedding,
name='FeatureExtractor' 
)
 fe_model.build(input_shape=[(None, max_len),(None, max_len)])
 return cls_model, fe_model
class ImageTransform:
def __init__(self, img_size=224, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) :
self.transform = {
'train': albu.Compose([
albu.RandomShadow(p=0.5),
albu.RandomResizedCrop(img_size, img_size, interpolation=cv2.INTER_AREA),
albu.ColorJitter(p=0.5),
albu.CLAHE(p=0.5),
albu.HorizontalFlip(p=0.5),
albu.VerticalFlip(p=0.5),
albu.Transpose(p=0.5),
albu.ShiftScaleRotate(p=0.5),
albu.OneOf([
albu.Blur(p=1.0),
albu.GaussianBlur(p=1.0)
 ], p=0.5),
albu.CoarseDropout(max_height=15, max_width=15, min_holes=3, p=0.5),
albu.Normalize(mean, std),
ToTensorV2() ,
], p=1.0),
'val': albu.Compose([
albu.Resize(img_size, img_size),
albu.Normalize(mean, std),
ToTensorV2() ,
], p=1.0),
'test': albu.Compose([
albu.Resize(img_size, img_size),
albu.HorizontalFlip(p=0.5),
albu.VerticalFlip(p=0.5),
albu.Normalize(mean, std),
ToTensorV2() ,
], p=1.0)
 }
def __call__(self, img, phase='train'):
augmented = self.transform[phase](image=img)
 augmented = augmented['image']
return augmented<categorify>
def euclidean_distance(vects):
x, y = vects
sum_square = tf.keras.backend.sum(tf.keras.backend.square(x - y),
axis=1, keepdims=True)
 return tf.keras.backend.sqrt(
 tf.keras.backend.maximum(sum_square, tf.keras.backend.epsilon())
 )
transform = ImageTransform()
dataset = CassavaDataset(data_dir, transform, phase='train', df=train)
 img, label = dataset.__getitem__(0)
 print(img.size() , label)
 print(img.max())
 print(img.min() )<define_variables>
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return(shape1[0], 1 )
dataloader = DataLoader(dataset, batch_size=8)
 imgs, labels = next(iter(dataloader))
print(imgs.size() )<categorify>
def build_siamese_nn(transformer_name: str, hidden_state_size: int, max_len: int,
max_lr: float, base_lr: float, steps_per_epoch: int)-> \
Tuple[tf.keras.Model, tf.keras.Model, tf.keras.Model]:
left_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_word_ids")
 left_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_attention_mask")
 right_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_word_ids")
 right_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_attention_mask")
 cls_, fe_ = build_feature_extractor(transformer_name, hidden_state_size, max_len)
 left_text_embedding = fe_([left_word_ids, left_attention_mask])
 right_text_embedding = fe_([right_word_ids, right_attention_mask])
 distance_layer = tf.keras.layers.Lambda(
 function=euclidean_distance,
output_shape=eucl_dist_output_shape,
name='L2DistLayer'
)([left_text_embedding, right_text_embedding])
 left_cls_layer = cls_([left_word_ids, left_attention_mask])
 right_cls_layer = cls_([right_word_ids, right_attention_mask])
 nn = tf.keras.Model(
 inputs=[left_word_ids, left_attention_mask, right_word_ids, right_attention_mask],
outputs=[distance_layer, left_cls_layer, right_cls_layer],
name='SiameseXLMR' 
)
 nn.compile(
 optimizer=tf.keras.optimizers.Adam(
 learning_rate=tfa.optimizers.Triangular2CyclicalLearningRate(
 initial_learning_rate=base_lr,
maximal_learning_rate=max_lr,
step_size=int(round(steps_per_epoch * 0.75)) 
)
),
loss=[
DBLLogLoss() ,
tf.keras.losses.BinaryCrossentropy() ,
tf.keras.losses.BinaryCrossentropy()
],
loss_weights=[
1.0,
0.3,
0.3
] 
)
 fe_.summary()
nn.summary()
return nn, cls_, fe_
transform = ImageTransform()
dataset = CassavaDataset(data_dir, transform, phase='test', df=None)
 img, label = dataset.__getitem__(0)
 print(img.size() , label)
 print(img.max())
 print(img.min() )<load_pretrained>
def build_classifier(config: XLMRobertaConfig, hidden_state_size: int, max_len: int,
max_lr: float, base_lr: float, steps_per_epoch: int,
language: str)-> tf.keras.Model:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="word_ids_{0}".format(language))
attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="attention_mask_{0}".format(language))
transformer_layer = TFXLMRobertaModel(
 config=config,
name='Transformer_{0}'.format(language.title())
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
output_mask = MaskCalculator(
 output_dim=hidden_state_size, trainable=False,
name='OutMaskCalculator_{0}'.format(language)
 )(attention_mask)
 masked_sequence_output = tf.keras.layers.Multiply(
 name='OutMaskMultiplicator_{0}'.format(language)
 )([output_mask, sequence_output])
 masked_sequence_output = tf.keras.layers.Masking(
 name='OutMasking_{0}'.format(language)
 )(masked_sequence_output)
 pooled_output = tf.keras.layers.GlobalAvgPool1D(
 name='AvePool_{0}'.format(language)
 )(masked_sequence_output)
 cls_layer = tf.keras.layers.Dropout(rate=0.3, name='ClsDropout' )(pooled_output)
 cls_layer = tf.keras.layers.Dense(
 units=1, activation='sigmoid', use_bias=True,
kernel_initializer=tf.keras.initializers.GlorotNormal(seed=42),
kernel_regularizer=tf.keras.regularizers.l2(l=1e-4),
name='ClsOutput_{0}'.format(language.title())
 )(cls_layer)
 cls_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=cls_layer,
name='Classifier_{0}'.format(language.title())
)
 cls_model.compile(
 optimizer=tf.keras.optimizers.Adam(
 learning_rate=tfa.optimizers.Triangular2CyclicalLearningRate(
 initial_learning_rate=base_lr,
maximal_learning_rate=max_lr,
step_size=int(round(steps_per_epoch * 2.0)) 
)
),
loss=DiceLoss() ,
metrics=[tf.keras.metrics.AUC() ] 
)
 cls_model.summary()
return cls_model
dataloader = DataLoader(dataset, batch_size=1)
 imgs, labels = next(iter(dataloader))
print(imgs.size() )<load_from_csv>
def train_siamese_nn(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,
validset: tf.data.Dataset, steps_per_validset: int,
max_duration: int, model_weights_path: str):
assert steps_per_validset >= 2
assert steps_per_trainset >= 10 * steps_per_validset
steps_per_epoch = 10 * steps_per_validset
n_epochs =(3 * steps_per_trainset)// steps_per_epoch
print('Maximal duration of the Siamese NN training is {0} seconds.'.format(max_duration))
print('n_epochs = {0}, steps_per_epoch = {1}'.format(n_epochs, steps_per_epoch))
print('steps_per_trainset = {0}, steps_per_validset = {1}'.format(
 steps_per_trainset, steps_per_validset 
))
callbacks = [
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True),
tf.keras.callbacks.ModelCheckpoint(model_weights_path, monitor='val_loss',
mode='min', save_best_only=True,
save_weights_only=True, verbose=True)
 ]
history = nn.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs, callbacks=callbacks 
)
 show_training_process(history, 'loss' )
class CassavaDataModule(pl.LightningDataModule):
def __init__(self, data_dir, cfg, transform, cv, fold):
super(CassavaDataModule, self ).__init__()
self.data_dir = data_dir
self.cfg = cfg
self.transform = transform
self.cv = cv
self.fold = fold
def prepare_data(self):
self.df = pd.read_csv(os.path.join(self.data_dir, 'train.csv'))
def setup(self, stage=None):
self.df['fold'] = -1
for i,(trn_idx, val_idx)in enumerate(self.cv.split(self.df, self.df['label'])) :
self.df.loc[val_idx, 'fold'] = i
train = self.df[self.df['fold'] != self.fold].reset_index(drop=True)
 val = self.df[self.df['fold'] == self.fold].reset_index(drop=True)
 self.train_dataset = CassavaDataset(self.data_dir, self.transform, phase='train', df=train)
 self.val_dataset = CassavaDataset(self.data_dir, self.transform, phase='val', df=val)
 self.test_dataset = CassavaDataset(self.data_dir, self.transform, phase='test', df=None)
 def train_dataloader(self):
return DataLoader(self.train_dataset,
batch_size=self.cfg.train['batch_size'],
pin_memory=True,
num_workers=4,
shuffle=True)
 def val_dataloader(self):
return DataLoader(self.val_dataset,
batch_size=self.cfg.train['batch_size'],
pin_memory=True,
num_workers=4,
shuffle=False)
 def test_dataloader(self):
batch_size = min(len(self.test_dataset), self.cfg.train['batch_size'])
 return DataLoader(self.test_dataset,
batch_size=batch_size,
pin_memory=True,
num_workers=4,
shuffle=False )<choose_model_class>
def calculate_features_of_texts(texts: Dict[str, List[Tuple[str, int]]],
tokenizer: XLMRobertaTokenizer, maxlen: int,
fe: tf.keras.Model, batch_size: int,
max_dataset_size: int = 0)-> \
Dict[str, Tuple[np.ndarray, np.ndarray]]:
languages = sorted(list(texts.keys()))
datasets_by_languages = dict()
if max_dataset_size > 0:
max_size_per_lang = max_dataset_size // len(languages)
 err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)
 assert max_size_per_lang > 0, err_msg
else:
max_size_per_lang = 0
for cur_lang in languages:
selected_indices = list(range(len(texts[cur_lang])))
 if max_size_per_lang > 0:
if len(selected_indices)> max_size_per_lang:
selected_indices = random.sample(
 population=selected_indices,
k=max_size_per_lang 
)
 tokens_of_texts, mask_of_texts = regular_encode(
 texts=[texts[cur_lang][idx][0] for idx in selected_indices],
tokenizer=tokenizer, maxlen=maxlen 
)
 X = []
n_batches = int(np.ceil(len(selected_indices)/ float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(len(selected_indices), batch_start + batch_size)
 res = fe.predict_on_batch(
 [
tokens_of_texts[batch_start:batch_end],
mask_of_texts[batch_start:batch_end]
] 
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
X.append(res)
 del res
X = np.vstack(X)
 y = np.array([texts[cur_lang][idx][1] for idx in selected_indices], dtype=np.int32)
 datasets_by_languages[cur_lang] =(X, y)
 del X, y, selected_indices
return datasets_by_languages
class Timm_model(nn.Module):
def __init__(self, model_name='efficientnet_b0', pretrained=True, out_dim=5):
super(Timm_model, self ).__init__()
self.base = create_model(model_name, pretrained=pretrained)
 if 'efficientnet' in model_name:
self.base.classifier = nn.Linear(in_features=self.base.classifier.in_features, out_features=out_dim)
 elif 'vit' in model_name:
self.base.head = nn.Linear(in_features=self.base.head.in_features, out_features=out_dim)
 else:
self.base.fc = nn.Linear(in_features=self.base.fc.in_features, out_features=out_dim)
 def forward(self, x):
return self.base(x )<define_search_space>
def predict_with_model(classifier: tf.keras.Model,
input_data: Tuple[np.ndarray, np.ndarray],
batch_size: int)-> np.ndarray:
predicted = []
n_batches = int(np.ceil(input_data[0].shape[0] / float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(batch_start + batch_size, input_data[0].shape[0])
 res = classifier.predict_on_batch(
(
 input_data[0][batch_start:batch_end],
input_data[1][batch_start:batch_end] 
)
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
predicted.append(res.flatten())
 del res
return np.concatenate(predicted )
z = torch.randn(4, 3, 224, 224)
 model = Timm_model(pretrained=False)
 out = model(z)
 print(out.size() )<choose_model_class>
def do_submit(data_for_training: Dict[str, Tuple[Tuple[np.ndarray, np.ndarray], np.ndarray]],
data_for_submission: Tuple[np.ndarray, np.ndarray],
language_for_validation: str, maxlen: int, base_classifier: tf.keras.Model,
max_duration: int, batch_size: int)-> Tuple[np.ndarray, int]:
assert language_for_validation in data_for_training
print('Duration of the training procedure must be less ' \
'than {0} seconds.'.format(max_duration))
print('')
 start_time = time.time()
X_train1 = []
X_train2 = []
y_train = []
X_val1 = []
X_val2 = []
y_val = []
for cur_lang in sorted(list(data_for_training.keys())) :
if cur_lang == language_for_validation:
X_val1.append(data_for_training[cur_lang][0][0])
 X_val2.append(data_for_training[cur_lang][0][1])
 y_val.append(data_for_training[cur_lang][1])
 else:
X_train1.append(data_for_training[cur_lang][0][0])
 X_train2.append(data_for_training[cur_lang][0][1])
 y_train.append(data_for_training[cur_lang][1])
 X_train =(np.vstack(X_train1), np.vstack(X_train2))
del X_train1, X_train2
y_train = np.concatenate(y_train)
 trainset = tf.data.Dataset.from_tensor_slices(
(X_train, y_train)
 ).repeat().batch(batch_size)
 del X_train
print('Labeled dataset for training contains {0} samples.'.format(y_train.shape[0]))
print('Number of positive samples is {0} from {1}.'.format(
 int(sum(y_train)) , y_train.shape[0]))
print('')
 steps_per_epoch = int(np.ceil(y_train.shape[0] / float(batch_size)))
 del y_train
X_val =(np.vstack(X_val1), np.vstack(X_val2))
del X_val1, X_val2
y_val = np.concatenate(y_val)
 validset = tf.data.Dataset.from_tensor_slices(
(X_val, y_val)
 ).batch(batch_size)
 print('Dataset for validation contains {0} samples.'.format(y_val.shape[0]))
print('Number of positive samples is {0} from {1}.'.format(int(sum(y_val)) ,
y_val.shape[0]))
print('')
 steps_per_validset = int(np.ceil(y_val.shape[0] / float(batch_size)))
 if steps_per_epoch <=(3 * steps_per_validset):
n_epochs = 25
else:
n_epochs =(25 * steps_per_epoch)//(3 * steps_per_validset)
 steps_per_epoch = 3 * steps_per_validset
callbacks = [
tf.keras.callbacks.EarlyStopping(patience=5, monitor='val_auc', mode='max',
restore_best_weights=True, verbose=True),
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)
 ]
print('n_epochs = {0}, steps_per_epoch = {1}, val_steps = {2}'.format(
 n_epochs, steps_per_epoch, steps_per_validset))
history = base_classifier.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs, callbacks=callbacks 
)
 postprocessing_start_time = time.time()
show_training_process(history, 'auc', figure_id=1)
 del history, trainset, validset
print('')
 probabilities = predict_with_model(
 classifier=base_classifier,
input_data=X_val,
batch_size=batch_size 
)
 show_roc_auc(y_true=y_val, probabilities=probabilities,
label='language "{0}"'.format(language_for_validation),
figure_id=2)
 del probabilities, y_val, X_val
gc.collect()
probabilities = predict_with_model(
 classifier=base_classifier,
input_data=data_for_submission,
batch_size=batch_size 
)
 assert probabilities.shape[0] == len(identifiers_for_submission)
 gc.collect()
end_time = time.time()
print('Duration of the final submission for language '\
'"{0}" is {1:.3f} seconds'.format(
 language_for_validation, end_time - start_time))
print('Postprocessing duration(after training)is ' \
'{0:.3f} seconds.'.format(end_time - postprocessing_start_time))
return probabilities, int(np.ceil(end_time - postprocessing_start_time))
class CassavaLightningSystem(pl.LightningModule):
def __init__(self, net, cfg, experiment=None):
super(CassavaLightningSystem, self ).__init__()
self.net = net
self.cfg = cfg
self.experiment = experiment
self.criterion = nn.CrossEntropyLoss()
self.best_loss = 1e+9
self.best_acc = None
self.epoch_num = 0
self.acc_fn = metrics.Accuracy()
self.loss_list = []
self.acc_list = []
def configure_optimizers(self):
self.optimizer = optim.AdamW(self.parameters() , lr=self.cfg.train['lr'], weight_decay=1e-5)
 self.scheduler = StepLR(self.optimizer, step_size=2, gamma=0.5)
 return [self.optimizer], [self.scheduler]
def forward(self, x):
return self.net(x)
 def step(self, batch):
inp, label = batch
out = self.forward(inp)
 loss = self.criterion(out, label.squeeze())
 return loss, label, torch.sigmoid(out)
 def training_step(self, batch, batch_idx):
loss, label, logits = self.step(batch)
 if self.experiment is not None:
logs = {'train/loss': loss.item() }
self.experiment.log_metrics(logs, step=batch_idx)
 return {'loss': loss, 'logits': logits, 'labels': label}
def validation_step(self, batch, batch_idx):
loss, label, logits = self.step(batch)
 if self.experiment is not None:
val_logs = {'val/loss': loss.item() }
self.experiment.log_metrics(val_logs, step=batch_idx)
 return {'val_loss': loss, 'logits': logits, 'labels': label}
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs] ).mean()
logits = torch.cat([x['logits'] for x in outputs])
 labels = torch.cat([x['labels'] for x in outputs])
 acc = self.acc_fn(logits, labels.squeeze())
 print(f'Epoch: {self.epoch_num} Loss: {avg_loss.item() :.4f} Acc {acc.item() :.4f}')
 self.loss_list.append(avg_loss.item())
 self.acc_list.append(acc.item())
 if self.experiment is not None:
logs = {'val/epoch_loss': avg_loss, 'val/epoch_acc': acc}
self.experiment.log_metrics(logs, step=self.epoch_num)
 if self.best_loss > avg_loss:
self.best_loss = avg_loss.item()
self.best_acc = acc.item()
expname = self.cfg.exp['exp_name']
filename = f'{expname}_epoch_{self.epoch_num}_loss_{self.best_loss:.3f}_acc_{self.best_acc:.3f}.pth'
torch.save(self.net.state_dict() , filename)
 if self.experiment is not None:
self.experiment.log_model(name=filename, file_or_folder='./'+filename)
 os.remove(filename)
 self.epoch_num += 1
return {'avg_val_loss': avg_loss}
def test_step(self, batch, batch_idx):
inp, img_id = batch
out = self.forward(inp)
 logits = torch.sigmoid(out)
 return {'preds': logits, 'image_id': img_id}
def test_epoch_end(self, outputs):
preds = torch.cat([x['preds'] for x in outputs])
 preds = preds.detach().cpu().numpy()
preds = pd.DataFrame(preds, columns=[f'label_{c}' for c in range(5)])
 img_ids = [x['image_id'] for x in outputs]
img_ids = [list(x)for x in img_ids]
img_ids = list(itertools.chain.from_iterable(img_ids))
self.sub = preds
self.sub.insert(0, 'image_id', img_ids)
 return None
def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx,
optimizer_closure, on_tpu, using_native_amp, using_lbfgs):
if self.trainer.global_step < 500:
lr_scale = min(1., float(self.trainer.global_step + 1)/ float(500))
for pg in optimizer.param_groups:
pg['lr'] = lr_scale * self.cfg.train['lr']
optimizer.step(closure=optimizer_closure)
 optimizer.zero_grad()<init_hyperparams>
experiment_start_time = time.time()
class cfg:
exp = {
'exp_name': 'test'
}
data = {
'img_size': 256,
'n_splits': 5
}
train = {
'batch_size': 64,
'epoch': 10,
'seed': 42,
'lr': 0.005,
'model_name': 'efficientnet_b0'
}<categorify>
random.seed(42)
 np.random.seed(42)
 tf.random.set_seed(42 )
data_dir = '/kaggle/input/cassava-leaf-disease-classification'
seed_everything(cfg.train['seed'])
 cv = StratifiedKFold(n_splits=cfg.data['n_splits'], shuffle=True, random_state=cfg.train['seed'])
 transform = ImageTransform(img_size=cfg.data['img_size'] )<choose_model_class>
siamese_learning_rate_max = 1e-5
siamese_learning_rate_min = 5e-6
classifier_learning_rate_max = 2e-6
classifier_learning_rate_min = 6e-7
dataset_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'
tmp_roberta_name = '/kaggle/working/base_nn.h5'
def main(data_dir, transform, cfg, cv, fold, TTA=5):
net = Timm_model(model_name=cfg.train['model_name'], pretrained=False)
 dm = CassavaDataModule(data_dir, cfg, transform, cv, fold=fold)
 model = CassavaLightningSystem(net, cfg, experiment=None)
 trainer = Trainer(
 logger=False,
max_epochs=cfg.train['epoch'],
gpus=1,
amp_backend='apex',
amp_level='O2', 
)
 trainer.fit(model, datamodule=dm)
 if TTA > 0:
for i in range(TTA):
trainer.test(model, datamodule=dm)
 if i == 0:
res = model.sub
else:
for j in range(5):
res[f'label_{j}'] += model.sub[f'label_{j}']
else:
trainer.test(model, datamodule=dm)
 res = model.sub
res.to_csv(f'submission_fold{fold}.csv', index=False)
 del net, dm
torch.cuda.empty_cache()
return model<categorify>
xlmroberta_tokenizer = AutoTokenizer.from_pretrained(model_name)
 xlmroberta_config = XLMRobertaConfig.from_pretrained(model_name)
 print(xlmroberta_config )
TTA = 3
models = []
for fold in range(cfg.data['n_splits']):
m = main(data_dir, transform, cfg, cv, fold, TTA)
 models.append(m)
 del m<save_to_csv>
sentence_embedding_size = xlmroberta_config.hidden_size
print('Sentence embedding size is {0}'.format(sentence_embedding_size))
assert max_seq_len <= xlmroberta_config.max_position_embeddings
sub_paths = glob.glob('submission_fold*')
 for i, path in enumerate(sub_paths):
tmp = pd.read_csv(path)
 if i == 0:
res = tmp
else:
for j in range(5):
res[f'label_{j}'] += tmp[f'label_{j}']
label_cols = [c for c in res.columns if c != 'image_id']
res['label'] = np.argmax(res[label_cols].values, axis=1)
 res = res[['image_id', 'label']]
res.to_csv('submission.csv', index=False )<install_modules>
corpus_for_training = load_train_set(
 os.path.join(dataset_dir, "jigsaw-unintended-bias-train.csv"),
text_field="comment_text", lang_field="lang",
sentiment_fields=["toxic", "severe_toxicity", "obscene", "identity_attack", "insult",
"threat"] 
)
 assert 'en' in corpus_for_training
%%capture
sys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')
 ! pip install -e /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle<import_modules>
corpus_for_validation = load_train_set(
 os.path.join(dataset_dir, "jigsaw-toxic-comment-train.csv"),
text_field="comment_text", lang_field="lang",
sentiment_fields=["toxic", "severe_toxic", "obscene", "threat", "insult",
"identity_hate"] 
)
 assert 'en' in corpus_for_validation
print("Tensorflow version " + tf.__version__ )<define_variables>
multilingual_corpus = load_train_set(
 os.path.join(dataset_dir, "validation.csv"),
text_field="comment_text", lang_field="lang", sentiment_fields=["toxic", ] 
)
 assert 'en' not in multilingual_corpus
max_size = 0
print('Multilingual data:')
 for language in sorted(list(multilingual_corpus.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(multilingual_corpus[language])))
 assert set(map(lambda cur: cur[1], multilingual_corpus[language])) == {0, 1}
if len(multilingual_corpus[language])> max_size:
max_size = len(multilingual_corpus[language] )
DIM = 512
IMAGE_SIZE = [DIM, DIM]
EFFNET = 6
PHASE = 'inference'
PRETRAINED_WEIGHTS = 'imagenet'
GCS_PATH1 = KaggleDatasets().get_gcs_path(f'cassava-tfrecords-{DIM}x{DIM}')if PHASE=='train' else f"/kaggle/input/cassava-tfrecords-{DIM}x{DIM}"
GCS_PATH2 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')if PHASE=='train' else "/kaggle/input/cassava-leaf-disease-classification"
AUTOTUNE = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
CLASSES = ['0', '1', '2', '3', '4']
EPOCHS = 12
FOLDS = 5
print(GCS_PATH1)
 print(GCS_PATH2 )<load_from_csv>
texts_for_submission = load_test_set(
 os.path.join(dataset_dir, "test.csv"),
text_field="content", lang_field="lang", id_field="id" 
)
 print('Data for submission:')
 for language in sorted(list(texts_for_submission.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(texts_for_submission[language])) )
test_df = pd.read_csv(GCS_PATH2 + '/sample_submission.csv')
 train_df = pd.read_csv(GCS_PATH2 + '/train.csv' )<define_variables>
dataset_for_training, n_train_batches = build_siamese_dataset(
 texts=corpus_for_training, dataset_size=300000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, for_training=True
)
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
del corpus_for_training
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform_mat(image, DIM=IMAGE_SIZE[0]):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d, [DIM, DIM,3] )<data_type_conversions>
dataset_for_validation, n_val_batches = build_siamese_dataset(
 texts=multilingual_corpus, dataset_size=2000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, for_training=False
)
def to_float32(image, label):
return tf.cast(image, tf.float32), label<categorify>
del corpus_for_validation
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image<categorify>
gc.collect()
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum<create_dataframe>
preparing_duration = int(round(time.time() - experiment_start_time))
print("Duration of data loading and preparing to the Siamese NN training is "
"{0} seconds.".format(preparing_duration))
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)
 return dataset<train_model>
with strategy.scope() :
siamese_network, neural_classifier, feature_extractor = build_siamese_nn(
 transformer_name=model_name,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
max_lr=siamese_learning_rate_max,
base_lr=siamese_learning_rate_min,
steps_per_epoch=n_val_batches * 10
)
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH1 + '/ld_train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH2 + '/test_tfrecords/ld_test*.tfrec')
 print('Train Files:',len(TRAINING_FILENAMES))
print('Test Files:',len(TEST_FILENAMES))<define_variables>
train_siamese_nn(nn=siamese_network,
trainset=dataset_for_training, steps_per_trainset=n_train_batches,
validset=dataset_for_validation, steps_per_validset=n_val_batches,
max_duration=int(round(3600 * 0.7 - preparing_duration)) ,
model_weights_path=tmp_roberta_name )
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(FOLDS-1.) /FOLDS)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)*(1./FOLDS))
NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<split>
del dataset_for_training
del dataset_for_validation
gc.collect()
TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(
 tf.io.gfile.glob(GCS_PATH1 + '/ld_train*.tfrec'),
test_size=0.35, random_state=5 
)
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH2 + '/test_tfrecords/ld_test*.tfrec' )<prepare_x_and_y>
siamese_network.load_weights(tmp_roberta_name)
 neural_classifier.save_weights(tmp_roberta_name, overwrite=True, save_format='h5' )
def dropout(image, DIM=DIM, PROBABILITY = 0.75, CT = 8, SZ = 0.2):
P = tf.cast(tf.random.uniform([],0,1)<PROBABILITY, tf.int32)
 if(P==0)|(CT==0)|(SZ==0): return image
for k in range(CT):
x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 WIDTH = tf.cast(SZ*DIM,tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[ya:yb,0:xa,:]
two = tf.zeros([yb-ya,xb-xa,3])
 three = image[ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)
 image = tf.reshape(image,[DIM,DIM,3])
 return image
def data_augment(image, label, dim=DIM, droprate=0.65, dropsize=0.15, dropct=8):
image = tf.image.random_flip_left_right(image)
 if(droprate!=0)&(dropct!=0)&(dropsize!=0):
image = dropout(image, DIM=dim, PROBABILITY=droprate, CT=dropct, SZ=dropsize)
 return image, label<prepare_x_and_y>
del neural_classifier, siamese_network
def get_training_dataset(training_fikenames=TRAINING_FILENAMES):
dataset = load_dataset(training_fikenames, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset<create_dataframe>
dataset_for_training = calculate_features_of_texts(
 texts=multilingual_corpus,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
fe=feature_extractor,
batch_size=batch_size_for_siamese,
max_dataset_size=min(600, batch_size_for_siamese * 9)
)
 assert len(dataset_for_training)== 4
def get_validation_dataset(valid_filenames=VALID_FILENAMES, ordered=False):
dataset = load_dataset(valid_filenames, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTOTUNE)
 return dataset 
<create_dataframe>
dataset_for_submission = calculate_features_of_texts(
 texts=texts_for_submission,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
fe=feature_extractor,
batch_size=batch_size_for_siamese,
max_dataset_size=min(300, batch_size_for_siamese * 3)
 )
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset<groupby>
X_embedded = []
y_embedded = []
for cur_lang in dataset_for_training:
X_embedded.append(dataset_for_training[cur_lang][0])
 y_embedded.append(dataset_for_training[cur_lang][1])
 for cur_lang in dataset_for_submission:
X_embedded.append(dataset_for_submission[cur_lang][0])
 y_embedded.append(
 np.array(
 [-1 for _ in range(dataset_for_submission[cur_lang][0].shape[0])],
dtype=np.int32 
)
)
 X_embedded = np.vstack(X_embedded)
 y_embedded = np.concatenate(y_embedded )
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<categorify>
del dataset_for_training, dataset_for_submission, feature_extractor
np.set_printoptions(threshold=15, linewidth=80)
 label2name = {"0": "Bacterial Blight",
"1": "Brown Streak Disease",
"2": "Green Mottle",
"3": "Mosaic Disease",
"4": "Healthy"}
def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_plant(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
plt.xticks([])
 plt.yticks([])
 else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
plt.xticks([])
 plt.yticks([])
 for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else label2name[CLASSES[label]] + f'({str(label)})'
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_plant(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.tight_layout()
plt.show()<define_variables>
X_embedded = TSNE(n_components=2, n_jobs=-1 ).fit_transform(X_embedded )
testing_dataset = get_test_dataset()
testing_dataset = testing_dataset.unbatch().batch(20)
 test_batch = iter(testing_dataset )<define_variables>
del indices_of_negative_classes
del indices_of_positive_classes
del indices_of_unknown_classes
del X_embedded, y_embedded
display_batch_of_images(next(test_batch))<categorify>
gc.collect()
tf.keras.backend.clear_session()
if tpu:
tf.tpu.experimental.shutdown_tpu_system(tpu)
 del strategy
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )
def onehot(image,label):
CLASSES = 5
return image,tf.one_hot(label,CLASSES )<categorify>
training_set_size = 0
for cur_lang in all_languages_for_training:
multilingual_corpus[cur_lang] =(
 regular_encode(
 texts=[cur[0] for cur in multilingual_corpus[cur_lang]],
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len 
),
np.array(
 [cur[1] for cur in multilingual_corpus[cur_lang]],
dtype=np.int32 
)
)
 training_set_size += multilingual_corpus[cur_lang][1].shape[0]
def cutmix(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = 5
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
def mixup(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
CLASSES = 5
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
def transform(image,label):
DIM = IMAGE_SIZE[0]
CLASSES = 5
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
image1 = []
for j in range(AUG_BATCH):
img = transform_mat(image[j,])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 image1.append(img)
 image1 = tf.reshape(tf.stack(image1),(AUG_BATCH,DIM,DIM,3))
image2, label2 = cutmix(image1, label, CUTMIX_PROB)
 image3, label3 = mixup(image1, label, MIXUP_PROB)
 imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4<categorify>
texts_for_submission_ = []
identifiers_for_submission = []
for cur_lang in sorted(list(texts_for_submission.keys())) :
texts_for_submission_ += [
cur[0] for cur in texts_for_submission[cur_lang]
]
identifiers_for_submission += [
cur[1] for cur in texts_for_submission[cur_lang]
]
X_submit = regular_encode(
 texts=texts_for_submission_,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len 
)
 identifiers_for_submission = np.array(
 identifiers_for_submission,
dtype=np.int32
)
def get_training_dataset(dataset=TRAINING_FILENAMES, do_aug=True):
if do_aug:
dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)
 dataset = dataset.repeat()
dataset = dataset.batch(AUG_BATCH)
 if do_aug:
dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE)
 dataset = dataset.unbatch()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset<categorify>
del xlmroberta_tokenizer, texts_for_submission_
gc.collect()
def get_validation_dataset(dataset, do_onehot=True):
dataset = dataset.batch(BATCH_SIZE)
 if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTOTUNE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTOTUNE)
 return dataset<define_variables>
experiment_duration = int(round(time.time() - experiment_start_time))
print('Duration of the Siamese XLM-RoBERTa preparing is {0} seconds.'.format(
 experiment_duration))
class DataGenerator(tf.keras.utils.Sequence):
def __init__(self, path, list_IDs, labels, batch_size, img_size, img_channel):
self.path = path
self.list_IDs = list_IDs
self.labels = labels
self.batch_size = batch_size
self.img_size = img_size
self.img_channel = img_channel
self.indexes = np.arange(len(self.list_IDs))
def __len__(self):
return int(np.floor(len(self.list_IDs)/self.batch_size))
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
X, y = self.__data_generation(list_IDs_temp)
 return X, y
def __data_generation(self, list_IDs_temp):
X = np.empty(( self.batch_size, self.img_size, self.img_size, self.img_channel))
y = np.empty(( self.batch_size, 5), dtype=int)
 for i, ID in enumerate(list_IDs_temp):
img = cv2.imread(self.path+ID)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = cv2.resize(img,(self.img_size, self.img_size))
X[i, ] = img
y[i, ] = self.labels[i]
X = X.astype('float32')
 X /= 255.
 return X, y
def preprocess_image(image_path, desired_size=DIM):
img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 im = cv2.resize(img,(desired_size,desired_size), interpolation = cv2.INTER_AREA)
 im = np.array(im)
 return im
test_generator = DataGenerator('.. /input/cassava-leaf-disease-classification/'+'test_images/',
test_df['image_id'],
test_df['label'],
1,
DIM,
3 )<init_hyperparams>
with strategy.scope() :
neural_classifier = build_classifier(
 config=xlmroberta_config,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
max_lr=classifier_learning_rate_max,
base_lr=classifier_learning_rate_min,
steps_per_epoch=training_set_size // batch_size_for_cls,
language=all_languages_for_training[0] 
)
 neural_classifier.load_weights(tmp_roberta_name )
def get_lr_callback(batch_size=8, show = False):
lr_start = 0.000015000
lr_max = 0.000000250 * strategy.num_replicas_in_sync * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
if show:
plt.figure(figsize =(8, 5))
plt.plot(np.arange(1, 12), [lrfn(x)for x in np.arange(1, 12)], marker = 'o')
 plt.xlabel('epoch')
 plt.ylabel('learaning_rate');
plt.show()
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback 
<choose_model_class>
result_of_submission1, postprocessing_duration = do_submit(
 data_for_training=multilingual_corpus,
data_for_submission=X_submit,
language_for_validation=all_languages_for_training[0],
maxlen=max_seq_len, base_classifier=neural_classifier,
batch_size=batch_size_for_cls,
max_duration=int(round(2.7 * 3600)-(time.time() - experiment_start_time)) // 5
)
model_dict = {0: efn.EfficientNetB0,
1: efn.EfficientNetB1,
2: efn.EfficientNetB2,
3: efn.EfficientNetB3,
4: efn.EfficientNetB4,
5: efn.EfficientNetB5,
6: efn.EfficientNetB6,
7: efn.EfficientNetB7,}
def get_model() :
with strategy.scope() :
inp = tf.keras.layers.Input(shape=(DIM,DIM,3))
base = model_dict[EFFNET](input_shape=(DIM,DIM,3),weights=PRETRAINED_WEIGHTS,include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(64,activation='relu' )(x)
 x = tf.keras.layers.Dense(5,activation='softmax' )(x)
 model = tf.keras.Model(inputs=inp, outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.CategoricalCrossentropy(
 from_logits=False, label_smoothing=0.0001,
name='categorical_crossentropy' 
)
 model.compile(optimizer=opt,loss=loss,metrics=['categorical_accuracy'])
 return model<split>
del neural_classifier
gc.collect()
tf.keras.backend.clear_session()
if tpu:
tf.tpu.experimental.shutdown_tpu_system(tpu)
 del strategy
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH1 + '/ld_train*.tfrec')
 kfold = KFold(FOLDS, shuffle = True, random_state = 42)
 probabilities = []
for f,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print() ; print('='*50)
 print(f' fold: {f+1} | model: EfficientNetB{EFFNET} | image_size: {DIM}')
 print('='*50)
 if PHASE == 'train':
train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)
 val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)
 sv = tf.keras.callbacks.ModelCheckpoint(
 'effnet-b%i_img-%i_fold-%02d_best.h5'%(EFFNET, DIM, f+1), monitor='val_categorical_accuracy', verbose=0, save_best_only=True,
save_weights_only=True, mode='max', save_freq='epoch')
 model = get_model()
history = model.fit(
 get_training_dataset(train_dataset),
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [get_lr_callback(BATCH_SIZE), sv],
validation_data = get_validation_dataset(val_dataset), 
)
 plt.figure(figsize=(15,5))
plt.plot(np.arange(len(model.history.history['categorical_accuracy'])) ,model.history.history['categorical_accuracy'],'-o',label='Train categorical_accuracy',color='
plt.plot(np.arange(len(model.history.history['categorical_accuracy'])) ,model.history.history['val_categorical_accuracy'],'-o',label='Val categorical_accuracy',color='
x = np.argmax(model.history.history['val_categorical_accuracy']); y = np.max(model.history.history['val_categorical_accuracy'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('categorical_accuracy',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(len(model.history.history['loss'])) ,model.history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(len(model.history.history['loss'])) ,model.history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(model.history.history['val_loss']); y = np.min(model.history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('Fold: %i | Image Size: %i | model: EfficientNetB%i | Batch_size: %i'%(f+1, DIM ,EFFNET, BATCH_SIZE))
plt.legend(loc=3)
 plt.show()
model.save_weights('effnet-b%i_img-%i_fold-%02d_last.h5'%(EFFNET, DIM, f+1))
del model; z = gc.collect()
elif PHASE == 'inference':
model = get_model()
weight_path = '/kaggle/input/casava-efficientnetb6-cutmixupdropout-models/effnet-b6_img-512_fold-%02d_best.h5'%(f+1)
 print('loading weight:', weight_path.split('/')[-1])
 model.load_weights(weight_path)
 preds = model.predict(test_generator, verbose=1)
 probabilities.append(preds)
 else:
print('PHASE must be train or inference.')
 predictions = np.array(probabilities ).mean(0 ).argmax(-1)
 print(predictions )<save_to_csv>
with strategy.scope() :
neural_classifier = build_classifier(
 config=xlmroberta_config,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
max_lr=classifier_learning_rate_max,
base_lr=classifier_learning_rate_min,
steps_per_epoch=training_set_size // batch_size_for_cls,
language=all_languages_for_training[1] 
)
 neural_classifier.load_weights(tmp_roberta_name )
if PHASE == 'inference':
test_df['label'] = predictions
test_df = test_df[["image_id","label"]]
test_df.to_csv('submission.csv',index=False)
 test_df<set_options>
result_of_submission2, _ = do_submit(
 data_for_training=multilingual_corpus,
data_for_submission=X_submit,
language_for_validation=all_languages_for_training[1],
maxlen=max_seq_len, base_classifier=neural_classifier,
batch_size=batch_size_for_cls,
max_duration=(int(round(2.7 * 3600)-(time.time() - experiment_start_time)) // 3 -
postprocessing_duration)
 )
warnings.filterwarnings('ignore')
<define_variables>
del neural_classifier
gc.collect()
tf.keras.backend.clear_session()
if tpu:
tf.tpu.experimental.shutdown_tpu_system(tpu)
 del strategy
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )
training_folder = '.. /input/cassava-leaf-disease-classification/train_images/'<load_from_csv>
with strategy.scope() :
neural_classifier = build_classifier(
 config=xlmroberta_config,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
max_lr=classifier_learning_rate_max,
base_lr=classifier_learning_rate_min,
steps_per_epoch=training_set_size // batch_size_for_cls,
language=all_languages_for_training[2] 
)
 neural_classifier.load_weights(tmp_roberta_name )
samples_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 samples_df = shuffle(samples_df, random_state=42)
 samples_df["filepath"] = training_folder+samples_df["image_id"]
samples_df.head()<define_variables>
result_of_submission3, _ = do_submit(
 data_for_training=multilingual_corpus,
data_for_submission=X_submit,
language_for_validation=all_languages_for_training[2],
maxlen=max_seq_len, base_classifier=neural_classifier,
batch_size=batch_size_for_cls,
max_duration=(int(round(2.7 * 3600)-(time.time() - experiment_start_time)) // 2 -
postprocessing_duration)
 )
training_percentage = 0.8
training_item_count = int(len(samples_df)*training_percentage)
 validation_item_count = len(samples_df)-int(len(samples_df)*training_percentage)
 training_df = samples_df[:training_item_count]
validation_df = samples_df[training_item_count:]<define_variables>
del neural_classifier
gc.collect()
tf.keras.backend.clear_session()
if tpu:
tf.tpu.experimental.shutdown_tpu_system(tpu)
 del strategy
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )
batch_size = 8
image_size = 512
input_shape =(image_size, image_size, 3)
 dropout_rate = 0.4
classes_to_predict = sorted(training_df.label.unique() )<prepare_x_and_y>
with strategy.scope() :
neural_classifier = build_classifier(
 config=xlmroberta_config,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
max_lr=classifier_learning_rate_max,
base_lr=classifier_learning_rate_min,
steps_per_epoch=training_set_size // batch_size_for_cls,
language=all_languages_for_training[3] 
)
 neural_classifier.load_weights(tmp_roberta_name )
training_data = tf.data.Dataset.from_tensor_slices(( training_df.filepath.values, training_df.label.values))
validation_data = tf.data.Dataset.from_tensor_slices(( validation_df.filepath.values, validation_df.label.values))<categorify>
result_of_submission4, _ = do_submit(
 data_for_training=multilingual_corpus,
data_for_submission=X_submit,
language_for_validation=all_languages_for_training[3],
maxlen=max_seq_len, base_classifier=neural_classifier,
batch_size=batch_size_for_cls,
max_duration=(int(round(2.7 * 3600)-(time.time() - experiment_start_time)) -
postprocessing_duration)
 )
def load_image_and_label_from_path(image_path, label):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 return img, label
AUTOTUNE = tf.data.experimental.AUTOTUNE
training_data = training_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)
 validation_data = validation_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE )<define_variables>
with codecs.open('submission.csv', mode='w', encoding='utf-8', errors='ignore')as fp:
fp.write('id,toxic
')
 for sample_idx in range(identifiers_for_submission.shape[0]):
proba_val = result_of_submission1[sample_idx]
proba_val += result_of_submission2[sample_idx]
proba_val += result_of_submission3[sample_idx]
proba_val += result_of_submission4[sample_idx]
proba_val /= 4.0
id_val = identifiers_for_submission[sample_idx]
fp.write('{0},{1:.9f}
'.format(id_val, proba_val))
<normalization><EOS> 
print('Experiment duration is {0:.3f}.'.format(time.time() - experiment_start_time))