Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<compute_test_metric>
Flatten, LeakyReLU, BatchNormalization)
 LearningRateScheduler,EarlyStopping,
ReduceLROnPlateau,CSVLogger)
 warnings.simplefilter('ignore')
 sns.set_style('whitegrid' )
print(f"Premise: {train['premise'].values[0]}")
 print(f"Hypothesis: {train['hypothesis'].values[0]}")
 print(f"Label: {train['label'].values[0]}" )<compute_test_metric>
SEED = 2020
batch_size = 12
dim =(128, 128)
 SIZE = 128
stats =(0.0692, 0.2051)
 HEIGHT = 137
WIDTH = 236
def seed_all(SEED):
random.seed(SEED)
 np.random.seed(SEED)
 os.environ['PYTHONHASHSEED'] = str(SEED)
 seed_all(SEED)
 im_path = '.. /input/grapheme-imgs-128x128/'
train = pd.read_csv('.. /input/bengaliai-cv19/train.csv')
 test = pd.read_csv('.. /input/bengaliai-cv19/test.csv')
 train = train.sample(frac=1 ).reset_index(drop=True)
 train['filename'] = train.image_id.apply(lambda filename: im_path + filename + '.png')
 train.head()
print(f"Premise: {train['premise'].values[1]}")
 print(f"Hypothesis: {train['hypothesis'].values[1]}")
 print(f"Label: {train['label'].values[1]}" )<categorify>
class GridMask(DualTransform):

 def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):
super(GridMask, self ).__init__(always_apply, p)
 if isinstance(num_grid, int):
num_grid =(num_grid, num_grid)
 if isinstance(rotate, int):
rotate =(-rotate, rotate)
 self.num_grid = num_grid
self.fill_value = fill_value
self.rotate = rotate
self.mode = mode
self.masks = None
self.rand_h_max = []
self.rand_w_max = []
def init_masks(self, height, width):
if self.masks is None:
self.masks = []
n_masks = self.num_grid[1] - self.num_grid[0] + 1
for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)) :
grid_h = height / n_g
grid_w = width / n_g
this_mask = np.ones(( int(( n_g + 1)* grid_h), int(( n_g + 1)* grid_w)) ).astype(np.uint8)
 for i in range(n_g + 1):
for j in range(n_g + 1):
this_mask[
int(i * grid_h): int(i * grid_h + grid_h / 2),
int(j * grid_w): int(j * grid_w + grid_w / 2)
 ] = self.fill_value
if self.mode == 2:
this_mask[
int(i * grid_h + grid_h / 2): int(i * grid_h + grid_h),
int(j * grid_w + grid_w / 2): int(j * grid_w + grid_w)
 ] = self.fill_value
if self.mode == 1:
this_mask = 1 - this_mask
self.masks.append(this_mask)
 self.rand_h_max.append(grid_h)
 self.rand_w_max.append(grid_w)
 def apply(self, image, mask, rand_h, rand_w, angle, **params):
h, w = image.shape[:2]
mask = F.rotate(mask, angle)if self.rotate[1] > 0 else mask
mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask
image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)
 return image
def get_params_dependent_on_targets(self, params):
img = params['image']
height, width = img.shape[:2]
self.init_masks(height, width)
 mid = np.random.randint(len(self.masks))
mask = self.masks[mid]
rand_h = np.random.randint(self.rand_h_max[mid])
 rand_w = np.random.randint(self.rand_w_max[mid])
 angle = np.random.randint(self.rotate[0], self.rotate[1])if self.rotate[1] > 0 else 0
return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}
@property
def targets_as_params(self):
return ['image']
def get_transform_init_args_names(self):
return('num_grid', 'fill_value', 'rotate', 'mode' )
def back_translate(sequence, PROB = 1):
languages = ['en', 'fr', 'th', 'tr', 'ur', 'ru', 'bg', 'de', 'ar', 'zh-cn', 'hi',
'sw', 'vi', 'es', 'el']
translator = Translator()
org_lang = translator.detect(sequence ).lang
random_lang = np.random.choice([lang for lang in languages if lang is not org_lang])
 if org_lang in languages:
translated = translator.translate(sequence, dest = random_lang ).text
translated_back = translator.translate(translated, dest = org_lang ).text
if np.random.uniform(0, 1)<= PROB:
output_sequence = translated_back
else:
output_sequence = sequence
else:
output_sequence = sequence
return output_sequence
for i in range(5):
output = back_translate('I genuinely have no idea what the output of this sequence of words will be')
 print(output )<load_from_csv>
def int_parameter(level, maxval):

 return int(level * maxval / 10)
 def float_parameter(level, maxval):

 return float(level)* maxval / 10.
 def sample_level(n):
return np.random.uniform(low=0.1, high=n)
 def autocontrast(pil_img, _):
return ImageOps.autocontrast(pil_img)
 def equalize(pil_img, _):
return ImageOps.equalize(pil_img)
 def posterize(pil_img, level):
level = int_parameter(sample_level(level), 4)
 return ImageOps.posterize(pil_img, 4 - level)
 def rotate(pil_img, level):
degrees = int_parameter(sample_level(level), 30)
 if np.random.uniform() > 0.5:
degrees = -degrees
return pil_img.rotate(degrees, resample=Image.BILINEAR)
 def solarize(pil_img, level):
level = int_parameter(sample_level(level), 256)
 return ImageOps.solarize(pil_img, 256 - level)
 def shear_x(pil_img, level):
level = float_parameter(sample_level(level), 0.3)
 if np.random.uniform() > 0.5:
level = -level
return pil_img.transform(( SIZE, SIZE),
Image.AFFINE,(1, level, 0, 0, 1, 0),
resample=Image.BILINEAR)
 def shear_y(pil_img, level):
level = float_parameter(sample_level(level), 0.3)
 if np.random.uniform() > 0.5:
level = -level
return pil_img.transform(( SIZE, SIZE),
Image.AFFINE,(1, 0, 0, level, 1, 0),
resample=Image.BILINEAR)
 def translate_x(pil_img, level):
level = int_parameter(sample_level(level), SIZE / 3)
 if np.random.random() > 0.5:
level = -level
return pil_img.transform(( SIZE, SIZE),
Image.AFFINE,(1, 0, level, 0, 1, 0),
resample=Image.BILINEAR)
 def translate_y(pil_img, level):
level = int_parameter(sample_level(level), SIZE / 3)
 if np.random.random() > 0.5:
level = -level
return pil_img.transform(( SIZE, SIZE),
Image.AFFINE,(1, 0, 0, 0, 1, level),
resample=Image.BILINEAR)
 augmentations = [
autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,
translate_x, translate_y
]
MEAN = [ 0.06922848809290576, 0.06922848809290576, 0.06922848809290576]
STD = [ 0.20515700083327537, 0.20515700083327537, 0.20515700083327537]
def normalize(image):

 image = image.transpose(2, 0, 1)
 mean, std = np.array(MEAN), np.array(STD)
 image =(image - mean[:, None, None])/ std[:, None, None]
return image.transpose(1, 2, 0)
 def apply_op(image, op, severity):
image = np.clip(image * 255., 0, 255 ).astype(np.uint8)
 pil_img = Image.fromarray(image)
 pil_img = op(pil_img, severity)
 return np.asarray(pil_img)/ 255.
 def augment_and_mix(image, severity=1, width=3, depth=1, alpha=1.) :

 ws = np.float32(
 np.random.dirichlet([alpha] * width))
m = np.float32(np.random.beta(alpha, alpha))
mix = np.zeros_like(image)
 for i in range(width):
image_aug = image.copy()
depth = depth if depth > 0 else np.random.randint(1, 4)
 for _ in range(depth):
op = np.random.choice(augmentations)
 image_aug = apply_op(image_aug, op, severity)
 mix = np.add(mix, ws[i] * normalize(image_aug), out=mix,
casting="unsafe")
 mixed =(1 - m)* normalize(image)+ m * mix
return mixed
train_aug = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/translation_aug_train.csv')
 train_aug.head()<load_from_csv>
class GraphemeGenerator(Sequence):
def __init__(self, data, batch_size, dim, shuffle=True, transform=None):
self._data = data
self._label_1 = pd.get_dummies(self._data['grapheme_root'],
columns = ['grapheme_root'])
 self._label_2 = pd.get_dummies(self._data['vowel_diacritic'],
columns = ['vowel_diacritic'])
 self._label_3 = pd.get_dummies(self._data['consonant_diacritic'],
columns = ['consonant_diacritic'])
 self._list_idx = data.index.values
self._batch_size = batch_size
self._dim = dim
self._shuffle = shuffle
self.transform = transform
self.on_epoch_end()
def __len__(self):
return int(np.floor(len(self._data)/self._batch_size))
def __getitem__(self, index):
batch_idx = self._indices[index*self._batch_size:(index+1)*self._batch_size]
_idx = [self._list_idx[k] for k in batch_idx]
Data = np.empty(( self._batch_size, *self._dim, 1))
Target_1 = np.empty(( self._batch_size, 168), dtype = int)
 Target_2 = np.empty(( self._batch_size, 11), dtype = int)
 Target_3 = np.empty(( self._batch_size, 7), dtype = int)
 for i, k in enumerate(_idx):
image = cv2.imread(im_path + self._data['image_id'][k] + '.png')
 image = cv2.resize(image, self._dim)
 if self.transform is not None:
if np.random.rand() > 0.7:
res = self.transform(image=image)
 image = res['image']
else:
image = augment_and_mix(image)
 image =(image.astype(np.float32)/255.0 - stats[0])/stats[1]
gray = lambda rgb : np.dot(rgb[..., :3] , [0.299 , 0.587, 0.114])
 image = gray(image)
 image = image[:, :, np.newaxis]
Data[i,:, :, :] = image
Target_1[i,:] = self._label_1.loc[k, :].values
Target_2[i,:] = self._label_2.loc[k, :].values
Target_3[i,:] = self._label_3.loc[k, :].values
return Data, [Target_1, Target_2, Target_3]
def on_epoch_end(self):
self._indices = np.arange(len(self._list_idx))
if self._shuffle:
np.random.shuffle(self._indices )
train_twice_aug = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/twice_translated_aug_train.csv')
 train_twice_aug.head()<load_from_csv>
!pip install.. /input/efficientnet-keras-source-code/repository/qubvel-efficientnet-c993591
train_vi = pd.read_csv(".. /input/contradictorytranslatedtrain/train_vi.csv")
 train_hi = pd.read_csv(".. /input/contradictorytranslatedtrain/train_hi.csv")
 train_bg = pd.read_csv(".. /input/contradictorytranslatedtrain/train_bg.csv" )<define_variables>
wg = '.. /input/efficientnet-keras-weights-b0b5/efficientnet-b0_imagenet_1000_notop.h5'
efnet = efn.EfficientNetB0(weights=wg,
include_top = False,
input_shape=(128, 128, 3))
!pip install --quiet transformers
roberta_base = "jplu/tf-xlm-roberta-base"
roberta_large = 'jplu/tf-xlm-roberta-large'<load_from_csv>
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shap
test_bt = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/translation_aug_test.csv')
 test_bt_twice = pd.read_csv('.. /input/contradictorywatsontwicetranslatedaug/translation_aug_test.csv' )<categorify>
for i, layer in enumerate(efnet.layers):
if "batch_normalization" in layer.name:
efnet.layers[i] = GroupNormalization(groups=32,
axis=-1,
epsilon=0.00001 )
TOKENIZER = AutoTokenizer.from_pretrained(roberta_large)
 def to_tf_dataset(dataset, max_len, repeat = False, shuffle = False, labeled = True, batch_size = BATCH_SIZE):
dataset_text = dataset[['premise', 'hypothesis']].values.tolist()
dataset_enc = TOKENIZER.batch_encode_plus(dataset_text, pad_to_max_length = True, max_length = max_len)
 if labeled:
tf_dataset = tf.data.Dataset.from_tensor_slices(( dataset_enc['input_ids'], dataset['label']))
else:
tf_dataset = tf.data.Dataset.from_tensor_slices(( dataset_enc['input_ids']))
if repeat: tf_dataset = tf_dataset.repeat()
if shuffle:
tf_dataset = tf_dataset.shuffle(2048)
 opt = tf.data.Options()
opt.experimental_deterministic = False
tf_dataset = tf_dataset.with_options(opt)
 tf_dataset = tf_dataset.batch(batch_size)
 tf_dataset = tf_dataset.prefetch(AUTO)
 return tf_dataset<init_hyperparams>
def E0(input_dim, output_dim, base_model):
input_tensor = Input(input_dim)
 x = Conv2D(3,(3, 3), padding='same', kernel_initializer='he_uniform',
bias_initializer='zeros' )(input_tensor)
 curr_output = base_model(x)
 curr_output = GlobalAveragePooling2D()(curr_output)
 curr_output = Dropout(0.5 )(curr_output)
 curr_output = Dense(512, activation='elu' )(curr_output)
 curr_output = Dropout(0.5 )(curr_output)
 oputput1 = Dense(168, activation='softmax', name='gra' )(curr_output)
 oputput2 = Dense(11, activation='softmax', name='vow' )(curr_output)
 oputput3 = Dense(7, activation='softmax', name='cons' )(curr_output)
 output_tensor = [oputput1, oputput2, oputput3]
model = Model(input_tensor, output_tensor)
 return model
model_E0 = E0(input_dim=(128,128,1),
output_dim=(168,11,7), base_model = efnet )
LR_START = 1e-6
LR_MAX = 1e-6 * 8
LR_MIN = 1e-6
LR_RAMPUP_EPOCHS = 2
LR_SUSTAIN_EPOCHS = 0
LR_DECAY =.8
def lrfn_step(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr = LR_MAX * LR_DECAY**(( epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)//2)
 return lr
def lrfn_smooth(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback_step = tf.keras.callbacks.LearningRateScheduler(lrfn_step, verbose = True)
 lr_callback_smooth = tf.keras.callbacks.LearningRateScheduler(lrfn_smooth, verbose = True)
 rng = [i for i in range(25)]
y1 = [lrfn_step(x)for x in rng]
y2 = [lrfn_smooth(x)for x in rng]
fix, ax = plt.subplots(1,2, figsize =(15, 5))
ax[0].plot(rng, y1)
 ax[1].plot(rng, y2)
 plt.tight_layout()
print("Learning rate schedule for step schedule: {:.3g} to {:.3g} to {:.3g}".format(y1[0], max(y1), y1[-1]))
print("Learning rate schedule for smooth schedule: {:.3g} to {:.3g} to {:.3g}".format(y2[0], max(y2), y2[-1]))<choose_model_class>
def D169(input_dim, output_dim, base_model):
input_tensor = Input(input_dim)
 x = Conv2D(3,(3, 3), padding='same', kernel_initializer='he_uniform',
bias_initializer='zeros' )(input_tensor)
 curr_output = base_model(x)
 curr_output = BatchNormalization()(curr_output)
 curr_output = GlobalAveragePooling2D()(curr_output)
 curr_output = Dense(512, activation='relu' )(curr_output)
 curr_output = Dropout(0.5 )(curr_output)
 curr_output = Dense(1024, activation='relu' )(curr_output)
 oputput1 = Dense(168, activation='softmax', name='gra' )(curr_output)
 oputput2 = Dense(11, activation='softmax', name='vow' )(curr_output)
 oputput3 = Dense(7, activation='softmax', name='cons' )(curr_output)
 output_tensor = [oputput1, oputput2, oputput3]
model = Model(input_tensor, output_tensor)
 return model
densenet = DenseNet169(include_top=False, weights=None,
input_shape=(128, 128, 3))
model_D169 = D169(input_dim=(128,128,1),
output_dim=(168,11,7), base_model = densenet )
def build_model(transformer_layer, max_len, learning_rate):
with strategy.scope() :
input_ids = tf.keras.Input(shape =(max_len,), dtype = tf.int32)
 roberta = TFAutoModel.from_pretrained(transformer_layer)
 roberta = roberta(input_ids)[0]
out = roberta[:, 0, :]
out = tf.keras.layers.Dense(3, activation = 'softmax' )(out)
 model = tf.keras.Model(inputs = input_ids, outputs = out)
 model.compile(
 optimizer = tf.keras.optimizers.Adam(lr = learning_rate),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 return model<init_hyperparams>
class AdaBound(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, final_lr=0.1, beta_1=0.9, beta_2=0.999, gamma=1e-3,
epsilon=None, decay=0., amsbound=False, weight_decay=0.0, **kwargs):
super(AdaBound, self ).__init__(**kwargs)
 if not 0.<= gamma <= 1.:
raise ValueError("Invalid `gamma` parameter.Must lie in [0, 1] range.")
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.final_lr = final_lr
self.gamma = gamma
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsbound = amsbound
self.weight_decay = float(weight_decay)
 self.base_lr = float(learning_rate)
 def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
learning_rate = self.learning_rate
if self.initial_decay > 0:
learning_rate = learning_rate *(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
step_size = learning_rate *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 final_lr = self.final_lr * learning_rate / self.base_lr
lower_bound = final_lr *(1.- 1./(self.gamma * t + 1.))
 upper_bound = final_lr *(1.+ 1./(self.gamma * t))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsbound:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
if self.weight_decay != 0.:
g += self.weight_decay * K.stop_gradient(p)
 m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 if self.amsbound:
vhat_t = K.maximum(vhat, v_t)
 denom =(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
denom =(K.sqrt(v_t)+ self.epsilon)
 step_size_p = step_size * K.ones_like(denom)
 step_size_p_bound = step_size_p / denom
bounded_lr_t = m_t * K.minimum(K.maximum(step_size_p_bound,
lower_bound), upper_bound)
 p_t = p - bounded_lr_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'learning_rate': float(K.get_value(self.learning_rate)) ,
'final_lr': float(self.final_lr),
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'gamma': float(self.gamma),
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'weight_decay': self.weight_decay,
'amsbound': self.amsbound}
base_config = super(AdaBound, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
LR_RATE = 1e-5
EPOCHS = 10
FOLDS = 4
MAX_LEN = 85
STEPS_PER_EPOCH = len(train)// BATCH_SIZE
TTA = 3
VERBOSE = 2
preds = np.zeros(( len(test), 3))
preds_tta = np.zeros(( len(test), 3))
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(train_index,val_index)in enumerate(skf.split(train)) :
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 K.clear_session()
model = build_model(roberta_large, max_len = MAX_LEN, learning_rate = LR_RATE)
 sv = tf.keras.callbacks.ModelCheckpoint(f'fold-{fold}.h5', monitor = 'val_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 train_ds = to_tf_dataset(train.loc[train_index], labeled = True, shuffle = True, repeat = True, max_len = MAX_LEN)
 val_ds = to_tf_dataset(train.loc[val_index], labeled = True, shuffle = False, repeat = False, max_len = MAX_LEN)
 print('
print('Training...'); print('')
 history = model.fit(train_ds, validation_data = val_ds, callbacks = [sv],
epochs = EPOCHS, steps_per_epoch = STEPS_PER_EPOCH,
verbose = VERBOSE); print('')
 print('Loading best model...')
 model.load_weights(f'fold-{fold}.h5')
 print('Predicting validation with TTA...')
 val_df = train.loc[val_index]
val_df_bt = train_aug.loc[val_index]
val_df_bt_twice = train_twice_aug.loc[val_index]
val_tta1 = to_tf_dataset(val_df, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 val_tta2 = to_tf_dataset(val_df_bt, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 val_tta3 = to_tf_dataset(val_df_bt_twice, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 val_pred1 = model.predict(val_tta1, verbose = VERBOSE)
 val_pred2 = model.predict(val_tta2, verbose = VERBOSE)
 val_pred3 = model.predict(val_tta3, verbose = VERBOSE)
 val_preds =(val_pred1 + val_pred2 + val_pred3)/ TTA
print(f"Without TTA: {accuracy_score(val_pred1.argmax(axis = 1), val_df['label'])}")
 print(f"With TTA: {accuracy_score(val_preds.argmax(axis = 1), val_df['label'])}")
 print('')
 print('Predicting OOF with TTA...')
 test_tta1 = to_tf_dataset(test, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 test_tta2 = to_tf_dataset(test_bt, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 test_tta3 = to_tf_dataset(test_bt_twice, shuffle = False, labeled = False, repeat = False, max_len = MAX_LEN)
 pred1 = model.predict(test_tta1, verbose = VERBOSE)
 pred2 = model.predict(test_tta2, verbose = VERBOSE)
 pred3 = model.predict(test_tta3, verbose = VERBOSE)
 preds_tta +=(pred1 + pred2 + pred3)/ TTA / FOLDS
preds += pred1 / FOLDS
os.remove(f"/kaggle/working/fold-{fold}.h5")
 del model ; z = gc.collect()<define_variables>
for model in [model_E0, model_D169]:
model.compile(
 optimizer = AdaBound(learning_rate=0.001,
final_lr=.1,
gamma=1e-3,
weight_decay=5e-4,
amsbound=False),
loss = {'gra' : 'categorical_crossentropy',
'vow' : 'categorical_crossentropy',
'cons': 'categorical_crossentropy'},
loss_weights = {'gra' : 1.0,
'vow' : 1.0,
'cons': 1.0},
metrics={'gra' : 'accuracy',
'vow' : 'accuracy',
'cons': 'accuracy'} 
)
USE_TTA = False<prepare_output>
plot_model(model_E0, to_file='model.png' )
if USE_TTA:
submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = preds_tta.argmax(axis = 1)
 else:
submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = preds.argmax(axis = 1)
 submission.head()<save_to_csv>
plot_model(model_D169, to_file='model.png' )
submission.to_csv('submission.csv', index = False)
 print('Submission saved' )<import_modules>
transforms_train = albumentations.Compose([
GridMask(num_grid=3, rotate=15, p=1),
])
 train_labels, val_labels = train_test_split(train, test_size = 0.10, random_state = SEED,
stratify = train[['grapheme_root',
'vowel_diacritic',
'consonant_diacritic']])
 train_generator = GraphemeGenerator(train_labels, batch_size, dim,
shuffle = True, transform=transforms_train)
 val_generator = GraphemeGenerator(val_labels, batch_size, dim,
shuffle = False )
import os
import numpy as np
import pandas as pd
from kaggle_datasets import KaggleDatasets
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
import plotly.express as px<set_options>
EarlyStopping, ReduceLROnPlateau, TensorBoard, CSVLogger)
 def Call_Back() :
checkpoint = ModelCheckpoint('E0Train_B0.h5',
monitor = 'val_loss',
verbose = 0, save_best_only=True,
mode = 'min',
save_weights_only = True)
 csv_logger = CSVLogger('E0.csv')
 early = EarlyStopping(monitor='val_loss',
mode='min', patience=5)
 return [checkpoint, csv_logger, early]
epochs = 60
training = False
callbacks = Call_Back()
if training:
train_history = model.fit_generator(
 train_generator,
steps_per_epoch=int(len(train_labels)/batch_size),
validation_data=val_generator,
validation_steps = int(len(val_labels)/batch_size),
epochs=epochs,
callbacks=callbacks 
)
 else:
model_E0.load_weights('.. /input/0909e0/0.909.h5')
 model_D169.load_weights('.. /input/newdensenet/newde69.h5' )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
def test_batch_generator(df, batch_size):
num_imgs = len(df)
 for batch_start in range(0, num_imgs, batch_size):
curr_batch_size = min(num_imgs, batch_start + batch_size)- batch_start
idx = np.arange(batch_start, batch_start + curr_batch_size)
 names_batch = df.iloc[idx, 0].values
imgs_batch = 255 - df.iloc[idx, 1:].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 X_batch = np.zeros(( curr_batch_size, SIZE, SIZE, 1))
for j in range(curr_batch_size):
img =(imgs_batch[j,]*(255.0/imgs_batch[j,].max())).astype(np.uint8)
 img = crop_resize(img, size=SIZE)
 img =(img.astype(np.float32)/255.0 - stats[0])/stats[1]
img = img[:, :, np.newaxis]
X_batch[j,] = img
yield X_batch, names_batch
TEST = [
".. /input/bengaliai-cv19/test_image_data_0.parquet",
".. /input/bengaliai-cv19/test_image_data_1.parquet",
".. /input/bengaliai-cv19/test_image_data_2.parquet",
".. /input/bengaliai-cv19/test_image_data_3.parquet",
]
row_id = []
target = []
for fname in tqdm(TEST):
test_ = pd.read_parquet(fname)
 test_gen = test_batch_generator(test_, batch_size=batch_size)
 for batch_x, batch_name in test_gen:
batch_predict_1 = model_E0.predict(batch_x, batch_size = 128)
 batch_predict_2 = model_D169.predict(batch_x, batch_size = 128)
 for idx, name in enumerate(batch_name):
row_id += [
f"{name}_consonant_diacritic",
f"{name}_grapheme_root",
f"{name}_vowel_diacritic",
]
target += [
np.argmax(( batch_predict_1[2] + batch_predict_2[2])/2, axis=1)[idx],
np.argmax(( batch_predict_1[0] + batch_predict_2[0])/2, axis=1)[idx],
np.argmax(( batch_predict_1[1] + batch_predict_2[1])/2, axis=1)[idx],
]
del test_
gc.collect()
df_sample = pd.DataFrame(
 {
'row_id': row_id,
'target':target
},
columns = ['row_id','target'] 
)
 df_sample.to_csv('submission.csv',index=False)
 gc.collect()