Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<categorify>
%%writefile /opt/conda/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/hdf5_format.py

 try:
HDF5_OBJECT_HEADER_LIMIT = 64512
except ImportError:
h5py = None
def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True):

 if h5py is None:
raise ImportError('`save_model` requires h5py.')
 if len(model.weights)!= len(model._undeduplicated_weights):
logging.warning('Found duplicated `Variable`s in Model's `weights`.'
'This is usually caused by `Variable`s being shared by '
'Layers in the Model.These `Variable`s will be treated '
'as separate `Variable`s when the Model is restored.To '
'avoid this, please save with `save_format="tf"`.')
 if not isinstance(filepath, h5py.File):
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
 if not proceed:
return
f = h5py.File(filepath, mode='w')
 opened_new_file = True
else:
f = filepath
opened_new_file = False
try:
model_metadata = saving_utils.model_metadata(model, include_optimizer)
 for k, v in model_metadata.items() :
if isinstance(v,(dict, list, tuple)) :
f.attrs[k] = json.dumps(
 v, default=serialization.get_json_type ).encode('utf8')
 else:
f.attrs[k] = v
model_weights_group = f.create_group('model_weights')
 model_layers = model.layers
save_weights_to_hdf5_group(model_weights_group, model_layers)
 if(include_optimizer and model.optimizer and
not isinstance(model.optimizer, optimizers.TFOptimizer)) :
save_optimizer_weights_to_hdf5_group(f, model.optimizer)
 f.flush()
finally:
if opened_new_file:
f.close()
def load_model_from_hdf5(filepath, custom_objects=None, compile=True):

 if h5py is None:
raise ImportError('`load_model` requires h5py.')
 if not custom_objects:
custom_objects = {}
opened_new_file = not isinstance(filepath, h5py.File)
 if opened_new_file:
f = h5py.File(filepath, mode='r')
 else:
f = filepath
model = None
try:
model_config = f.attrs.get('model_config')
 if model_config is None:
raise ValueError('No model found in config file.')
 model_config = json.loads(model_config.decode('utf-8'))
model = model_config_lib.model_from_config(model_config,
custom_objects=custom_objects)
 load_weights_from_hdf5_group(f['model_weights'], model.layers)
 if compile:
training_config = f.attrs.get('training_config')
 if training_config is None:
logging.warning('No training configuration found in the save file, so '
'the model was *not* compiled.Compile it manually.')
 return model
training_config = json.loads(training_config.decode('utf-8'))
model.compile(**saving_utils.compile_args_from_training_config(
 training_config, custom_objects))
if 'optimizer_weights' in f:
if model._is_graph_network:
if not ops.executing_eagerly_outside_functions() :
model._make_train_function()
optimizer_weight_values = load_optimizer_weights_from_hdf5_group(f)
 try:
model.optimizer.set_weights(optimizer_weight_values)
 except ValueError:
logging.warning('Error in loading the saved optimizer '
'state.As a result, your model is '
'starting with a freshly initialized '
'optimizer.')
 else:
logging.warning('Sequential models without an `input_shape` '
'passed to the first layer cannot reload their '
'optimizer state.As a result, your model is'
'starting with a freshly initialized optimizer.')
 finally:
if opened_new_file:
f.close()
return model
def preprocess_weights_for_loading(layer,
weights,
original_keras_version=None,
original_backend=None):

 def convert_nested_bidirectional(weights):

 num_weights_per_layer = len(weights)// 2
forward_weights = preprocess_weights_for_loading(
 layer.forward_layer, weights[:num_weights_per_layer],
original_keras_version, original_backend)
 backward_weights = preprocess_weights_for_loading(
 layer.backward_layer, weights[num_weights_per_layer:],
original_keras_version, original_backend)
 return forward_weights + backward_weights
def convert_nested_time_distributed(weights):

 return preprocess_weights_for_loading(
 layer.layer, weights, original_keras_version, original_backend)
 def convert_nested_model(weights):

 trainable_weights = weights[:len(layer.trainable_weights)]
non_trainable_weights = weights[len(layer.trainable_weights):]
new_trainable_weights = []
new_non_trainable_weights = []
for sublayer in layer.layers:
num_trainable_weights = len(sublayer.trainable_weights)
 num_non_trainable_weights = len(sublayer.non_trainable_weights)
 if sublayer.weights:
preprocessed = preprocess_weights_for_loading(
 layer=sublayer,
weights=(trainable_weights[:num_trainable_weights] +
non_trainable_weights[:num_non_trainable_weights]),
original_keras_version=original_keras_version,
original_backend=original_backend)
 new_trainable_weights.extend(preprocessed[:num_trainable_weights])
 new_non_trainable_weights.extend(preprocessed[num_trainable_weights:])
 trainable_weights = trainable_weights[num_trainable_weights:]
non_trainable_weights = non_trainable_weights[
num_non_trainable_weights:]
return new_trainable_weights + new_non_trainable_weights
if layer.__class__.__name__ == 'Bidirectional':
weights = convert_nested_bidirectional(weights)
 if layer.__class__.__name__ == 'TimeDistributed':
weights = convert_nested_time_distributed(weights)
 elif layer.__class__.__name__ in ['Model', 'Sequential']:
weights = convert_nested_model(weights)
 if original_keras_version == '1':
if layer.__class__.__name__ == 'TimeDistributed':
weights = preprocess_weights_for_loading(
 layer.layer, weights, original_keras_version, original_backend)
 if layer.__class__.__name__ == 'Conv1D':
shape = weights[0].shape
if shape[:2] !=(layer.kernel_size[0], 1)or shape[3] != layer.filters:
assert shape[0] == layer.filters and shape[2:] ==(layer.kernel_size[0],
1)
 weights[0] = np.transpose(weights[0],(2, 3, 1, 0))
weights[0] = weights[0][:, 0, :, :]
if layer.__class__.__name__ == 'Conv2D':
if layer.data_format == 'channels_first':
weights[0] = np.transpose(weights[0],(2, 3, 1, 0))
if layer.__class__.__name__ == 'Conv2DTranspose':
if layer.data_format == 'channels_last':
weights[0] = np.transpose(weights[0],(0, 1, 3, 2))
if layer.data_format == 'channels_first':
weights[0] = np.transpose(weights[0],(2, 3, 0, 1))
if layer.__class__.__name__ == 'Conv3D':
if layer.data_format == 'channels_first':
weights[0] = np.transpose(weights[0],(2, 3, 4, 1, 0))
if layer.__class__.__name__ == 'GRU':
if len(weights)== 9:
kernel = np.concatenate([weights[0], weights[3], weights[6]], axis=-1)
 recurrent_kernel = np.concatenate(
 [weights[1], weights[4], weights[7]], axis=-1)
 bias = np.concatenate([weights[2], weights[5], weights[8]], axis=-1)
 weights = [kernel, recurrent_kernel, bias]
if layer.__class__.__name__ == 'LSTM':
if len(weights)== 12:
kernel = np.concatenate(
 [weights[0], weights[6], weights[3], weights[9]], axis=-1)
 recurrent_kernel = np.concatenate(
 [weights[1], weights[7], weights[4], weights[10]], axis=-1)
 bias = np.concatenate(
 [weights[2], weights[8], weights[5], weights[11]], axis=-1)
 weights = [kernel, recurrent_kernel, bias]
if layer.__class__.__name__ == 'ConvLSTM2D':
if len(weights)== 12:
kernel = np.concatenate(
 [weights[0], weights[6], weights[3], weights[9]], axis=-1)
 recurrent_kernel = np.concatenate(
 [weights[1], weights[7], weights[4], weights[10]], axis=-1)
 bias = np.concatenate(
 [weights[2], weights[8], weights[5], weights[11]], axis=-1)
 if layer.data_format == 'channels_first':
kernel = np.transpose(kernel,(2, 3, 1, 0))
recurrent_kernel = np.transpose(recurrent_kernel,(2, 3, 1, 0))
weights = [kernel, recurrent_kernel, bias]
conv_layers = ['Conv1D', 'Conv2D', 'Conv3D', 'Conv2DTranspose', 'ConvLSTM2D']
if layer.__class__.__name__ in conv_layers:
if original_backend == 'theano':
weights[0] = conv_utils.convert_kernel(weights[0])
 if layer.__class__.__name__ == 'ConvLSTM2D':
weights[1] = conv_utils.convert_kernel(weights[1])
 if K.int_shape(layer.weights[0])!= weights[0].shape:
weights[0] = np.transpose(weights[0],(3, 2, 0, 1))
if layer.__class__.__name__ == 'ConvLSTM2D':
weights[1] = np.transpose(weights[1],(3, 2, 0, 1))
return _convert_rnn_weights(layer, weights)
 def _convert_rnn_weights(layer, weights):

 def transform_kernels(kernels, func, n_gates):

 return np.hstack([func(k)for k in np.hsplit(kernels, n_gates)])
 def transpose_input(from_cudnn):

 order = 'F' if from_cudnn else 'C'
def transform(kernel):
return kernel.T.reshape(kernel.shape, order=order)
 return transform
target_class = layer.__class__.__name__
if target_class in ['LSTM', 'CuDNNLSTM'] and len(weights)== 3:
units = weights[1].shape[0]
bias_shape = weights[2].shape
n_gates = 4
if bias_shape ==(2 * units * n_gates,):
source = 'CuDNNLSTM'
elif bias_shape ==(units * n_gates,):
source = 'LSTM'
else:
raise ValueError('Invalid bias shape: ' + str(bias_shape))
def convert_lstm_weights(weights, from_cudnn=True):

 kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
n_gates)
 recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
 if from_cudnn:
biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
 else:
biases = np.tile(0.5 * weights[2], 2)
 return [kernels, recurrent_kernels, biases]
if source != target_class:
weights = convert_lstm_weights(weights, from_cudnn=source == 'CuDNNLSTM')
 if target_class in ['GRU', 'CuDNNGRU'] and len(weights)== 3:
units = weights[1].shape[0]
bias_shape = weights[2].shape
n_gates = 3
def convert_gru_weights(weights, from_cudnn=True):

 kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
n_gates)
 recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
 biases = np.array(weights[2] ).reshape(( 2, -1)if from_cudnn else -1)
 return [kernels, recurrent_kernels, biases]
if bias_shape ==(2 * units * n_gates,):
source = 'CuDNNGRU'
elif bias_shape ==(2, units * n_gates):
source = 'GRU(reset_after=True)'
elif bias_shape ==(units * n_gates,):
source = 'GRU(reset_after=False)'
else:
raise ValueError('Invalid bias shape: ' + str(bias_shape))
if target_class == 'CuDNNGRU':
target = 'CuDNNGRU'
elif layer.reset_after:
target = 'GRU(reset_after=True)'
else:
target = 'GRU(reset_after=False)'
if source != target:
types =(source, target)
 if 'GRU(reset_after=False)' in types:
raise ValueError('%s is not compatible with %s' % types)
 if source == 'CuDNNGRU':
weights = convert_gru_weights(weights, from_cudnn=True)
 elif source == 'GRU(reset_after=True)':
weights = convert_gru_weights(weights, from_cudnn=False)
 return weights
def save_optimizer_weights_to_hdf5_group(hdf5_group, optimizer):

 symbolic_weights = getattr(optimizer, 'weights')
 if symbolic_weights:
weights_group = hdf5_group.create_group('optimizer_weights')
 weight_names = [str(w.name ).encode('utf8')for w in symbolic_weights]
save_attributes_to_hdf5_group(weights_group, 'weight_names', weight_names)
 weight_values = K.batch_get_value(symbolic_weights)
 for name, val in zip(weight_names, weight_values):
param_dset = weights_group.create_dataset(
 name, val.shape, dtype=val.dtype)
 if not val.shape:
param_dset[() ] = val
else:
param_dset[:] = val
def load_optimizer_weights_from_hdf5_group(hdf5_group):

 weights_group = hdf5_group['optimizer_weights']
optimizer_weight_names = load_attributes_from_hdf5_group(
 weights_group, 'weight_names')
 return [weights_group[weight_name] for weight_name in optimizer_weight_names]
def save_weights_to_hdf5_group(f, layers):

 save_attributes_to_hdf5_group(
 f, 'layer_names', [layer.name.encode('utf8')for layer in layers])
 f.attrs['backend'] = K.backend().encode('utf8')
 f.attrs['keras_version'] = str(keras_version ).encode('utf8')
 for layer in sorted(layers, key=lambda x: x.name):
g = f.create_group(layer.name)
 weights = _legacy_weights(layer)
 weight_values = K.batch_get_value(weights)
 weight_names = [w.name.encode('utf8')for w in weights]
save_attributes_to_hdf5_group(g, 'weight_names', weight_names)
 for name, val in zip(weight_names, weight_values):
param_dset = g.create_dataset(name, val.shape, dtype=val.dtype)
 if not val.shape:
param_dset[() ] = val
else:
param_dset[:] = val
def load_weights_from_hdf5_group(f, layers):

 if 'keras_version' in f.attrs:
original_keras_version = f.attrs['keras_version'].decode('utf8')
 else:
original_keras_version = '1'
if 'backend' in f.attrs:
original_backend = f.attrs['backend'].decode('utf8')
 else:
original_backend = None
filtered_layers = []
for layer in layers:
weights = _legacy_weights(layer)
 if weights:
filtered_layers.append(layer)
 layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
 filtered_layer_names = []
for name in layer_names:
g = f[name]
weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
 if weight_names:
filtered_layer_names.append(name)
 layer_names = filtered_layer_names
if len(layer_names)!= len(filtered_layers):
raise ValueError('You are trying to load a weight file '
'containing ' + str(len(layer_names)) +
' layers into a model with ' + str(len(filtered_layers)) +
' layers.')
 weight_value_tuples = []
for k, name in enumerate(layer_names):
g = f[name]
weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
 weight_values = [np.asarray(g[weight_name])for weight_name in weight_names]
layer = filtered_layers[k]
symbolic_weights = _legacy_weights(layer)
 weight_values = preprocess_weights_for_loading(
 layer, weight_values, original_keras_version, original_backend)
 if len(weight_values)!= len(symbolic_weights):
raise ValueError('Layer
'" in the current model)was found to '
'correspond to layer ' + name + ' in the save file.'
'However the new layer ' + layer.name + ' expects ' +
str(len(symbolic_weights)) +
' weights, but the saved weights have ' +
str(len(weight_values)) + ' elements.')
 weight_value_tuples += zip(symbolic_weights, weight_values)
 K.batch_set_value(weight_value_tuples)
 def load_weights_from_hdf5_group_by_name(
 f, layers, skip_mismatch=False):

 if 'keras_version' in f.attrs:
original_keras_version = f.attrs['keras_version'].decode('utf8')
 else:
original_keras_version = '1'
if 'backend' in f.attrs:
original_backend = f.attrs['backend'].decode('utf8')
 else:
original_backend = None
layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
 index = {}
for layer in layers:
if layer.name:
index.setdefault(layer.name, [] ).append(layer)
 weight_value_tuples = []
for k, name in enumerate(layer_names):
g = f[name]
weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
 weight_values = [np.asarray(g[weight_name])for weight_name in weight_names]
for layer in index.get(name, []):
symbolic_weights = _legacy_weights(layer)
 weight_values = preprocess_weights_for_loading(
 layer, weight_values, original_keras_version, original_backend)
 if len(weight_values)!= len(symbolic_weights):
if skip_mismatch:
logging.warning('Skipping loading of weights for '
'layer {}'.format(layer.name)+ ' due to mismatch '
'in number of weights({} vs {} ).'.format(
 len(symbolic_weights), len(weight_values)))
 continue
raise ValueError('Layer
'")expects ' + str(len(symbolic_weights)) +
' weight(s), but the saved weights' + ' have ' +
str(len(weight_values)) + ' element(s ).')
 for i in range(len(weight_values)) :
if K.int_shape(symbolic_weights[i])!= weight_values[i].shape:
if skip_mismatch:
logging.warning('Skipping loading of weights for '
'layer {}'.format(layer.name)+ ' due to '
'mismatch in shape({} vs {} ).'.format(
 symbolic_weights[i].shape,
weight_values[i].shape))
continue
raise ValueError('Layer
'"), weight ' + str(symbolic_weights[i])+
' has shape {}'.format(K.int_shape(
 symbolic_weights[i])) +
', but the saved weight has shape ' +
str(weight_values[i].shape)+ '.')
 else:
weight_value_tuples.append(( symbolic_weights[i], weight_values[i]))
K.batch_set_value(weight_value_tuples)
 def save_attributes_to_hdf5_group(group, name, data):

 bad_attributes = [x for x in data if len(x)> HDF5_OBJECT_HEADER_LIMIT]
if bad_attributes:
raise RuntimeError('The following attributes cannot be saved to HDF5 '
'file because they are larger than %d bytes: %s' % 
(HDF5_OBJECT_HEADER_LIMIT, ', '.join(bad_attributes)))
 data_npy = np.asarray(data)
 num_chunks = 1
chunked_data = np.array_split(data_npy, num_chunks)
 while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
num_chunks += 1
chunked_data = np.array_split(data_npy, num_chunks)
 if num_chunks > 1:
for chunk_id, chunk_data in enumerate(chunked_data):
group.attrs['%s%d' %(name, chunk_id)] = chunk_data
else:
group.attrs[name] = data
def load_attributes_from_hdf5_group(group, name):

 if name in group.attrs:
data = [n.decode('utf8')for n in group.attrs[name]]
else:
data = []
chunk_id = 0
while '%s%d' %(name, chunk_id)in group.attrs:
data.extend(
 [n.decode('utf8')for n in group.attrs['%s%d' %(name, chunk_id)]])
 chunk_id += 1
return data
def _legacy_weights(layer):

 weights = layer.trainable_weights + layer.non_trainable_weights
if any([not isinstance(w, variables_module.Variable)for w in weights]):
raise NotImplementedError(
 'Save or restore weights that is not an instance of `tf.Variable` is '
'not supported in h5, use `save_format='tf'` instead.Got a model '
'or layer {} with weights {}'.format(layer.__class__.__name__, weights))
return weights
NROW, NCOL = 25, 25
def generate_samples(delta=1, n=32):
batch = np.split(np.random.binomial(1, 0.5,(NROW * n, NCOL)).astype('uint8'), n)
 Yy = [life.make_move(state, 5)for state in batch]
Xx = [life.make_move(state, 1)for state in Yy]
Y = np.array([y.ravel() for y in Yy])
 X = np.array([x.ravel() for x in Xx])
 return X, Y
def data_generator(delta=1, batch_size=32, ravel=True):
while True:
batch = np.split(np.random.binomial(1, 0.5,(NROW * batch_size, NCOL)).astype('uint8'),
batch_size)
 Yy = [make_move(state, 5)for state in batch]
Xx = [make_move(state, delta)for state in Yy]
if ravel:
Y = np.array([y.ravel() for y in Yy])
 X = np.array([x.ravel() for x in Xx])
 yield X, Y
else:
yield np.array(Xx)[:,:, :, np.newaxis], np.array(Yy)[:, :, :, np.newaxis]<choose_model_class>
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl
EarlyStopping, ReduceLROnPlateau, TensorBoard, CSVLogger)
def create_model(n_hidden_convs=2, n_hidden_filters=128, kernel_size=5):
nn = Sequential()
nn.add(Conv2D(n_hidden_filters, kernel_size, padding='same', activation='relu',
input_shape=(25, 25, 1)))
 nn.add(BatchNormalization())
 for i in range(n_hidden_convs):
nn.add(Conv2D(n_hidden_filters, kernel_size, padding='same', activation='relu'))
nn.add(BatchNormalization())
 nn.add(Conv2D(1, kernel_size, padding='same', activation='sigmoid'))
nn.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 return nn<train_model>
def normalize(image):
image -= tf.constant([0.485 * 255, 0.456 * 255, 0.406 * 255])
 image /= tf.constant([0.229 * 255, 0.224 * 255, 0.225 * 255])
 return image
def one_hot(image, label):
label['root'] = tf.one_hot(label['root'], 168)
 label['vowel'] = tf.one_hot(label['vowel'], 11)
 label['consonant'] = tf.one_hot(label['consonant'], 7)
 label['unique'] = tf.one_hot(label['unique'], 1292)
 label['root2'] = tf.one_hot(label['root2'], 168)
 label['vowel2'] = tf.one_hot(label['vowel2'], 11)
 label['consonant2'] = tf.one_hot(label['consonant2'], 7)
 label['unique2'] = tf.one_hot(label['unique2'], 1292)
 label['root3'] = tf.one_hot(label['root3'], 168)
 label['vowel3'] = tf.one_hot(label['vowel3'], 11)
 label['consonant3'] = tf.one_hot(label['consonant3'], 7)
 label['unique3'] = tf.one_hot(label['unique3'], 1292)
 return image, label
def prepare_metric_learning(image, label, mode='train'):
if model == 'train':
return(image, label['root'], label['vowel'], label['consonant'], label['unique']), label
else:
return(image, tf.zeros_like(label['root']), tf.zeros_like(label['vowel']), tf.zeros_like(label['consonant']), tf.zeros_like(label['unique'])) , label
def get_callbacks(work_dir):
checkpoint = ModelCheckpoint(work_dir + '/best.h5',
monitor = 'val_loss',
verbose = 0, save_best_only=True,
mode = 'min',
save_weights_only = True)
 csv_logger = CSVLogger(work_dir + '/log.csv')
 early = EarlyStopping(monitor='val_loss', mode='min', patience=20)
 scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10)
 return [checkpoint, csv_logger, early, scheduler]
def read_tfrecords(example, input_size):
features = {
'img': tf.io.FixedLenFeature([], tf.string),
'image_id': tf.io.FixedLenFeature([], tf.int64),
'grapheme_root': tf.io.FixedLenFeature([], tf.int64),
'vowel_diacritic': tf.io.FixedLenFeature([], tf.int64),
'consonant_diacritic': tf.io.FixedLenFeature([], tf.int64),
'unique_tuple': tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, features)
 img = tf.image.decode_image(example['img'])
 img = tf.reshape(img, input_size +(1,))
img = tf.cast(img, tf.float32)
 img = tf.repeat(img, 3, -1)
 grapheme_root = tf.cast(example['grapheme_root'], tf.int32)
 vowel_diacritic = tf.cast(example['vowel_diacritic'], tf.int32)
 consonant_diacritic = tf.cast(example['consonant_diacritic'], tf.int32)
 unique_tuple = tf.cast(example['unique_tuple'], tf.int32)
 return img, {'root': grapheme_root, 'vowel': vowel_diacritic, 'consonant': consonant_diacritic, 'unique': unique_tuple, 'root2': grapheme_root, 'vowel2': vowel_diacritic, 'consonant2': consonant_diacritic, 'unique2': unique_tuple, 'root3': grapheme_root, 'vowel3': vowel_diacritic, 'consonant3': consonant_diacritic, 'unique3': unique_tuple}
models = []
for delta in range(1, 6):
model = create_model(n_hidden_convs=6, n_hidden_filters=256)
 es = EarlyStopping(monitor='loss', patience=9, min_delta=0.001)
 model.fit_generator(data_generator(delta=delta, ravel=False),
steps_per_epoch=500, epochs=50, verbose=1, callbacks=[es])
 models.append(model )<load_from_csv>
class Generalized_mean_pooling2D(tf.keras.layers.Layer):
def __init__(self, p=3, epsilon=1e-6, name='', **kwargs):
super(Generalized_mean_pooling2D, self ).__init__(name, **kwargs)
 self.init_p = p
self.epsilon = epsilon
def build(self, input_shape):
if isinstance(input_shape, list)or len(input_shape)!= 4:
raise ValueError('`GeM` pooling layer only allow 1 input with 4 dimensions(b, h, w, c)')
 self.build_shape = input_shape
self.p = self.add_weight(
 name='p',
shape=[1,],
initializer=tf.keras.initializers.Constant(value=self.init_p),
regularizer=None,
trainable=True,
dtype=tf.float32 
)
 self.built=True
def call(self, inputs):
input_shape = inputs.get_shape()
if isinstance(inputs, list)or len(input_shape)!= 4:
raise ValueError('`GeM` pooling layer only allow 1 input with 4 dimensions(b, h, w, c)')
 return(tf.reduce_mean(tf.abs(inputs**self.p), axis=[1,2], keepdims=False)+ self.epsilon)**(1.0/self.p)
 class CosFace(Layer):
def __init__(self, n_classes=10, s=30.0, m=0.35, regularizer=None, **kwargs):
super(CosFace, self ).__init__(**kwargs)
 self.n_classes = n_classes
self.s = s
self.m = m
self.regularizer = regularizers.get(regularizer)
 def build(self, input_shape):
super(CosFace, self ).build(input_shape[0])
 self.W = self.add_weight(shape=(input_shape[0][-1], self.n_classes),
initializer='glorot_uniform',
trainable=True,
regularizer=self.regularizer)
 def call(self, inputs):
x, y = inputs
c = K.shape(x)[-1]
W = tf.nn.l2_normalize(self.W, axis=0)
 logits = x @ W
target_logits = logits - self.m
logits = logits *(1 - y)+ target_logits * y
logits *= self.s
out = tf.nn.softmax(logits)
 return out
def compute_output_shape(self, input_shape):
return(None, self.n_classes)
 def get_model(input_size, backbone='efficientnet-b0', weights='imagenet', tta=False):
print(f'Using backbone {backbone} and weights {weights}')
 x_input = L.Input(shape=input_size, name='imgs', dtype='float32')
 r_label = L.Input(shape=(168,))
v_label = L.Input(shape=(11,))
c_label = L.Input(shape=(7,))
u_label = L.Input(shape=(1292,))
if backbone.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone[-1]}')
 x = model_fn(input_shape=input_size, weights=weights, include_top=False )(x_input)
 x = Generalized_mean_pooling2D()(x)
 weight_decay = 1e-4
x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Flatten()(x)
 x1 = Dense(512, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x1 = BatchNormalization()(x1)
 x1 = tf.nn.l2_normalize(x1, axis=1)
 root = CosFace(168, regularizer=regularizers.l2(weight_decay), name='root' )([x1, r_label])
 x1 = Dense(168, use_bias=False )(x1)
 root2 = Lambda(lambda x: K.softmax(x), name='root2' )(x1)
 x2 = Dense(512, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x2 = BatchNormalization()(x2)
 x2 = tf.nn.l2_normalize(x2, axis=1)
 vowel = CosFace(11, regularizer=regularizers.l2(weight_decay), name='vowel' )([x2, v_label])
 x2 = Dense(11, use_bias=False )(x2)
 vowel2 = Lambda(lambda x: K.softmax(x), name='vowel2' )(x2)
 x3 = Dense(512, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x3 = BatchNormalization()(x3)
 x3 = tf.nn.l2_normalize(x3, axis=1)
 consonant = CosFace(7, regularizer=regularizers.l2(weight_decay), name='consonant' )([x3, c_label])
 x3 = Dense(7, use_bias=False )(x3)
 consonant2 = Lambda(lambda x: K.softmax(x), name='consonant2' )(x3)
 x4 = Dense(1024, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x4 = BatchNormalization()(x4)
 x4 = tf.nn.l2_normalize(x4, axis=1)
 unique = CosFace(1292, regularizer=regularizers.l2(weight_decay), name='unique' )([x4, u_label])
 x4 = Dense(1292, use_bias=False )(x4)
 unique2 = Lambda(lambda x: K.softmax(x), name='unique2' )(x4)
 xx = tf.concat([x1, x2, x3, x4], axis=1)
 xx = tf.keras.activations.relu(xx)
 xx1 = Dense(168 )(xx)
 xx2 = Dense(11 )(xx)
 xx3 = Dense(7 )(xx)
 xx4 = Dense(1292 )(xx)
 xx1 = Dense(168, use_bias=False )(xx1)
 root3 = Lambda(lambda x: K.softmax(x), name='root3' )(xx1)
 xx2 = Dense(11, use_bias=False )(xx2)
 vowel3 = Lambda(lambda x: K.softmax(x), name='vowel3' )(xx2)
 xx3 = Dense(7, use_bias=False )(xx3)
 consonant3 = Lambda(lambda x: K.softmax(x), name='consonant3' )(xx3)
 xx4 = Dense(1292, use_bias=False )(xx4)
 unique3 = Lambda(lambda x: K.softmax(x), name='unique3' )(xx4)
 model = tf.keras.Model(
 inputs = [x_input,r_label,v_label,c_label,u_label],
outputs = [root, vowel, consonant, unique, root2, vowel2, consonant2, unique2, root3, vowel3, consonant3, unique3] 
)
 return model
def get_r_model(input_size, backbone='efficientnet-b0', weights='imagenet', tta=False):
print(f'Using backbone {backbone} and weights {weights}')
 x_input = L.Input(shape=input_size, name='imgs', dtype='float32')
 r_label = L.Input(shape=(168,))
if backbone.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone[-1]}')
 x = model_fn(input_shape=input_size, weights=weights, include_top=False )(x_input)
 x = Generalized_mean_pooling2D()(x)
 weight_decay = 1e-4
x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Flatten()(x)
 x1 = Dense(512, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x1 = BatchNormalization()(x1)
 x1 = tf.nn.l2_normalize(x1, axis=1)
 root = CosFace(168, regularizer=regularizers.l2(weight_decay), name='root' )([x1, r_label])
 x1 = Dense(168, use_bias=False )(x1)
 root2 = Lambda(lambda x: K.softmax(x), name='root2' )(x1)
 model = tf.keras.Model(
 inputs = [x_input,r_label],
outputs = [root, root2] 
)
 return model
def get_v_model(input_size, backbone='efficientnet-b0', weights='imagenet', tta=False):
print(f'Using backbone {backbone} and weights {weights}')
 x_input = L.Input(shape=input_size, name='imgs', dtype='float32')
 v_label = L.Input(shape=(11,))
if backbone.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone[-1]}')
 x = model_fn(input_shape=input_size, weights=weights, include_top=False )(x_input)
 x = Generalized_mean_pooling2D()(x)
 weight_decay = 1e-4
x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Flatten()(x)
 x2 = Dense(512, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x2 = BatchNormalization()(x2)
 x2 = tf.nn.l2_normalize(x2, axis=1)
 vowel = CosFace(11, regularizer=regularizers.l2(weight_decay), name='vowel' )([x2, v_label])
 x2 = Dense(11, use_bias=False )(x2)
 vowel2 = Lambda(lambda x: K.softmax(x), name='vowel2' )(x2)
 model = tf.keras.Model(
 inputs = [x_input,v_label],
outputs = [vowel, vowel2] 
)
 return model
def get_c_model(input_size, backbone='efficientnet-b0', weights='imagenet', tta=False):
print(f'Using backbone {backbone} and weights {weights}')
 x_input = L.Input(shape=input_size, name='imgs', dtype='float32')
 c_label = L.Input(shape=(7,))
if backbone.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone[-1]}')
 x = model_fn(input_shape=input_size, weights=weights, include_top=False )(x_input)
 x = Generalized_mean_pooling2D()(x)
 weight_decay = 1e-4
x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Flatten()(x)
 x3 = Dense(512, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(weight_decay))(x)
 x3 = BatchNormalization()(x3)
 x3 = tf.nn.l2_normalize(x3, axis=1)
 consonant = CosFace(7, regularizer=regularizers.l2(weight_decay), name='consonant' )([x3, c_label])
 x3 = Dense(7, use_bias=False )(x3)
 consonant2 = Lambda(lambda x: K.softmax(x), name='consonant2' )(x3)
 model = tf.keras.Model(
 inputs = [x_input,c_label],
outputs = [consonant, consonant2] 
)
 return model
test = pd.read_csv('.. /input/conways-reverse-game-of-life-2020/test.csv', index_col='id' )<create_dataframe>
def categorical_focal_loss(num_classes, gamma=2., alpha=.25, smooth_alpha=0.05):

 def categorical_focal_loss_fixed(y_true, y_pred):

 if smooth_alpha > 0:
y_true = y_true *(1 - smooth_alpha)+ smooth_alpha / num_classes
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
 epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 cross_entropy = -y_true * K.log(y_pred)
 loss = alpha * K.pow(1 - y_pred, gamma)* cross_entropy
return K.sum(loss, axis=1)
 return categorical_focal_loss_fixed
submit_df = pd.DataFrame(index=test.index, columns=['start_' + str(i)for i in range(625)] )<categorify>
def cutmix(images, labels, batch_size, image_size):
DIM = image_size[0]
new_image_indices = tf.cast(tf.random.uniform([batch_size], 0, batch_size), tf.int32)
 new_x = tf.cast(tf.random.uniform([batch_size], 0, DIM), tf.int32)
 new_y = tf.cast(tf.random.uniform([batch_size], 0, DIM), tf.int32)
 b = tf.random.uniform([batch_size], 0, 1)
 new_width = tf.cast(DIM * tf.math.sqrt(1-b), tf.int32)
 new_y0 = tf.math.maximum(0, new_y - new_width // 2)
 new_y1 = tf.math.minimum(DIM, new_y + new_width // 2)
 new_x0 = tf.math.maximum(0, new_x - new_width // 2)
 new_x1 = tf.math.minimum(DIM, new_x + new_width // 2)
 target = tf.broadcast_to(tf.range(DIM), shape=(batch_size, DIM))
mask_y = tf.math.logical_and(new_y0[:, tf.newaxis] <= target, target <= new_y1[:, tf.newaxis])
 mask_x = tf.math.logical_and(new_x0[:, tf.newaxis] <= target, target <= new_x1[:, tf.newaxis])
 mask = tf.cast(tf.math.logical_and(mask_y[:, :, tf.newaxis], mask_x[:, tf.newaxis, :]), tf.float32)
 new_images = images * tf.broadcast_to(1 - mask[:, :, :, tf.newaxis], [batch_size, DIM, DIM, 3])+ \
tf.gather(images, new_image_indices)* tf.broadcast_to(mask[:, :, :, tf.newaxis], [batch_size, DIM, DIM, 3])
 a = tf.cast(new_width ** 2 / DIM ** 2, tf.float32)
 for c in ['root', 'vowel', 'consonant', 'unique', 'root2', 'vowel2', 'consonant2', 'unique2', 'root3', 'vowel3', 'consonant3', 'unique3']:
y1, y2 = labels[c], tf.gather(labels[c], new_image_indices)
 labels[c] = y1 *(1-a)[:, tf.newaxis] + y2 * a[:, tf.newaxis]
return new_images, labels
def transform(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape):
AugParams = {
'd1' : 10,
'd2': 100,
'rotate' : 15,
'ratio' : 0.5
}
mask = GridMask(image_shape[0],
image_shape[1],
AugParams['d1'],
AugParams['d2'],
AugParams['rotate'],
tf.random.uniform(shape=[], minval=0.3, maxval=0.6)
)
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, tf.float32)
 def augmentation(image, label, input_size):
image = apply_grid_mask(image,(*input_size,3))
return image, label
<save_to_csv><EOS> 
def normalize_image(img, org_width, org_height, new_width, new_height):
img = 255 - img
img =(img *(255.0 / img.max())).astype(np.uint8)
 img = img.reshape(org_height, org_width)
 image_resized = cv2.resize(img,(new_width, new_height), interpolation=cv2.INTER_LINEAR)
 return image_resized
def predict_batch(img_batch, model, all_preds, unseen_th=0.5):
img_batch = np.float32(img_batch)
 bs = img_batch.shape[0]
r,v,c,u = np.zeros(( bs,168)) , np.zeros(( bs,11)) , np.zeros(( bs,7)) , np.zeros(( bs,1292))
if img_batch.ndim != 4:
img_batch = np.expand_dims(img_batch, 0)
 y_pred = [
model[0].predict([img_batch, r,v,c,u]),
model[1].predict([img_batch, r]),
model[2].predict([img_batch, v]),
model[3].predict([img_batch, c]),
]
for k in range(len(y_pred[0][0])) :
uu_p = y_pred[0][3][k].max(-1)
 if uu_p > unseen_th:
rr=y_pred[0][0][k].astype('float16')
 vv=y_pred[0][1][k].astype('float16')
 cc=y_pred[0][2][k].astype('float16')
 else:
rr=y_pred[1][0][k].astype('float16')
 vv=y_pred[2][0][k].astype('float16')
 cc=y_pred[3][0][k].astype('float16')
 all_preds['r'].append(rr)
 all_preds['v'].append(vv)
 all_preds['c'].append(cc)
 def postprocess(preds, num_classes, EXP = -1.2):
p0 = np.argmax(preds,axis=1)
 s = pd.Series(p0)
 vc = s.value_counts().sort_index()
df = pd.DataFrame({'a':np.arange(num_classes),'b':np.ones(num_classes)})
 df.b = df.a.map(vc)
 df.fillna(df.b.min() ,inplace=True)
 mat1 = np.diag(df.b.astype('float32')**EXP)
 p0 = np.argmax(preds.dot(mat1), axis=1)
 return p0
def main() :
parser = argparse.ArgumentParser()
parser.add_argument('--seed', type=int, default=123)
 parser.add_argument('--input_size', type=str, default='224,224')
 parser.add_argument('--batch_size', type=int, default=32)
 parser.add_argument('--backbone', type=str, default='efficientnet-b3')
 parser.add_argument('--weights', type=str, default='.. /input/efnet-b3-cosface-val9-3/best.h5')
 parser.add_argument('--r_weights', type=str, default='.. /input/efnet-b3-rcv-models/r_best.h5')
 parser.add_argument('--v_weights', type=str, default='.. /input/efnet-b3-rcv-models/v_best.h5')
 parser.add_argument('--c_weights', type=str, default='.. /input/efnet-b3-rcv-models/c_best.h5')
 parser.add_argument('--mode',type=str, default='all')
 args, _ = parser.parse_known_args()
org_height = 137
org_width = 236
args.input_size = tuple(int(x)for x in args.input_size.split(','))
np.random.seed(args.seed)
 tf.random.set_seed(args.seed)
 u_model = get_model(input_size=args.input_size +(3,), backbone=args.backbone,weights=None)
 u_model.load_weights(args.weights)
 r_model = get_r_model(input_size=args.input_size +(3,), backbone=args.backbone,weights=None)
 r_model.load_weights(args.r_weights)
 v_model = get_v_model(input_size=args.input_size +(3,), backbone=args.backbone,weights=None)
 v_model.load_weights(args.v_weights)
 c_model = get_c_model(input_size=args.input_size +(3,), backbone=args.backbone,weights=None)
 c_model.load_weights(args.c_weights)
 model = [u_model, r_model, v_model, c_model]
all_preds = dict(r = [], v = [], c = [])
 all_image_ids = []
img_batch = []
for i in tqdm(range(4)) :
parquet_fn = f'.. /input/bengaliai-cv19/test_image_data_{i}.parquet'
all_images = pd.read_parquet(parquet_fn)
 image_ids = all_images['image_id'].values
all_images = all_images.iloc[:, 1:].values
for k in tqdm(range(len(image_ids))):
all_image_ids.append(image_ids[k])
 img = all_images[k]
img = normalize_image(img, org_width, org_height, args.input_size[1], args.input_size[0])
 img_batch.append(np.dstack([img] * 3))
if len(img_batch)>= args.batch_size:
predict_batch(img_batch, model, all_preds)
 img_batch = []
if len(img_batch)> 0:
predict_batch(img_batch, model, all_preds)
 img_batch = []
np.save(f'r_preds{i}', all_preds['r'])
 np.save(f'v_preds{i}', all_preds['v'])
 np.save(f'c_preds{i}', all_preds['c'])
 all_preds = dict(r = [], v = [], c = [])
 del all_images
gc.collect()
del u_model, r_model, v_model, c_model, model
gc.collect()
r_preds = np.concatenate([np.load(f'r_preds{i}.npy')for i in range(4)] ,axis=0)
 r_preds = postprocess(r_preds, 168, -1.2)
 v_preds = np.concatenate([np.load(f'v_preds{i}.npy')for i in range(4)] ,axis=0)
 v_preds = postprocess(v_preds, 11, -1.2)
 c_preds = np.concatenate([np.load(f'c_preds{i}.npy')for i in range(4)] ,axis=0)
 c_preds = postprocess(c_preds, 7, -0.5)
 row_id, target = [], []
for iid, r, v, c in zip(all_image_ids, r_preds, v_preds, c_preds):
row_id.append(iid + '_grapheme_root')
 target.append(r)
 row_id.append(iid + '_vowel_diacritic')
 target.append(v)
 row_id.append(iid + '_consonant_diacritic')
 target.append(c)
 sub_fn = 'submission.csv'
sub = pd.DataFrame({'row_id': row_id, 'target': target})
 sub.to_csv(sub_fn, index=False)
 print(f'Done wrote to {sub_fn}')
 main()