Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<prepare_x_and_y>

 GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b0-355c32eb.pth',
'efficientnet-b1': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b1-f1951068.pth',
'efficientnet-b2': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b2-8bb594d6.pth',
'efficientnet-b3': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b3-5fb5a3c3.pth',
'efficientnet-b4': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b4-6ed6700e.pth',
'efficientnet-b5': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b5-b6417697.pth',
'efficientnet-b6': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b6-c76e70fd.pth',
'efficientnet-b7': 'https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b7-dcc49843.pth',
}
url_map_advprop = {
'efficientnet-b0': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b0-b64d5a18.pth',
'efficientnet-b1': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b1-0f3ce85a.pth',
'efficientnet-b2': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b2-6e9d97e5.pth',
'efficientnet-b3': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b3-cdd7c0f4.pth',
'efficientnet-b4': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b4-44fb3a87.pth',
'efficientnet-b5': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b5-86493f6b.pth',
'efficientnet-b6': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b6-ac80338e.pth',
'efficientnet-b7': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b7-4652b6dd.pth',
'efficientnet-b8': 'https://publicmodels.blob.core.windows.net/container/advprop/efficientnet-b8-22a8fe65.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True, advprop=False):

 url_map_ = url_map_advprop if advprop else url_map
state_dict = model_zoo.load_url(url_map_[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert set(res.missing_keys)== set(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
X_train = [delta_train, stop_train]
X_valid = [delta_validation, stop_validation]
X_test = [test_df["delta"].values, test_df[stop_features].values.reshape(-1, 25, 25, 1 ).astype(float)]
X_all_train = [train_df["delta"].values, train_df[stop_features].values.reshape(-1, 25, 25, 1 ).astype(float)]
Y_all_train = train_df[start_features].values.reshape(-1, 25, 25, 1 ).astype(float )<choose_model_class>
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
 x = self._swish(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.extract_features(inputs)
 x = self._avg_pooling(x)
 x = x.view(bs, -1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return cls(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, advprop=False, num_classes=1000, in_channels=3):
model = cls.from_name(model_name, override_params={'num_classes': num_classes})
 load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000), advprop=advprop)
 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)
 out_channels = round_filters(32, model._global_params)
 model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 valid_models = ['efficientnet-b'+str(i)for i in range(9)]
if model_name not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))
def conv_block(inputs, filters, index, activation='relu'):

 x = layers.Conv2D(filters, kernel_size=(3,3), padding="SAME", name=f'conv{index}' )(inputs)
 x = layers.BatchNormalization()(x)
 return layers.Activation(activation, name=activation + str(index))(x )<choose_model_class>
import pandas as pd
import numpy as np
from tqdm import tqdm
import os
import cv2
import torch
from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
import torchvision.models as tvm
import torch.nn.functional as F
import random
def create_model(dropout_prob=0.3):

 input_delta = layers.Input(shape=(1,), name="input_delta")
 dense_delta = layers.Dense(25*25, name='dense_delta' )(input_delta)
 dense_reshape = layers.Reshape(( 25,25,1), name='reshape_delta' )(dense_delta)
 input_image = layers.Input(shape=(25,25,1), name="input_images")
 all_inputs = layers.Concatenate(axis=3, name='concatenate' )([input_image, dense_reshape])
 x = conv_block(all_inputs, 32, index=1)
 x = layers.Dropout(dropout_prob )(x)
 x = conv_block(x, 128, index=2)
 x = layers.Dropout(dropout_prob )(x)
 x = conv_block(x, 256, index=3)
 x = layers.Dropout(dropout_prob )(x)
 x = conv_block(x, 64, index=4)
 x = layers.Dropout(dropout_prob )(x)
 out = conv_block(x, 1, index=5, activation='sigmoid')
 return Model(inputs=[input_delta, input_image], outputs=out )<choose_model_class>
def set_random_state(seed_value):
random.seed(seed_value)
 torch.manual_seed(seed_value)
 torch.cuda.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
np.random.seed(seed_value)
 set_random_state(4690 )
model = create_model()
model.compile(loss="bce", optimizer=tf.keras.optimizers.Adam() , metrics=["accuracy"] )<train_model>
package_path = '.. /input/myscripts/'
sys.path.append(package_path )
history = model.fit(x=X_train,
y=Y_train,
batch_size=128,
epochs=25,
validation_data=(X_valid, Y_valid))<choose_model_class>
from bengali_optimizers import *
model = create_model()
model.compile(loss="bce", optimizer=tf.keras.optimizers.Adam() , metrics=["accuracy"] )<train_model>
class CNN_Model(nn.Module):
def __init__(self,model_name='efficientnet-b5',global_dim=2048):
super(CNN_Model,self ).__init__()
self.cnn_backbone = EfficientNet.from_name(model_name)
 self.cnn_backbone.set_swish(memory_efficient=False)
 self.cnn_head_grapheme_root = CNN_Head(168,global_dim,1024,512)
 self.cnn_head_vowel_diacritic = CNN_Head(11,global_dim,1024,512)
 self.cnn_head_consonant_diacritic = CNN_Head(7,global_dim,1024,512)
 def forward(self, x):
global_feat = self.cnn_backbone.extract_features(x)
 global_feat =(F.adaptive_avg_pool2d(global_feat,1)+F.adaptive_max_pool2d(global_feat,1)) *0.5
global_feat = global_feat.view(global_feat.size(0), -1)
 grapheme_output = self.cnn_head_grapheme_root(global_feat)
 vowel_diacritic_output = self.cnn_head_vowel_diacritic(global_feat)
 consonant_diacritic_output = self.cnn_head_consonant_diacritic(global_feat)
 return grapheme_output,vowel_diacritic_output,consonant_diacritic_output
class CNN_Head(nn.Sequential):
def __init__(self, classes, input_dim, linear1_output_dim, linear2_output_dim,
use_batch_norm=True, use_activation_function=False,
global_dropout=0.0, lin1_dropout=0.0, lin2_dropout=0.00):
super(CNN_Head, self ).__init__()
if global_dropout > 0.0:
self.add_module('drop_1',nn.Dropout(p=global_dropout,inplace=False))
self.add_module('linear_1',nn.Linear(input_dim,linear1_output_dim))
if use_batch_norm:
self.add_module('norm_1', nn.BatchNorm1d(linear1_output_dim))
if use_activation_function:
self.add_module('activ_1',nn.ReLU(inplace=False))
if lin1_dropout > 0.0:
self.add_module('drop_2',nn.Dropout(p=lin1_dropout,inplace=False))
self.add_module('linear_2',nn.Linear(linear1_output_dim,linear2_output_dim))
if use_batch_norm:
self.add_module('norm_2', nn.BatchNorm1d(linear2_output_dim))
if use_activation_function:
self.add_module('activ_2',nn.ReLU(inplace=False))
if lin2_dropout > 0.0:
self.add_module('drop_3',nn.Dropout(p=lin2_dropout,inplace=False))
self.add_module('linear_final',nn.Linear(linear2_output_dim,classes))
history = model.fit(x=X_all_train,
y=Y_all_train,
batch_size=128,
epochs=20,
)<predict_on_test>
model_1 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission46/checkpoint_best_Average_recall.pth')
 model_1.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_1 = model_1.cuda()
model_1.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))
test_prediction = model.predict(X_test)
 test_prediction =(test_prediction > 0.5 ).astype(int ).reshape(test_df.shape[0], -1)
 sub = test_df[["id"]].copy()
tmp = pd.DataFrame(test_prediction, columns=start_features)
 submission = sub.join(tmp )<save_to_csv>
model_2 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission58/checkpoint_best_Average_recall.pth')
 model_2.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_2 = model_2.cuda()
model_2.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))
submission.to_csv("submission.csv", index=False )<save_to_csv>
model_3 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission57/checkpoint_best_Average_recall.pth')
 model_3.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_3 = model_3.cuda()
model_3.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))
submission.to_csv("submission.csv", index=False )<define_search_model>
model_4 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission73/checkpoint_best_Average_recall.pth')
 model_4.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_4 = model_4.cuda()
model_4.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))
class OneIterationReverseNet(nn.Module):
def __init__(self, info_ch, ch):
super().__init__()
self.relu = nn.ReLU()
self.conv1 = nn.Conv2d(info_ch, ch, 5, padding=4, padding_mode='circular')
 self.conv2 = nn.Conv2d(ch, ch, 3,)
 self.conv3 = nn.Conv2d(ch, info_ch, 3)
 def forward(self, inp):
x = self.relu(self.conv1(inp))
x = self.relu(self.conv2(x))
x = self.relu(self.conv3(x))
return x
class ReverseModel(nn.Module):
def __init__(self, info_ch=64, ch=128):
super().__init__()
self.relu = nn.ReLU()
self.encoder = nn.Conv2d(1, info_ch, 7, padding=3, padding_mode='circular')
 self.reverse_one_iter = OneIterationReverseNet(info_ch, ch)
 self.decoder = nn.Conv2d(info_ch, 1, 3, padding=1, padding_mode='circular')
 def forward(self, stop, delta):
x = self.relu(self.encoder(stop-0.5))
for i in range(delta.max().item()):
y = self.reverse_one_iter(x)
 mask =(delta > i ).reshape(-1,1,1,1)
 x = x*(~mask ).float() + y*mask.float()
x = self.decoder(x)
 return x<import_modules>
model_5 = CNN_Model()
checkpoint_dict = torch.load('/kaggle/input/modelsubmission74/checkpoint_best_Average_recall.pth')
 model_5.load_state_dict(checkpoint_dict['Model_state_dict'])
 model_5 = model_5.cuda()
model_5.eval()
print('Validation Average Recall {} \
Validation Grapheme Root Recall {} \
Validation Vowel Diacritic Recall {} \
Validation Consonant Diacritic Recall {} \
Validation Loss {} '.format(checkpoint_dict['Current_average_recall'],checkpoint_dict['Current_grapheme_root_recall'],checkpoint_dict['Current_vowel_diacritic_recall'], \
checkpoint_dict['Current_consonant_diacritic_recall'],checkpoint_dict['Current_val_Loss']))
import pandas as pd
from sklearn.model_selection import train_test_split<load_from_csv>
class BengaliAiDataset(Dataset):
def __init__(self, all_imgs):
super(BengaliAiDataset, self ).__init__()
self.all_imgs = all_imgs
def __len__(self):
return self.all_imgs.shape[0]
def __getitem__(self, index):
img = self.all_imgs[index,:,:]
img = img[:,:,np.newaxis]
img = np.concatenate([img,img,img],axis=2)
 img =(img/255 ).astype(np.float32)
 img = np.transpose(img,(2, 0, 1))
img = torch.from_numpy(img ).float()
return img
train_val = pd.read_csv('/kaggle/input/conways-reverse-game-of-life-2020/train.csv', index_col='id')
 test = pd.read_csv('/kaggle/input/conways-reverse-game-of-life-2020/test.csv', index_col='id')
 train, val = train_test_split(train_val, test_size=0.2, shuffle=True, random_state=42, stratify=train_val['delta'] )<data_type_conversions>
HEIGHT = 137
WIDTH = 236
TEST = ['/kaggle/input/bengaliai-cv19/test_image_data_0.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_1.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_2.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_3.parquet']
all_grapheme_labels = []
all_vowel_diacritic_labels = []
all_consonant_diacritic_labels = []
torch.set_grad_enabled(False)
 with torch.no_grad() :
for fname in TEST:
df = pd.read_parquet(fname)
 data = df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 all_imgs = []
for idx in tqdm(range(len(df))):
img = data[idx].astype(np.uint8)
 img = img[np.newaxis,...]
all_imgs.append(img)
 all_imgs = np.concatenate(all_imgs,axis=0)
 ds_test = BengaliAiDataset(all_imgs)
 dl_test = DataLoader(ds_test, shuffle=False, batch_size=100, num_workers=4, pin_memory=True)
 for i,(imgs)in enumerate(dl_test):
imgs = imgs.cuda()
grapheme_output_1,vowel_diacritic_output_1,consonant_diacritic_output_1 = model_1(imgs)
 grapheme_output_2,vowel_diacritic_output_2,consonant_diacritic_output_2 = model_2(imgs)
 grapheme_output_3,vowel_diacritic_output_3,consonant_diacritic_output_3 = model_3(imgs)
 grapheme_output_4,vowel_diacritic_output_4,consonant_diacritic_output_4 = model_4(imgs)
 grapheme_output_5,vowel_diacritic_output_5,consonant_diacritic_output_5 = model_5(imgs)
 grapheme_output =(grapheme_output_1+grapheme_output_2+grapheme_output_3+grapheme_output_4+grapheme_output_5)/5
vowel_diacritic_output =(vowel_diacritic_output_1+vowel_diacritic_output_2+vowel_diacritic_output_3+vowel_diacritic_output_4+vowel_diacritic_output_5)/5
consonant_diacritic_output =(consonant_diacritic_output_1+consonant_diacritic_output_2+consonant_diacritic_output_3+consonant_diacritic_output_4+consonant_diacritic_output_5)/5
submission_grapheme_root_labels = torch.argmax(grapheme_output.clone() ,dim=1)
 submission_vowel_diacritic_labels = torch.argmax(vowel_diacritic_output.clone() ,dim=1)
 submission_consonant_diacritic_labels = torch.argmax(consonant_diacritic_output.clone() ,dim=1)
 submission_grapheme_root_labels = submission_grapheme_root_labels.tolist()
submission_vowel_diacritic_labels = submission_vowel_diacritic_labels.tolist()
submission_consonant_diacritic_labels = submission_consonant_diacritic_labels.tolist()
all_grapheme_labels = all_grapheme_labels+submission_grapheme_root_labels
all_vowel_diacritic_labels = all_vowel_diacritic_labels+submission_vowel_diacritic_labels
all_consonant_diacritic_labels = all_consonant_diacritic_labels+submission_consonant_diacritic_labels
<load_pretrained><EOS> 
row_id = []
target = []
for i in tqdm(range(len(all_vowel_diacritic_labels))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [all_grapheme_labels[i], all_vowel_diacritic_labels[i], all_consonant_diacritic_labels[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False )