Kaggle competition: Bengali.AI Handwritten Grapheme Classification
XLM_model = "jplu/tf-xlm-roberta-large"
xlm_tokenizer = AutoTokenizer.from_pretrained(XLM_model)
 X_train_ids, X_train_masks, _ = tokeniZer(shuffled_data,xlm_tokenizer )<choose_model_class>
import torch
import numpy as np
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
from torchvision import transforms as torchtransforms
import cv2
import torch.nn as nn
from tqdm import tqdm_notebook as tqdm
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import time
def create_xlm(transformer_layer, random_seed, learning_rate = 1e-5):
tf.keras.backend.clear_session()
tf.random.set_seed(random_seed)
 with tpu_strategy.scope() :
input_ids = Input(shape =(max_len,), dtype = tf.int32)
 input_masks = Input(shape =(max_len,), dtype = tf.int32)
 roberta = TFAutoModel.from_pretrained(transformer_layer)
 roberta = roberta([input_ids, input_masks])[0]
out = GlobalAveragePooling1D()(roberta)
 out = Dense(3, activation = 'softmax' )(out)
 model = Model(inputs = [input_ids, input_masks], outputs = out)
 model.compile(
 optimizer = Adam(lr = learning_rate),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 model.summary()
return model
Xlm = create_xlm(XLM_model ,123443334, 1e-5 )<train_model>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 modelpath = "/kaggle/input/se-resnext50-32x4d-fold2/se_resnext50_32x4d_fold2.pkl"
root_path="/kaggle/input/bengaliai-cv19"
history_xlm = Xlm.fit([X_train_ids, X_train_masks], shuffled_data['label'],
batch_size = batch_size,
validation_split = 0.2,
epochs = 39, callbacks = callbacks )<define_variables>
simple_transform_valid = torchtransforms.Compose([
torchtransforms.ToTensor() ,
torchtransforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ] )
input_ids_test_xml, input_masks_test_xml, _ = tokeniZer(test_df, xlm_tokenizer)
 predictions_xlm = Xlm.predict([input_ids_test_xml, input_masks_test_xml])
 predictions = predictions_xlm
final = np.argmax(predictions, axis = 1)
 submission = pd.DataFrame()
submission['id'] = test_df['id']
submission['prediction'] = final.astype(np.int32)
 submission.to_csv('submission.csv', index = False )<set_options>
__all__ = ['SENet', 'se_resnext50_32x4d']
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):
super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
!pip install --quiet googletrans
warnings.filterwarnings('ignore')
 print("Currently using Tensorflow version " + tf.__version__ )<define_variables>
model = se_resnext50_32x4d(pretrained=None)
 model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.last_linear = nn.Linear(model.last_linear.in_features, 186)
 modelvalue = torch.load(modelpath, map_location='cuda:0')
 newmodelvalue = {}
for kv in modelvalue:
newmodelvalue[kv[4:]]=modelvalue[kv]
model.load_state_dict(newmodelvalue)
 model = model.to(device )
SEED = 34
def seed_everything(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything(SEED )<choose_model_class>
allpathes=[]
allpreds_root = []
allpreds_vowel = []
allpreds_consonant = []
tAllBegin = time.time()
for i in range(4):
test_csv = pd.read_parquet(os.path.join(root_path, f'test_image_data_{i}.parquet'))
tBegin = time.time()
dataloaders = make_loader(data_folder = test_csv,
batch_size=8,
num_workers = 2,
is_shuffle = False)
 pathes, logit1, logit2, logit3 = getmodeleval(model, dataloaders)
 preds_root = np.argmax(logit1, axis=1)
 preds_vowel = np.argmax(logit2, axis=1)
 preds_consonant = np.argmax(logit3, axis=1)
 allpathes.extend(pathes)
 allpreds_root.extend(preds_root.tolist())
 allpreds_vowel.extend(preds_vowel.tolist())
 allpreds_consonant.extend(preds_consonant.tolist())
 tEnd = time.time()
print(i, int(round(tEnd * 1000)) - int(round(tBegin * 1000)) , "ms")
 tAllEnd = time.time()
print(len(allpathes), len(allpreds_root), len(allpreds_vowel), len(allpreds_consonant), int(round(tAllEnd * 1000)) - int(round(tAllBegin * 1000)) , "ms" )
<load_from_csv><EOS> 
row_id=[]
target=[]
for idx, image_id in enumerate(allpathes):
target.extend([allpreds_consonant[idx]])
 target.extend([allpreds_root[idx]])
 target.extend([allpreds_vowel[idx]])
 row_id.extend([str(image_id)+ '_consonant_diacritic'])
 row_id.extend([str(image_id)+ '_grapheme_root'])
 row_id.extend([str(image_id)+ '_vowel_diacritic'])
 submission_df = pd.read_csv(root_path + '/sample_submission.csv')
 submission_df.target = np.hstack(np.array(target ).astype(np.int))
print(submission_df.head(10))
submission_df.to_csv('submission.csv', index=False )