Kaggle competition: Bengali.AI Handwritten Grapheme Classification
n_epochs = 250
max_len = 80
batch_size = 16 * strategy.num_replicas_in_sync<load_from_csv>
import torch
import numpy as np
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
from torchvision import transforms as torchtransforms
import cv2
import torch.nn as nn
from tqdm import tqdm_notebook as tqdm
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import time
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv' )<load_pretrained>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 modelpath = "/kaggle/input/se-resnext50-32x4d-fold2/se_resnext50_32x4d_fold2.pkl"
root_path="/kaggle/input/bengaliai-cv19"
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )<categorify>
simple_transform_valid = torchtransforms.Compose([
torchtransforms.ToTensor() ,
torchtransforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ] )
train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=max_len
)<split>
__all__ = ['SENet', 'se_resnext50_32x4d']
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):
super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'], train.label.values,
test_size=0.20, random_state=2020 
)
 x_test = test_encoded['input_ids']<split>
model = se_resnext50_32x4d(pretrained=None)
 model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.last_linear = nn.Linear(model.last_linear.in_features, 186)
 modelvalue = torch.load(modelpath, map_location='cuda:0')
 newmodelvalue = {}
for kv in modelvalue:
newmodelvalue[kv[4:]]=modelvalue[kv]
model.load_state_dict(newmodelvalue)
 model = model.to(device )
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
 )<categorify>
allpathes=[]
allpreds_root = []
allpreds_vowel = []
allpreds_consonant = []
tAllBegin = time.time()
for i in range(4):
test_csv = pd.read_parquet(os.path.join(root_path, f'test_image_data_{i}.parquet'))
tBegin = time.time()
dataloaders = make_loader(data_folder = test_csv,
batch_size=8,
num_workers = 2,
is_shuffle = False)
 pathes, logit1, logit2, logit3 = getmodeleval(model, dataloaders)
 preds_root = np.argmax(logit1, axis=1)
 preds_vowel = np.argmax(logit2, axis=1)
 preds_consonant = np.argmax(logit3, axis=1)
 allpathes.extend(pathes)
 allpreds_root.extend(preds_root.tolist())
 allpreds_vowel.extend(preds_vowel.tolist())
 allpreds_consonant.extend(preds_consonant.tolist())
 tEnd = time.time()
print(i, int(round(tEnd * 1000)) - int(round(tBegin * 1000)) , "ms")
 tAllEnd = time.time()
print(len(allpathes), len(allpreds_root), len(allpreds_vowel), len(allpreds_consonant), int(round(tAllEnd * 1000)) - int(round(tAllBegin * 1000)) , "ms" )
<train_model><EOS> 
row_id=[]
target=[]
for idx, image_id in enumerate(allpathes):
target.extend([allpreds_consonant[idx]])
 target.extend([allpreds_root[idx]])
 target.extend([allpreds_vowel[idx]])
 row_id.extend([str(image_id)+ '_consonant_diacritic'])
 row_id.extend([str(image_id)+ '_grapheme_root'])
 row_id.extend([str(image_id)+ '_vowel_diacritic'])
 submission_df = pd.read_csv(root_path + '/sample_submission.csv')
 submission_df.target = np.hstack(np.array(target ).astype(np.int))
print(submission_df.head(10))
submission_df.to_csv('submission.csv', index=False)
 print("================end ======================" )