Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<compute_test_metric>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
one_way = calculate_one_way_anova('stopword_count')
 print(f'The F score when comaring all the authors: {one_way.statistic}, which reflect pvalue of: {one_way.pvalue}' )<filter>
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )
t_EAP_HPL = stats.ttest_ind(train['stopword_count'][train['author'] == 'EAP'], train['stopword_count'][train['author'] == 'HPL'])
 t_MWS_HPL = stats.ttest_ind(train['stopword_count'][train['author'] == 'MWS'], train['stopword_count'][train['author'] == 'HPL'])
 t_MWS_EAP = stats.ttest_ind(train['stopword_count'][train['author'] == 'MWS'], train['stopword_count'][train['author'] == 'EAP'])
 print(f'The T score when comaring EAP and HPL: {t_EAP_HPL.statistic} which reflect pvalue of: {t_EAP_HPL.pvalue}')
 print(f'The T score when comaring MWS and HPL: {t_MWS_HPL.statistic} which reflect pvalue of: {t_MWS_HPL.pvalue}')
 print(f'The T score when comaring MWS and EAP: {t_MWS_EAP.statistic} which reflect pvalue of: {t_MWS_EAP.pvalue}' )<categorify>
debug=False
submission=True
batch_size=256
device='cuda:0'
out='.'
def author_to_label(author):
labels = {'EAP': 0,'HPL': 1,'MWS': 2}
return labels[author]
def label_to_author(label):
authors = ['EAP','HPL','MWS']
return authors[int(label)]<categorify>
datadir = Path('/kaggle/input/bengaliai-cv19')
 featherdir = Path('/kaggle/input/bengaliaicv19feather')
 outdir = Path('.' )
train['label'] = train['author'].apply(author_to_label)
 dummies = pd.get_dummies(train['author'])
 train = pd.concat([train,dummies], axis=1)
 train.head(1 )<categorify>
class TrainDataSet(data.Dataset):
def __init__(self, excerpts, labels, tokenizer, max_len):
self.excerpts = excerpts
self.labels = labels
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.excerpts)
 def __getitem__(self, item):
excerpt = str(self.excerpts[item])
 encoding = self.tokenizer.encode_plus(
 excerpt,
max_length = self.max_len,
add_special_tokens=True,
pad_to_max_length=True,
return_attention_mask=True,
return_token_type_ids=False,
return_tensors='pt' 
)
 return {
'excerpt_text': excerpt,
'input_ids': encoding['input_ids'].flatten() ,
'attention_mask': encoding['attention_mask'].flatten() ,
'targets': torch.tensor(self.labels[item], dtype=torch.long)
 }<categorify>

 class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError
class TestDataSet(data.Dataset):
def __init__(self, ids, excerpts, tokenizer, max_len):
self.ids = ids
self.excerpts = excerpts
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.excerpts)
 def __getitem__(self, item):
excerpt = str(self.excerpts[item])
 excerpt_id = str(self.ids[item])
 encoding = self.tokenizer.encode_plus(
 excerpt,
max_length = self.max_len,
add_special_tokens=True,
pad_to_max_length=True,
return_attention_mask=True,
return_token_type_ids=False,
return_tensors='pt' 
)
 return {
'excerpt_id': excerpt_id,
'excerpt_text': excerpt,
'input_ids': encoding['input_ids'].flatten() ,
'attention_mask': encoding['attention_mask'].flatten() ,
}<categorify>
class BengaliAIDataset(DatasetMixin):
def __init__(self, images, labels=None, transform=None, indices=None):
super(BengaliAIDataset, self ).__init__(transform=transform)
 self.images = images
self.labels = labels
if indices is None:
indices = np.arange(len(images))
self.indices = indices
self.train = labels is not None
def __len__(self):

 return len(self.indices)
 def get_example(self, i):

 i = self.indices[i]
x = self.images[i]
x =(255 - x ).astype(np.float32)/ 255.
 if self.train:
y = self.labels[i]
return x, y
else:
return x
def create_train_data_loader(df, tokenizer, max_len, batch_size):
excerpts = df['text'].to_numpy() ,
print(f'Excerpts size: {len(excerpts)}')
 labels = df['label'].to_numpy() ,
dataset = TrainDataSet(excerpts=excerpts[0], labels=labels[0], tokenizer=tokenizer, max_len=max_len)
 print(f'Dataset size: {len(dataset)}')
 return data.DataLoader(dataset, batch_size=batch_size, num_workers=4, shuffle=True)
 def create_test_data_loader(df, tokenizer, max_len, batch_size):
excerpts = df['text'].to_numpy() ,
ids = df['id'].to_numpy() ,
print(f'Excerpts size: {len(excerpts)}')
 dataset = TestDataSet(ids= ids[0], excerpts=excerpts[0], tokenizer=tokenizer, max_len=max_len)
 print(f'Dataset size: {len(dataset)}')
 return data.DataLoader(dataset, batch_size=batch_size, num_workers=4 )<split>
def add_gaussian_noise(x, sigma):
x += np.random.randn(*x.shape)* sigma
x = np.clip(x, 0., 1.)
return x
class Transform:
def __init__(self, affine=True, crop=True, size=(64, 64),
normalize=True, train=True, threshold=40.,
sigma=-1.) :
self.affine = affine
self.crop = crop
self.size = size
self.normalize = normalize
self.train = train
self.threshold = threshold / 255.
 self.sigma = sigma / 255.
 def __call__(self, example):
if self.train:
x, y = example
else:
x = example
if self.affine:
x = affine_image(x)
 if self.crop:
x = crop_char_image(x, threshold=self.threshold)
 if self.size is not None:
x = resize(x, size=self.size)
 if self.sigma > 0.:
x = add_gaussian_noise(x, sigma=self.sigma)
 if self.normalize:
x =(x.astype(np.float32)- 0.0692)/ 0.2051
if x.ndim == 2:
x = x[None, :, :]
x = x.astype(np.float32)
 if self.train:
y = y.astype(np.int64)
 return x, y
else:
return x
train_set, val_set = train_test_split(train, test_size=0.2 )<define_variables>
def residual_add(lhs, rhs):
lhs_ch, rhs_ch = lhs.shape[1], rhs.shape[1]
if lhs_ch < rhs_ch:
out = lhs + rhs[:, :lhs_ch]
elif lhs_ch > rhs_ch:
out = torch.cat([lhs[:, :rhs_ch] + rhs, lhs[:, rhs_ch:]], dim=1)
 else:
out = lhs + rhs
return out
BATCH_SIZE = 16
MAX_LEN = 160<load_pretrained>
class LazyLoadModule(nn.Module):

 _lazy_buffer_keys: List[str] = []
_lazy_parameter_keys: List[str] = []
def __init__(self):
super(LazyLoadModule, self ).__init__()
for k in self._lazy_buffer_keys:
self.register_buffer(k, torch.tensor([]))
for k in self._lazy_parameter_keys:
self.register_parameter(k, None)
 self._register_load_state_dict_pre_hook(self._hook)
 def _hook(self, state_dict, prefix, local_metadata, strict, missing_keys,
unexpected_keys, error_msgs):
for key in self._lazy_buffer_keys:
self.register_buffer(key, state_dict[prefix + key])
 for key in self._lazy_parameter_keys:
self.register_parameter(key, Parameter(state_dict[prefix + key]))
train_data_loader = create_train_data_loader(train_set, tokenizer, max_len=MAX_LEN, batch_size=BATCH_SIZE)
 val_data_loader = create_train_data_loader(val_set, tokenizer, max_len=MAX_LEN, batch_size=BATCH_SIZE )<choose_model_class>
class LazyLinear(LazyLoadModule):

 __constants__ = ['bias', 'in_features', 'out_features']
_lazy_parameter_keys = ['weight']
def __init__(self, in_features, out_features, bias=True):
super(LazyLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
if bias:
self.bias = Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 if in_features is not None:
self.weight = Parameter(torch.Tensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight, a=math.sqrt(5))
if self.bias is not None:
fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
 bound = 1 / math.sqrt(fan_in)
 init.uniform_(self.bias, -bound, bound)
 def forward(self, input):
if self.weight is None:
self.in_features = input.shape[-1]
self.weight = Parameter(torch.Tensor(self.out_features, self.in_features))
self.reset_parameters()
self.to(input.device)
 return F.linear(input, self.weight, self.bias)
 def extra_repr(self):
return 'in_features={}, out_features={}, bias={}'.format(
 self.in_features, self.out_features, self.bias is not None 
)
class DistilBertAuthorClassifier(nn.Module):
def __init__(self):
super(DistilBertAuthorClassifier, self ).__init__()
self.num_labels = 3
self.softmax = nn.Softmax(dim=1)
 self.distilbert = DistilBertModel.from_pretrained('distilbert-base-cased')
 self.classifier = nn.Linear(self.distilbert.config.dim, 3)
 self.dropout = nn.Dropout(0.3)
 nn.init.xavier_normal_(self.classifier.weight)
 def forward(self, input_ids, attention_mask):
distilbert_output = self.distilbert(input_ids=input_ids,
attention_mask=attention_mask)
 hidden_state = distilbert_output[0]
pooled_output = hidden_state[:, 0, :]
pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits<choose_model_class>
class LinearBlock(nn.Module):
def __init__(self, in_features, out_features, bias=True,
use_bn=True, activation=F.relu, dropout_ratio=-1, residual=False,):
super(LinearBlock, self ).__init__()
if in_features is None:
self.linear = LazyLinear(in_features, out_features, bias=bias)
 else:
self.linear = nn.Linear(in_features, out_features, bias=bias)
 if use_bn:
self.bn = nn.BatchNorm1d(out_features)
 if dropout_ratio > 0.:
self.dropout = nn.Dropout(p=dropout_ratio)
 else:
self.dropout = None
self.activation = activation
self.use_bn = use_bn
self.dropout_ratio = dropout_ratio
self.residual = residual
def __call__(self, x):
h = self.linear(x)
 if self.use_bn:
h = self.bn(h)
 if self.activation is not None:
h = self.activation(h)
 if self.residual:
h = residual_add(h, x)
 if self.dropout_ratio > 0:
h = self.dropout(h)
 return h
gc.collect()
model = DistilBertAuthorClassifier()
model = model.to(device )<train_model>
class PretrainedCNN(nn.Module):
def __init__(self, model_name='se_resnext101_32x4d',
in_channels=1, out_dim=10, use_bn=True,
pretrained=None):
super(PretrainedCNN, self ).__init__()
self.conv0 = nn.Conv2d(
 in_channels, 3, kernel_size=3, stride=1, padding=1, bias=True)
 self.base_model = pretrainedmodels.__dict__[model_name](pretrained=pretrained)
 activation = F.leaky_relu
self.do_pooling = True
if self.do_pooling:
inch = self.base_model.last_linear.in_features
else:
inch = None
hdim = 512
lin1 = LinearBlock(inch, hdim, use_bn=use_bn, activation=activation, residual=False)
 lin2 = LinearBlock(hdim, out_dim, use_bn=use_bn, activation=None, residual=False)
 self.lin_layers = Sequential(lin1, lin2)
 def forward(self, x):
h = self.conv0(x)
 h = self.base_model.features(h)
 if self.do_pooling:
h = torch.sum(h, dim=(-1, -2))
else:
bs, ch, height, width = h.shape
h = h.view(bs, ch*height*width)
 for layer in self.lin_layers:
h = layer(h)
 return h
input_ids = sample['input_ids'].to(device)
 attention_mask = sample['attention_mask'].to(device)
 print(input_ids.shape)
 print(attention_mask.shape)
 prob, pred = torch.max(model(input_ids=input_ids, attention_mask=attention_mask),dim=1)
 print(prob)
 print(pred)
 print(sample['targets'] )<set_options>
def accuracy(y, t):
pred_label = torch.argmax(y, dim=1)
 count = pred_label.shape[0]
correct =(pred_label == t ).sum().type(torch.float32)
 acc = correct / count
return acc
class BengaliClassifier(nn.Module):
def __init__(self, predictor, n_grapheme=168, n_vowel=11, n_consonant=7):
super(BengaliClassifier, self ).__init__()
self.n_grapheme = n_grapheme
self.n_vowel = n_vowel
self.n_consonant = n_consonant
self.n_total_class = self.n_grapheme + self.n_vowel + self.n_consonant
self.predictor = predictor
self.metrics_keys = [
'loss', 'loss_grapheme', 'loss_vowel', 'loss_consonant',
'acc_grapheme', 'acc_vowel', 'acc_consonant']
def forward(self, x, y=None):
pred = self.predictor(x)
 if isinstance(pred, tuple):
assert len(pred)== 3
preds = pred
else:
assert pred.shape[1] == self.n_total_class
preds = torch.split(pred, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 loss_grapheme = F.cross_entropy(preds[0], y[:, 0])
 loss_vowel = F.cross_entropy(preds[1], y[:, 1])
 loss_consonant = F.cross_entropy(preds[2], y[:, 2])
 loss = loss_grapheme + loss_vowel + loss_consonant
metrics = {
'loss': loss.item() ,
'loss_grapheme': loss_grapheme.item() ,
'loss_vowel': loss_vowel.item() ,
'loss_consonant': loss_consonant.item() ,
'acc_grapheme': accuracy(preds[0], y[:, 0]),
'acc_vowel': accuracy(preds[1], y[:, 1]),
'acc_consonant': accuracy(preds[2], y[:, 2]),
}
return loss, metrics, pred
def calc(self, data_loader):
device: torch.device = next(self.parameters() ).device
self.eval()
output_list = []
with torch.no_grad() :
for batch in tqdm(data_loader):
batch = batch.to(device)
 pred = self.predictor(batch)
 output_list.append(pred)
 output = torch.cat(output_list, dim=0)
 preds = torch.split(output, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 return preds
def predict_proba(self, data_loader):
preds = self.calc(data_loader)
 return [F.softmax(p, dim=1)for p in preds]
def predict(self, data_loader):
preds = self.calc(data_loader)
 pred_labels = [torch.argmax(p, dim=1)for p in preds]
return pred_labels
model.distilbert.config<choose_model_class>
def build_predictor(arch, out_dim, model_name=None):
if arch == 'pretrained':
predictor = PretrainedCNN(in_channels=1, out_dim=out_dim, model_name=model_name)
 else:
raise ValueError("[ERROR] Unexpected value arch={}".format(arch))
return predictor
def build_classifier(arch, load_model_path, n_total, model_name='', device='cuda:0'):
if isinstance(device, str):
device = torch.device(device)
 predictor = build_predictor(arch, out_dim=n_total, model_name=model_name)
 print('predictor', type(predictor))
classifier = BengaliClassifier(predictor)
 if load_model_path:
predictor.load_state_dict(torch.load(load_model_path))
else:
print("[WARNING] Unexpected value load_model_path={}" 
.format(load_model_path))
classifier.to(device)
 return classifier
EPOCHES = 3
if torch.cuda.is_available() :
optimizer = transformers.AdamW(model.parameters() , lr=2e-5, correct_bias=False)
 total_steps = len(train_data_loader)* EPOCHES
scheduler = transformers.get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=total_steps 
)
 loss_fn = nn.CrossEntropyLoss().to(device )<train_model>
def prepare_image(datadir, featherdir, data_type='train',
submission=False, indices=[0, 1, 2, 3]):
assert data_type in ['train', 'test']
if submission:
image_df_list = [pd.read_parquet(datadir / f'{data_type}_image_data_{i}.parquet')
 for i in indices]
else:
image_df_list = [pd.read_feather(featherdir / f'{data_type}_image_data_{i}.feather')
 for i in indices]
print('image_df_list', len(image_df_list))
HEIGHT = 137
WIDTH = 236
images = [df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH)for df in image_df_list]
del image_df_list
gc.collect()
images = np.concatenate(images, axis=0)
 return images
def train_epoch(model, data_loader, loss_fn, optimizer, scheduler, n_examples):
model = model.train()
losses = []
correct_predictions = 0
for d in data_loader:
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 targets = d['targets'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 _, preds = torch.max(outputs, dim=1)
 loss = loss_fn(outputs, targets)
 correct_predictions += torch.sum(preds == targets)
 losses.append(loss.item())
 loss.backward()
optimizer.step()
scheduler.step()
optimizer.zero_grad()
return correct_predictions.double() / n_examples, np.mean(losses )<predict_on_test>
def predict_core(test_images, image_size, threshold,
arch, n_total, model_name, load_model_path, batch_size=512, device='cuda:0', **kwargs):
classifier = build_classifier(arch, load_model_path, n_total, model_name, device=device)
 test_dataset = BengaliAIDataset(
 test_images, None,
transform=Transform(affine=False, crop=True, size=(image_size, image_size),
threshold=threshold, train=False))
print('test_dataset', len(test_dataset))
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 test_pred_proba = classifier.predict_proba(test_loader)
 return test_pred_proba
def eval_model(model, data_loader, loss_fn, n_examples):
model = model.eval()
losses = []
correct_predictions = 0
with torch.no_grad() :
for d in data_loader:
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 targets = d['targets'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 _, preds = torch.max(outputs, dim=1)
 loss = loss_fn(outputs, targets)
 correct_predictions += torch.sum(preds == targets)
 losses.append(loss.item())
 return correct_predictions.double() / n_examples, np.mean(losses )<predict_on_test>
device = torch.device(device)
 n_grapheme = 168
n_vowel = 11
n_consonant = 7
n_total = n_grapheme + n_vowel + n_consonant
print('n_total', n_total )
def predict_authors(model, data_loader, submission_df):
model = model.eval()
with torch.no_grad() :
for d in data_loader:
input_ids = d['input_ids'].to(device)
 attention_mask = d['attention_mask'].to(device)
 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
 print(outputs)
<choose_model_class>
traindir = '/kaggle/input/bengaliaicv19-trainedmodels/'
data_type = 'test'
test_preds_list = []
for i in range(4):
indices = [i]
test_images = prepare_image(
 datadir, featherdir, data_type=data_type, submission=submission, indices=indices)
 n_dataset = len(test_images)
 print(f'n_dataset={n_dataset}')
 model_preds_list = []
for j in range(4):
train_args_dict = load_json(os.path.join(traindir, f'args_{j}.json'))
train_args_dict.update({
'load_model_path': os.path.join(traindir, f'predictor_{j}.pt'),
'device': device,
'batch_size': batch_size,
'debug': debug,
})
 print(f'j {j} updated train_args_dict {train_args_dict}')
 test_preds = predict_core(
 test_images=test_images, n_total=n_total,
**train_args_dict)
 model_preds_list.append(test_preds)
 proba0 = torch.mean(torch.stack([test_preds[0] for test_preds in model_preds_list], dim=0), dim=0)
 proba1 = torch.mean(torch.stack([test_preds[1] for test_preds in model_preds_list], dim=0), dim=0)
 proba2 = torch.mean(torch.stack([test_preds[2] for test_preds in model_preds_list], dim=0), dim=0)
 p0 = torch.argmax(proba0, dim=1 ).cpu().numpy()
p1 = torch.argmax(proba1, dim=1 ).cpu().numpy()
p2 = torch.argmax(proba2, dim=1 ).cpu().numpy()
print('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 test_preds_list.append([p0, p1, p2])
 if debug:
break
del test_images
gc.collect()
if is_colab and torch.cuda.is_available() :
optimizer = transformers.AdamW(model.parameters() , lr=2e-5, correct_bias=False)
 total_steps = len(train_data_loader)* EPOCHES
scheduler = transformers.get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=total_steps 
)
 loss_fn = nn.CrossEntropyLoss().to(device )<train_model>
p0 = np.concatenate([test_preds[0] for test_preds in test_preds_list], axis=0)
 p1 = np.concatenate([test_preds[1] for test_preds in test_preds_list], axis=0)
 p2 = np.concatenate([test_preds[2] for test_preds in test_preds_list], axis=0)
 print('concat:', 'p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 row_id = []
target = []
for i in tqdm(range(len(p0))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [p0[i], p1[i], p2[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False)
filename = 'finalized_model.pt'
if is_colab and torch.cuda.is_available() :
model = model.to(device)
 history = defaultdict(list)
 best_accuracy = 0
for epoch in range(EPOCHES):
print(f'Epoch {epoch + 1}/{EPOCHES}')
 print('-'*10)
 train_acc, train_loss = train_epoch(
 model,
train_data_loader,
loss_fn,
optimizer,
scheduler,
len(train_set)
)
 print(f'Train loss: {train_loss}, accuracy: {train_acc}')
 val_acc, val_loss = eval_model(
 model,
val_data_loader,
loss_fn,
len(val_set)
)
 print(f'Validation loss: {val_loss}, accuracy: {val_acc}')
 torch.save(model.state_dict() , DIR_NAME+filename )<load_pretrained>
train = pd.read_csv(datadir/'train.csv' )
if not is_colab:
model.load_state_dict(torch.load('/kaggle/input/spookydistilbert/finalized_model.pt', map_location=device))
model.eval()<categorify>
pred_df = pd.DataFrame({
'grapheme_root': p0,
'vowel_diacritic': p1,
'consonant_diacritic': p2
} )
<predict_on_test><EOS> 
train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values