Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<load_from_csv>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train_dataset = nlp.load_dataset('csv', data_files=['.. /input/contradictory-my-dear-watson/train.csv'])['train']
print(train_dataset.num_rows)
 print(train_dataset.column_names)
 drop_columns = train_dataset.column_names[:-1]
encoded_train_dataset = train_dataset.map(convert_to_features, batched=True, remove_columns=drop_columns)
 encoded_train_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'])
 print(encoded_train_dataset.num_rows)
 print(encoded_train_dataset.column_names )<concatenate>
train_df_ = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')
 test_df_ = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')
 class_map_df = pd.read_csv('/kaggle/input/bengaliai-cv19/class_map.csv')
 sample_sub_df = pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv' )
train_dataset = nlp.concatenate_datasets([mnli_encoded,
xnli_encoded,
snli_encoded,
encoded_train_dataset
])
 print(train_dataset.num_rows)
 print(train_dataset.column_names )<drop_column>
print(f'Number of unique grapheme roots: {train_df_["grapheme_root"].nunique() }')
 print(f'Number of unique vowel diacritic: {train_df_["vowel_diacritic"].nunique() }')
 print(f'Number of unique consonant diacritic: {train_df_["consonant_diacritic"].nunique() }' )
train_dataset.cleanup_cache_files()
del mnli, mnli_encoded
del xnli, xnli_encoded, xnli_processed
del snli, snli_encoded
gc.collect()<feature_engineering>
top_10_roots = get_n(train_df_, 'grapheme_root', 10)
 top_10_roots
class DatasetRetriever(Dataset):
def __init__(self, dataset:nlp.arrow_dataset.Dataset):
self.dataset = dataset
self.ids = self.dataset['input_ids']
self.mask = self.dataset['attention_mask']
self.type_ids = self.dataset['token_type_ids']
self.targets = self.dataset["label"]
def __len__(self):
return self.dataset.num_rows
def __getitem__(self, index):
ids = self.ids[index]
mask = self.mask[index]
type_ids = self.type_ids[index]
targets = self.targets[index]
return {
'ids':torch.tensor(ids),
'mask':torch.tensor(mask),
'type_ids':torch.tensor(type_ids),
'targets':targets
}<load_pretrained>
bottom_10_roots = get_n(train_df_, 'grapheme_root', 10, False)
 bottom_10_roots
class XLMRoberta(nn.Module):
def __init__(self, num_labels, multisample):
super(XLMRoberta, self ).__init__()
output_hidden_states = False
self.num_labels = num_labels
self.multisample= multisample
self.roberta = XLMRobertaModel.from_pretrained("xlm-roberta-large",
output_hidden_states=output_hidden_states,
num_labels=1)
 self.layer_norm = nn.LayerNorm(1024*2)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.classifier = nn.Linear(1024*2, self.num_labels)
 def forward(self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None):
outputs = self.roberta(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
 average_pool = torch.mean(outputs[0], 1)
 max_pool, _ = torch.max(outputs[0], 1)
 concatenate_layer = torch.cat(( average_pool, max_pool), 1)
 normalization = self.layer_norm(concatenate_layer)
 if self.multisample:
logits = torch.mean(
 torch.stack(
 [self.classifier(self.dropout(normalization)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits = self.dropout(normalization)
 logits = self.classifier(logits)
 outputs = logits
return outputs<normalization>
train_df_ = train_df_.drop(['grapheme'], axis=1, inplace=False )
class AverageMeter(object):
def __init__(self, name, fmt=':f'):
self.name = name
self.fmt = fmt
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def __str__(self):
fmtstr = '{name} {val' + self.fmt + '}({avg' + self.fmt + '})'
return fmtstr.format(**self.__dict__)
 class ProgressMeter(object):
def __init__(self, num_batches, meters, prefix=""):
self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
 self.meters = meters
self.prefix = prefix
def display(self, batch):
entries = [self.prefix + self.batch_fmtstr.format(batch)]
entries += [str(meter)for meter in self.meters]
print('\t'.join(entries))
def _get_batch_fmtstr(self, num_batches):
num_digits = len(str(num_batches // 1))
fmt = '{:' + str(num_digits)+ 'd}'
return '[' + fmt + '/' + fmt.format(num_batches)+ ']'
def accuracy(output, target, topk=(1,)) :
with torch.no_grad() :
maxk = max(topk)
 batch_size = target.size(0)
 _, pred = output.topk(maxk, 1, True, True)
 pred = pred.t()
correct = pred.eq(target.view(1, -1 ).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1 ).float().sum(0, keepdim=True)
 res.append(correct_k.mul_(100.0 / batch_size))
return res<choose_model_class>
train_df_[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']] = train_df_[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].astype('uint8' )
def get_model_optimizer(model):
def is_backbone(name):
return "roberta" in name
optimizer_grouped_parameters = [
{'params': [param for name, param in model.named_parameters() if is_backbone(name)], 'lr': LR},
{'params': [param for name, param in model.named_parameters() if not is_backbone(name)], 'lr': 1e-3}
]
optimizer = AdamW(
 optimizer_grouped_parameters, lr=LR, weight_decay=1e-2 
)
 return optimizer<compute_test_metric>
IMG_SIZE=64
N_CHANNELS=1
def loss_fn(outputs, targets):
return nn.CrossEntropyLoss()(outputs, targets )<init_hyperparams>
HEIGHT = 137
WIDTH = 236
SIZE = 64
CROP_SIZE = 64
def train_loop_fn(train_loader, model, optimizer, device, scheduler, epoch=None):
batch_time = AverageMeter('Time', ':6.3f')
 data_time = AverageMeter('Data', ':6.3f')
 losses = AverageMeter('Loss', ':.4e')
 top1 = AverageMeter('Acc@1', ':6.2f')
 progress = ProgressMeter(
 len(train_loader),
[batch_time, data_time, losses, top1],
prefix="[xla:{}]Train: Epoch: [{}]".format(xm.get_ordinal() , epoch)
)
 model.train()
end = time.time()
for i, data in enumerate(train_loader):
data_time.update(time.time() -end)
 ids, mask, type_ids, targets = data["input_ids"], data["attention_mask"], data['token_type_ids'], data["label"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 type_ids = type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 optimizer.zero_grad()
outputs = model(
 input_ids = ids,
attention_mask = mask,
token_type_ids = type_ids 
)
 loss = loss_fn(outputs, targets)
 loss.backward()
xm.optimizer_step(optimizer)
 loss = loss_fn(outputs, targets)
 acc1= accuracy(outputs, targets, topk=(1,))
losses.update(loss.item() , ids.size(0))
top1.update(acc1[0].item() , ids.size(0))
scheduler.step()
batch_time.update(time.time() - end)
 end = time.time()
if i % 50 == 0:
progress.display(i)
 del loss
del outputs
del ids
del mask
del targets
gc.collect()<init_hyperparams>
def get_dummies(df):
cols = []
for col in df:
cols.append(pd.get_dummies(df[col].astype(str)))
 return pd.concat(cols, axis=1 )
def eval_loop_fn(validation_loader, model, device):
model.eval()
batch_time = AverageMeter('Time', ':6.3f')
 losses = AverageMeter('Loss', ':.4e')
 top1 = AverageMeter('Acc@1', ':6.2f')
 learning_rate = AverageMeter('LR',':2.8f')
 progress = ProgressMeter(
 len(validation_loader),
[batch_time, losses, top1],
prefix='[xla:{}]Validation: '.format(xm.get_ordinal()))
with torch.no_grad() :
end = time.time()
for i, data in enumerate(validation_loader):
ids, mask, type_ids, targets = data["input_ids"], data["attention_mask"], data['token_type_ids'], data["label"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 type_ids = type_ids.to(device, dtype=torch.long)
 targets = targets.to(device, dtype=torch.float)
 outputs = model(
 input_ids = ids,
attention_mask = mask,
token_type_ids = type_ids 
)
 loss = loss_fn(outputs, targets)
 acc1= accuracy(outputs, targets, topk=(1,))
losses.update(loss.item() , ids.size(0))
top1.update(acc1[0].item() , ids.size(0))
batch_time.update(time.time() - end)
 end = time.time()
if i % 50 == 0:
progress.display(i)
 del loss
del outputs
del ids
del mask
del targets
gc.collect()<train_model>
inputs = Input(shape =(IMG_SIZE, IMG_SIZE, 1))
model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 1))(inputs)
 model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=32, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=128, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=256, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.3 )(model)
 model = Flatten()(model)
 model = Dense(1024, activation = "relu" )(model)
 model = Dropout(rate=0.3 )(model)
 dense = Dense(512, activation = "relu" )(model)
 head_root = Dense(168, activation = 'softmax' )(dense)
 head_vowel = Dense(11, activation = 'softmax' )(dense)
 head_consonant = Dense(7, activation = 'softmax' )(dense)
 model = Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant] )
WRAPPED_MODEL = xmp.MpModelWrapper(XLMRoberta(num_labels=3, multisample=False))
dataset = train_dataset.train_test_split(test_size=0.1)
 train_dataset = dataset['train']
valid_dataset = dataset['test']
train_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'])
 valid_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'] )<load_pretrained>
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
def _run() :
xm.master_print('Starting Run...')
 train_sampler = DistributedSampler(
 train_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 train_data_loader = DataLoader(
 train_dataset,
batch_size=TRAIN_BATCH_SIZE,
sampler=train_sampler,
drop_last=True,
num_workers=0 
)
 xm.master_print('Train Loader Created.')
 valid_sampler = DistributedSampler(
 valid_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 valid_data_loader = DataLoader(
 valid_dataset,
batch_size=VALID_BATCH_SIZE,
sampler=valid_sampler,
drop_last=True,
num_workers=0 
)
 xm.master_print('Valid Loader Created.')
 num_train_steps = int(len(train_dataset)/ TRAIN_BATCH_SIZE / xm.xrt_world_size())
 device = xm.xla_device()
model = WRAPPED_MODEL.to(device)
 xm.master_print('Done Model Loading.')
 optimizer = get_model_optimizer(model)
 scheduler = get_cosine_schedule_with_warmup(
 optimizer,
num_warmup_steps = 0,
num_training_steps = num_train_steps * EPOCHS 
)
 xm.master_print(f'Num Train Steps= {num_train_steps}, XRT World Size= {xm.xrt_world_size() }.')
 for epoch in range(EPOCHS):
para_loader = pl.ParallelLoader(train_data_loader, [device])
 xm.master_print('Parallel Loader Created.Training...')
 train_loop_fn(para_loader.per_device_loader(device),
model,
optimizer,
device,
scheduler,
epoch 
)
 xm.master_print("Finished training epoch {}".format(epoch))
para_loader = pl.ParallelLoader(valid_data_loader, [device])
 xm.master_print('Parallel Loader Created.Validating...')
 eval_loop_fn(para_loader.per_device_loader(device),
model,
device 
)
 if epoch == EPOCHS-1:
xm.master_print('Saving Model.. ')
 xm.save(model.state_dict() , "model.bin")
 xm.master_print('Model Saved.')
 if METRICS_DEBUG:
xm.master_print(met.metrics_report() , flush=True )<train_model>
learning_rate_reduction_root = ReduceLROnPlateau(monitor='dense_3_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 learning_rate_reduction_vowel = ReduceLROnPlateau(monitor='dense_4_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 learning_rate_reduction_consonant = ReduceLROnPlateau(monitor='dense_5_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
def _mp_fn(rank, flags):
_run()
FLAGS={}
xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork' )<set_options>
batch_size = 256
epochs = 30
!pip install --quiet efficientnet
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )<set_options>
del histories
gc.collect()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
preds_dict = {
'grapheme_root': [],
'vowel_diacritic': [],
'consonant_diacritic': []
}
BATCH_SIZE = 16 * REPLICAS
LEARNING_RATE = 3e-5 * REPLICAS
EPOCHS = 20
HEIGHT = 331
WIDTH = 331
CHANNELS = 3
N_CLASSES = 104
ES_PATIENCE = 5
N_FOLDS = 5
FOLDS_USED = 5<define_variables>
components = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']
target=[]
row_id=[]
for i in range(4):
df_test_img = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_{}.parquet'.format(i))
df_test_img.set_index('image_id', inplace=True)
 X_test = resize(df_test_img, need_progress_bar=False)/255
X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)
 preds = model.predict(X_test)
 for i, p in enumerate(preds_dict):
preds_dict[p] = np.argmax(preds[i], axis=1)
 for k,id in enumerate(df_test_img.index.values):
for i,comp in enumerate(components):
id_sample=id+'_'+comp
row_id.append(id_sample)
 target.append(preds_dict[comp][k])
 del df_test_img
del X_test
gc.collect()
df_sample = pd.DataFrame(
 {
'row_id': row_id,
'target':target
},
columns = ['row_id','target'] 
)
 df_sample.to_csv('submission.csv',index=False)
 df_sample.head()