Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<predict_on_test>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
test_preds = model.predict(test_dataset, verbose=1)
 submission['prediction'] = test_preds.argmax(axis=1 )<save_to_csv>
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )
submission.to_csv('submission.csv', index=False)
<set_options>
debug=False
submission=True
batch_size=256
device='cuda:0'
out='.'
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
<load_from_csv>
datadir = Path('/kaggle/input/bengaliai-cv19')
 featherdir = Path('/kaggle/input/bengaliaicv19feather')
 outdir = Path('.' )
df_train=pd.read_csv(os.path.join(path,"train.csv"))
df_test=pd.read_csv(os.path.join(path,"test.csv"))<set_options>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_pretrained>

 class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError
MODEL = 'jplu/tf-xlm-roberta-large'
EPOCHS = 10
MAX_LEN = 96
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE
tokenizer = AutoTokenizer.from_pretrained(MODEL )<prepare_x_and_y>
class BengaliAIDataset(DatasetMixin):
def __init__(self, images, labels=None, transform=None, indices=None):
super(BengaliAIDataset, self ).__init__(transform=transform)
 self.images = images
self.labels = labels
if indices is None:
indices = np.arange(len(images))
self.indices = indices
self.train = labels is not None
def __len__(self):

 return len(self.indices)
 def get_example(self, i):

 i = self.indices[i]
x = self.images[i]
x =(255 - x ).astype(np.float32)/ 255.
 if self.train:
y = self.labels[i]
return x, y
else:
return x
def quick_encode(df,maxlen=100):
values = df[['premise','hypothesis']].values.tolist()
tokens=tokenizer.batch_encode_plus(values,max_length=maxlen,pad_to_max_length=True)
 return np.array(tokens['input_ids'])
 x_train = quick_encode(df_train)
 x_test = quick_encode(df_test)
 y_train = df_train.label.values 
<define_variables>
def add_gaussian_noise(x, sigma):
x += np.random.randn(*x.shape)* sigma
x = np.clip(x, 0., 1.)
return x
class Transform:
def __init__(self, affine=True, crop=True, size=(64, 64),
normalize=True, train=True, threshold=40.,
sigma=-1.) :
self.affine = affine
self.crop = crop
self.size = size
self.normalize = normalize
self.train = train
self.threshold = threshold / 255.
 self.sigma = sigma / 255.
 def __call__(self, example):
if self.train:
x, y = example
else:
x = example
if self.affine:
x = affine_image(x)
 if self.crop:
x = crop_char_image(x, threshold=self.threshold)
 if self.size is not None:
x = resize(x, size=self.size)
 if self.sigma > 0.:
x = add_gaussian_noise(x, sigma=self.sigma)
 if self.normalize:
x =(x.astype(np.float32)- 0.0692)/ 0.2051
if x.ndim == 2:
x = x[None, :, :]
x = x.astype(np.float32)
 if self.train:
y = y.astype(np.int64)
 return x, y
else:
return x
def create_dist_dataset(X, y,val,batch_size= BATCH_SIZE):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(len(X))
if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(AUTO)
 else:
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test)) 
.batch(BATCH_SIZE)
)
<choose_model_class>
def residual_add(lhs, rhs):
lhs_ch, rhs_ch = lhs.shape[1], rhs.shape[1]
if lhs_ch < rhs_ch:
out = lhs + rhs[:, :lhs_ch]
elif lhs_ch > rhs_ch:
out = torch.cat([lhs[:, :rhs_ch] + rhs, lhs[:, rhs_ch:]], dim=1)
 else:
out = lhs + rhs
return out
def build_model(transformer,max_len):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
cls_token = Dropout(0.4 )(cls_token)
 cls_token = Dense(32,activation='relu' )(cls_token)
 cls_token = Dropout(0.4 )(cls_token)
 out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model 
<split>
class LazyLoadModule(nn.Module):

 _lazy_buffer_keys: List[str] = []
_lazy_parameter_keys: List[str] = []
def __init__(self):
super(LazyLoadModule, self ).__init__()
for k in self._lazy_buffer_keys:
self.register_buffer(k, torch.tensor([]))
for k in self._lazy_parameter_keys:
self.register_parameter(k, None)
 self._register_load_state_dict_pre_hook(self._hook)
 def _hook(self, state_dict, prefix, local_metadata, strict, missing_keys,
unexpected_keys, error_msgs):
for key in self._lazy_buffer_keys:
self.register_buffer(key, state_dict[prefix + key])
 for key in self._lazy_parameter_keys:
self.register_parameter(key, Parameter(state_dict[prefix + key]))
pred_test=np.zeros(( df_test.shape[0],3))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
history=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_train,y_train)) :
if fold < 4:
print("fold",fold+1)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data = create_dist_dataset(x_train[train_ind],y_train[train_ind],val=False)
 valid_data = create_dist_dataset(x_train[valid_ind],y_train[valid_ind],val=True)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base.h5", monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps = len(train_ind)//BATCH_SIZE
print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=1)
 print("Loading model...")
 model.load_weights(f"roberta_base.h5")
 print("fold {} validation accuracy {}".format(fold+1,np.mean(train_history.history['val_accuracy'])))
 print("fold {} validation loss {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 history.append(train_history)
 val_score.append(np.mean(train_history.history['val_accuracy']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/4 
<load_from_csv>
class LazyLinear(LazyLoadModule):

 __constants__ = ['bias', 'in_features', 'out_features']
_lazy_parameter_keys = ['weight']
def __init__(self, in_features, out_features, bias=True):
super(LazyLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
if bias:
self.bias = Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 if in_features is not None:
self.weight = Parameter(torch.Tensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight, a=math.sqrt(5))
if self.bias is not None:
fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
 bound = 1 / math.sqrt(fan_in)
 init.uniform_(self.bias, -bound, bound)
 def forward(self, input):
if self.weight is None:
self.in_features = input.shape[-1]
self.weight = Parameter(torch.Tensor(self.out_features, self.in_features))
self.reset_parameters()
self.to(input.device)
 return F.linear(input, self.weight, self.bias)
 def extra_repr(self):
return 'in_features={}, out_features={}, bias={}'.format(
 self.in_features, self.out_features, self.bias is not None 
)
submission = pd.read_csv(os.path.join(path,'sample_submission.csv'))
submission['prediction'] = np.argmax(pred_test,axis=1)
 submission.head()<save_to_csv>
class LinearBlock(nn.Module):
def __init__(self, in_features, out_features, bias=True,
use_bn=True, activation=F.relu, dropout_ratio=-1, residual=False,):
super(LinearBlock, self ).__init__()
if in_features is None:
self.linear = LazyLinear(in_features, out_features, bias=bias)
 else:
self.linear = nn.Linear(in_features, out_features, bias=bias)
 if use_bn:
self.bn = nn.BatchNorm1d(out_features)
 if dropout_ratio > 0.:
self.dropout = nn.Dropout(p=dropout_ratio)
 else:
self.dropout = None
self.activation = activation
self.use_bn = use_bn
self.dropout_ratio = dropout_ratio
self.residual = residual
def __call__(self, x):
h = self.linear(x)
 if self.use_bn:
h = self.bn(h)
 if self.activation is not None:
h = self.activation(h)
 if self.residual:
h = residual_add(h, x)
 if self.dropout_ratio > 0:
h = self.dropout(h)
 return h
submission.to_csv('submission.csv',index=False )<find_best_params>
class PretrainedCNN(nn.Module):
def __init__(self, model_name='se_resnext101_32x4d',
in_channels=1, out_dim=10, use_bn=True,
pretrained=None):
super(PretrainedCNN, self ).__init__()
self.conv0 = nn.Conv2d(
 in_channels, 3, kernel_size=3, stride=1, padding=1, bias=True)
 self.base_model = pretrainedmodels.__dict__[model_name](pretrained=pretrained)
 activation = F.leaky_relu
self.do_pooling = True
if self.do_pooling:
inch = self.base_model.last_linear.in_features
else:
inch = None
hdim = 512
lin1 = LinearBlock(inch, hdim, use_bn=use_bn, activation=activation, residual=False)
 lin2 = LinearBlock(hdim, out_dim, use_bn=use_bn, activation=None, residual=False)
 self.lin_layers = Sequential(lin1, lin2)
 def forward(self, x):
h = self.conv0(x)
 h = self.base_model.features(h)
 if self.do_pooling:
h = torch.sum(h, dim=(-1, -2))
else:
bs, ch, height, width = h.shape
h = h.view(bs, ch*height*width)
 for layer in self.lin_layers:
h = layer(h)
 return h
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()<install_modules>
def accuracy(y, t):
pred_label = torch.argmax(y, dim=1)
 count = pred_label.shape[0]
correct =(pred_label == t ).sum().type(torch.float32)
 acc = correct / count
return acc
class BengaliClassifier(nn.Module):
def __init__(self, predictor, n_grapheme=168, n_vowel=11, n_consonant=7):
super(BengaliClassifier, self ).__init__()
self.n_grapheme = n_grapheme
self.n_vowel = n_vowel
self.n_consonant = n_consonant
self.n_total_class = self.n_grapheme + self.n_vowel + self.n_consonant
self.predictor = predictor
self.metrics_keys = [
'loss', 'loss_grapheme', 'loss_vowel', 'loss_consonant',
'acc_grapheme', 'acc_vowel', 'acc_consonant']
def forward(self, x, y=None):
pred = self.predictor(x)
 if isinstance(pred, tuple):
assert len(pred)== 3
preds = pred
else:
assert pred.shape[1] == self.n_total_class
preds = torch.split(pred, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 loss_grapheme = F.cross_entropy(preds[0], y[:, 0])
 loss_vowel = F.cross_entropy(preds[1], y[:, 1])
 loss_consonant = F.cross_entropy(preds[2], y[:, 2])
 loss = loss_grapheme + loss_vowel + loss_consonant
metrics = {
'loss': loss.item() ,
'loss_grapheme': loss_grapheme.item() ,
'loss_vowel': loss_vowel.item() ,
'loss_consonant': loss_consonant.item() ,
'acc_grapheme': accuracy(preds[0], y[:, 0]),
'acc_vowel': accuracy(preds[1], y[:, 1]),
'acc_consonant': accuracy(preds[2], y[:, 2]),
}
return loss, metrics, pred
def calc(self, data_loader):
device: torch.device = next(self.parameters() ).device
self.eval()
output_list = []
with torch.no_grad() :
for batch in tqdm(data_loader):
batch = batch.to(device)
 pred = self.predictor(batch)
 output_list.append(pred)
 output = torch.cat(output_list, dim=0)
 preds = torch.split(output, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 return preds
def predict_proba(self, data_loader):
preds = self.calc(data_loader)
 return [F.softmax(p, dim=1)for p in preds]
def predict(self, data_loader):
preds = self.calc(data_loader)
 pred_labels = [torch.argmax(p, dim=1)for p in preds]
return pred_labels
!pip install nlp
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev<set_options>
def build_predictor(arch, out_dim, model_name=None):
if arch == 'pretrained':
predictor = PretrainedCNN(in_channels=1, out_dim=out_dim, model_name=model_name)
 else:
raise ValueError("[ERROR] Unexpected value arch={}".format(arch))
return predictor
def build_classifier(arch, load_model_path, n_total, model_name='', device='cuda:0'):
if isinstance(device, str):
device = torch.device(device)
 predictor = build_predictor(arch, out_dim=n_total, model_name=model_name)
 print('predictor', type(predictor))
classifier = BengaliClassifier(predictor)
 if load_model_path:
predictor.load_state_dict(torch.load(load_model_path))
else:
print("[WARNING] Unexpected value load_model_path={}" 
.format(load_model_path))
classifier.to(device)
 return classifier
%%time
%autosave 60
os.environ['XLA_USE_BF16'] = "1"
os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'
gc.enable()
XLMRobertaTokenizer,
XLMRobertaModel,
get_cosine_schedule_with_warmup)
 warnings.filterwarnings("ignore")
 print('PYTORCH:', xv.__torch_gitrev__)
 print('XLA:', xv.__xla_gitrev__ )<load_from_csv>
def prepare_image(datadir, featherdir, data_type='train',
submission=False, indices=[0, 1, 2, 3]):
assert data_type in ['train', 'test']
if submission:
image_df_list = [pd.read_parquet(datadir / f'{data_type}_image_data_{i}.parquet')
 for i in indices]
else:
image_df_list = [pd.read_feather(featherdir / f'{data_type}_image_data_{i}.feather')
 for i in indices]
print('image_df_list', len(image_df_list))
HEIGHT = 137
WIDTH = 236
images = [df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH)for df in image_df_list]
del image_df_list
gc.collect()
images = np.concatenate(images, axis=0)
 return images
train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv')
 sample_submission = pd.read_csv('.. /input/contradictory-my-dear-watson/sample_submission.csv' )<load_pretrained>
def predict_core(test_images, image_size, threshold,
arch, n_total, model_name, load_model_path, batch_size=512, device='cuda:0', **kwargs):
classifier = build_classifier(arch, load_model_path, n_total, model_name, device=device)
 test_dataset = BengaliAIDataset(
 test_images, None,
transform=Transform(affine=False, crop=True, size=(image_size, image_size),
threshold=threshold, train=False))
print('test_dataset', len(test_dataset))
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 test_pred_proba = classifier.predict_proba(test_loader)
 return test_pred_proba
TRAIN_BATCH_SIZE = 16
VALID_BATCH_SIZE = 16
EPOCHS = 4
MAX_LEN = 80
LR = 2e-5 * xm.xrt_world_size()
METRICS_DEBUG = True
tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large' )<create_dataframe>
device = torch.device(device)
 n_grapheme = 168
n_vowel = 11
n_consonant = 7
n_total = n_grapheme + n_vowel + n_consonant
print('n_total', n_total )
mnli = nlp.load_dataset(path='glue', name='mnli', split='train[:5%]')
 xnli = nlp.load_dataset(path='xnli')
 xnli = nlp.concatenate_datasets([xnli['test'], xnli['validation']])
 snli = nlp.load_dataset(path='snli', split='train[:5%]' )<categorify>
traindir = '/kaggle/input/bengaliaicv19-trainedmodels/'
data_type = 'test'
test_preds_list = []
for i in range(4):
indices = [i]
test_images = prepare_image(
 datadir, featherdir, data_type=data_type, submission=submission, indices=indices)
 n_dataset = len(test_images)
 print(f'n_dataset={n_dataset}')
 model_preds_list = []
for j in range(4):
train_args_dict = load_json(os.path.join(traindir, f'args_{j}.json'))
train_args_dict.update({
'load_model_path': os.path.join(traindir, f'predictor_{j}.pt'),
'device': device,
'batch_size': batch_size,
'debug': debug,
})
 print(f'j {j} updated train_args_dict {train_args_dict}')
 test_preds = predict_core(
 test_images=test_images, n_total=n_total,
**train_args_dict)
 model_preds_list.append(test_preds)
 proba0 = torch.mean(torch.stack([test_preds[0] for test_preds in model_preds_list], dim=0), dim=0)
 proba1 = torch.mean(torch.stack([test_preds[1] for test_preds in model_preds_list], dim=0), dim=0)
 proba2 = torch.mean(torch.stack([test_preds[2] for test_preds in model_preds_list], dim=0), dim=0)
 p0 = torch.argmax(proba0, dim=1 ).cpu().numpy()
p1 = torch.argmax(proba1, dim=1 ).cpu().numpy()
p2 = torch.argmax(proba2, dim=1 ).cpu().numpy()
print('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 test_preds_list.append([p0, p1, p2])
 if debug:
break
del test_images
gc.collect()
def convert_to_features(batch):
input_pairs = list(zip(batch['premise'], batch['hypothesis']))
encodings = tokenizer.batch_encode_plus(input_pairs,
add_special_tokens=True,
pad_to_max_length=True,
max_length=MAX_LEN,
truncation=True,
return_attention_mask=True,
return_token_type_ids=True)
 return encodings<define_variables>
p0 = np.concatenate([test_preds[0] for test_preds in test_preds_list], axis=0)
 p1 = np.concatenate([test_preds[1] for test_preds in test_preds_list], axis=0)
 p2 = np.concatenate([test_preds[2] for test_preds in test_preds_list], axis=0)
 print('concat:', 'p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 row_id = []
target = []
for i in tqdm(range(len(p0))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [p0[i], p1[i], p2[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False)
def preprocess_xnli(example):
premise_output = []
hypothesis_output = []
label_output = []
for prem, hyp, lab in zip(example['premise'], example['hypothesis'], example["label"]):
label = lab
langs = hyp['language']
translations = hyp['translation']
hypothesis = {k: v for k, v in zip(langs, translations)}
for lang in prem:
if lang in hypothesis:
premise_output += [prem[lang]]
hypothesis_output += [hypothesis[lang]]
label_output += [label]
return {'premise':premise_output, 'hypothesis':hypothesis_output, 'label':label_output}<categorify>
train = pd.read_csv(datadir/'train.csv' )
mnli_encoded = mnli.map(convert_to_features, batched=True, remove_columns=['idx', 'premise', 'hypothesis'])
 mnli_encoded.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'] )<categorify>
pred_df = pd.DataFrame({
'grapheme_root': p0,
'vowel_diacritic': p1,
'consonant_diacritic': p2
} )
<categorify><EOS> 
train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values