Kaggle competition: Bengali.AI Handwritten Grapheme Classification
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.75
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
!pip install /kaggle/input/efficientnet-pytorch/efficientnet_pytorch-0.6.1-py3-none-any.whl > /dev/null
def get_model(use_model):
base_model = use_model(weights='imagenet',
include_top=False, pooling='avg',
input_shape=(*IMAGE_SIZE, 3))
x = base_model.output
predictions = Dense(104, activation='softmax' )(x)
 return Model(inputs=base_model.input, outputs=predictions)
 with strategy.scope() :
model = get_model(ResNet50)
 model.compile(
 optimizer='nadam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
<train_model>
BATCH_SIZE = 96
N_WORKERS = 4
HEIGHT = 137
WIDTH = 236
INPUT_PATH="/kaggle/input/bengaliai-cv19"
MODEL_BASE_PATH_EFNET_B3 = '/kaggle/input/bengaliaieffnetb3'
MODEL_BASE_PATH_DN161FULL = '/kaggle/input/densenet161full'
history = model.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback, ModelCheckpoint(filepath='my_ef_net_b7.h5', monitor='val_loss',
save_best_only=True)],
validation_data=get_validation_dataset() ,
workers = 3 )<load_pretrained>
ENSEMBLES = [
{
'model': 'efficientnet_b3',
'model_state_file': MODEL_BASE_PATH_EFNET_B3 + '/efficientNet_fold2.pth',
},
{
'model': 'efficientnet_b3',
'model_state_file': MODEL_BASE_PATH_EFNET_B3 + '/efficientNet_fold4.pth',
},
{
'model': 'densenet161',
'model_state_file': MODEL_BASE_PATH_DN161FULL + '/densenet161_b30',
},
]
model = tf.keras.models.load_model('my_ef_net_b7.h5' )<predict_on_test>
NUM_ENSEMBLE = len(ENSEMBLES )
test_ds = get_test_dataset(ordered=True)
 print('Вычисляем предсказания...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 predictions = np.argmax(probabilities, axis=-1)
 print(predictions)
 print('Создание файла submission.csv...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<install_modules>
for ensemble in ENSEMBLES:
ensemble['w_grapheme'] = 1 / NUM_ENSEMBLE
ensemble['w_vowel'] = 1 / NUM_ENSEMBLE
ensemble['w_conso'] = 1 / NUM_ENSEMBLE
!pip install transformers==3.0.2
!pip install nlp<create_dataframe>
test_df = pd.read_csv(INPUT_PATH +('/test.csv'))
submission_df = pd.read_csv(INPUT_PATH + '/sample_submission.csv' )
mnli = load_dataset(path='glue', name='mnli')
<load_from_csv>
class Identity(nn.Module):
def __init__(self):
super(Identity, self ).__init__()
def forward(self, x):
return x
class EfficientNetWrapper(nn.Module):
def __init__(self):
super(EfficientNetWrapper, self ).__init__()
self.effNet = EfficientNet.from_name('efficientnet-b3')
 self.fc_root = nn.Linear(in_features=1000, out_features=168)
 self.fc_vowel = nn.Linear(in_features=1000, out_features=11)
 self.fc_consonant = nn.Linear(in_features=1000, out_features=7)
 def forward(self, X):
output = self.effNet(X)
 output_root = self.fc_root(output)
 output_vowel = self.fc_vowel(output)
 output_consonant = self.fc_consonant(output)
 return output_vowel, output_root, output_consonant
class DenseNet161(nn.Module):
def __init__(self, pretrained):
super(DenseNet161, self ).__init__()
if pretrained == True:
self.model = pretrainedmodels.__dict__["densenet161"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["densenet161"](pretrained=None)
 self.model.last_linear = Identity()
self.l0 = nn.Linear(2208, 11)
 self.l1 = nn.Linear(2208, 168)
 self.l2 = nn.Linear(2208, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2
xnli = pd.read_csv('.. /input/xnli-organized/xnli_df.csv')
 xnli = xnli.rename(columns = {'Unnamed: 0': 'lang_abv', '0' : 'premise', '1': 'hypothesis', '0.1': 'label' })
 xnli.head()<load_from_csv>
class ClsTestDataset(Dataset):
def __init__(self, num_samples=1, model='densenet161'):
self.num_samples = num_samples
self.images = np.zeros([num_samples, WIDTH * HEIGHT], dtype=np.uint8)
 self.aug = albumentations.Compose([
albumentations.Resize(int(HEIGHT), int(WIDTH), always_apply=True),
albumentations.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], always_apply=True)
 ])
 img_id = 0
for i in tqdm(range(4)) :
datafile = INPUT_PATH + '/test_image_data_{}.parquet'.format(i)
 parq = pq.read_pandas(datafile, columns=[str(x)for x in range(32332)] ).to_pandas()
parq = parq.iloc[:, :].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 for idx, image in enumerate(parq):
self.images[img_id,...] = image.reshape(-1 ).astype(np.uint8)
 img_id = img_id + 1
del parq
def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
img = self.images[idx]
img = img.reshape(HEIGHT, WIDTH ).astype(float)
 img = Image.fromarray(img ).convert("RGB")
 img = self.aug(image=np.array(img)) ["image"]
img = np.transpose(img,(2, 0, 1)).astype(np.float32)
 img = torch.tensor(img, dtype=torch.float)
 return {
"image": img,
"image_id": idx
}
train_df = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 print('Traning Data, the size of the dataset is: {}
'.format(train_df.shape))
test_df = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )<concatenate>
bengali_dataset = ClsTestDataset(num_samples = test_df.shape[0] // 3 )
train_df = pd.concat([train_df, xnli, mnli_df])
 train_df = train_df[train_df['label'] != -1]
mnli_df = None
snli_df = None
print('the shape of the whole DF to be used is: ' + str(train_df.shape))<count_duplicates>
data_loader_test = torch.utils.data.DataLoader(
 bengali_dataset,
batch_size=BATCH_SIZE,
num_workers=N_WORKERS,
sampler=SequentialSampler(bengali_dataset),
shuffle=False
)
train_df = train_df[train_df.duplicated() == False]
print('the shape of the whole DF to be used is: ' + str(train_df.shape))<load_pretrained>
size = submission_df.shape[0] // 3
results = {
'grapheme_root': np.zeros(( len(ENSEMBLES), size, 168), dtype=np.float),
'vowel_diacritic': np.zeros(( len(ENSEMBLES), size, 11), dtype=np.float),
'consonant_diacritic': np.zeros(( len(ENSEMBLES), size, 7), dtype=np.float),
}
np.random.seed(123)
 max_len = 50
Bert_model = "bert-large-uncased"
Bert_tokenizer = BertTokenizer.from_pretrained(Bert_model)
 def tokeniZer(dataset,tokenizer):
encoded_list = []
type_id_list = np.zeros(( dataset.shape[0], max_len))
mask_list = np.zeros(( dataset.shape[0], max_len))
for i in range(dataset.shape[0]):
datapoint = '[CLS] ' + dataset['premise'][i] + ' [SEP]' + dataset['hypothesis'][i] + ' [SEP]'
datapoint = tokenizer.tokenize(datapoint)
 datapoint = tokenizer.convert_tokens_to_ids(datapoint)
 encoded_list.append(datapoint)
 encoded_list = pad_sequences(encoded_list, maxlen = max_len, padding = 'post')
 for i in range(encoded_list.shape[0]):
flag = 0
a = encoded_list[i]
for j in range(len(a)) :
if flag == 0:
type_id_list[i,j] = 0
else:
type_id_list[i,j] = 1
if encoded_list[i,j] == 102:
flag = 1
if encoded_list[i,j] == 0:
mask_list[i,j] = 0
else:
mask_list[i,j] = 1
return encoded_list,mask_list,type_id_list 
<feature_engineering>
for model_idx, ensemble in enumerate(ENSEMBLES):
if ensemble['model'].lower() == 'densenet161':
model = DenseNet161(pretrained=False)
 elif ensemble['model'].lower() == 'efficientnet_b3':
model = EfficientNetWrapper()
else:
raise ValueError
model_state = torch.load(ensemble['model_state_file'], map_location='cuda:0')
 model.load_state_dict(model_state)
 model.eval()
if torch.cuda.is_available() :
model.cuda()
del model_state
for batch_idx, data in enumerate(data_loader_test):
images = data['image']
image_idx = data['image_id']
if torch.cuda.is_available() :
images = images.float().cuda()
else:
images = images.float()
with torch.no_grad() :
out_vowel, out_graph, out_conso = model(images)
 out_graph = F.softmax(out_graph, dim=1 ).data.cpu().numpy() * ensemble['w_grapheme']
out_vowel = F.softmax(out_vowel, dim=1 ).data.cpu().numpy() * ensemble['w_vowel']
out_conso = F.softmax(out_conso, dim=1 ).data.cpu().numpy() * ensemble['w_conso']
start = batch_idx * BATCH_SIZE
end = min(( batch_idx + 1)* BATCH_SIZE, submission_df.shape[0] // 3)
 results['grapheme_root'][model_idx, start:end, :] = out_graph
results['vowel_diacritic'][model_idx, start:end, :] = out_vowel
results['consonant_diacritic'][model_idx, start:end, :] = out_conso
del images
del out_graph, out_vowel, out_conso
del model
def mish(x):
return x*tanh(softplus(x))
get_custom_objects() ["mish"] = Activation(mish )<define_search_model>
del data_loader_test
del bengali_dataset
del test_df
gc.collect()
%reset -f out
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<categorify>
submission_df = pd.read_csv(INPUT_PATH + '/sample_submission.csv')
 submission_df.head()
def create_BERT(random_seed):
tf.random.set_seed(random_seed)
 with tpu_strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(Bert_model)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_layer")
 input_masks = Input(shape =(max_len,), dtype = tf.int32, name = 'input_mask')
 input_type_id = Input(shape =(max_len,), dtype = tf.int32, name = 'input_type_id')
 sequence_output = transformer_encoder([input_ids, input_masks, input_type_id])[0]
cls_token = sequence_output[:, 0, :]
output_layer = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=[input_ids, input_masks, input_type_id], outputs = output_layer)
 model.summary()
model.compile(Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model<define_variables>
for l in ['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']:
idx = submission_df[submission_df['row_id'].str.contains(l)].index
res = results[l].sum(axis=0)
 submission_df.iloc[idx, 1] =(res / res.mean(axis=0)).argmax(axis=1 )
callbacks = [tf.keras.callbacks.EarlyStopping(patience = 2, monitor = 'val_loss', \
restore_best_weights = True, mode = 'min')]
shuffled_data = shuffle(train_df ).reset_index(drop = True)
 train_df = None
batch_size = 128 
<load_pretrained>
submission_df.to_csv('./submission.csv', index=False)