Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<train_model>
%%writefile create_folds.py
if __name__ == "__main__":
df = pd.read_csv(".. /input/train.csv")
 print(df.head())
 df.loc[:, 'kfold'] = -1
df = df.sample(frac=1 ).reset_index(drop=True)
 X = df.image_id.values
y = df[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values
mskf = MultilabelStratifiedKFold(n_splits=5)
 for fold,(trn_, val_)in enumerate(mskf.split(X, y)) :
print("TRAIN: ", trn_, "VAL: ", val_)
 df.loc[val_, "kfold"] = fold
print(df.kfold.value_counts())
 df.to_csv(".. /input/train_folds.csv", index=False )
if not SKIP_VALIDATION:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
validation_data=get_validation_dataset() ,
callbacks = [lr_callback])
 else:
history2 = model2.fit(get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks = [lr_callback] )<predict_on_test>
%%writefile create_image_pickles.py
if __name__ == "__main__":
files = glob.glob(".. /input/train_*.parquet")
 for f in files:
df = pd.read_parquet(f, engine='fastparquet')
 image_ids = df.image_id.values
df = df.drop("image_id", axis=1)
 image_array = df.values
for j, image_id in tqdm(enumerate(image_ids), total=len(image_ids)) :
joblib.dump(image_array[j, :], f".. /input/image_pickles/{image_id}.pkl")
if not SKIP_VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
m1 = model1.predict(images_ds)
 m2 = model2.predict(images_ds)
 scores = []
for alpha in np.linspace(0,1,100):
cm_probabilities = alpha*m1+(1-alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro'))
best_alpha = np.argmax(scores)/100
else:
best_alpha = 0.51
print('Best alpha: ' + str(best_alpha))<compute_test_metric>
%%writefile dataset.py
class BengaliDatasetTrain:
def __init__(self, folds, img_height, img_width, mean, std):
df = pd.read_csv(".. /input/train_folds.csv")
 df = df[["image_id", "grapheme_root", "vowel_diacritic", "consonant_diacritic", "kfold"]]
df = df[df.kfold.isin(folds)].reset_index(drop=True)
 self.image_ids = df.image_id.values
self.grapheme_root = df.grapheme_root.values
self.vowel_diacritic = df.vowel_diacritic.values
self.consonant_diacritic = df.consonant_diacritic.values
if len(folds)== 1:
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 else:
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = joblib.load(f".. /input/image_pickles/{self.image_ids[item]}.pkl")
 image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"grapheme_root": torch.tensor(self.grapheme_root[item], dtype=torch.long),
"vowel_diacritic": torch.tensor(self.vowel_diacritic[item], dtype=torch.long),
"consonant_diacritic": torch.tensor(self.consonant_diacritic[item], dtype=torch.long)
 }
if not SKIP_VALIDATION:
cm_probabilities = best_alpha*m1 +(1-best_alpha)*m2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<predict_on_test>
%%writefile models.py
class ResNet34(nn.Module):
def __init__(self, pretrained):
super(ResNet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.l0 = nn.Linear(512, 168)
 self.l1 = nn.Linear(512, 11)
 self.l2 = nn.Linear(512, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2
def predict_tta(model, n_iter):
probs = []
for i in range(n_iter):
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs.append(model.predict(test_images_ds,verbose=0))
return probs<predict_on_test>
%%writefile train.py
DEVICE = "cuda"
TRAINING_FOLDS_CSV = os.environ.get("TRAINING_FOLDS_CSV")
 IMG_HEIGHT = int(os.environ.get("IMG_HEIGHT"))
IMG_WIDTH = int(os.environ.get("IMG_WIDTH"))
EPOCHS = int(os.environ.get("EPOCHS"))
TRAIN_BATCH_SIZE = int(os.environ.get("TRAIN_BATCH_SIZE"))
TEST_BATCH_SIZE = int(os.environ.get("TEST_BATCH_SIZE"))
MODEL_MEAN = ast.literal_eval(os.environ.get("MODEL_MEAN"))
MODEL_STD = ast.literal_eval(os.environ.get("MODEL_STD"))
TRAINING_FOLDS = ast.literal_eval(os.environ.get("TRAINING_FOLDS"))
VALIDATION_FOLDS = ast.literal_eval(os.environ.get("VALIDATION_FOLDS"))
BASE_MODEL = os.environ.get("BASE_MODEL")
 def macro_recall(pred_y, y, n_grapheme=168, n_vowel=11, n_consonant=7):
pred_y = torch.split(pred_y, [n_grapheme, n_vowel, n_consonant], dim=1)
 pred_labels = [torch.argmax(py, dim=1 ).cpu().numpy() for py in pred_y]
y = y.cpu().numpy()
recall_grapheme = sklearn.metrics.recall_score(pred_labels[0], y[:, 0], average='macro')
 recall_vowel = sklearn.metrics.recall_score(pred_labels[1], y[:, 1], average='macro')
 recall_consonant = sklearn.metrics.recall_score(pred_labels[2], y[:, 2], average='macro')
 scores = [recall_grapheme, recall_vowel, recall_consonant]
final_score = np.average(scores, weights=[2, 1, 1])
 print(f'recall: grapheme {recall_grapheme}, vowel {recall_vowel}, consonant {recall_consonant}, 'f'total {final_score}, y {y.shape}')
 return final_score
def loss_fn(outputs, targets):
o1, o2, o3 = outputs
t1, t2, t3 = targets
l1 = nn.CrossEntropyLoss()(o1, t1)
 l2 = nn.CrossEntropyLoss()(o2, t2)
 l3 = nn.CrossEntropyLoss()(o3, t3)
 return(l1 + l2 + l3)/ 3
def train(dataset, data_loader, model, optimizer):
model.train()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)) :
counter = counter + 1
image = d["image"]
grapheme_root = d["grapheme_root"]
vowel_diacritic = d["vowel_diacritic"]
consonant_diacritic = d["consonant_diacritic"]
image = image.to(DEVICE, dtype=torch.float)
 grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)
 vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)
 consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)
 print(image.shape)
 optimizer.zero_grad()
outputs = model(image)
 targets =(grapheme_root, vowel_diacritic, consonant_diacritic)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
final_loss += loss
o1, o2, o3 = outputs
t1, t2, t3 = targets
final_outputs.append(torch.cat(( o1,o2,o3), dim=1))
final_targets.append(torch.stack(( t1,t2,t3), dim=1))
final_outputs = torch.cat(final_outputs)
 final_targets = torch.cat(final_targets)
 print("=================Train=================")
 macro_recall_score = macro_recall(final_outputs, final_targets)
 return final_loss/counter , macro_recall_score
def evaluate(dataset, data_loader, model):
with torch.no_grad() :
model.eval()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)) :
counter = counter + 1
image = d["image"]
grapheme_root = d["grapheme_root"]
vowel_diacritic = d["vowel_diacritic"]
consonant_diacritic = d["consonant_diacritic"]
image = image.to(DEVICE, dtype=torch.float)
 grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)
 vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)
 consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)
 outputs = model(image)
 targets =(grapheme_root, vowel_diacritic, consonant_diacritic)
 loss = loss_fn(outputs, targets)
 final_loss += loss
o1, o2, o3 = outputs
t1, t2, t3 = targets
final_outputs.append(torch.cat(( o1,o2,o3), dim=1))
final_targets.append(torch.stack(( t1,t2,t3), dim=1))
final_outputs = torch.cat(final_outputs)
 final_targets = torch.cat(final_targets)
 print("=================Train=================")
 macro_recall_score = macro_recall(final_outputs, final_targets)
 return final_loss/counter , macro_recall_score
def main() :
model = MODEL_DISPATCHER[BASE_MODEL](pretrained=True)
 model.to(DEVICE)
 train_dataset = BengaliDatasetTrain(
 folds=TRAINING_FOLDS,
img_height = IMG_HEIGHT,
img_width = IMG_WIDTH,
mean = MODEL_MEAN,
std = MODEL_STD 
)
 train_loader = torch.utils.data.DataLoader(
 dataset=train_dataset,
batch_size= TRAIN_BATCH_SIZE,
shuffle=True,
num_workers=4 
)
 valid_dataset = BengaliDatasetTrain(
 folds=VALIDATION_FOLDS,
img_height = IMG_HEIGHT,
img_width = IMG_WIDTH,
mean = MODEL_MEAN,
std = MODEL_STD 
)
 valid_loader = torch.utils.data.DataLoader(
 dataset=valid_dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=True,
num_workers=4 
)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="min",
patience=5,
factor=0.3,verbose=True)
 early_stopping = EarlyStopping(patience=5, verbose=True)
 best_score = -1
print("FOLD : ", VALIDATION_FOLDS[0])
 for epoch in range(1, EPOCHS+1):
train_loss, train_score = train(train_dataset,train_loader, model, optimizer)
 val_loss, val_score = evaluate(valid_dataset, valid_loader, model)
 scheduler.step(val_loss)
 if val_score > best_score:
best_score = val_score
torch.save(model.state_dict() , f"{BASE_MODEL}_fold{VALIDATION_FOLDS[0]}.pth")
 epoch_len = len(str(EPOCHS))
print_msg =(f'[{epoch:>{epoch_len}}/{EPOCHS:>{epoch_len}}] ' +
f'train_loss: {train_loss:.5f} ' +
f'train_score: {train_score:.5f} ' +
f'valid_loss: {val_loss:.5f} ' +
f'valid_score: {val_score:.5f}' 
)
 print(print_msg)
 early_stopping(val_score, model)
 if early_stopping.early_stop:
print("Early stopping")
 break
if __name__ == "__main__":
main()
test_ds = get_test_dataset(ordered=True)
 print('Calculating predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probs1 = np.mean(predict_tta(model1, TTA_NUM), axis=0)
 probs2 = np.mean(predict_tta(model2, TTA_NUM), axis=0)
 probabilities = best_alpha*probs1 +(1-best_alpha)*probs2
predictions = np.argmax(probabilities, axis=-1)
 print('Generating submission file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<import_modules>
%%writefile run.sh
export IMG_HEIGHT=137
export IMG_WIDTH=236
export EPOCHS=50
export TRAIN_BATCH_SIZE=64
export TEST_BATCH_SIZE=64
export MODEL_MEAN="(0.485, 0.456, 0.406)"
export MODEL_STD="(0.229, 0.224, 0.225)"
export BASE_MODEL="resnet34"
export TRAINING_FOLDS_CSV=".. /input/train_folds.csv"
export TRAINING_FOLDS="(0,1,2,3)"
export VALIDATION_FOLDS="(4,)"
python3 train.py
export TRAINING_FOLDS="(0,1,2,4)"
export VALIDATION_FOLDS="(3,)"
python3 train.py
export TRAINING_FOLDS="(0,1,3,4)"
export VALIDATION_FOLDS="(2,)"
python3 train.py
export TRAINING_FOLDS="(0,2,3,4)"
export VALIDATION_FOLDS="(1,)"
python3 train.py
export TRAINING_FOLDS="(1,2,3,4)"
export VALIDATION_FOLDS="(0,)"
python3 train.py
!pip install efficientnet
print("Tensorflow version " + tf.__version__ )<set_options>
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, pt_models)
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("TPU REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
import glob
import torch
import albumentations
import pandas as pd
import numpy as np
from tqdm import tqdm
from PIL import Image
import joblib
import torch.nn as nn
from torch.nn import functional as F
GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_variables>
MODEL_MEAN =(0.485, 0.456, 0.406)
 MODEL_STD =(0.229, 0.224, 0.225)
 IMG_HEIGHT = 137
IMG_WIDTH = 236
DEVICE="cuda"
IMAGE_SIZE = [224, 224]
EPOCHS = 10
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
NUM_TRAINING_IMAGES = 12753
NUM_TEST_IMAGES = 7382
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE<categorify>
class ResNet34(nn.Module):
def __init__(self, pretrained):
super(ResNet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.l0 = nn.Linear(512, 168)
 self.l1 = nn.Linear(512, 11)
 self.l2 = nn.Linear(512, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum 
<normalization>
class BengaliDatasetTest:
def __init__(self, df, img_height, img_width, mean, std):
self.image_ids = df.image_id.values
self.img_arr = df.iloc[:, 1:].values
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = self.img_arr[item, :]
img_id = self.image_ids[item]
image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"image_id": img_id
}
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 15.* tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label 
<create_dataframe>
def model_predict() :
g_pred, v_pred, c_pred = [], [], []
img_ids_list = []
for file_idx in range(4):
df = pd.read_parquet(f".. /input/bengaliai-cv19/test_image_data_{file_idx}.parquet")
 dataset = BengaliDatasetTest(df=df,
img_height=IMG_HEIGHT,
img_width=IMG_WIDTH,
mean=MODEL_MEAN,
std=MODEL_STD)
 data_loader = torch.utils.data.DataLoader(
 dataset=dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=False,
num_workers=4 
)
 for bi, d in enumerate(data_loader):
image = d["image"]
img_id = d["image_id"]
image = image.to(DEVICE, dtype=torch.float)
 g, v, c = model(image)
 for ii, imid in enumerate(img_id):
g_pred.append(g[ii].cpu().detach().numpy())
 v_pred.append(v[ii].cpu().detach().numpy())
 c_pred.append(c[ii].cpu().detach().numpy())
 img_ids_list.append(imid)
 return g_pred, v_pred, c_pred, img_ids_list
def load_dataset(filenames, labeled=True, ordered=False, is_train=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)
 return dataset
def get_training_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/train/*.tfrec'), labeled=True, is_train=True)
 dataset = dataset.map(transform)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 return dataset
def get_validation_dataset() :
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/val/*.tfrec'), labeled=True, ordered=False, is_train=False)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-224x224/test/*.tfrec'), labeled=False, ordered=ordered, is_train=False)
 dataset = dataset.batch(BATCH_SIZE)
 return dataset
training_dataset = get_training_dataset()
validation_dataset = get_validation_dataset()<choose_model_class>
model = ResNet34(pretrained=False)
 TEST_BATCH_SIZE = 32
final_g_pred = []
final_v_pred = []
final_c_pred = []
final_img_ids = []
for i in range(5):
model.load_state_dict(torch.load(f".. /input/resnet34weights/resnet34_fold{i}.pth"))
model.to(DEVICE)
 model.eval()
g_pred, v_pred, c_pred, img_ids_list = model_predict()
final_g_pred.append(g_pred)
 final_v_pred.append(v_pred)
 final_c_pred.append(c_pred)
 if i == 0:
final_img_ids.extend(img_ids_list )
LR_START = 0.00001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
learning_rate_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
final_g = np.argmax(np.mean(np.array(final_g_pred), axis=0), axis=1)
 final_v = np.argmax(np.mean(np.array(final_v_pred), axis=0), axis=1)
 final_c = np.argmax(np.mean(np.array(final_c_pred), axis=0), axis=1 )
with strategy.scope() :
effb3_model = efn.EfficientNetB3(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 effb3_model.trainable = True
model1 = tf.keras.Sequential([
effb3_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 opt = optimizers.Adam(lr=0.01)
 model1.compile(
 optimizer=opt,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical1 = model1.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[learning_rate_callback],
validation_data=validation_dataset )<choose_model_class>
predictions = []
for ii, imid in enumerate(final_img_ids):
predictions.append(( f"{imid}_grapheme_root", final_g[ii]))
predictions.append(( f"{imid}_vowel_diacritic", final_v[ii]))
predictions.append(( f"{imid}_consonant_diacritic", final_c[ii]))
with strategy.scope() :
effb4_model = efn.EfficientNetB4(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 effb4_model.trainable = True
model2 = tf.keras.Sequential([
effb4_model,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(104, activation='softmax')
 ])
 opt = optimizers.Adam(lr=0.01)
 model2.compile(
 optimizer=opt,
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 historical2 = model2.fit(training_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[learning_rate_callback],
validation_data=validation_dataset)
<choose_model_class>
sub = pd.DataFrame(predictions, columns=["row_id", "target"] )
<choose_model_class><EOS> 
sub.to_csv("submission.csv", index=False )