Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<load_pretrained>
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, pt_models)
create_gif('/kaggle/working/monet/*.png', 'monet.gif' )<load_pretrained>
def normalize_image(img):
return(img*(255.0/img.max())).astype(np.uint8)
 def get_min_indices(img, min_writing_value=80):
min_value = img > min_writing_value
h_min, h_max = np.where(np.any(min_value, axis=0)) [0][[0, -1]]
v_min, v_max = np.where(np.any(min_value, axis=1)) [0][[0, -1]]
return(h_min, h_max, v_min, v_max)
 def get_min_indices_with_border(img, min_writing_value=80, border=20):
h_min, h_max, v_min, v_max = get_min_indices(img[border:-border, border:-border], min_writing_value=min_writing_value)
 return(h_min + border, h_max + border, v_min + border, v_max + border)
 def cut_and_denoise_image(img, border=20, min_writing_value=80, max_noise=28):
h_min, h_max, v_min, v_max = get_min_indices_with_border(img, border=border, min_writing_value=min_writing_value)
 h_min=(h_min-border)if h_min>border else 0
v_min=(v_min-border)if v_min>border else 0
h_max=(h_max+border)if ORIGINAL_WIDTH-h_max>border else ORIGINAL_WIDTH
v_max=(v_max+border)if ORIGINAL_HEIGHT-v_max>border else ORIGINAL_HEIGHT
img = img[v_min:v_max, h_min:h_max]
img[img < max_noise] = 0
longer_side_length = max(np.ma.size(img, axis=0), np.ma.size(img, axis=1))
padding = [(( longer_side_length - np.ma.size(img, axis=0)) // 2,), 
(( longer_side_length - np.ma.size(img, axis=1)) // 2,)]
img = np.pad(img, padding, mode="constant")
 return cv2.resize(img,(PROCESSED_HEIGHT, PROCESSED_WIDTH))
create_gif('/kaggle/working/photo/*.png', 'photo.gif' )<compute_test_metric>
def load_data(file_name):
print("Dealing with ", file_name)
 df = pd.read_parquet(file_name)
 names = df.iloc[:,0].values
images = 255 - df.iloc[:,1:].values.reshape(-1, ORIGINAL_HEIGHT, ORIGINAL_WIDTH)
 del df
print("Freed after loading: ", gc.collect())
 for image_index, img in enumerate(images):
img = normalize_image(img)
 img = cut_and_denoise_image(img)
 images[image_index,0:PROCESSED_HEIGHT, 0:PROCESSED_WIDTH] = img
images = images[:,0:PROCESSED_HEIGHT, 0:PROCESSED_WIDTH]
images = images.reshape(-1, PROCESSED_HEIGHT, PROCESSED_WIDTH)
 print("Freed after processing: ", gc.collect())
 return names, images
evaluate_cycle(photo_ds_eval.take(2), monet_generator, photo_generator, n_samples=2 )<compute_test_metric>
with open("submission.csv", "a")as submission_file:
submission_file.write("row_id,target
")
 def write_predictions(names, prediction):
chosen_root = np.argmax(prediction[0].detach().numpy() , axis=1)
 chosen_vowel = np.argmax(prediction[1].detach().numpy() , axis=1)
 chosen_consonant = np.argmax(prediction[2].detach().numpy() , axis=1)
 with open("submission.csv", "a")as submission_file:
for index, name in enumerate(names):
submission_file.write(f"{name}_consonant_diacritic,{chosen_consonant[index]}
")
 submission_file.write(f"{name}_grapheme_root,{chosen_root[index]}
")
 submission_file.write(f"{name}_vowel_diacritic,{chosen_vowel[index]}
" )
evaluate_cycle(monet_ds_eval.take(2), photo_generator, monet_generator, n_samples=2 )<split>
class SubmissionDataSet() :
transform = albumentations.Compose([albumentations.Normalize(always_apply=True)])
 def __init__(self, names, images):
self.names = names
self.images = images
def __len__(self):
return len(self.names)
 def __getitem__(self, index):
img = self.images[index]
img = Image.fromarray(img ).convert("RGB")
 img = self.transform(image=np.array(img)) ["image"]
img = np.transpose(img,(2,0,1))
return(self.names[index], torch.tensor(img))
display_generated_samples(photo_ds_eval.take(8), monet_generator, 8 )<predict_on_test>
class Net(torch.nn.Module):
def __init__(self):
super(Net, self ).__init__()
self.transfer_model = pretrainedmodels.__dict__["resnet101"](pretrained=None)
 self.drop1 = torch.nn.Dropout(p=0.3)
 self.l1 = torch.nn.Linear(2048, 1024)
 self.drop2 = torch.nn.Dropout(p=0.3)
 self.l2 = torch.nn.Linear(1024, 512)
 self.root = torch.nn.Linear(512, 168)
 self.vowel = torch.nn.Linear(512, 11)
 self.consonant = torch.nn.Linear(512, 7)
 def forward(self, x):
batch_size = x.shape[0]
x = self.transfer_model.features(x)
 x = torch.nn.functional.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 x = self.drop1(x)
 x = self.l1(x)
 x = self.drop2(x)
 x = self.l2(x)
 root = self.root(x)
 vowel = self.vowel(x)
 consonant = self.consonant(x)
 return(root, vowel, consonant)
 return(root, vowel, consonant)
 model = Net()
model.load_state_dict(torch.load("/kaggle/input/bengalipytorchmodels/ResNet101_30Epochs.pt", map_location=torch.device('cpu')))
 model.eval()
<load_pretrained><EOS> 
for file_index in range(0, 4):
names, images = load_data(f"/kaggle/input/bengaliai-cv19/test_image_data_{file_index}.parquet")
 batch_data_set = SubmissionDataSet(names, images)
 data_loader = torch.utils.data.DataLoader(batch_data_set,
batch_size=8,
shuffle=False,
num_workers=4)
 for batch_index,(batch_names, batch_images)in enumerate(tqdm(data_loader, total=(len(batch_data_set)//data_loader.batch_size))):
prediction = model(batch_images)
 write_predictions(batch_names, prediction)
 print("Collected after writing predictions: ", gc.collect() )