Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<load_from_csv>
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0,pt_models)
submission = pd.read_csv('.. /input/cloudefficientnetb2/submission.csv')
 submission.head()<filter>
TEST_BATCH_SIZE = 32
MODEL_MEAN=(0.485,0.465,0.406)
 MODEL_STD=(0.229,0.224,0.225)
 IMG_HEIGHT=137
IMG_WIDTH=236
DEVICE="cuda"
image_labels_andrew_nonempty = submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values
image_labels_andrew_empty = submission.loc[submission['EncodedPixels'].isnull() , 'Image_Label'].values<load_from_csv>
class ResNet34(nn.Module):
def __init__(self ,pretrained):
super(ResNet34,self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.layer0 = nn.Linear(512,168)
 self.layer1 = nn.Linear(512,11)
 self.layer2 = nn.Linear(512,7)
 def forward(self,x):
batch_size ,_,_,_ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x,1 ).reshape(batch_size, -1)
 layer0 = self.layer0(x)
 layer1 = self.layer1(x)
 layer2 = self.layer2(x)
 return layer0, layer1, layer2
images_to_refill = images_labels_full.intersection(set(image_labels_andrew_empty))
submission_no_minsize = pd.read_csv('.. /input/clouds-classifier-files/submission_no_minsize.csv')
 masks_no_minsize_refillers = submission_no_minsize.loc[submission_no_minsize['Image_Label'].isin(images_to_refill)&
~submission_no_minsize['EncodedPixels'].isnull() ]
print(f'{len(masks_no_minsize_refillers)} masks would be added.' )<concatenate>
class BengaliDatasetTest:
def __init__(self, df, img_height, img_width, mean, std):
self.image_ids = df.image_id.values
self.img_arr = df.iloc[:, 1:].values
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = self.img_arr[item, :]
img_id = self.image_ids[item]
image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"image_id": img_id
}
print(f'{len(image_labels_empty.intersection(set(image_labels_andrew_nonempty)))} masks would be removed' )<save_to_csv>
def model_predict() :
g_pred, v_pred, c_pred = [], [], []
img_ids_list = []
for file_idx in range(4):
df = pd.read_parquet(f".. /input/bengaliai-cv19/test_image_data_{file_idx}.parquet")
 dataset = BengaliDatasetTest(df=df,
img_height=IMG_HEIGHT,
img_width=IMG_WIDTH,
mean=MODEL_MEAN,
std=MODEL_STD)
 data_loader = torch.utils.data.DataLoader(
 dataset=dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=False,
num_workers=4 
)
 for bi, d in enumerate(data_loader):
image = d["image"]
img_id = d["image_id"]
image = image.to(DEVICE, dtype=torch.float)
 g, v, c = model(image)
 for ii, imid in enumerate(img_id):
g_pred.append(g[ii].cpu().detach().numpy())
 v_pred.append(v[ii].cpu().detach().numpy())
 c_pred.append(c[ii].cpu().detach().numpy())
 img_ids_list.append(imid)
 return g_pred, v_pred, c_pred, img_ids_list
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
img_labels_to_refill = set(masks_no_minsize_refillers['Image_Label'].values)
 submission = submission[~submission['Image_Label'].isin(img_labels_to_refill)]
submission = pd.concat(( submission, masks_no_minsize_refillers))
submission.to_csv('submission.csv', index=None )<load_from_csv>
model = ResNet34(pretrained=False)
 TEST_BATCH_SIZE = 32
Start_fold=3
final_g_pred = []
final_v_pred = []
final_c_pred = []
final_img_ids = []
for i in range(Start_fold,5):
model.load_state_dict(torch.load(f".. /input/bengali-models/50 Epoch/resnet34_fold{i}.bin"))
model.to(DEVICE)
 model.eval()
g_pred, v_pred, c_pred, img_ids_list = model_predict()
final_g_pred.append(g_pred)
 final_v_pred.append(v_pred)
 final_c_pred.append(c_pred)
 if i == Start_fold:
final_img_ids.extend(img_ids_list )
submission = pd.read_csv('/kaggle/input/104-flowers-submissions/0.96523.csv')
 submission<save_to_csv>
final_g = np.argmax(np.mean(np.array(final_g_pred), axis=0), axis=1)
 final_v = np.argmax(np.mean(np.array(final_v_pred), axis=0), axis=1)
 final_c = np.argmax(np.mean(np.array(final_c_pred), axis=0), axis=1)
 predictions = []
for ii, imid in enumerate(final_img_ids):
predictions.append(( f"{imid}_grapheme_root", final_g[ii]))
predictions.append(( f"{imid}_vowel_diacritic", final_v[ii]))
predictions.append(( f"{imid}_consonant_diacritic", final_c[ii]))
submission.to_csv('submission.csv', index=False )<save_to_csv>
sub = pd.DataFrame(predictions,columns=["row_id","target"])
 print(sub )
submission.to_csv('submission.csv', index=False )<install_modules>
sub.to_csv("submission.csv",index=False )