Kaggle competition: Bengali.AI Handwritten Grapheme Classification
<SOS> metric: WeightedCategorizationAccuracy Kaggle data source: bengali.ai-handwritten-grapheme-classification<load_from_csv>
%matplotlib inline
start_time = time.time()
results_df = pd.read_csv('./sample_submission.csv')
 results_df.head()<train_model>
dataset = '/kaggle/input/bengaliai-cv19'
pretrained = '.. /input/bangla-graphemes-pretrained-weights'
results_df = test_model(model, test_data_loader, results_df)
 results_df.head()<save_to_csv>
if os.path.isfile(os.path.join(pretrained,"GraphemeDenseNet121.h5")) \
and os.path.isfile(os.path.join(pretrained,"hist.csv")) :
print('Model is present')
 else:
print("Error.No Model Found" )
results_df.to_csv("submission.csv", index=False )<init_hyperparams>
SIZE = 100
N_ch = 1
class LM:
def __init__(self, n):
self.n_gram = n
def train(self, text):
self.n_counts = defaultdict(Counter)
 for i in range(0, len(text)- self.n_gram + 1):
t = text[i:i+self.n_gram-1]
n_char = text[i+self.n_gram-1]
self.n_counts[t][n_char] += 1
def generate(self, init_text, n):
text = init_text
while len(text)< n:
lookup_text = text[-self.n_gram+1:]
if lookup_text not in self.n_counts:
break
counter = self.n_counts[text[-self.n_gram+1:]]
keys = list(counter.keys())
 values = list(counter.values())
 probs = [v/sum(values)for v in values]
cummulative_probs = np.cumsum(probs)
 p = np.random.rand()
for i in range(len(cummulative_probs)) :
if p <= cummulative_probs[i]:
text += keys[i]
break
return text<train_model>
model = load_model(os.path.join(pretrained, 'GraphemeDenseNet121.h5'))
def get_trained_lm(n, excerpts):
lm = LM(n)
 text = ' '.join(list(excerpts))
lm.train(text)
 return lm<normalization>
def resize(df, size=100):
resized = {}
resize_size=100
angle=0
for i in range(df.shape[0]):
image=df.loc[df.index[i]].values.reshape(137,236)
 image_center = tuple(np.array(image.shape[1::-1])/ 2)
 matrix = cv2.getRotationMatrix2D(image_center, angle, 1.0)
 image = cv2.warpAffine(image, matrix, image.shape[1::-1], flags=cv2.INTER_AREA,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
matrix = cv2.getRotationMatrix2D(image_center, 0, 1.0)
 image = cv2.warpAffine(image, matrix, image.shape[1::-1], flags=cv2.INTER_AREA,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
augBright=A.RandomBrightnessContrast(p=1.0)
 image = augBright(image=image)['image']
_, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
 contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
idx = 0
ls_xmin = []
ls_ymin = []
ls_xmax = []
ls_ymax = []
for cnt in contours:
idx += 1
x,y,w,h = cv2.boundingRect(cnt)
 ls_xmin.append(x)
 ls_ymin.append(y)
 ls_xmax.append(x + w)
 ls_ymax.append(y + h)
 xmin = min(ls_xmin)
 ymin = min(ls_ymin)
 xmax = max(ls_xmax)
 ymax = max(ls_ymax)
 roi = image[ymin:ymax,xmin:xmax]
resized_roi = cv2.resize(roi,(resize_size, resize_size),interpolation=cv2.INTER_AREA)
 resized[df.index[i]] = resized_roi.reshape(-1)
 resized_df = pd.DataFrame(resized ).T
return resized_df
n=7
lm_EAP = get_trained_lm(n, train[train['author']=='EAP']['text'])
 lm_HPL = get_trained_lm(n, train[train['author']=='HPL']['text'])
 lm_MWS = get_trained_lm(n, train[train['author']=='MWS']['text'])
 print(f"lm_EAP size: {len(lm_EAP.n_counts)}, lm_HPL size: {len(lm_HPL.n_counts)}, lm_MWS size: {len(lm_MWS.n_counts)}" )<define_variables>
tgt_cols = ['grapheme_root','vowel_diacritic','consonant_diacritic']
seed_string = "Dark night"
l = 120
models = {'Poe':lm_EAP, 'Lovecraft': lm_HPL, 'Shelley': lm_MWS}
for model_name in models.keys() :
generated = models[model_name].generate(seed_string, l)
 print(f""{generated}...", The bot {model_name}, 2020
" )<import_modules>
row_ids = []
targets = []
id = 0
for i in range(4):
img_df = pd.read_parquet(os.path.join(
 dataset, 'test_image_data_'+str(i)+'.parquet'))
img_df = img_df.drop('image_id', axis = 1)
 img_df = resize(img_df, SIZE)/ 255
X_test = img_df.values.reshape(-1, SIZE, SIZE, N_ch)
 preds = model.predict(X_test)
 for j in range(len(X_test)) :
for k in range(3):
row_ids.append('Test_'+str(id)+'_'+tgt_cols[k])
 targets.append(np.argmax(preds[k][j]))
id += 1
def np_resize(img, input_shape):

 height, width = input_shape
return cv2.resize(img,(width, height))
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape, reshape=None):
depth = len(rles)
 if reshape is None:
masks = np.zeros(( *input_shape, depth))
else:
masks = np.zeros(( *reshape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
if reshape is None:
masks[:, :, i] = rle2mask(rle, input_shape)
 else:
mask = rle2mask(rle, input_shape)
 reshaped_mask = np_resize(mask, reshape)
 masks[:, :, i] = reshaped_mask
return masks
def build_rles(masks, reshape=None):
width, height, depth = masks.shape
rles = []
for i in range(depth):
mask = masks[:, :, i]
if reshape:
mask = mask.astype(np.float32)
 mask = np_resize(mask, reshape ).astype(np.int64)
 rle = mask2rle(mask)
 rles.append(rle)
 return rles
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 350, 525), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
def post_process_minsize(mask, min_size):

 num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(mask.shape)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions
def draw_convex_hull(mask, mode='convex'):
img = np.zeros(mask.shape)
 contours, hier = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
 for c in contours:
if mode=='rect':
x, y, w, h = cv2.boundingRect(c)
 cv2.rectangle(img,(x, y),(x+w, y+h),(255, 255, 255), -1)
 elif mode=='convex':
hull = cv2.convexHull(c)
 cv2.drawContours(img, [hull], 0,(255, 255, 255),-1)
 elif mode=='approx':
epsilon = 0.02*cv2.arcLength(c,True)
 approx = cv2.approxPolyDP(c,epsilon,True)
 cv2.drawContours(img, [approx], 0,(255, 255, 255),-1)
 else:
rect = cv2.minAreaRect(c)
 box = cv2.boxPoints(rect)
 box = np.int0(box)
 cv2.drawContours(img, [box], 0,(255, 255, 255),-1)
 return img/255.
 def dice(img1, img2):
img1 = np.asarray(img1 ).astype(np.bool)
 img2 = np.asarray(img2 ).astype(np.bool)
 intersection = np.logical_and(img1, img2)
 return 2.* intersection.sum() /(img1.sum() + img2.sum())
 def rle_decode(mask_rle, shape=(1400, 2100)) :
s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape, order='F' )<load_from_csv>
df_submit = pd.DataFrame({'row_id':row_ids,'target':targets},
columns = ['row_id','target'])
 df_submit.to_csv('submission.csv',index=False)
 df_submit.head(10 )