Kaggle competition: Kannada MNIST
frame.to_csv('submission.csv', index=False )<import_modules>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score<define_variables>
from fastai import *
from fastai.vision import *
import imageio
PATH = './'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE = '.. /input/sample_submission.csv'<define_variables>
path = Path('.. /input/Kannada-MNIST')
 train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test =pd.read_csv('.. /input/Kannada-MNIST/test.csv' )
name_label_dict = {
0: 'Nucleoplasm',
1: 'Nuclear membrane',
2: 'Nucleoli',
3: 'Nucleoli fibrillar center',
4: 'Nuclear speckles',
5: 'Nuclear bodies',
6: 'Endoplasmic reticulum',
7: 'Golgi apparatus',
8: 'Peroxisomes',
9: 'Endosomes',
10: 'Lysosomes',
11: 'Intermediate filaments',
12: 'Actin filaments',
13: 'Focal adhesion sites',
14: 'Microtubules',
15: 'Microtubule ends',
16: 'Cytokinetic bridge',
17: 'Mitotic spindle',
18: 'Microtubule organizing center',
19: 'Centrosome',
20: 'Lipid droplets',
21: 'Plasma membrane',
22: 'Cell junctions',
23: 'Mitochondria',
24: 'Aggresome',
25: 'Cytosol',
26: 'Cytoplasmic bodies',
27: 'Rods & rings' }<choose_model_class>
def to_img_shape(data_X, data_y=[]):
data_X = np.array(data_X ).reshape(-1,28,28)
 data_X = np.stack(( data_X,)*3, axis=-1)
 data_y = np.array(data_y)
 return data_X,data_y
nw = 2
arch = resnet34<split>
data_X, data_y = train.loc[:,'pixel0':'pixel783'], train['label']
train_X, val_X, train_y, val_y = train_test_split(data_X, data_y, test_size=0.01,random_state=7,stratify=data_y )
train_names = list({f[:36] for f in os.listdir(TRAIN)})
 test_names = list({f[:36] for f in os.listdir(TEST)})
 tr_n, val_n = train_test_split(train_names, test_size=0.1, random_state=42 )<data_type_conversions>
train_X,train_y = to_img_shape(train_X, train_y)
 val_X,val_y = to_img_shape(val_X,val_y )
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return np.stack(img, axis=-1 )<prepare_x_and_y>
tfms = get_transforms(do_flip=False)
 data =(ImageList.from_folder('/data/')
.split_by_folder() 
.label_from_folder() 
.add_test_folder() 
.transform(tfms, size=64)
.databunch() )
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.labels = pd.read_csv(LABELS ).set_index('Id')
 self.labels['Target'] = [[int(i)for i in s.split() ] for s in self.labels['Target']]
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_rgby(self.path,self.fnames[i])
 if self.sz == 512: return img
else: return cv2.resize(img,(self.sz, self.sz),cv2.INTER_AREA)
 def get_y(self, i):
if(self.path == TEST): return np.zeros(len(name_label_dict),dtype=np.int)
 else:
labels = self.labels.loc[self.fnames[i]]['Target']
return np.eye(len(name_label_dict),dtype=np.float)[labels].sum(axis=0)
 @property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return len(name_label_dict )<create_dataframe>
data.show_batch(3,figsize=(6,6))
def get_data(sz,bs):
aug_tfms = [RandomRotate(30, tfm_y=TfmType.NO),
RandomDihedral(tfm_y=TfmType.NO)]
stats = A([0.00505, 0.00331, 0.00344, 0.00519], [0.10038, 0.08131, 0.08284, 0.10179])
 tfms = tfms_from_stats(stats, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(pdFilesDataset,(tr_n[:-(len(tr_n)%bs)],TRAIN), 
(val_n,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md<train_model>
!mkdir -p /tmp/.cache/torch/checkpoints
!cp /kaggle/input/fastai-pretrained-models/resnet50-19c8e357.pth /tmp/.cache/torch/checkpoints/resnet50-19c8e357.pth
learn = cnn_learner(data, models.resnet50, metrics=[error_rate, accuracy], model_dir = Path('.. /kaggle/working'),path = Path("."))
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()<compute_test_metric>
learn.fit_one_cycle(4 )
def acc(preds,targs,thresh=0.0):
preds =(preds > thresh ).int()
targs = targs.int()
return(preds==targs ).float().mean()<choose_model_class>
learn.lr_find()
class ConvnetBuilder_custom() :
def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0,
custom_head=None, pretrained=True):
self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut
if xtra_fc is None: xtra_fc = [512]
if ps is None: ps = [0.25]*len(xtra_fc)+ [0.5]
self.ps,self.xtra_fc = ps,xtra_fc
if f in model_meta: cut,self.lr_cut = model_meta[f]
else: cut,self.lr_cut = 0,0
cut-=xtra_cut
layers = cut_model(f(pretrained), cut)
 w = layers[0].weight
layers[0] = nn.Conv2d(4,64,kernel_size=(7,7),stride=(2,2),padding=(3, 3), bias=False)
 layers[0].weight = torch.nn.Parameter(torch.cat(( w,torch.zeros(64,1,7,7)) ,dim=1))
self.nf = model_features[f] if f in model_features else(num_features(layers)*2)
 if not custom_head: layers += [AdaptiveConcatPool2d() , Flatten() ]
self.top_model = nn.Sequential(*layers)
 n_fc = len(self.xtra_fc)+1
if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
if custom_head: fc_layers = [custom_head]
else: fc_layers = self.get_fc_layers()
self.n_fc = len(fc_layers)
 self.fc_model = to_gpu(nn.Sequential(*fc_layers))
if not custom_head: apply_init(self.fc_model, kaiming_normal)
 self.model = to_gpu(nn.Sequential(*(layers+fc_layers)))
 @property
def name(self): return f'{self.f.__name__}_{self.xtra_cut}'
def create_fc_layer(self, ni, nf, p, actn=None):
res=[nn.BatchNorm1d(num_features=ni)]
if p: res.append(nn.Dropout(p=p))
res.append(nn.Linear(in_features=ni, out_features=nf))
if actn: res.append(actn)
 return res
def get_fc_layers(self):
res=[]
ni=self.nf
for i,nf in enumerate(self.xtra_fc):
res += self.create_fc_layer(ni, nf, p=self.ps[i], actn=nn.ReLU())
 ni=nf
final_actn = nn.Sigmoid() if self.is_multi else nn.LogSoftmax()
if self.is_reg: final_actn = None
res += self.create_fc_layer(ni, self.c, p=self.ps[-1], actn=final_actn)
 return res
def get_layer_groups(self, do_fc=False):
if do_fc:
return [self.fc_model]
idxs = [self.lr_cut]
c = children(self.top_model)
 if len(c)==3: c = children(c[0])+c[1:]
lgs = list(split_by_idxs(c,idxs))
return lgs+[self.fc_model]
class ConvLearner(Learner):
def __init__(self, data, models, precompute=False, **kwargs):
self.precompute = False
super().__init__(data, models, **kwargs)
 if hasattr(data, 'is_multi')and not data.is_reg and self.metrics is None:
self.metrics = [accuracy_thresh(0.5)] if self.data.is_multi else [accuracy]
if precompute: self.save_fc1()
self.freeze()
self.precompute = precompute
def _get_crit(self, data):
if not hasattr(data, 'is_multi'): return super()._get_crit(data)
 return F.l1_loss if data.is_reg else F.binary_cross_entropy if data.is_multi else F.nll_loss
@classmethod
def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
pretrained=True, **kwargs):
models = ConvnetBuilder_custom(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=pretrained)
 return cls(data, models, precompute, **kwargs)
 @classmethod
def lsuv_learner(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False, **kwargs):
models = ConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=False)
 convlearn=cls(data, models, precompute, **kwargs)
 convlearn.lsuv_init()
return convlearn
@property
def model(self): return self.models.fc_model if self.precompute else self.models.model
def half(self):
if self.fp16: return
self.fp16 = True
if type(self.model)!= FP16: self.models.model = FP16(self.model)
 if not isinstance(self.models.fc_model, FP16): self.models.fc_model = FP16(self.models.fc_model)
 def float(self):
if not self.fp16: return
self.fp16 = False
if type(self.models.model)== FP16: self.models.model = self.model.module.float()
if type(self.models.fc_model)== FP16: self.models.fc_model = self.models.fc_model.module.float()
@property
def data(self): return self.fc_data if self.precompute else self.data_
def create_empty_bcolz(self, n, name):
return bcolz.carray(np.zeros(( 0,n), np.float32), chunklen=1, mode='w', rootdir=name)
 def set_data(self, data, precompute=False):
super().set_data(data)
 if precompute:
self.unfreeze()
self.save_fc1()
self.freeze()
self.precompute = True
else:
self.freeze()
def get_layer_groups(self):
return self.models.get_layer_groups(self.precompute)
 def summary(self):
precompute = self.precompute
self.precompute = False
res = super().summary()
self.precompute = precompute
return res
def get_activations(self, force=False):
tmpl = f'_{self.models.name}_{self.data.sz}.bc'
names = [os.path.join(self.tmp_path, p+tmpl)for p in('x_act', 'x_act_val', 'x_act_test')]
if os.path.exists(names[0])and not force:
self.activations = [bcolz.open(p)for p in names]
else:
self.activations = [self.create_empty_bcolz(self.models.nf,n)for n in names]
def save_fc1(self):
self.get_activations()
act, val_act, test_act = self.activations
m=self.models.top_model
if len(self.activations[0])!=len(self.data.trn_ds):
predict_to_bcolz(m, self.data.fix_dl, act)
 if len(self.activations[1])!=len(self.data.val_ds):
predict_to_bcolz(m, self.data.val_dl, val_act)
 if self.data.test_dl and(len(self.activations[2])!=len(self.data.test_ds)) :
if self.data.test_dl: predict_to_bcolz(m, self.data.test_dl, test_act)
 self.fc_data = ImageClassifierData.from_arrays(self.data.path, 
(act, self.data.trn_y),(val_act, self.data.val_y), self.data.bs, classes=self.data.classes,
test = test_act if self.data.test_dl else None, num_workers=8)
 def freeze(self):
self.freeze_to(-1)
 def unfreeze(self):
self.freeze_to(0)
 self.precompute = False
def predict_array(self, arr):
precompute = self.precompute
self.precompute = False
pred = super().predict_array(arr)
 self.precompute = precompute
return pred<choose_model_class>
lr = slice(2e-05 )
sz = 256
bs = 64
md = get_data(sz,bs)
 learner = ConvLearner.pretrained(arch, md, ps=0.5)
 learner.opt_fn = optim.Adam
learner.crit = FocalLoss()
learner.metrics = [acc]<train_model>
learn.save('stage-1' )
lr = 2e-2
learner.fit(lr,1 )<define_search_space>
learn.fit_one_cycle(5,lr )
learner.unfreeze()
lrs=np.array([lr/10,lr/3,lr] )<train_model>
learn.save('stage-2' )
learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20))<train_model>
learn.fit_one_cycle(10,lr )
learner.fit(lrs/4,2,cycle_len=4,use_clr=(10,20))<train_model>
learn.save('stage-3' )
learner.fit(lrs/8,1,cycle_len=8,use_clr=(5,20))<save_model>
learn.load('stage-3' )
learner.save('ResNet34_256_1' )<compute_test_metric>
test_csv = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_csv.drop('id',axis = 'columns',inplace = True)
 sub_df = pd.DataFrame(columns=['id','label'] )
def sigmoid_np(x):
return 1.0/(1.0 + np.exp(-x))
preds,y = learner.TTA()
preds = np.stack(preds, axis=-1)
 preds = sigmoid_np(preds)
 pred = preds.max(axis=-1 )<compute_test_metric>
def get_img(data):
t1 = data.reshape(28,28)/255
t1 = np.stack([t1]*3,axis=0)
 img = Image(FloatTensor(t1))
return img
th = np.array([0.50,0.56,0.50,0.50,0.50,0.47,0.39,0.56,0.36,0.45,
0.41,0.48,0.45,0.44,0.63,0.50,0.38,0.33,0.45,0.41,
0.33,0.49,0.43,0.52,0.51,0.48,0.43,0.1])
 print(f1_score(y, pred>0.5, average='macro'),f1_score(y, pred>th, average='macro'))<concatenate>
from fastprogress import progress_bar
preds_t,y_t = learner.TTA(is_test=True)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = sigmoid_np(preds_t)
 pred_t = preds_t.max(axis=-1 )<define_variables>
def decr(ido):
return ido-1
sub_df['id'] = sub_df['id'].map(decr)
 sub_df.to_csv('submission.csv',index=False )