Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<drop_column>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
all_data = all_data.drop(['Name'], axis=1)
 all_data.head()<sort_values>
PATH = './'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE = '.. /input/sample_submission.csv'
all_data[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
name_label_dict = {
0: 'Nucleoplasm',
1: 'Nuclear membrane',
2: 'Nucleoli',
3: 'Nucleoli fibrillar center',
4: 'Nuclear speckles',
5: 'Nuclear bodies',
6: 'Endoplasmic reticulum',
7: 'Golgi apparatus',
8: 'Peroxisomes',
9: 'Endosomes',
10: 'Lysosomes',
11: 'Intermediate filaments',
12: 'Actin filaments',
13: 'Focal adhesion sites',
14: 'Microtubules',
15: 'Microtubule ends',
16: 'Cytokinetic bridge',
17: 'Mitotic spindle',
18: 'Microtubule organizing center',
19: 'Centrosome',
20: 'Lipid droplets',
21: 'Plasma membrane',
22: 'Cell junctions',
23: 'Mitochondria',
24: 'Aggresome',
25: 'Cytosol',
26: 'Cytoplasmic bodies',
27: 'Rods & rings' }
all_data['family_size'] = all_data.SibSp + all_data.Parch+1<sort_values>
nw = 2
arch = resnet34
all_data[['family_size', 'Survived']].groupby(['family_size'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<categorify>
train_names = list({f[:36] for f in os.listdir(TRAIN)})
 test_names = list({f[:36] for f in os.listdir(TEST)})
 tr_n, val_n = train_test_split(train_names, test_size=0.1, random_state=42 )
all_data['family_group'] = all_data['family_size'].map(family_group )<drop_column>
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return np.stack(img, axis=-1 )
all_data = all_data.drop(['Parch', 'SibSp'], axis=1)
 all_data.head()<feature_engineering>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.labels = pd.read_csv(LABELS ).set_index('Id')
 self.labels['Target'] = [[int(i)for i in s.split() ] for s in self.labels['Target']]
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_rgby(self.path,self.fnames[i])
 if self.sz == 512: return img
else: return cv2.resize(img,(self.sz, self.sz),cv2.INTER_AREA)
 def get_y(self, i):
if(self.path == TEST): return np.zeros(len(name_label_dict),dtype=np.int)
 else:
labels = self.labels.loc[self.fnames[i]]['Target']
return np.eye(len(name_label_dict),dtype=np.float)[labels].sum(axis=0)
 @property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return len(name_label_dict )
all_data['calculated_fare'] = all_data["Fare"]/all_data["family_size"]
all_data.head()<drop_column>
def get_data(sz,bs):
aug_tfms = [RandomRotate(30, tfm_y=TfmType.NO),
RandomDihedral(tfm_y=TfmType.NO)]
stats = A([0.00505, 0.00331, 0.00344, 0.00519], [0.10038, 0.08131, 0.08284, 0.10179])
 tfms = tfms_from_stats(stats, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(pdFilesDataset,(tr_n[:-(len(tr_n)%bs)],TRAIN), 
(val_n,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md
all_data = all_data.drop(['Fare', 'family_size'], axis=1 )<feature_engineering>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
all_data.loc[all_data['calculated_fare'] <= 7.775, 'Fare_group'] = 1
all_data.loc[(all_data['calculated_fare'] > 7.775)&(all_data['calculated_fare'] <= 13), 'Fare_group'] =2
all_data.loc[ all_data['calculated_fare'] > 13, 'Fare_group'] = 3
all_data['Fare_group'] = all_data['Fare_group'].astype(int )<drop_column>
def acc(preds,targs,thresh=0.0):
preds =(preds > thresh ).int()
targs = targs.int()
return(preds==targs ).float().mean()
all_data = all_data.drop(['FareBand', 'calculated_fare'], axis=1 )<count_missing_values>
class ConvnetBuilder_custom() :
def __init__(self, f, c, is_multi, is_reg, ps=None, xtra_fc=None, xtra_cut=0,
custom_head=None, pretrained=True):
self.f,self.c,self.is_multi,self.is_reg,self.xtra_cut = f,c,is_multi,is_reg,xtra_cut
if xtra_fc is None: xtra_fc = [512]
if ps is None: ps = [0.25]*len(xtra_fc)+ [0.5]
self.ps,self.xtra_fc = ps,xtra_fc
if f in model_meta: cut,self.lr_cut = model_meta[f]
else: cut,self.lr_cut = 0,0
cut-=xtra_cut
layers = cut_model(f(pretrained), cut)
 w = layers[0].weight
layers[0] = nn.Conv2d(4,64,kernel_size=(7,7),stride=(2,2),padding=(3, 3), bias=False)
 layers[0].weight = torch.nn.Parameter(torch.cat(( w,torch.zeros(64,1,7,7)) ,dim=1))
self.nf = model_features[f] if f in model_features else(num_features(layers)*2)
 if not custom_head: layers += [AdaptiveConcatPool2d() , Flatten() ]
self.top_model = nn.Sequential(*layers)
 n_fc = len(self.xtra_fc)+1
if not isinstance(self.ps, list): self.ps = [self.ps]*n_fc
if custom_head: fc_layers = [custom_head]
else: fc_layers = self.get_fc_layers()
self.n_fc = len(fc_layers)
 self.fc_model = to_gpu(nn.Sequential(*fc_layers))
if not custom_head: apply_init(self.fc_model, kaiming_normal)
 self.model = to_gpu(nn.Sequential(*(layers+fc_layers)))
 @property
def name(self): return f'{self.f.__name__}_{self.xtra_cut}'
def create_fc_layer(self, ni, nf, p, actn=None):
res=[nn.BatchNorm1d(num_features=ni)]
if p: res.append(nn.Dropout(p=p))
res.append(nn.Linear(in_features=ni, out_features=nf))
if actn: res.append(actn)
 return res
def get_fc_layers(self):
res=[]
ni=self.nf
for i,nf in enumerate(self.xtra_fc):
res += self.create_fc_layer(ni, nf, p=self.ps[i], actn=nn.ReLU())
 ni=nf
final_actn = nn.Sigmoid() if self.is_multi else nn.LogSoftmax()
if self.is_reg: final_actn = None
res += self.create_fc_layer(ni, self.c, p=self.ps[-1], actn=final_actn)
 return res
def get_layer_groups(self, do_fc=False):
if do_fc:
return [self.fc_model]
idxs = [self.lr_cut]
c = children(self.top_model)
 if len(c)==3: c = children(c[0])+c[1:]
lgs = list(split_by_idxs(c,idxs))
return lgs+[self.fc_model]
class ConvLearner(Learner):
def __init__(self, data, models, precompute=False, **kwargs):
self.precompute = False
super().__init__(data, models, **kwargs)
 if hasattr(data, 'is_multi')and not data.is_reg and self.metrics is None:
self.metrics = [accuracy_thresh(0.5)] if self.data.is_multi else [accuracy]
if precompute: self.save_fc1()
self.freeze()
self.precompute = precompute
def _get_crit(self, data):
if not hasattr(data, 'is_multi'): return super()._get_crit(data)
 return F.l1_loss if data.is_reg else F.binary_cross_entropy if data.is_multi else F.nll_loss
@classmethod
def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
pretrained=True, **kwargs):
models = ConvnetBuilder_custom(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=pretrained)
 return cls(data, models, precompute, **kwargs)
 @classmethod
def lsuv_learner(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, custom_head=None, precompute=False,
needed_std=1.0, std_tol=0.1, max_attempts=10, do_orthonorm=False, **kwargs):
models = ConvnetBuilder(f, data.c, data.is_multi, data.is_reg,
ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut, custom_head=custom_head, pretrained=False)
 convlearn=cls(data, models, precompute, **kwargs)
 convlearn.lsuv_init()
return convlearn
@property
def model(self): return self.models.fc_model if self.precompute else self.models.model
def half(self):
if self.fp16: return
self.fp16 = True
if type(self.model)!= FP16: self.models.model = FP16(self.model)
 if not isinstance(self.models.fc_model, FP16): self.models.fc_model = FP16(self.models.fc_model)
 def float(self):
if not self.fp16: return
self.fp16 = False
if type(self.models.model)== FP16: self.models.model = self.model.module.float()
if type(self.models.fc_model)== FP16: self.models.fc_model = self.models.fc_model.module.float()
@property
def data(self): return self.fc_data if self.precompute else self.data_
def create_empty_bcolz(self, n, name):
return bcolz.carray(np.zeros(( 0,n), np.float32), chunklen=1, mode='w', rootdir=name)
 def set_data(self, data, precompute=False):
super().set_data(data)
 if precompute:
self.unfreeze()
self.save_fc1()
self.freeze()
self.precompute = True
else:
self.freeze()
def get_layer_groups(self):
return self.models.get_layer_groups(self.precompute)
 def summary(self):
precompute = self.precompute
self.precompute = False
res = super().summary()
self.precompute = precompute
return res
def get_activations(self, force=False):
tmpl = f'_{self.models.name}_{self.data.sz}.bc'
names = [os.path.join(self.tmp_path, p+tmpl)for p in('x_act', 'x_act_val', 'x_act_test')]
if os.path.exists(names[0])and not force:
self.activations = [bcolz.open(p)for p in names]
else:
self.activations = [self.create_empty_bcolz(self.models.nf,n)for n in names]
def save_fc1(self):
self.get_activations()
act, val_act, test_act = self.activations
m=self.models.top_model
if len(self.activations[0])!=len(self.data.trn_ds):
predict_to_bcolz(m, self.data.fix_dl, act)
 if len(self.activations[1])!=len(self.data.val_ds):
predict_to_bcolz(m, self.data.val_dl, val_act)
 if self.data.test_dl and(len(self.activations[2])!=len(self.data.test_ds)) :
if self.data.test_dl: predict_to_bcolz(m, self.data.test_dl, test_act)
 self.fc_data = ImageClassifierData.from_arrays(self.data.path, 
(act, self.data.trn_y),(val_act, self.data.val_y), self.data.bs, classes=self.data.classes,
test = test_act if self.data.test_dl else None, num_workers=8)
 def freeze(self):
self.freeze_to(-1)
 def unfreeze(self):
self.freeze_to(0)
 self.precompute = False
def predict_array(self, arr):
precompute = self.precompute
self.precompute = False
pred = super().predict_array(arr)
 self.precompute = precompute
return pred
missing_percentage(all_data )<predict_on_test>
sz = 256
bs = 64
md = get_data(sz,bs)
 learner = ConvLearner.pretrained(arch, md, ps=0.5)
 learner.opt_fn = optim.Adam
learner.crit = FocalLoss()
learner.metrics = [acc]
def completing_age(df):
age_df = df.loc[:,:]
temp_train = age_df.loc[age_df.Age.notnull() ]
temp_test = age_df.loc[age_df.Age.isnull() ]
y = temp_train.Age.values
x = temp_train.loc[:,"family_group":].values
rfr = RandomForestRegressor(n_estimators=1500, n_jobs=-1)
 rfr.fit(x, y)
 predicted_age = rfr.predict(temp_test.loc[:,"family_group":])
 df.loc[df.Age.isnull() , "Age"] = predicted_age
return df
completing_age(all_data)
<sort_values>
lr = 2e-2
learner.fit(lr,1 )
all_data['AgeBand'] = pd.cut(all_data['Age'], 5)
 all_data[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )<feature_engineering>
learner.unfreeze()
lrs=np.array([lr/10,lr/3,lr] )
all_data.loc[ all_data['Age'] <= 16.136, 'Age_group'] = 1
all_data.loc[(all_data['Age'] > 16.136)&(all_data['Age'] <= 32.102), 'Age_group'] = 2
all_data.loc[(all_data['Age'] > 32.102)&(all_data['Age'] <= 48.068), 'Age_group'] = 3
all_data.loc[(all_data['Age'] > 48.068)&(all_data['Age'] <= 64.034), 'Age_group'] = 4
all_data.loc[ all_data['Age'] > 64.034, 'Age_group'] = 5<drop_column>
learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20))
all_data = all_data.drop(['Age', 'AgeBand'], axis=1)
 all_data.head()<categorify>
learner.fit(lrs/4,2,cycle_len=4,use_clr=(10,20))
all_data = pd.get_dummies(all_data, columns=['Title',"Pclass",'Embarked','Sex'], drop_first=False)
 all_data.head()<data_type_conversions>
learner.fit(lrs/8,1,cycle_len=8,use_clr=(5,20))
all_data['Age_group'] = all_data['Age_group'].astype(int)
 all_data.head()<prepare_x_and_y>
learner.save('ResNet34_256_1' )
X = train_df.drop(["Survived","PassengerId"], axis=1)
 Y = train_df["Survived"]
test = test_df.drop(["Survived","PassengerId"], axis=1 ).copy()
X.shape, Y.shape, test.shape<split>
def sigmoid_np(x):
return 1.0/(1.0 + np.exp(-x))
preds,y = learner.TTA()
preds = np.stack(preds, axis=-1)
 preds = sigmoid_np(preds)
 pred = preds.max(axis=-1 )
X_train, X_test, Y_train, Y_test = train_test_split(X, Y,test_size =.33, random_state=0 )<normalization>
th = np.array([0.50,0.56,0.50,0.50,0.50,0.47,0.39,0.56,0.36,0.45,
0.41,0.48,0.45,0.44,0.63,0.50,0.38,0.33,0.45,0.41,
0.33,0.49,0.43,0.52,0.51,0.48,0.43,0.1])
 print(f1_score(y, pred>0.5, average='macro'),f1_score(y, pred>th, average='macro'))
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test)
 test= sc.transform(test )<create_dataframe>
preds_t,y_t = learner.TTA(is_test=True)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = sigmoid_np(preds_t)
 pred_t = preds_t.max(axis=-1 )
X_train = pd.DataFrame(X_train, columns=headers)
 X_test = pd.DataFrame(X_test, columns=headers )<compute_train_metric>
pred_list = []
for line in pred_t:
s = ' '.join(list([str(i)for i in np.nonzero(line>th_t)[0]]))
pred_list.append(s )
<train_on_grid><EOS> 
sample_df = pd.read_csv(SAMPLE)
 sample_list = list(sample_df.Id)
 pred_dic = dict(( key, value)for(key, value)
 in zip(learner.data.test_ds.fnames,pred_list))
pred_list_cor = [pred_dic[id] for id in sample_list]
df = pd.DataFrame({'Id':sample_list,'Predicted':pred_list_cor})
 df.to_csv('protein_classification.csv', header=True, index=False )