Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<filter>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
def is_oldest_children(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
match =(data.loc[i].Age >= max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(match_data['Title'] == 5)].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list<filter>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import scipy.optimize as opt
import matplotlib.pyplot as plt
def is_the_only_child(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
match = len(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(match_data['Title'] == 5)])
 if match == 1:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list<categorify>
PATH = './'
TRAIN = '.. /input/human-protein-atlas-image-classification/train/'
TEST = '.. /input/human-protein-atlas-image-classification/test/'
LABELS = '.. /input/human-protein-atlas-image-classification/train.csv'
SPLIT = '.. /input/protein-trainval-split/'
nw = 2
train['Is_youngest_children'] = is_youngest_children(train, total_data)
 test['Is_youngest_children'] = is_youngest_children(test, total_data )<categorify>
name_label_dict = {
0: 'Nucleoplasm',
1: 'Nuclear membrane',
2: 'Nucleoli',
3: 'Nucleoli fibrillar center',
4: 'Nuclear speckles',
5: 'Nuclear bodies',
6: 'Endoplasmic reticulum',
7: 'Golgi apparatus',
8: 'Peroxisomes',
9: 'Endosomes',
10: 'Lysosomes',
11: 'Intermediate filaments',
12: 'Actin filaments',
13: 'Focal adhesion sites',
14: 'Microtubules',
15: 'Microtubule ends',
16: 'Cytokinetic bridge',
17: 'Mitotic spindle',
18: 'Microtubule organizing center',
19: 'Centrosome',
20: 'Lipid droplets',
21: 'Plasma membrane',
22: 'Cell junctions',
23: 'Mitochondria',
24: 'Aggresome',
25: 'Cytosol',
26: 'Cytoplasmic bodies',
27: 'Rods & rings' }
train['Is_oldest_children'] = is_oldest_children(train, total_data)
 test['Is_oldest_children'] = is_oldest_children(test, total_data )<categorify>
with open(os.path.join(SPLIT,'tr_names.txt'), 'r')as text_file:
tr_n = text_file.read().split(',')
 with open(os.path.join(SPLIT,'val_names.txt'), 'r')as text_file:
val_n = text_file.read().split(',')
 test_names = sorted({f[:36] for f in os.listdir(TEST)})
 print(len(tr_n),len(val_n))
train['Is_the_only_child'] = is_the_only_child(train, total_data)
 test['Is_the_only_child'] = is_the_only_child(test, total_data )<feature_engineering>
class Oversampling:
def __init__(self,path):
self.train_labels = pd.read_csv(path ).set_index('Id')
 self.train_labels['Target'] = [[int(i)for i in s.split() ]
for s in self.train_labels['Target']]
self.multi = [1,1,1,1,1,1,1,1,
4,4,4,1,1,1,1,4,
1,1,1,1,2,1,1,1,
1,1,1,4]
def get(self,image_id):
labels = self.train_labels.loc[image_id,'Target'] if image_id \
in self.train_labels.index else []
m = 1
for l in labels:
if m < self.multi[l]: m = self.multi[l]
return m
s = Oversampling(os.path.join(PATH,LABELS))
tr_n = [idx for idx in tr_n for _ in range(s.get(idx)) ]
print(len(tr_n),flush=True )
for column in ['Is_youngest_children', 'Is_oldest_children', 'Is_the_only_child']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )<filter>
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return np.stack(img, axis=-1 )
def family_survived(data, match_data):
match_list = []
for i in range(len(data)) :
if data.Family_size[i] > 1:
survived =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(data['PassengerId'][i] != match_data['PassengerId'])].Survived ).sum()
if survived > 0:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list<filter>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.labels = pd.read_csv(LABELS ).set_index('Id')
 self.labels['Target'] = [[int(i)for i in s.split() ] for s in self.labels['Target']]
super().__init__(fnames, transform, path)
 def get_x(self, i):
return open_rgby(self.path,self.fnames[i])
 def get_y(self, i):
if(self.path == TEST): return np.zeros(len(name_label_dict),dtype=np.int)
 else:
labels = self.labels.loc[self.fnames[i]]['Target']
return np.eye(len(name_label_dict),dtype=np.float)[labels].sum(axis=0)
 @property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return len(name_label_dict )
def is_a_wife_with_survived_children(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Name[i].find('(')> -1)&(data.Sex[i] == 1):
survived =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(data['PassengerId'][i] != match_data['PassengerId'])\
&(match_data['Title'] == 5)].Survived ).sum()
if survived > 0:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list<filter>
bs = 32
sz = 256
md = get_data(sz,bs,is_test=True)
 x,y = next(iter(md.aug_dl))
x.shape, y.shape
def is_a_child_with_other_candw_survived(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
survived =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(data['PassengerId'][i] != match_data['PassengerId'])\
&(( match_data['Title'] == 5)|(match_data['Sex'] == 0)) ].Survived ).sum()
if survived > 0:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list<feature_engineering>
class Resnet34_4(nn.Module):
def __init__(self, pre=True):
super().__init__()
encoder = resnet34(pretrained=pre)
 self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3, bias=False)
 if(pre):
w = encoder.conv1.weight
self.conv1.weight = nn.Parameter(torch.cat(( w,
0.5*(w[:,:1,:,:]+w[:,2:,:,:])) ,dim=1))
self.bn1 = encoder.bn1
self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer0 = nn.Sequential(self.conv1,self.relu,self.bn1,self.maxpool)
 self.layer1 = encoder.layer1
self.layer2 = encoder.layer2
self.layer3 = encoder.layer3
self.layer4 = encoder.layer4
def forward(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
train['Family_survived'] = family_survived(train, total_data)
 test['Family_survived'] = family_survived(test, total_data )<feature_engineering>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
train['Is_a_wife_with_sc'] = is_a_wife_with_survived_children(train, total_data)
 test['Is_a_wife_with_sc'] = is_a_wife_with_survived_children(test, total_data )<feature_engineering>
def acc(preds,targs,th=0.0):
preds =(preds > th ).int()
targs = targs.int()
return(preds==targs ).float().mean()
train['Is_a_child_with_wandc_survived'] = is_a_child_with_other_candw_survived(train, total_data)
 test['Is_a_child_with_wandc_survived'] = is_a_child_with_other_candw_survived(test, total_data )<prepare_x_and_y>
class F1:
__name__ = 'F1 macro'
def __init__(self,n=28):
self.n = n
self.TP = np.zeros(self.n)
 self.FP = np.zeros(self.n)
 self.FN = np.zeros(self.n)
 def __call__(self,preds,targs,th=0.0):
preds =(preds > th ).int()
targs = targs.int()
self.TP +=(preds*targs ).float().sum(dim=0)
 self.FP +=(preds > targs ).float().sum(dim=0)
 self.FN +=(preds < targs ).float().sum(dim=0)
 score =(2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6)).mean()
return score
def reset(self):
score =(2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6))
print('F1 macro:',score.mean() ,flush=True)
 self.TP = np.zeros(self.n)
 self.FP = np.zeros(self.n)
 self.FN = np.zeros(self.n)
 class F1_callback(Callback):
def __init__(self, n=28):
self.f1 = F1(n)
 def on_epoch_end(self, metrics):
self.f1.reset()
for column in ['Family_survived', 'Is_a_wife_with_sc', 'Is_a_child_with_wandc_survived']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )<create_dataframe>
sz = 256
bs = 100
md = get_data(sz,bs)
 learner = ConvLearner.pretrained(Resnet34_4, md, ps=0.7)
 learner.opt_fn = optim.Adam
learner.clip = 1.0
learner.crit = FocalLoss()
f1_callback = F1_callback()
learner.metrics = [acc,f1_callback.f1]
learner.summary
for value in [(0,15),(15,35),(35,50),(50,65),(65,100)]:
get_continuous_feature_info(train, column='Age', y_column='Survived', xlim=value )<feature_engineering>
lr = 0.5e-3
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lr,1,callbacks=[f1_callback] )
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 14, 'Age'] = 0
dataset.loc[(dataset['Age'] > 14)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4<count_missing_values>
learner.unfreeze()
lrs=np.array([lr/10,lr/3,lr] )
test.isna().sum() , train.isna().sum()<count_values>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20),callbacks=[f1_callback] )
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))<data_type_conversions>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/16,2,cycle_len=4,use_clr=(10,20),callbacks=[f1_callback] )
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )<categorify>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/32,1,cycle_len=8,use_clr=(10,20),callbacks=[f1_callback] )
embarked_mapping = {"C": "Cherbourg", "S": "Southampton", "Q": "Queenstown"}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )<feature_engineering>
learner.save('ResNet34_256_1' )
for dataset in train_test_data:
dataset['Is_AloneandPoor'] = 0
dataset.loc[(dataset['Family_size'] == 1)&(dataset['Pclass'] == 3),'Is_AloneandPoor'] = 1<create_dataframe>
md = get_data(sz,bs,is_test=True)
 learner.set_data(md)
 preds,y = learner.TTA(n_aug=8)
 preds = np.stack(preds, axis=-1)
 pred = preds.mean(axis=-1 )
get_categorical_feature_info(train, column='Is_AloneandPoor', y_column='Survived', positive_state='Survived', negative_state='Dead' )<feature_engineering>
def sigmoid_np(x):
return 1.0/(1.0 + np.exp(-x))
def F1_soft(preds,targs,th=0.0,d=25.0):
preds = sigmoid_np(d*(preds - th))
targs = targs.astype(np.float)
 score = 2.0*(preds*targs ).sum(axis=0)/(( preds+targs ).sum(axis=0)+ 1e-6)
 return score
def fit_val(x,y):
params = np.zeros(len(name_label_dict))
wd = 1e-5
error = lambda p: np.concatenate(( F1_soft(x,y,p)- 1.0,
wd*p), axis=None)
 p, success = opt.leastsq(error, params)
 return p
for dataset in train_test_data:
dataset['IsChildandRich'] = 0
dataset.loc[(dataset['Title'] == 5)&(dataset['Pclass'] < 3),'IsChildandRich'] = 1<create_dataframe>
th = fit_val(pred,y)
 print('Thresholds: ',th)
 print('F1 macro: ',f1_score(y, pred>th, average='macro'))
print('F1 macro(th = 0.0): ',f1_score(y, pred>0.0, average='macro'))
print('F1 micro: ',f1_score(y, pred>th, average='micro'))
get_categorical_feature_info(train, column='IsChildandRich', y_column='Survived', positive_state='Survived', negative_state='Dead' )<feature_engineering>
th, score, cv = 0,0,10
for i in range(cv):
xt,xv,yt,yv = train_test_split(pred,y,test_size=0.5,random_state=i)
 th_i = fit_val(xt,yt)
 th += th_i
score += f1_score(yv, xv>th_i, average='macro')
 th/=cv
score/=cv
print('Thresholds: ',th)
 print('F1 macro avr:',score)
 print('F1 macro: ',f1_score(y, pred>th, average='macro'))
print('F1 micro: ',f1_score(y, pred>th, average='micro'))
for data in train_test_data:
data['Cabin'] = data['Cabin'].fillna('X')
 data['Cabin'] = data['Cabin'].apply(lambda x: str(x)[0])
 data['Cabin'] = data['Cabin'].replace(['A', 'D', 'E', 'T'], 'M')
 data['Cabin'] = data['Cabin'].replace(['B', 'C'], 'H')
 data['Cabin'] = data['Cabin'].replace(['F', 'G'], 'L' )<create_dataframe>
preds_t,y_t = learner.TTA(n_aug=8,is_test=True)
 preds_t = np.stack(preds_t, axis=-1)
 pred_t = preds_t.mean(axis=-1 )
get_categorical_feature_info(train, column='Cabin', y_column='Survived', positive_state='Survived', negative_state='Dead' )<feature_engineering>
def save_pred(pred, th=0.0, fname='protein_classification.csv'):
pred_list = []
for line in pred:
s = ' '.join(list([str(i)for i in np.nonzero(line>th)[0]]))
pred_list.append(s)
 df = pd.DataFrame({'Id':learner.data.test_ds.fnames,'Predicted':pred_list})
 df.sort_values(by='Id' ).to_csv(fname, header=True, index=False )
for dataset in train_test_data:
dataset['Family_size_group'] = 'Small'
dataset.loc[dataset['Family_size'] == 1, 'Family_size_group'] = 'Alone'
dataset.loc[dataset['Family_size'] > 2, 'Family_size_group'] = 'Big'<feature_engineering>
save_pred(pred_t,th,'protein_classification_v.csv')
 save_pred(pred_t,0.0,'protein_classification_0.csv' )
train['IsMaleandPoor'] = 0
train.loc[(train['Sex'] == 0)&(train['Pclass'] == 3),'IsMaleandPoor'] = 1
test['IsMaleandPoor'] = 0
test.loc[(test['Sex'] == 0)&(test['Pclass'] == 3),'IsMaleandPoor'] = 1<create_dataframe>
lb_prob = [
0.362397820,0.043841336,0.075268817,0.059322034,0.075268817,
0.075268817,0.043841336,0.075268817,0.010000000,0.010000000,
0.010000000,0.043841336,0.043841336,0.014198783,0.043841336,
0.010000000,0.028806584,0.014198783,0.028806584,0.059322034,
0.010000000,0.126126126,0.028806584,0.075268817,0.010000000,
0.222493880,0.028806584,0.010000000]
for column in ['Family_size_group', 'IsMaleandPoor']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )<drop_column>
def Count_soft(preds,th=0.0,d=50.0):
preds = sigmoid_np(d*(preds - th))
return preds.mean(axis=0)
 def fit_test(x,y):
params = np.zeros(len(name_label_dict))
wd = 1e-5
error = lambda p: np.concatenate(( Count_soft(x,p)- y,
wd*p), axis=None)
 p, success = opt.leastsq(error, params)
 return p
for dataset in train_test_data:
dataset.drop(['PassengerId', 'Name', 'Ticket', 'Last_Name', 'SibSp', 'Ticket'\
, 'Family_size'], axis=1, inplace=True )<categorify>
th_t = fit_test(pred_t,lb_prob)
 print('Thresholds: ',th_t)
 print('Fractions: ',(pred_t > th_t ).mean(axis=0))
print('Fractions(th = 0.0): ',(pred_t > 0.0 ).mean(axis=0))
def get_dummies_from_list(categorical_list):
dummies = pd.get_dummies(pd.Categorical(categorical_list), sparse=True)
 return dummies<categorify>
save_pred(pred_t,th_t,'protein_classification_f.csv' )
def transform_dummies(data, column_list=[]):
for column in column_list:
dummy_df = get_dummies_from_list(data[column])
 data = data.merge(dummy_df, how='left', left_index=True, right_index=True)
 data.drop(labels=column_list, axis='columns', inplace=True)
 return data<categorify>
class_list = [8,9,10,15,20,24,27]
for i in class_list:
th_t[i] = th[i]
save_pred(pred_t,th_t,'protein_classification_c.csv' )
train = transform_dummies(train, column_list=['Embarked', 'Family_size_group', 'Cabin'])
 test = transform_dummies(test, column_list=['Embarked', 'Family_size_group', 'Cabin'] )<feature_engineering>
labels = pd.read_csv(LABELS ).set_index('Id')
 label_count = np.zeros(len(name_label_dict))
for label in labels['Target']:
l = [int(i)for i in label.split() ]
label_count += np.eye(len(name_label_dict)) [l].sum(axis=0)
 label_fraction = label_count.astype(np.float)/len(labels)
 label_count, label_fraction
for column in train.columns:
get_indipendent_variables_mix_info(train, x_column=column, y_column='Age', dependent_variable='Survived' )<sort_values>
th_t = fit_test(pred_t,label_fraction)
 print('Thresholds: ',th_t)
 print('Fractions: ',(pred_t > th_t ).mean(axis=0))
save_pred(pred_t,th_t,'protein_classification_t.csv' )