Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<predict_on_test>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
rfc_prediction = logreg.predict(X_test)
 rfc_score=accuracy_score(Y_test, rfc_prediction)
 print(rfc_score )<compute_train_metric>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import scipy.optimize as opt
import matplotlib.pyplot as plt
cv = StratifiedShuffleSplit(n_splits = 10, test_size =.25, random_state = 0)
 column_names = X.columns
X = sc.fit_transform(X)
 accuracies = cross_val_score(LogisticRegression(solver='liblinear'),X,Y, cv = cv)
 print("Cross-Validation accuracy scores:{}".format(accuracies))
print("Mean Cross-Validation accuracy score: {}".format(round(accuracies.mean() ,5)) )<train_on_grid>
PATH = './'
TRAIN = '.. /input/human-protein-atlas-image-classification/train/'
TEST = '.. /input/human-protein-atlas-image-classification/test/'
LABELS = '.. /input/human-protein-atlas-image-classification/train.csv'
SPLIT = '.. /input/protein-trainval-split/'
nw = 2
C_vals = [0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,16.5,17,17.5,18]
penalties = ['l1','l2']
cv = StratifiedShuffleSplit(n_splits = 10, test_size =.25)
 param = {'penalty': penalties, 'C': C_vals}
grid = GridSearchCV(estimator=LogisticRegression() ,
param_grid = param,
scoring = 'accuracy',
n_jobs =-1,
cv = cv 
)
 grid.fit(X, Y )<find_best_params>
name_label_dict = {
0: 'Nucleoplasm',
1: 'Nuclear membrane',
2: 'Nucleoli',
3: 'Nucleoli fibrillar center',
4: 'Nuclear speckles',
5: 'Nuclear bodies',
6: 'Endoplasmic reticulum',
7: 'Golgi apparatus',
8: 'Peroxisomes',
9: 'Endosomes',
10: 'Lysosomes',
11: 'Intermediate filaments',
12: 'Actin filaments',
13: 'Focal adhesion sites',
14: 'Microtubules',
15: 'Microtubule ends',
16: 'Cytokinetic bridge',
17: 'Mitotic spindle',
18: 'Microtubule organizing center',
19: 'Centrosome',
20: 'Lipid droplets',
21: 'Plasma membrane',
22: 'Cell junctions',
23: 'Mitochondria',
24: 'Aggresome',
25: 'Cytosol',
26: 'Cytoplasmic bodies',
27: 'Rods & rings' }
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )<compute_train_metric>
with open(os.path.join(SPLIT,'tr_names.txt'), 'r')as text_file:
tr_n = text_file.read().split(',')
 with open(os.path.join(SPLIT,'val_names.txt'), 'r')as text_file:
val_n = text_file.read().split(',')
 test_names = sorted({f[:36] for f in os.listdir(TEST)})
 print(len(tr_n),len(val_n))
logreg_grid = grid.best_estimator_
logreg_grid.score(X,Y )<compute_train_metric>
class Oversampling:
def __init__(self,path):
self.train_labels = pd.read_csv(path ).set_index('Id')
 self.train_labels['Target'] = [[int(i)for i in s.split() ]
for s in self.train_labels['Target']]
self.multi = [1,1,1,1,1,1,1,1,
4,4,4,1,1,1,1,4,
1,1,1,1,2,1,1,1,
1,1,1,4]
def get(self,image_id):
labels = self.train_labels.loc[image_id,'Target'] if image_id \
in self.train_labels.index else []
m = 1
for l in labels:
if m < self.multi[l]: m = self.multi[l]
return m
s = Oversampling(os.path.join(PATH,LABELS))
tr_n = [idx for idx in tr_n for _ in range(s.get(idx)) ]
print(len(tr_n),flush=True )
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_svc<train_on_grid>
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return np.stack(img, axis=-1 )
Cs = [0.001, 0.01, 0.1, 1,1.5,2,2.5,3,4,5, 10]
gammas = [0.0001,0.001, 0.01, 0.1, 1]
param_grid = {'C': Cs, 'gamma' : gammas}
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 grid_search = GridSearchCV(SVC(kernel = 'rbf', probability=True), param_grid, cv=cv)
 grid_search.fit(X,Y )<find_best_params>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.labels = pd.read_csv(LABELS ).set_index('Id')
 self.labels['Target'] = [[int(i)for i in s.split() ] for s in self.labels['Target']]
super().__init__(fnames, transform, path)
 def get_x(self, i):
return open_rgby(self.path,self.fnames[i])
 def get_y(self, i):
if(self.path == TEST): return np.zeros(len(name_label_dict),dtype=np.int)
 else:
labels = self.labels.loc[self.fnames[i]]['Target']
return np.eye(len(name_label_dict),dtype=np.float)[labels].sum(axis=0)
 @property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return len(name_label_dict )
print(grid_search.best_score_)
 print(grid_search.best_params_)
 print(grid_search.best_estimator_ )<compute_train_metric>
bs = 16
sz = 256
md = get_data(sz,bs,is_test=True)
 x,y = next(iter(md.aug_dl))
x.shape, y.shape
svc_grid = grid.best_estimator_
svc_grid.score(X,Y )<compute_train_metric>
class Resnet34_4(nn.Module):
def __init__(self, pre=True):
super().__init__()
encoder = resnet34(pretrained=pre)
 self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3, bias=False)
 if(pre):
w = encoder.conv1.weight
self.conv1.weight = nn.Parameter(torch.cat(( w,
0.5*(w[:,:1,:,:]+w[:,2:,:,:])) ,dim=1))
self.bn1 = encoder.bn1
self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer0 = nn.Sequential(self.conv1,self.relu,self.bn1,self.maxpool)
 self.layer1 = encoder.layer1
self.layer2 = encoder.layer2
self.layer3 = encoder.layer3
self.layer4 = encoder.layer4
def forward(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_linear_svc<choose_model_class>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
dectree = DecisionTreeClassifier()
dectree.fit(X_train, Y_train)
 Y_pred = dectree.predict(X_test)
 acc_dectree = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_dectree<train_on_grid>
def acc(preds,targs,th=0.0):
preds =(preds > th ).int()
targs = targs.int()
return(preds==targs ).float().mean()
max_depth = range(1,20)
 criterion=["entropy", "gini"]
param = {'max_depth':max_depth,
'criterion': criterion}
grid = GridSearchCV(DecisionTreeClassifier() ,
param_grid = param,
verbose=False,
cv=StratifiedKFold(n_splits=20, random_state=15, shuffle=True),
n_jobs = -1)
 grid.fit(X,Y )<find_best_params>
class F1:
__name__ = 'F1 macro'
def __init__(self,n=28):
self.n = n
self.TP = np.zeros(self.n)
 self.FP = np.zeros(self.n)
 self.FN = np.zeros(self.n)
 def __call__(self,preds,targs,th=0.0):
preds =(preds > th ).int()
targs = targs.int()
self.TP +=(preds*targs ).float().sum(dim=0)
 self.FP +=(preds > targs ).float().sum(dim=0)
 self.FN +=(preds < targs ).float().sum(dim=0)
 score =(2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6)).mean()
return score
def reset(self):
score =(2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6))
print('F1 macro:',score.mean() ,flush=True)
 self.TP = np.zeros(self.n)
 self.FP = np.zeros(self.n)
 self.FN = np.zeros(self.n)
 class F1_callback(Callback):
def __init__(self, n=28):
self.f1 = F1(n)
 def on_epoch_end(self, metrics):
self.f1.reset()
print(grid.best_params_)
 print(grid.best_score_)
 print(grid.best_estimator_ )<find_best_params>
sz = 256
bs = 64
md = get_data(sz,bs)
 learner = ConvLearner.pretrained(Resnet34_4, md, ps=0.5)
 learner.opt_fn = optim.Adam
learner.clip = 1.0
learner.crit = FocalLoss()
f1_callback = F1_callback()
learner.metrics = [acc,f1_callback.f1]
learner.summary
dectree_grid = grid.best_estimator_
dectree_grid.score(X,Y )<save_to_csv>
lr = 0.5e-2
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lr,1,callbacks=[f1_callback] )
dot_data = StringIO()
export_graphviz(dectree_grid, out_file=dot_data,
feature_names=column_names, class_names =(["Survived" if int(i)is 1 else "Not_survived" for i in Y.unique() ]),
filled=True, rounded=True,
proportion=True,
special_characters=True)
(graph,)= pydot.graph_from_dot_data(dot_data.getvalue())
 Image(graph.create_png() )<predict_on_test>
learner.unfreeze()
lrs=np.array([lr/10,lr/3,lr] )
rf = RandomForestClassifier(n_estimators=100)
 rf.fit(X_train, Y_train)
 Y_pred = rf.predict(X_test)
 acc_rf = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_rf<define_search_space>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20),callbacks=[f1_callback] )
n_estimators = [140,145,150,155,160];
max_depth = range(1,10);
criterions = ['gini', 'entropy'];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 parameters = {'n_estimators':n_estimators,
'max_depth':max_depth,
'criterion': criterions
}
grid = GridSearchCV(estimator=RandomForestClassifier(max_features='auto'),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 grid.fit(X,Y )<find_best_params>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/16,2,cycle_len=4,use_clr=(10,20),callbacks=[f1_callback] )
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )<find_best_params>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/32,1,cycle_len=8,use_clr=(10,20),callbacks=[f1_callback] )
rf_grid = grid.best_estimator_
rf_grid.score(X,Y )<choose_model_class>
learner.save('ResNet34_256_1' )
n_estimators = [10,30,50,70,80,150,160, 170,175,180,185];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 parameters = {'n_estimators':n_estimators,
}
grid = GridSearchCV(BaggingClassifier(base_estimator= None,
bootstrap_features=False),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 grid.fit(X,Y )<find_best_params>
md = get_data(sz,bs,is_test=True)
 learner.set_data(md)
 preds,y = learner.TTA(n_aug=8)
 preds = np.stack(preds, axis=-1)
 pred = preds.mean(axis=-1 )
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )<compute_test_metric>
def sigmoid_np(x):
return 1.0/(1.0 + np.exp(-x))
def F1_soft(preds,targs,th=0.0,d=25.0):
preds = sigmoid_np(d*(preds - th))
targs = targs.astype(np.float)
 score = 2.0*(preds*targs ).sum(axis=0)/(( preds+targs ).sum(axis=0)+ 1e-6)
 return score
def fit_val(x,y):
params = np.zeros(len(name_label_dict))
wd = 1e-5
error = lambda p: np.concatenate(( F1_soft(x,y,p)- 1.0,
wd*p), axis=None)
 p, success = opt.leastsq(error, params)
 return p
bagging_grid = grid.best_estimator_
bagging_grid.score(X,Y )<choose_model_class>
th = fit_val(pred,y)
 print('Thresholds: ',th)
 print('F1 macro: ',f1_score(y, pred>th, average='macro'))
print('F1 macro(th = 0.0): ',f1_score(y, pred>0.0, average='macro'))
print('F1 micro: ',f1_score(y, pred>th, average='micro'))
n_estimators = [100,140,145,150,160, 170,175,180,185];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 learning_r = [0.1,1,0.01,0.5]
parameters = {'n_estimators':n_estimators,
'learning_rate':learning_r
}
grid = GridSearchCV(AdaBoostClassifier(base_estimator= None, 
),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 grid.fit(X,Y )<find_best_params>
th, score, cv = 0,0,10
for i in range(cv):
xt,xv,yt,yv = train_test_split(pred,y,test_size=0.5,random_state=i)
 th_i = fit_val(xt,yt)
 th += th_i
score += f1_score(yv, xv>th_i, average='macro')
 th/=cv
score/=cv
print('Thresholds: ',th)
 print('F1 macro avr:',score)
 print('F1 macro: ',f1_score(y, pred>th, average='macro'))
print('F1 micro: ',f1_score(y, pred>th, average='micro'))
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )<compute_test_metric>
preds_t,y_t = learner.TTA(n_aug=8,is_test=True)
 preds_t = np.stack(preds_t, axis=-1)
 pred_t = preds_t.mean(axis=-1 )
adaBoost_grid = grid.best_estimator_
adaBoost_grid.score(X,Y )<predict_on_test>
def save_pred(pred, th=0.0, fname='protein_classification.csv'):
pred_list = []
for line in pred:
s = ' '.join(list([str(i)for i in np.nonzero(line>th)[0]]))
pred_list.append(s)
 df = pd.DataFrame({'Id':learner.data.test_ds.fnames,'Predicted':pred_list})
 df.sort_values(by='Id' ).to_csv(fname, header=True, index=False )
gradient_boost = GradientBoostingClassifier()
gradient_boost.fit(X_train, Y_train)
 Y_pred = gradient_boost.predict(X_test)
 gradient_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(gradient_accy )<choose_model_class>
save_pred(pred_t,th,'protein_classification_v.csv')
 save_pred(pred_t,0.0,'protein_classification_0.csv' )
XGBClassifier = XGBClassifier()
XGBClassifier.fit(X_train, Y_train)
 Y_pred = XGBClassifier.predict(X_test)
 XGBClassifier_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(XGBClassifier_accy )<train_model>
lb_prob = [
0.362397820,0.043841336,0.075268817,0.059322034,0.075268817,
0.075268817,0.043841336,0.075268817,0.010000000,0.010000000,
0.010000000,0.043841336,0.043841336,0.014198783,0.043841336,
0.010000000,0.028806584,0.014198783,0.028806584,0.059322034,
0.010000000,0.126126126,0.028806584,0.075268817,0.010000000,
0.222493880,0.028806584,0.010000000]
ExtraTreesClassifier = ExtraTreesClassifier()
ExtraTreesClassifier.fit(X_train, Y_train)
 Y_pred = ExtraTreesClassifier.predict(X_test)
 extraTree_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(extraTree_accy )<predict_on_test>
def Count_soft(preds,th=0.0,d=50.0):
preds = sigmoid_np(d*(preds - th))
return preds.mean(axis=0)
 def fit_test(x,y):
params = np.zeros(len(name_label_dict))
wd = 1e-5
error = lambda p: np.concatenate(( Count_soft(x,p)- y,
wd*p), axis=None)
 p, success = opt.leastsq(error, params)
 return p
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn<compute_train_metric>
th_t = fit_test(pred_t,lb_prob)
 print('Thresholds: ',th_t)
 print('Fractions: ',(pred_t > th_t ).mean(axis=0))
print('Fractions(th = 0.0): ',(pred_t > 0.0 ).mean(axis=0))
knn_k = KNeighborsClassifier(metric='minkowski', p=2)
 cv = StratifiedShuffleSplit(n_splits=10, test_size=.25, random_state=2)
 accuracies = cross_val_score(knn_k, X,Y, cv = cv, scoring='accuracy')
 print("Cross-Validation accuracy scores:{}".format(accuracies))
print("Mean Cross-Validation accuracy score: {}".format(round(accuracies.mean() ,3)) )<compute_train_metric>
save_pred(pred_t,th_t,'protein_classification_f.csv' )
cv = StratifiedShuffleSplit(n_splits=10, test_size=.25, random_state=2)
 k_range = range(1,31)
 k_scores = []
for k in k_range:
knn_kk = KNeighborsClassifier(n_neighbors=k)
 scores = cross_val_score(knn_kk, X,Y, cv = cv, scoring = 'accuracy')
 k_scores.append(scores.mean())
 print("Accuracy scores are: {}
".format(k_scores))
print("Mean accuracy score: {}".format(np.mean(k_scores)) )<train_on_grid>
class_list = [8,9,10,15,20,24,27]
for i in class_list:
th_t[i] = th[i]
save_pred(pred_t,th_t,'protein_classification_c.csv' )
k_range = range(1,31)
 weights_options=['uniform','distance']
param = {'n_neighbors':k_range, 'weights':weights_options}
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 grid = GridSearchCV(KNeighborsClassifier() , param,cv=cv,verbose = False, n_jobs=-1)
 grid.fit(X,Y )<find_best_params>
labels = pd.read_csv(LABELS ).set_index('Id')
 label_count = np.zeros(len(name_label_dict))
for label in labels['Target']:
l = [int(i)for i in label.split() ]
label_count += np.eye(len(name_label_dict)) [l].sum(axis=0)
 label_fraction = label_count.astype(np.float)/len(labels)
 label_count, label_fraction
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )<find_best_params>
th_t = fit_test(pred_t,label_fraction)
 print('Thresholds: ',th_t)
 print('Fractions: ',(pred_t > th_t ).mean(axis=0))
save_pred(pred_t,th_t,'protein_classification_t.csv' )