Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<count_missing_values>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
test.isnull().sum()<count_missing_values>
import os
from pathlib import Path
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastai import *
from fastai.vision import *
from fastai.vision.image import *
test.isnull().sum()<feature_engineering>
RESNET_ENCODERS = {
34: torchvision.models.resnet34,
50: torchvision.models.resnet50,
101: torchvision.models.resnet101,
152: torchvision.models.resnet152,
}
class Resnet4Channel(nn.Module):
def __init__(self, encoder_depth=34, pretrained=True, num_classes=28, copy_extra_channel=False, adjust_first_layer=False):
super().__init__()
encoder = RESNET_ENCODERS[encoder_depth](pretrained=pretrained)
 w = encoder.conv1.weight
self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 if copy_extra_channel:
to_concat = w[:, :1, :, :].clone()
else:
to_concat = torch.zeros(64,1,7,7)
 self.conv1.weight = nn.Parameter(torch.cat(( w,to_concat),dim=1)*(0.75 if adjust_first_layer else 1.))
 self.bn1 = encoder.bn1
self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = encoder.layer1
self.layer2 = encoder.layer2
self.layer3 = encoder.layer3
self.layer4 = encoder.layer4
self.avgpool = encoder.avgpool
self.fc = nn.Linear(512 *(1 if encoder_depth==34 else 4), num_classes)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
for item in combo:
item['Cabin'].fillna('N', inplace=True)
 item['Cabin'] = item['Cabin'].map(lambda c: c[0] )<categorify>
bs = 64
for item in combo:
item['Cabin'] = item['Cabin'].replace(['A', 'B','C',"D",'E','T'], 0)
 item['Cabin'] = item['Cabin'].replace(['F','G'], 0)
 item['Cabin'] = item['Cabin'].replace(['N'], 1 )<groupby>
path = Path('/kaggle/input/' )
train[['Cabin','Survived']].groupby(['Cabin'], as_index=False ).mean()<data_type_conversions>
df = pd.read_csv(path/'train.csv')
 df.head()
train['Embarked'] = train['Embarked'].fillna('S')
 combo = [train, test]
for item in combo:
item['Embarked'] = item['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )<feature_engineering>
np.random.seed(42)
 src =(ImageItemList.from_csv(path, 'train.csv', folder='train', suffix='.png')
.random_split_by_pct(0.2)
.label_from_df(sep=' ', classes=[str(i)for i in range(28)]))
combo = [train, test]
for item in combo:
item['Age'] = item.groupby(['Pclass','Title'])['Age'].apply(lambda x: x.fillna(x.mean()))<correct_missing_values>
src.train.x.create_func = open_4_channel
src.train.x.open = open_4_channel
test['Fare'].fillna(test['Fare'].median() , inplace = True )<categorify>
src.valid.x.create_func = open_4_channel
src.valid.x.open = open_4_channel
for item in combo:
item.loc[ item['Fare'] <= 7.55, 'Fare'] = 0
item.loc[(item['Fare'] > 7.55)&(item['Fare'] <= 10.5), 'Fare'] = 1
item.loc[(item['Fare'] > 10.5)&(item['Fare'] <= 14.454), 'Fare'] = 2
item.loc[(item['Fare'] > 14.454)&(item['Fare'] <= 21.67), 'Fare'] = 3
item.loc[(item['Fare'] > 21.67)&(item['Fare'] <= 77.958), 'Fare'] = 4
item.loc[ item['Fare'] > 77.958, 'Fare'] = 5
item['Fare'] = item['Fare'].astype(int )<groupby>
test_ids = list(sorted({fname.split('_')[0] for fname in os.listdir(path/'test')}))
train[['Pclass','Survived']].groupby(['Pclass'], as_index=False ).mean()<groupby>
test_fnames = [path/'test'/test_id for test_id in test_ids]
train[['Fare','Survived']].groupby(['Fare'], as_index=False ).mean()<drop_column>
src.add_test(test_fnames, label='0');
for item in combo:
item = item.drop(['Ticket'], axis=1, inplace=True )<feature_engineering>
src.test.x.create_func = open_4_channel
src.test.x.open = open_4_channel
train['Age2']=-1
test['Age2']=-1
combo=[train,test]
for item in combo:
item.loc[(item['Age'] <=4), 'Age2'] =0
item.loc[(item['Age']<=17)&(item['Age'] >4), 'Age2'] =1
item.loc[(item['Age'] >=18)&(item['Age'] <=32), 'Age2'] =2
item.loc[(item['Age'] >32)&(item['Age'] <=42), 'Age2']= 3
item.loc[(item['Age'] >42)&(item['Age'] <=52), 'Age2']= 4
item.loc[(item['Age'] >52)&(item['Age'] <=60), 'Age2']= 5
item.loc[(item['Age'] >60), 'Age2']= 6<groupby>
protein_stats =([0.08069, 0.05258, 0.05487, 0.08282], [0.13704, 0.10145, 0.15313, 0.13814] )
train[['Survived','Age2']].groupby(['Age2'], as_index=False ).mean()<compute_train_metric>
trn_tfms,_ = get_transforms(do_flip=True, flip_vert=True, max_rotate=30., max_zoom=1,
max_lighting=0.05, max_warp=0.)
pearson_coef1 , p_value1 =scipy.stats.pearsonr(train["Cabin"],train["Pclass"])
 print("Cabin - Pclass", pearson_coef1," Correlation certainty =",p_value1 )<drop_column>
data =(src.transform(( trn_tfms, _), size=224)
.databunch(num_workers=0 ).normalize(protein_stats))
for item in combo:
item = item.drop(['Sex'], axis=1, inplace=True )<feature_engineering>
data.show_batch(rows=3, figsize=(12,9))
for item in combo:
item.loc[(( item['Title']=='Miss')&(item['Age'] <=4)) , 'Title'] = 'child'
item.loc[(( item['Title']=='Master')&(item['Age'] <=4)) , 'Title'] = 'child'
item.loc[(( item['Title']=='Miss')&(item['Age'] <=17)) , 'Title'] = 'kid'
item.loc[(( item['Title']=='Master')&(item['Age'] <=17)) , 'Title'] = 'kid'
item.loc[(( item['Title']=='Master')&(item['Age'] >17)) , 'Title'] = 'Mr'
item.loc[(( item['Title']=='Mrs')&(item['Parch']!=0)) ,'Title']='MrsYesCh'
item.loc[(( item['Title']=='Mrs')&(item['Parch']==0)) ,'Title']='MrsNoCh'
item = item.drop(['SibSp','Parch'], axis=1, inplace=True )<groupby>
def resnet50(pretrained):
return Resnet4Channel(encoder_depth=50 )
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<data_type_conversions>
def _resnet_split(m): return(m[0][6],m[1] )
train['Title'] = train['Title'].map({ 'Mr': 0 ,'Miss':1, 'MrsNoCh':2 ,'MrsYesCh':3,'kid':4,'child':5} ).astype(int)
 test['Title'] = test['Title'].map({'Mr': 0 ,'Miss':1, 'MrsNoCh':2 ,'MrsYesCh':3,'kid':4, 'child':5} ).astype(int )<drop_column>
f1_score = partial(fbeta, thresh=0.2, beta=1 )
for item in combo:
item = item.drop(['Age'], axis=1, inplace=True )<prepare_x_and_y>
learn = create_cnn(
 data,
resnet50,
cut=-2,
split_on=_resnet_split,
loss_func=F.binary_cross_entropy_with_logits,
path=path,
metrics=[f1_score],
model_dir="/tmp/models/"
)
X_train = train.drop(['Survived','PassengerId','FamilySize','FareBin'], axis=1)
 col=(X_train.columns)
 Y_train = train["Survived"]
X_test=test.drop(["PassengerId","FamilySize"], axis=1 )<normalization>
learn.lr_find()
scaler = StandardScaler()
scaler.fit_transform(X_train)
 scaler.transform(X_test )<train_model>
lr = 3e-2
test1 = SelectKBest(score_func= f_classif, k=4)
 fit = test1.fit(X_train, Y_train)
 set_printoptions(precision=3)
 print(col)
 print(' ',fit.scores_ )<train_model>
learn.fit_one_cycle(1, slice(lr))
model = ExtraTreesClassifier(n_estimators=10)
 model.fit(X_train, Y_train)
 print(col)
 print('Feature importance')
 print(model.feature_importances_ )<train_model>
learn.save('stage-1-rn50-test' )
rfe = RFE(estimator=DecisionTreeClassifier() , n_features_to_select=4)
 fit = rfe.fit(X_train, Y_train)
 print('Features : ',col)
 print("Num Features: ",fit.n_features_)
 print("Selected Features: ",fit.support_)
 print("Feature Ranking: ",fit.ranking_ )<categorify>
learn.lr_find()
X_train = pd.get_dummies(X_train, columns = ["Title"],prefix="N")
 X_train = pd.get_dummies(X_train, columns = ["Fare"],prefix="F",drop_first=True)
 X_test = pd.get_dummies(X_test, columns = ["Title"],prefix="N")
 X_test = pd.get_dummies(X_test, columns = ["Fare"],prefix="F",drop_first=True)
 Y_train = train["Survived"].values<train_model>
learn.fit_one_cycle(2, slice(3e-5, lr/5))
X_train.drop(["Embarked","Cabin",'Age2'], axis=1, inplace=True)
 X_test.drop(["Embarked","Cabin",'Age2'], axis=1,inplace=True)
 print("Training set X: ", X_train.shape, "Training set Y:",Y_train.shape, "Testing set X", X_test.shape )<import_modules>
learn.save('stage-2-rn50-test' )
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier,AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
import lightgbm as lgb
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier
from sklearn.ensemble import VotingClassifier, StackingClassifier
from sklearn.linear_model import LogisticRegression
import keras
from keras import models,layers,optimizers
from keras.optimizers import Adam
from keras.layers import Activation, Dense ,Dropout
from keras.callbacks import EarlyStopping
from numpy import mean , std
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression
from numpy import arange, argmax
from sklearn import metrics
from sklearn.metrics import auc,confusion_matrix,classification_report,roc_curve,accuracy_score,roc_auc_score 
<define_search_space>
preds,_ = learn.get_preds(DatasetType.Test )
kfld = StratifiedKFold(n_splits=5, shuffle=True, random_state=2509)
 GB_param = {"n_estimators":[100,150,250,500], "learning_rate": [0.001,0.01,0.05,0.1,0.2],
"min_samples_split":[3,10,31],"max_depth":[3,5,7], "max_features":[4,7,8,9,10,11,13],
"subsample":[0.8,1], "criterion": ["mae"]}
AdaB_param ={'n_estimators': [10,20,50,100,200,500],'learning_rate':[0.001,0.01,0.03,0.05,0.07,0.1,0.2,0.3] }
SVC_param = { 'kernel' : ['rbf', 'linear', 'poly', 'sigmoid'],
'gamma': ['auto'],
'C': [0.01,0.1,0.2,0.5,1,2,3,4,5,6,7,8,9,10,11,12,20,50,100],
'shrinking' : [True, False]}
RF_param = {'n_estimators' : [20,50,70,100,500], 'max_features' : [3,4,5,6,7,9,10,13],
'max_depth' : [3,5,7],'criterion' : ['gini'], 'min_samples_split' : [4,10,31,64]}
XGB_param= {"learning_rate" :[0.001,0.01,0.05,0.1,0.2], "max_depth":[3,5,7],
"gamma": [0,0.5,1],"subsample":[0.8,1], "alpha" : [0,0.5,1], "n_estimators" : [100,500,800],
'colsample_bytree': [0.6,0.8,1], "objective": ['binary:logistic']}
LGparam= {'boosting_type': ['gbdt'],'num_leaves': [4,10,31],'subsample': [0.8,1],
'colsample_bytree': [0.6,0.8,1],'reg_alpha': [0,0.5,1],'reg_lambda':[0,0.5,1],
'max_depth':[3,5,7],'n_estimators' :[100,500,800],'learning_rate': [0.001,0.005,0.01,0.05,0.1],
'objective': ['binary'],'metric': ['binary_logloss']}
MPL_param = {'hidden_layer_sizes': [(16,8,1),(32,16,1),(32,32)],
'activation': ['tanh', 'relu'],'solver': ['adam'], 'alpha': [0.01, 0.05]}
Knn_param = dict(leaf_size=list(range(1,30)) , n_neighbors=list(range(1,50)) , weights = ['uniform','distance'],p=[1,2])
 clfxgb =XGBClassifier(random_state=seed_value)
 clfgb = GradientBoostingClassifier(random_state=seed_value)
 clfAdaB=AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=5),random_state=seed_value)
 clfsvc = SVC(random_state=seed_value)
 clfRf=RandomForestClassifier(random_state=seed_value)
 clfxgb =XGBClassifier(random_state=seed_value)
 clfknn=KNeighborsClassifier()
clfLGB = LGBMClassifier(seed=seed_value)
 clfmlp = MLPClassifier(max_iter=200)
 classfrs={clfxgb:XGB_param,clfknn:Knn_param,clfLGB:LGparam,clfRf:RF_param,clfsvc:SVC_param,
clfmlp:MPL_param,clfAdaB:AdaB_param, clfgb:GB_param} 
<choose_model_class>
pred_labels = [' '.join(list([str(i)for i in np.nonzero(row>0.2)[0]])) for row in np.array(preds)]
df = pd.DataFrame({'Id':test_ids,'Predicted':pred_labels})
 df.to_csv('submission.csv', header=True, index=False )