Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<drop_column>
import sys
import numpy as np
import keras
from keras.utils import Sequence
from PIL import Image
from matplotlib import pyplot as plt
import pandas as pd
from tqdm import tqdm
import os
import imgaug as ia
from imgaug import augmenters as iaa
import cv2
X=train.drop('Survived',axis=1 )<split>
BATCH_SIZE = 32
SEED = 777
SHAPE =(512, 512, 4)
 DIR = '.. /input'
VAL_RATIO = 0.1
THRESHOLD = 0.5
ia.seed(SEED )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=2 )<train_model>
def getTrainDataset() :
path_to_train = DIR + '/train/'
data = pd.read_csv(DIR + '/train.csv')
 paths = []
labels = []
for name, lbl in zip(data['Id'], data['Target'].str.split(' ')) :
y = np.zeros(28)
 for key in lbl:
y[int(key)] = 1
paths.append(os.path.join(path_to_train, name))
labels.append(y)
 return np.array(paths), np.array(labels)
 def getTestDataset() :
path_to_test = DIR + '/test/'
data = pd.read_csv(DIR + '/sample_submission.csv')
 paths = []
labels = []
for name in data['Id']:
y = np.ones(28)
 paths.append(os.path.join(path_to_test, name))
labels.append(y)
 return np.array(paths), np.array(labels)
logmodel = LogisticRegression()
logmodel.fit(X_train,y_train )<predict_on_test>
class ProteinDataGenerator(keras.utils.Sequence):
def __init__(self, paths, labels, batch_size, shape, shuffle = False, use_cache = False, augment = False):
self.paths, self.labels = paths, labels
self.batch_size = batch_size
self.shape = shape
self.shuffle = shuffle
self.use_cache = use_cache
self.augment = augment
if use_cache == True:
self.cache = np.zeros(( paths.shape[0], shape[0], shape[1], shape[2]), dtype=np.float16)
 self.is_cached = np.zeros(( paths.shape[0]))
self.on_epoch_end()
def __len__(self):
return int(np.ceil(len(self.paths)/ float(self.batch_size)))
 def __getitem__(self, idx):
indexes = self.indexes[idx * self.batch_size :(idx+1)* self.batch_size]
paths = self.paths[indexes]
X = np.zeros(( paths.shape[0], self.shape[0], self.shape[1], self.shape[2]))
if self.use_cache == True:
X = self.cache[indexes]
for i, path in enumerate(paths[np.where(self.is_cached[indexes] == 0)]):
image = self.__load_image(path)
 self.is_cached[indexes[i]] = 1
self.cache[indexes[i]] = image
X[i] = image
else:
for i, path in enumerate(paths):
X[i] = self.__load_image(path)
 y = self.labels[indexes]
if self.augment == True:
seq = iaa.Sequential([
iaa.OneOf([
iaa.Fliplr(0.5),
iaa.Crop(percent=(0, 0.1)) ,
iaa.Sometimes(0.5,
iaa.GaussianBlur(sigma=(0, 0.5)) 
),
iaa.ContrastNormalization(( 0.75, 1.5)) ,
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5),
iaa.Multiply(( 0.8, 1.2), per_channel=0.2),
iaa.Affine(
 scale={"x":(0.8, 1.2), "y":(0.8, 1.2)},
translate_percent={"x":(-0.2, 0.2), "y":(-0.2, 0.2)},
rotate=(-180, 180),
shear=(-8, 8)
)
 ])], random_order=True)
 X = np.concatenate(( X, seq.augment_images(X), seq.augment_images(X), seq.augment_images(X)) , 0)
 y = np.concatenate(( y, y, y, y), 0)
 return X, y
def on_epoch_end(self):
self.indexes = np.arange(len(self.paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __iter__(self):

 for item in(self[i] for i in range(len(self))):
yield item
def __load_image(self, path):
R = Image.open(path + '_red.png')
 G = Image.open(path + '_green.png')
 B = Image.open(path + '_blue.png')
 Y = Image.open(path + '_yellow.png')
 im = np.stack((
np.array(R),
np.array(G),
np.array(B),
np.array(Y)) , -1)
 im = cv2.resize(im,(SHAPE[0], SHAPE[1]))
im = np.divide(im, 255)
 return im
predictions = logmodel.predict(X_test )<import_modules>
set_random_seed(SEED )
from sklearn.metrics import classification_report<compute_test_metric>
def f1(y_true, y_pred):
tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return K.mean(f1)
 def f1_loss(y_true, y_pred):
tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return 1-K.mean(f1 )
classification_report(y_test,predictions )<import_modules>
def create_model(input_shape):
dropRate = 0.25
init = Input(input_shape)
 x = BatchNormalization(axis=-1 )(init)
 x = Conv2D(32,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 ginp1 = Dropout(dropRate )(x)
 x = BatchNormalization(axis=-1 )(ginp1)
 x = Conv2D(64,(3, 3), strides=(2,2))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Conv2D(64,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Conv2D(64,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 ginp2 = Dropout(dropRate )(x)
 x = BatchNormalization(axis=-1 )(ginp2)
 x = Conv2D(128,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Conv2D(128,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Conv2D(128,(3, 3))(x)
 x = ReLU()(x)
 ginp3 = Dropout(dropRate )(x)
 gap1 = GlobalAveragePooling2D()(ginp1)
 gap2 = GlobalAveragePooling2D()(ginp2)
 gap3 = GlobalAveragePooling2D()(ginp3)
 x = Concatenate()([gap1, gap2, gap3])
 x = BatchNormalization(axis=-1 )(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(dropRate )(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.1 )(x)
 x = Dense(28 )(x)
 x = Activation('sigmoid' )(x)
 model = Model(init, x)
 return model
from sklearn.metrics import confusion_matrix<compute_test_metric>
model = create_model(SHAPE)
 model.compile(
 loss='binary_crossentropy',
optimizer=Adam(1e-03),
metrics=['acc',f1])
 model.summary()
confusion_matrix(y_test,predictions )<define_variables>
paths, labels = getTrainDataset()
keys = np.arange(paths.shape[0], dtype=np.int)
 np.random.seed(SEED)
 np.random.shuffle(keys)
 lastTrainIndex = int(( 1-VAL_RATIO)* paths.shape[0])
 pathsTrain = paths[0:lastTrainIndex]
labelsTrain = labels[0:lastTrainIndex]
pathsVal = paths[lastTrainIndex:]
labelsVal = labels[lastTrainIndex:]
print(paths.shape, labels.shape)
 print(pathsTrain.shape, labelsTrain.shape, pathsVal.shape, labelsVal.shape)
 tg = ProteinDataGenerator(pathsTrain, labelsTrain, BATCH_SIZE, SHAPE, use_cache=False, augment = False, shuffle = False)
 vg = ProteinDataGenerator(pathsVal, labelsVal, BATCH_SIZE, SHAPE, use_cache=False, shuffle = False)
 checkpoint = ModelCheckpoint('./base.model', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='min', period=1)
 reduceLROnPlato = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1, mode='min' )
( 145+67)/(41+67+15+145 )<predict_on_test>
epochs = 20
use_multiprocessing = False
workers = 1
hist = model.fit_generator(
 tg,
steps_per_epoch=len(tg),
validation_data=vg,
validation_steps=8,
epochs=epochs,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=1,
callbacks=[checkpoint] )
gbk = GradientBoostingClassifier()
gbk.fit(X_train, y_train)
 y_pred = gbk.predict(X_test)
 acc_gbk = round(accuracy_score(y_pred, y_test)* 100, 2)
 print(acc_gbk )<count_missing_values>
for layer in model.layers:
layer.trainable = False
model.layers[-1].trainable = True
model.layers[-2].trainable = True
model.layers[-3].trainable = True
model.layers[-4].trainable = True
model.layers[-5].trainable = True
model.layers[-6].trainable = True
model.layers[-7].trainable = True
model.compile(loss=f1_loss,
optimizer=Adam(lr=1e-4),
metrics=['accuracy', f1])
 model.fit_generator(
 tg,
steps_per_epoch=len(tg),
validation_data=vg,
validation_steps=8,
epochs=1,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=1,
max_queue_size=4
)
test.isnull().sum()<count_missing_values>
bestModel = load_model('./base.model', custom_objects={'f1': f1})
test.isnull().sum()<save_to_csv>
fullValGen = vg
predictions1 = gbk.predict(test)
 sample_sub['Survived']= predictions1
sample_sub.to_csv("submit.csv", index=False)
 sample_sub.head()<import_modules>
def getOptimalT(mdl, fullValGen):
lastFullValPred = np.empty(( 0, 28))
lastFullValLabels = np.empty(( 0, 28))
for i in tqdm(range(len(fullValGen))):
im, lbl = fullValGen[i]
scores = mdl.predict(im)
 lastFullValPred = np.append(lastFullValPred, scores, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape)
 rng = np.arange(0, 1, 0.001)
 f1s = np.zeros(( rng.shape[0], 28))
for j,t in enumerate(tqdm(rng)) :
for i in range(28):
p = np.array(lastFullValPred[:,i]>t, dtype=np.int8)
 scoref1 = off1(lastFullValLabels[:,i], p, average='binary')
 f1s[j,i] = scoref1
print(np.max(f1s, axis=0))
print(np.mean(np.max(f1s, axis=0)))
 plt.plot(rng, f1s)
 T = np.empty(28)
 for i in range(28):
T[i] = rng[np.where(f1s[:,i] == np.max(f1s[:,i])) [0][0]]
print(T)
 return T, np.mean(np.max(f1s, axis=0))
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
fullValGen = ProteinDataGenerator(paths[lastTrainIndex:], labels[lastTrainIndex:], BATCH_SIZE, SHAPE)
 print('Last model after fine-tuning')
 T1, ff1 = getOptimalT(model, fullValGen )
df_train = pd.read_csv(".. /input/train.csv", sep=",")
 df_test = pd.read_csv(".. /input/test.csv", sep=",")
 df_train.head()<sort_values>
print('Best save model')
 T2, ff2 = getOptimalT(bestModel, fullValGen )
df_train[['Sex','Survived']].groupby(['Sex'],as_index=False ).mean().sort_values(by='Survived',ascending=False ).round(2 )<drop_column>
if ff1 > ff2:
T = T1
bestModel = model
else:
T = T2
bestModel = bestModel
df_train = df_train.drop(["PassengerId", "Name"], axis=1)
 df_test = df_test.drop(["Name"], axis=1 )<count_missing_values>
pathsTest, labelsTest = getTestDataset()
testg = ProteinDataGenerator(pathsTest, labelsTest, BATCH_SIZE, SHAPE)
 submit = pd.read_csv(DIR + '/sample_submission.csv')
 P = np.zeros(( pathsTest.shape[0], 28))
for i in tqdm(range(len(testg))):
images, labels = testg[i]
score = bestModel.predict(images)
 P[i*BATCH_SIZE:i*BATCH_SIZE+score.shape[0]] = score
df_train['Cabin'].isna().sum()<drop_column>
PP = np.array(P )
<data_type_conversions><EOS> 
prediction = []
for row in tqdm(range(submit.shape[0])) :
str_label = ''
for col in range(PP.shape[1]):
if(PP[row, col] < T[col]):
str_label += ''
else:
str_label += str(col)+ ' '
prediction.append(str_label.strip())
 submit['Predicted'] = np.array(prediction)
 submit.to_csv('4channels_cnn_from_scratch.csv', index=False )