Kaggle competition: Human Protein Atlas Image Classification
<SOS> metric: MacroFScore Kaggle data source: human-protein-atlas-image-classification<feature_engineering>
import sys
import numpy as np
import keras
from keras.utils import Sequence
from PIL import Image
from matplotlib import pyplot as plt
import pandas as pd
from tqdm import tqdm
import os
import imgaug as ia
from imgaug import augmenters as iaa
import cv2
outliers_bounds = {}
for column in continuous_features:
outliers_bounds[column] = get_outliers_info(total_data, column=column )<drop_column>
BATCH_SIZE = 128
SEED = 777
SHAPE =(192, 192, 4)
 DIR = '.. /input'
VAL_RATIO = 0.1
THRESHOLD = 0.05
ia.seed(SEED )
for dataset in train_test_data:
dataset.drop('Parch', axis='columns', inplace=True )<drop_column>
def getTrainDataset() :
path_to_train = DIR + '/train/'
data = pd.read_csv(DIR + '/train.csv')
 paths = []
labels = []
for name, lbl in zip(data['Id'], data['Target'].str.split(' ')) :
y = np.zeros(28)
 for key in lbl:
y[int(key)] = 1
paths.append(os.path.join(path_to_train, name))
labels.append(y)
 return np.array(paths), np.array(labels)
 def getTestDataset() :
path_to_test = DIR + '/test/'
data = pd.read_csv(DIR + '/sample_submission.csv')
 paths = []
labels = []
for name in data['Id']:
y = np.ones(28)
 paths.append(os.path.join(path_to_test, name))
labels.append(y)
 return np.array(paths), np.array(labels)
del outliers_bounds['Parch']
del outliers_bounds['Pclass']<categorify>
class ProteinDataGenerator(keras.utils.Sequence):
def __init__(self, paths, labels, batch_size, shape, shuffle = False, use_cache = False, augment = False):
self.paths, self.labels = paths, labels
self.batch_size = batch_size
self.shape = shape
self.shuffle = shuffle
self.use_cache = use_cache
self.augment = augment
if use_cache == True:
self.cache = np.zeros(( paths.shape[0], shape[0], shape[1], shape[2]), dtype=np.float16)
 self.is_cached = np.zeros(( paths.shape[0]))
self.on_epoch_end()
def __len__(self):
return int(np.ceil(len(self.paths)/ float(self.batch_size)))
 def __getitem__(self, idx):
indexes = self.indexes[idx * self.batch_size :(idx+1)* self.batch_size]
paths = self.paths[indexes]
X = np.zeros(( paths.shape[0], self.shape[0], self.shape[1], self.shape[2]))
if self.use_cache == True:
X = self.cache[indexes]
for i, path in enumerate(paths[np.where(self.is_cached[indexes] == 0)]):
image = self.__load_image(path)
 self.is_cached[indexes[i]] = 1
self.cache[indexes[i]] = image
X[i] = image
else:
for i, path in enumerate(paths):
X[i] = self.__load_image(path)
 y = self.labels[indexes]
if self.augment == True:
seq = iaa.Sequential([
iaa.OneOf([
iaa.Fliplr(0.5),
iaa.Crop(percent=(0, 0.1)) ,
iaa.Sometimes(0.5,
iaa.GaussianBlur(sigma=(0, 0.5)) 
),
iaa.ContrastNormalization(( 0.75, 1.5)) ,
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5),
iaa.Multiply(( 0.8, 1.2), per_channel=0.2),
iaa.Affine(
 scale={"x":(0.8, 1.2), "y":(0.8, 1.2)},
translate_percent={"x":(-0.2, 0.2), "y":(-0.2, 0.2)},
rotate=(-180, 180),
shear=(-8, 8)
)
 ])], random_order=True)
 X = np.concatenate(( X, seq.augment_images(X), seq.augment_images(X), seq.augment_images(X)) , 0)
 y = np.concatenate(( y, y, y, y), 0)
 return X, y
def on_epoch_end(self):
self.indexes = np.arange(len(self.paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __iter__(self):

 for item in(self[i] for i in range(len(self))):
yield item
def __load_image(self, path):
R = Image.open(path + '_red.png')
 G = Image.open(path + '_green.png')
 B = Image.open(path + '_blue.png')
 Y = Image.open(path + '_yellow.png')
 im = np.stack((
np.array(R),
np.array(G),
np.array(B),
np.array(Y)) , -1)
 im = cv2.resize(im,(SHAPE[0], SHAPE[1]))
im = np.divide(im, 255)
 return im
def normalize_outliers(data, outliers_bounds={}):
for key in outliers_bounds.keys() :
median = data[key].median()
new_values = []
for i in range(len(data)) :
value = data[key][i]
if value < outliers_bounds[key][0] or value > outliers_bounds[key][1]:
new_values.append(outliers_bounds[key][1])
 else:
new_values.append(value)
 data[key] = new_values
return data<categorify>
set_random_seed(SEED )
train = normalize_outliers(train, outliers_bounds=outliers_bounds)
 test = normalize_outliers(test, outliers_bounds=outliers_bounds )<concatenate>
def f1(y_true, y_pred):
y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())
 tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return K.mean(f1)
 def f1_loss(y_true, y_pred):
tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return 1-K.mean(f1 )
total_data = train.append(test, sort=False )<drop_column>
def create_model(input_shape):
dropRate = 0.25
init = Input(input_shape)
 x = BatchNormalization(axis=-1 )(init)
 x = Conv2D(8,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Conv2D(8,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Conv2D(16,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Dropout(dropRate )(x)
 c1 = Conv2D(16,(3, 3), padding='same' )(x)
 c1 = ReLU()(c1)
 c2 = Conv2D(16,(5, 5), padding='same' )(x)
 c2 = ReLU()(c2)
 c3 = Conv2D(16,(7, 7), padding='same' )(x)
 c3 = ReLU()(c3)
 c4 = Conv2D(16,(1, 1), padding='same' )(x)
 c4 = ReLU()(c4)
 x = Concatenate()([c1, c2, c3, c4])
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Dropout(dropRate )(x)
 x = Conv2D(32,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Dropout(dropRate )(x)
 x = Conv2D(64,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Dropout(dropRate )(x)
 x = Conv2D(128,(3, 3))(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Dropout(dropRate )(x)
 x = Flatten()(x)
 x = Dropout(0.5 )(x)
 x = Dense(28 )(x)
 x = ReLU()(x)
 x = BatchNormalization(axis=-1 )(x)
 x = Dropout(0.1 )(x)
 x = Dense(28 )(x)
 x = Activation('sigmoid' )(x)
 model = Model(init, x)
 return model
del outliers_bounds['SibSp']<create_dataframe>
model = create_model(SHAPE)
 model.compile(
 loss='binary_crossentropy',
optimizer=Adam(1e-03),
metrics=['acc',f1])
 model.summary()
for column in outliers_bounds.keys() :
get_continuous_feature_info(train, column=column, y_column='Survived' )<categorify>
paths, labels = getTrainDataset()
keys = np.arange(paths.shape[0], dtype=np.int)
 np.random.seed(SEED)
 np.random.shuffle(keys)
 lastTrainIndex = int(( 1-VAL_RATIO)* paths.shape[0])
 pathsTrain = paths[0:lastTrainIndex]
labelsTrain = labels[0:lastTrainIndex]
pathsVal = paths[lastTrainIndex:]
labelsVal = labels[lastTrainIndex:]
print(paths.shape, labels.shape)
 print(pathsTrain.shape, labelsTrain.shape, pathsVal.shape, labelsVal.shape)
 tg = ProteinDataGenerator(pathsTrain, labelsTrain, BATCH_SIZE, SHAPE, use_cache=True, augment = False, shuffle = False)
 vg = ProteinDataGenerator(pathsVal, labelsVal, BATCH_SIZE, SHAPE, use_cache=True, shuffle = False)
 checkpoint = ModelCheckpoint('./base.model', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='min', period=1)
 reduceLROnPlato = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1, mode='min' )
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 16), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 16)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[(dataset['Fare'] > 31)&(dataset['Fare'] <= 50), 'Fare'] = 3
dataset.loc[ dataset['Fare'] > 50, 'Fare'] = 4
dataset['Fare'] = dataset['Fare'].astype(int )<compute_test_metric>
epochs = 100
use_multiprocessing = False
workers = 1
hist = model.fit_generator(
 tg,
steps_per_epoch=len(tg),
validation_data=vg,
validation_steps=8,
epochs=epochs,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=1,
callbacks=[checkpoint] )
def is_youngest_girl(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Sex[i] == 1):
match =(data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list<feature_engineering>
bestModel = load_model('./base.model', custom_objects={'f1': f1} )
train['Is_Youngest_lady'] = is_youngest_girl(train, total_data)
 test['Is_Youngest_lady'] = is_youngest_girl(test, total_data )<feature_engineering>
fullValGen = vg
for dataset in train_test_data:
dataset['Is_WifeandRich'] =(( dataset['Name'].str.find('(')> -1)&(dataset['Sex'] == 1)&(dataset['Pclass'] <3)) *1<feature_engineering>
lastFullValPred = np.empty(( 0, 28))
lastFullValLabels = np.empty(( 0, 28))
for i in tqdm(range(len(fullValGen))):
im, lbl = fullValGen[i]
scores = bestModel.predict(im)
 lastFullValPred = np.append(lastFullValPred, scores, axis=0)
 lastFullValLabels = np.append(lastFullValLabels, lbl, axis=0)
 print(lastFullValPred.shape, lastFullValLabels.shape )
for column in ['Is_WifeandRich', 'Is_Youngest_lady']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )<compute_test_metric>
rng = np.arange(0, 1, 0.001)
 f1s = np.zeros(( rng.shape[0], 28))
for j,t in enumerate(tqdm(rng)) :
for i in range(28):
p = np.array(lastFullValPred[:,i]>t, dtype=np.int8)
 scoref1 = off1(lastFullValLabels[:,i], p, average='binary')
 f1s[j,i] = scoref1
def is_a_rich_Mother(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Pclass[i] < 3)&(data.Sex[i] == 1):
match =(14 >= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list<feature_engineering>
print('Individual F1-scores for each class:')
 print(np.max(f1s, axis=0))
print('Macro F1-score CV =', np.mean(np.max(f1s, axis=0)) )
train['Is_a_rich_Mother'] = is_a_rich_Mother(train, total_data)
 test['Is_a_rich_Mother'] = is_a_rich_Mother(test, total_data )<filter>
pathsTest, labelsTest = getTestDataset()
testg = ProteinDataGenerator(pathsTest, labelsTest, BATCH_SIZE, SHAPE)
 submit = pd.read_csv(DIR + '/sample_submission.csv')
 P = np.zeros(( pathsTest.shape[0], 28))
for i in tqdm(range(len(testg))):
images, labels = testg[i]
score = bestModel.predict(images)
 P[i*BATCH_SIZE:i*BATCH_SIZE+score.shape[0]] = score
def is_the_only_woman(data, match_data):
match_list = []
for i in range(len(data)) :
if data.Sex[i] == 1:
match =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Sex.sum())
 if match == 1:
match_list.append(match)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list<categorify>
PP = np.array(P )
<feature_engineering><EOS> 
prediction = []
for row in tqdm(range(submit.shape[0])) :
str_label = ''
for col in range(PP.shape[1]):
if(PP[row, col] < T[col]):
str_label += ''
else:
str_label += str(col)+ ' '
prediction.append(str_label.strip())
 submit['Predicted'] = np.array(prediction)
 submit.to_csv('4channels_cnn_from_scratch.csv', index=False )