Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<set_options>
!pip install livelossplot
warnings.filterwarnings("ignore")
<import_modules>
TRAINING_LOGS_FILE = "training_logs.csv"
MODEL_FILE = "histopathologic_cancer_detector.h5"
ROC_PLOT_FILE = "roc.png"
KAGGLE_SUBMISSION_FILE = "kaggle_submission.csv"
INPUT_DIR = '.. /input/'
from tensorflow.keras.utils import get_custom_objects<compute_test_metric>
SAMPLE_COUNT =50000
TRAINING_RATIO = 0.9
IMAGE_SIZE = 96
EPOCHS = 12
BATCH_SIZE = 128
VERBOSITY = 1
TESTING_BATCH_SIZE = 3000
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<choose_model_class>
training_dir = INPUT_DIR + 'train/'
data_frame = pd.DataFrame({'path': glob(os.path.join(training_dir,'*.tif')) })
 data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split('.')[0])
 labels = pd.read_csv(INPUT_DIR + 'train_labels.csv')
 data_frame = data_frame.merge(labels, on = 'id')
 negatives = data_frame[data_frame.label == 0].sample(SAMPLE_COUNT)
 positives = data_frame[data_frame.label == 1].sample(SAMPLE_COUNT)
 data_frame = pd.concat([negatives, positives] ).reset_index()
data_frame = data_frame[['path', 'id', 'label']]
data_frame['image'] = data_frame['path'].map(imread)
 training_path = '.. /training1'
validation_path = '.. /validation1'
for folder in [training_path, validation_path]:
for subfolder in ['0', '1']:
path = os.path.join(folder, subfolder)
 os.makedirs(path, exist_ok=True)
 training, validation = train_test_split(data_frame, train_size=TRAINING_RATIO, stratify=data_frame['label'])
 data_frame.set_index('id', inplace=True)
 for images_and_path in [(training, training_path),(validation, validation_path)]:
images = images_and_path[0]
path = images_and_path[1]
for image in images['id'].values:
file_name = image + '.tif'
label = str(data_frame.loc[image,'label'])
 destination = os.path.join(path, label, file_name)
 if not os.path.exists(destination):
source = os.path.join(INPUT_DIR + 'train', file_name)
 shutil.copyfile(source, destination )
def create_model(data, catcols):
inputs = []
outputs = []
for c in catcols:
num_unique_values = int(data[c].nunique())
 embed_dim = int(min(np.ceil(( num_unique_values)/2), 50))
inp = layers.Input(shape=(1,))
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c )(inp)
 out = layers.SpatialDropout1D(0.3 )(out)
 out = layers.Reshape(target_shape=(embed_dim,))(out)
 inputs.append(inp)
 outputs.append(out)
 x = layers.Concatenate()(outputs)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300 )(x)
 x = Activation('Mish', name="act1" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300 )(x)
 x = Activation('Mish', name="act2" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 y = layers.Dense(2, activation="softmax" )(x)
 model = Model(inputs=inputs, outputs=y)
 return model<load_from_csv>
training_data_generator = ImageDataGenerator(rescale=1./255,
horizontal_flip=True,
vertical_flip=True,
rotation_range=180,
zoom_range=[1, 1.5],
fill_mode='reflect',
width_shift_range=0.3,
height_shift_range=0.3,
shear_range=0.3,
channel_shift_range=0.3 )
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 sample = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv" )<concatenate>
training_generator = training_data_generator.flow_from_directory(training_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE,
class_mode='binary')
 validation_generator = ImageDataGenerator(rescale=1./255 ).flow_from_directory(validation_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE,
class_mode='binary' )
test["target"] = -1
data = pd.concat([train, test] ).reset_index(drop=True )<define_variables>
input_shape =(96, 96, 3)
 inputs = Input(input_shape)
 dense = DenseNet121(include_top=False, input_shape=input_shape)
 outputs = GlobalAveragePooling2D()(dense(inputs))
outputs = Dropout(0.5 )(outputs)
 outputs = Dense(1, activation='sigmoid' )(outputs)
 model = Model(inputs, outputs)
 model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),
loss='binary_crossentropy',
metrics=['accuracy'])
 model.summary()
features = [x for x in train.columns if x not in ["id", "target"]]<categorify>
history = model.fit_generator(training_generator,
steps_per_epoch=len(training_generator),
validation_data=validation_generator,
validation_steps=len(validation_generator),
epochs=EPOCHS,
verbose=VERBOSITY,shuffle=True,
callbacks=[PlotLossesKeras() ,
ModelCheckpoint(MODEL_FILE,
monitor='val_acc',
verbose=VERBOSITY,
save_best_only=True,
mode='max'),
CSVLogger(TRAINING_LOGS_FILE,
append=False,
separator=';')])
 model.load_weights(MODEL_FILE )
for feat in features:
lbl_enc = preprocessing.LabelEncoder()
data[feat] = lbl_enc.fit_transform(data[feat].values )<filter>
roc_validation_generator = ImageDataGenerator(rescale=1./255,shear_range=0.3,zoom_range=[1,1.5],
horizontal_flip=True,
rotation_range=10.,
width_shift_range = 0.3,
height_shift_range = 0.3 ).flow_from_directory(validation_path,target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE,
class_mode='binary',
shuffle=False)
 predictions = model.predict_generator(roc_validation_generator, steps=len(roc_validation_generator), verbose=VERBOSITY)
 false_positive_rate, true_positive_rate, threshold = roc_curve(roc_validation_generator.classes, predictions)
 area_under_curve = auc(false_positive_rate, true_positive_rate)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.plot(false_positive_rate, true_positive_rate, label='AUC = {:.3f}'.format(area_under_curve))
plt.xlabel('False positive rate')
 plt.ylabel('True positive rate')
 plt.title('ROC curve')
 plt.legend(loc='best')
 plt.savefig(ROC_PLOT_FILE, bbox_inches='tight')
 plt.show()
<split><EOS> 
testing_files = glob(os.path.join(INPUT_DIR+'test/','*.tif'))
submission = pd.DataFrame()
for index in range(0, len(testing_files), TESTING_BATCH_SIZE):
data_frame = pd.DataFrame({'path': testing_files[index:index+TESTING_BATCH_SIZE]})
 data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split(".")[0])
 data_frame['image'] = data_frame['path'].map(imread)
 images = np.stack(data_frame.image, axis=0)
 predicted_labels = [model.predict(np.expand_dims(image/255.0, axis=0)) [0][0] for image in images]
predictions = np.array(predicted_labels)
 data_frame['label'] = predictions
submission = pd.concat([submission, data_frame[["id", "label"]]])
 submission.to_csv(KAGGLE_SUBMISSION_FILE, index=False, header=True )