app.py

import keras
import numpy as np
import pandas as pd
import gradio as gr
import os

from keras.applications.densenet import DenseNet121
from keras.layers import Dense, GlobalAveragePooling2D
from keras.models import Model

med_labels = ['Cardiomegaly', 
          'Emphysema', 
          'Effusion', 
          'Hernia', 
          'Infiltration', 
          'Mass', 
          'Nodule', 
          'Atelectasis',
          'Pneumothorax',
          'Pleural_Thickening', 
          'Pneumonia', 
          'Fibrosis', 
          'Edema', 
          'Consolidation']

def get_weighted_loss(pos_weights, neg_weights, epsilon=1e-7):
    """
    Return weighted loss function given negative weights and positive weights.

    Args:
      pos_weights (np.array): array of positive weights for each class, size (num_classes)
      neg_weights (np.array): array of negative weights for each class, size (num_classes)
    
    Returns:
      weighted_loss (function): weighted loss function
    """
    def weighted_loss(y_true, y_pred):
        """
        Return weighted loss value. 

        Args:
            y_true (Tensor): Tensor of true labels, size is (num_examples, num_classes)
            y_pred (Tensor): Tensor of predicted labels, size is (num_examples, num_classes)
        Returns:
            loss (float): overall scalar loss summed across all classes
        """
        # initialize loss to zero
        loss = 0.0
        
        for i in range(len(pos_weights)):
            positive_term_loss = pos_weights[i] * tf.cast(y_true[:,i], tf.float32) * K.log(y_pred[:,i] + epsilon)
            negative_term_loss = neg_weights[i] * tf.cast((1-y_true[:,i]), tf.float32) * K.log(1-y_pred[:,i] + epsilon)
            loss +=  -K.mean(positive_term_loss + negative_term_loss)
        
        return loss

    return weighted_loss

freq_neg = np.loadtxt('freq_neg.txt')
freq_pos = np.loadtxt('freq_pos.txt')

pos_weights = freq_neg
neg_weights = freq_pos


# create the base pre-trained model
base_model = DenseNet121(weights='./nih/densenet.hdf5', include_top=False)

x = base_model.output

# add a global spatial average pooling layer
x = GlobalAveragePooling2D()(x)

# and a logistic layer
predictions = Dense(len(med_labels), activation="sigmoid")(x)

model = Model(inputs=base_model.input, outputs=predictions)
model.compile(optimizer='adam', loss=get_weighted_loss(pos_weights, neg_weights))


model.load_weights("./nih/pretrained_model.h5")


import os
import tensorflow as tf
from tensorflow import keras
from IPython.display import Image, display
import matplotlib.cm as cm


def convert_preds(preds):
    q = dict(zip(med_labels, preds[0]))
    return q



# The Grad-CAM algorithm

def make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index=None):
    # First, we create a model that maps the input image to the activations
    # of the last conv layer as well as the output predictions
    grad_model = keras.models.Model(
        model.inputs, [model.get_layer(last_conv_layer_name).output, model.output]
    )

    # Then, we compute the gradient of the top predicted class for our input image
    # with respect to the activations of the last conv layer
    with tf.GradientTape() as tape:
        last_conv_layer_output, preds = grad_model(img_array)
        if pred_index is None:
            pred_index = tf.argmax(preds[0])
        class_channel = preds[:, pred_index]

    # This is the gradient of the output neuron (top predicted or chosen)
    # with regard to the output feature map of the last conv layer
    grads = tape.gradient(class_channel, last_conv_layer_output)

    # This is a vector where each entry is the mean intensity of the gradient
    # over a specific feature map channel
    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))

    # We multiply each channel in the feature map array
    # by "how important this channel is" with regard to the top predicted class
    # then sum all the channels to obtain the heatmap class activation
    last_conv_layer_output = last_conv_layer_output[0]
    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
    heatmap = tf.squeeze(heatmap)

    # For visualization purpose, we will also normalize the heatmap between 0 & 1
    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
    return heatmap.numpy()


# Create a superimposed visualization
    
def superimpose_gradcam(img_path, heatmap, alpha=0.5):
    # Load the original image
    img = keras.utils.load_img(img_path)
    img = keras.utils.img_to_array(img)

    # Rescale heatmap to a range 0-255
    heatmap = np.uint8(255 * heatmap)

    # Use jet colormap to colorize heatmap
    jet = cm.get_cmap("jet")

    # Use RGB values of the colormap
    jet_colors = jet(np.arange(256))[:, :3]
    jet_heatmap = jet_colors[heatmap]

    # Create an image with RGB colorized heatmap
    jet_heatmap = keras.utils.array_to_img(jet_heatmap)
    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))
    jet_heatmap = keras.utils.img_to_array(jet_heatmap)

    # Superimpose the heatmap on original image
    superimposed_img = jet_heatmap * alpha + img * 0.4
    superimposed_img = keras.utils.array_to_img(superimposed_img)

    return superimposed_img

    # Save the superimposed image
    # superimposed_img.save(cam_path)

    # # Display Grad CAM
    # display(Image(cam_path,width=300))


def pil_to_np(pil):
    a = np.array(pil)
    return a

def np_to_pil(a):
    from PIL import Image
    im = Image.fromarray(a) #, mode="RGB"
    return im


from keras.preprocessing import image

def load_image_to_array(image_path, H=320, W=320):
    pil = image.load_img(
        image_path, 
        target_size=(H, W),
        color_mode = 'rgb',
        interpolation = 'nearest',
    )
    a = pil_to_np(pil)
    return a

def normalize_array(a):
    pil = np_to_pil(a)
    mean = np.mean(pil)
    std = np.std(pil)
    pil -= mean
    pil /= std
    a2 = pil_to_np(pil)
    a2 = np.expand_dims(a2, axis=0)
    return a2


selected_keys = ['Cardiomegaly','Mass','Pneumothorax','Edema']
# selected_keys.append('Infiltration')
def print_selected(preds):
    for k in selected_keys:
        print('{:15}\t{:6.3f}'.format(k, preds[k]))



IMAGE_DIR = "nih/images-small/"
last_conv_layer_name = 'bn'


def med_classify_image(inp):
    inp = load_image_to_array(inp)
    inp = normalize_array(inp)
    preds = model.predict(inp,verbose=0)
    preds = convert_preds(preds)
    preds = {key:value.item() for key, value in preds.items()}
    return preds

def gradcam(inp):
    selected_labels = [
        (idx, label) 
        for idx, label in enumerate(med_labels) 
        if label in selected_keys]
    img_array = load_image_to_array(inp)
    img_array = normalize_array(img_array)
    images = []
    for k, l in selected_labels:
        heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name, pred_index = k)
        superimposed_img = superimpose_gradcam(inp, heatmap)
        images.append((superimposed_img,l))
    return images


with gr.Blocks(css="footer {visibility: hidden}") as demo:
    gr.Markdown('# Chest X-Ray Medical Diagnosis with Deep Learning')
    with gr.Row():
        input_image = gr.Image(label='Chest X-Ray',type='filepath',image_mode='L')
        with gr.Column():
            gr.Examples(
                examples=[
                    "nih/images-small/00008270_015.png",
                    "nih/images-small/00011355_002.png",
                    "nih/images-small/00029855_001.png",
                    "nih/images-small/00005410_000.png",
                ],
                inputs=input_image,
                label='Examples'
                # fn=mirror,
                # cache_examples=True,
            )
            with gr.Column():
                b1 = gr.Button("Classify")
                b2 = gr.Button("Compute GradCam")
    with gr.Row():
        label = gr.Label(label='Classification',num_top_classes=5)
        gallery = gr.Gallery(
            label="GradCam", 
            show_label=True, 
            elem_id="gallery",
            object_fit="scale-down", 
            height=400)
    gr.Markdown(
    """
    [ChestX-ray8 dataset](https://arxiv.org/abs/1705.02315)
    [Download the entire dataset](https://nihcc.app.box.com/v/ChestXray-NIHCC)
    """)
    b1.click(med_classify_image, inputs=input_image, outputs=label)
    b2.click(gradcam, inputs=input_image, outputs=gallery)

if __name__ == "__main__":
    demo.launch()