img_extractor.py

import cv2
import matplotlib.pyplot as plt
from PIL import Image
from sklearn.pipeline import make_pipeline
from sklearn.base import BaseEstimator, TransformerMixin
import cv2
from keras.models import Model
from keras.layers import Input
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Concatenate
from keras.utils import plot_model
import tensorflow as tf
from keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, Reshape, Dense, Input, Flatten, Dropout
from keras.layers import Activation, Concatenate, Conv2D, Multiply
from keras.applications.inception_v3 import InceptionV3
import keras
from keras.applications.inception_v3 import preprocess_input
import tensorflow as tf
from keras import layers
from sklearn.model_selection import train_test_split
import pandas as pd

import numpy as np

class RoiExtractor:
    def __init__(self):
        # Model initialization
        self.model = build_image_model()
        self.model.load_weights("https://drive.google.com/file/d/1JqIQHYvW8q5DMjOpcVsfkwxaOZP2CmnL/view?usp=sharing")
        self.heatmap_model = Model(inputs=self.model.inputs, outputs=self.model.layers[-3].output)

        # Outputs
        self.heatmap_1 = None
        self.carpal_img = None
        
        self.heatmap_2 = None
        self.metacarpal_img = None
        
        self.img = None
        self.masked_img = None

    def generate_heatmap(self, img):
        
        # Preprocessing
        i = preprocess_input(img)
        preprocessed_img = np.expand_dims(i, axis=0)

        cbam_output = self.heatmap_model.predict(preprocessed_img)[0]

        #Heatmap generation
        heatmap = np.sum(cbam_output, axis=-1)
        heatmap = (heatmap - np.min(heatmap)) / (np.max(heatmap) - np.min(heatmap))
        upsampled_heatmap = cv2.resize(heatmap, (299, 299))
        return upsampled_heatmap


    def get_bounding_box(self, heatmap, threshold=0.7):
        binary_mask = (heatmap > threshold).astype(np.uint8)
        non_zero_indices = np.nonzero(binary_mask)
        if len(non_zero_indices[0]) == 0 or len(non_zero_indices[1]) == 0:
            return None
        min_row, min_col = np.min(non_zero_indices[0]), np.min(non_zero_indices[1])
        max_row, max_col = np.max(non_zero_indices[0]), np.max(non_zero_indices[1])

        bounding_box ={
            'min_row': min_row,
            'min_col': min_col,
            'max_row': max_row,
            'max_col': max_col
        }
        return bounding_box


    def crop_image_by_bounding_box(self, image, bounding_box):
        min_row, min_col = bounding_box['min_row'], bounding_box['min_col']
        max_row, max_col = bounding_box['max_row'], bounding_box['max_col']
        cropped_image = image[min_row:max_row + 1, min_col:max_col + 1, :]
        cropped_resized = cv2.resize(cropped_image, (224, 224))
        return cropped_resized


    def apply_black_rectangle_by_bounding_box(self, image, bounding_box):
        min_row, min_col = bounding_box['min_row'], bounding_box['min_col']
        max_row, max_col = 299, bounding_box['max_col']
        test_img = image.copy()
        test_img = cv2.rectangle(test_img, (min_col, min_row), (max_col, max_row), (0, 0, 0), thickness=cv2.FILLED)
        return test_img

    
    def process_img(self, img): 
        self.img = img
        self.heatmap_1 = self.generate_heatmap(img)
        bounding_box_1 = self.get_bounding_box(self.heatmap_1)
        self.carpal_img = self.crop_image_by_bounding_box(img, bounding_box_1)
        self.masked_img = self.apply_black_rectangle_by_bounding_box(img, bounding_box_1)
        self.heatmap_2 = self.generate_heatmap(self.masked_img)
        bounding_box_2 = self.get_bounding_box(self.heatmap_2)
        self.metacarpal_img = self.crop_image_by_bounding_box(img, bounding_box_2)
    

# Channel Attention Module
def channel_attention_module(x, ratio=8):

    batch,_,_,channel=x.shape
    # shared layers
    l1 = Dense(channel//ratio, activation="relu", use_bias=False)
    l2 = Dense(channel, use_bias= False)

    x1 = GlobalAveragePooling2D()(x)
    x1 = Flatten()(x1)
    x1 = l1(x1)
    x1 = l2(x1)

    x2 = GlobalMaxPooling2D()(x)
    x2 = Flatten()(x2)
    x2 = l1(x2)
    x2 = l2(x2)

    feats = x1 + x2
    feats = Activation("sigmoid")(feats)
    feats = Multiply()([x,feats])

    return feats

# spatical attention module

def spatial_attention_module(x):
    # Average Pooling
    x1 = tf.reduce_mean(x,axis = -1)
    x1 = tf.expand_dims(x1,axis = -1)

    # max pooling
    x2 = tf.reduce_max(x, axis = -1)
    x2 = tf.expand_dims(x2,axis=-1)

    feats = Concatenate()([x1,x2])

    feats = Conv2D(1,kernel_size=7, padding="same",activation="softmax")(feats)
    feats = Multiply()([x,feats])

    return feats

# Cbam module
def cbam(x):
    x = channel_attention_module(x)
    x = spatial_attention_module(x)

    return x


def build_image_model():
    base_model = InceptionV3(
        weights='imagenet',
        include_top=False,
        input_shape=(299,299,3)
    )
    for layer in base_model.layers:
        layer.trainable = False
    x = cbam(base_model.output)

    x= GlobalAveragePooling2D()(x)
    predictions = Dense(1, activation='sigmoid')(x)

    model = Model(inputs=base_model.input, outputs=predictions)

    return model