gradcam.py

from transformers import ViTFeatureExtractor, ViTForImageClassification
import warnings
from torchvision import transforms
from datasets import load_dataset
from pytorch_grad_cam import run_dff_on_image, GradCAM
from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
from pytorch_grad_cam.utils.image import show_cam_on_image
from PIL import Image
import numpy as np
import cv2 as cv
import torch
from typing import List, Callable, Optional
import logging
from face_grab import FaceGrabber

# original borrowed from https://github.com/jacobgil/pytorch-grad-cam/blob/master/tutorials/HuggingFace.ipynb
# thanks @jacobgil
# further mods beyond this commit by @simonSlamka

warnings.filterwarnings("ignore")

logging.basicConfig(level=logging.INFO)



class HuggingfaceToTensorModelWrapper(torch.nn.Module):
    def __init__(self, model):
        super(HuggingfaceToTensorModelWrapper, self).__init__()
        self.model = model

    def forward(self, x):
        return self.model(x).logits



class GradCam():
    def __init__(self):
        pass
    
    def category_name_to_index(self, model, category_name):
        name_to_index = dict((v, k) for k, v in model.config.id2label.items())
        return name_to_index[category_name]
        
    def run_grad_cam_on_image(self, model: torch.nn.Module,
                            target_layer: torch.nn.Module,
                            targets_for_gradcam: List[Callable],
                            reshape_transform: Optional[Callable],
                            input_tensor: torch.nn.Module,
                            input_image: Image,
                            method: Callable=GradCAM,
                            threshold: float=0.5):
        with method(model=HuggingfaceToTensorModelWrapper(model),
                    target_layers=[target_layer],
                    reshape_transform=reshape_transform) as cam:

            # Replicate the tensor for each of the categories we want to create Grad-CAM for:
            repeated_tensor = input_tensor[None, :].repeat(len(targets_for_gradcam), 1, 1, 1)

            batch_results = cam(input_tensor=repeated_tensor,
                                targets=targets_for_gradcam)
            results = []
            for grayscale_cam in batch_results:
                grayscale_cam[grayscale_cam < threshold] = 0
                visualization = show_cam_on_image(np.float32(input_image)/255,
                                                grayscale_cam,
                                                use_rgb=True)
                # Make it weight less in the notebook:
                visualization = cv.resize(visualization,
                                        (visualization.shape[1]//2, visualization.shape[0]//2))
                results.append(visualization)
            return np.hstack(results)
        
        
    def get_top_category(self, model, img_tensor, top_k=5):
        logits = model(img_tensor.unsqueeze(0)).logits
        probabilities = torch.nn.functional.softmax(logits, dim=1)
        topIdx = logits.cpu()[0, :].detach().numpy().argsort()[-1]
        topClass = model.config.id2label[topIdx]
        topScore = probabilities[0][topIdx].item()
        return [{"label": topClass, "score": topScore}]

    def reshape_transform_vit_huggingface(self, x):
        activations = x[:, 1:, :]
        activations = activations.view(activations.shape[0],
                                    14, 14, activations.shape[2])
        activations = activations.transpose(2, 3).transpose(1, 2)
        return activations



if __name__ == "__main__":

    faceGrabber = FaceGrabber()
    gradCam = GradCam()

    image = Image.open("Feature-Image-74.jpg").convert("RGB")
    face = faceGrabber.grab_faces(np.array(image))
    if face is not None:
        image = Image.fromarray(face)

    img_tensor = transforms.ToTensor()(image)

    model = ViTForImageClassification.from_pretrained("ongkn/attraction-classifier")
    targets_for_gradcam = [ClassifierOutputTarget(gradCam.category_name_to_index(model, "pos")),
                        ClassifierOutputTarget(gradCam.category_name_to_index(model, "neg"))]
    target_layer_dff = model.vit.layernorm
    target_layer_gradcam = model.vit.encoder.layer[-2].output
    image_resized = image.resize((224, 224))
    tensor_resized = transforms.ToTensor()(image_resized)

    dff_image = run_dff_on_image(model=model,
                                target_layer=target_layer_dff,
                                classifier=model.classifier,
                                img_pil=image_resized,
                                img_tensor=tensor_resized,
                                reshape_transform=gradCam.reshape_transform_vit_huggingface,
                                n_components=5,
                                top_k=10,
                                threshold=0,
                                output_size=None) #(500, 500))
    cv.namedWindow("DFF Image", cv.WINDOW_KEEPRATIO)
    cv.imshow("DFF Image", cv.cvtColor(dff_image, cv.COLOR_BGR2RGB))
    cv.resizeWindow("DFF Image", 2500, 700)
    # cv.waitKey(0)
    # cv.destroyAllWindows()
    grad_cam_image = gradCam.run_grad_cam_on_image(model=model,
                                        target_layer=target_layer_gradcam,
                                        targets_for_gradcam=targets_for_gradcam,
                                        input_tensor=tensor_resized,
                                        input_image=image_resized,
                                        reshape_transform=gradCam.reshape_transform_vit_huggingface,
                                        threshold=0)
    cv.namedWindow("Grad-CAM Image", cv.WINDOW_KEEPRATIO)
    cv.imshow("Grad-CAM Image", grad_cam_image)
    cv.resizeWindow("Grad-CAM Image", 2000, 1250)
    cv.waitKey(0)
    cv.destroyAllWindows()
    gradCam.print_top_categories(model, tensor_resized)