Heatmap generate

Mahmoud_Salem: <p>i am trying to generate a heatmap for image depending on weights from last conv layer in pretrained densenet121, but when i try to multiply the weights by the output of model in gives me this error:<br>
“ValueError: operands could not be broadcast together with shapes (1024,) (7,7)”</p>
<p>this is the code :</p>
<pre><code class="lang-auto">import os

import torch
import urllib
from PIL import Image
from torchvision import transforms

import numpy as np
import cv2

# load the model
model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet121', pretrained=True)
# or any of these variants
# model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet169', pretrained=True)
# model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet201', pretrained=True)
# model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet161', pretrained=True)
model.eval()

filename = 'dog.jpg'

# get input image and apply preprocessing
input_image = Image.open(filename)
preprocess = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
input_tensor = preprocess(input_image)
input_batch = input_tensor.unsqueeze(0)

# output = model(input_batch)
# print(output.shape)

# The output has unnormalized scores. To get probabilities, you can run a softmax on it.
# probabilities = torch.nn.functional.softmax(output[0], dim=0)

# print(probabilities)

named_layers = dict(model.named_modules())


# for layer in named_layers:
#     print(layer)
# print(named_layers['features.denseblock4.denselayer16'])


class densenet_last_layer(torch.nn.Module):
    def __init__(self, model):
        super(densenet_last_layer, self).__init__()
        self.features = torch.nn.Sequential(
            *list(model.children())[:-1]
        )

    def forward(self, x):
        x = self.features(x)
        x = torch.nn.functional.relu(x, inplace=True)
        return x


conv_model = densenet_last_layer(model)
# print(conv_model)
input_batch = torch.autograd.Variable(input_batch)
conv_output = conv_model(input_batch)

# print(conv_output.shape)

conv_output = conv_output.cpu().data.numpy()
# conv_output = np.squeeze(conv_output)

# for state in model.state_dict():
#     print(state)

weights = model.state_dict()['classifier.weight']
weights = weights.cpu().numpy()

bias = model.state_dict()['classifier.bias']
bias = bias.cpu().numpy()

# print(conv_output.shape)
heatmap = None

for i in range(0, len(weights)):
    map = conv_output[0, i, :, :]
    if i == 0:
        heatmap = weights[i] * map
    else:
        heatmap += weights[i] * map

# ---- Blend original and heatmap
npHeatmap = heatmap.cpu().data.numpy()

transCrop = 224

imgOriginal = cv2.imread(filename, 1)
imgOriginal = cv2.resize(imgOriginal, (transCrop, transCrop))

cam = npHeatmap / np.max(npHeatmap)
cam = cv2.resize(cam, (transCrop, transCrop))
heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)

img = heatmap * 0.5 + imgOriginal

cv2.imwrite(os.getcwd(), img)

print(model.state_dict()['classifier.weight'])

</code></pre>

ptrblck: <p>Based on the error message the <code>heatmap</code> and <code>imgOriginal</code> are two numpy arrays in different shapes, which cannot be accumulated:</p>
<pre><code class="lang-python">heatmap = np.random.randn(1024,)
imgOriginal = np.random.randn(7, 7)
img = heatmap * 0.5 + imgOriginal
&gt; ValueError: operands could not be broadcast together with shapes (1024,) (7,7) 
</code></pre>
<p>so you would have to make sure the <code>heatmap</code> has the same shape as <code>imgOriginal</code> (or can be broadcasted).</p>
<p>I’m also unsure, what exactly is happening in these lines of code:</p>
<pre><code class="lang-python">heatmap = weights[i] * map
</code></pre>
<p>as it seems you are indexing the <code>weight</code> matrix of the last linear layer and are multiplying it with a channel of an output activation of a conv layer.</p>

Mahmoud_Salem: <p>i am trying to generate heatmap but always have corrupted images…i am not sure about the logic of generating CAM but i will explain the steps i do so that if you can please help me…<br>
1-) the first thing i load my trained model then remove the last classification layer from it to get the last conv layer.<br>
2-) then i convert the image to tensor so i can feed it to the last convolutional layer and get the output feature map.<br>
3-) i loop through the weights of classification layer of my trained model and then multiply the classification weights by output feature map.</p>
<p>that was the logic i do…i am not pretty sure it is right because i am new to deep learning and don’t have a lot of experience with it.<br>
i would be very thankful for you help.</p>
<p>here is the code :</p>
<pre><code class="lang-auto">import os
import numpy as np
import time
import sys
from PIL import Image

import heatmap_test as ht

import cv2

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision
import torchvision.transforms as transforms

from densenet import densenet121
from densenet import densenet169
from densenet import densenet201
from densenet import DenseNet

from torchvision.transforms import ToTensor



class densenet_last_layer(torch.nn.Module):
    def __init__(self, model):
        super(densenet_last_layer, self).__init__()
        # remove last classification layer
        self.features = torch.nn.Sequential(
            *list(model.children())[:-1]
        )

    def forward(self, x):
        x = self.features(x)
        x = torch.nn.functional.relu(x, inplace=True)
        return x


class DrawHeatmap():
    def __init__(self, pathModel, img, pathOutputFile):
        model = densenet121(False)

        modelCheckpoint = torch.load(pathModel, map_location=torch.device('cpu'))
        model.load_state_dict(modelCheckpoint['state_dict'], strict=False)

        named_layers = dict(model.named_modules())

        model.eval()
        weights = None
        bias = None
        for name, params in model.named_parameters():
            print(name)
            if name == 'classifier.weight':
                weights = params
            if name == 'classifier.bias':
                bias = params

        conv_model = densenet_last_layer(model)

        # ---- Initialize the image transform - resize + normalize
        normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        transformList = []
        # transformList.append(transforms.Resize(transCrop))
        transformList.append(transforms.ToTensor())
        # transformList.append(normalize)

        transformSequence = transforms.Compose(transformList)

        imageData = Image.open(img).convert('RGB')
        imageData = transformSequence(imageData)
        imageData = imageData.unsqueeze(0)

        input_img = torch.autograd.Variable(imageData)
        output = conv_model(input_img)
        # print(output.shape)
        #
        # output = output.cpu().data.numpy()
        # output = np.squeeze(output)
        print(output.shape)

        heatmap = None
        for i in range(0, len(weights)):
            map = output[0, i, :, :]
            if i == 0:
                heatmap = weights[i] * map
            else:
                heatmap += weights[i] * map


        npHeatmap = heatmap.cpu().data.numpy()

        imgOriginal = cv2.imread(img, 1)
        imgOriginal = cv2.resize(imgOriginal, (224, 224))

        cam = npHeatmap / np.max(npHeatmap)
        cam = cv2.resize(cam, (224, 224))
        heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)

        img = heatmap * 0.5 + imgOriginal

        cv2.imwrite(pathOutputFile, img)




pathInputImage = 'C:/Users/mahmo/Desktop/best projects/Stanford Code/cxr-code/00000001_000.png'
pathModel = 'C:/Users/mahmo/Desktop/best projects/Stanford Code/cxr-code/m-25012018-123527.pth.tar'
pathOutputImage = 'C:/Users/mahmo/Desktop/best projects/Stanford Code/cxr-code/heatmap_test.png'
h = DrawHeatmap(pathModel, pathInputImage, pathOutputImage)



# ht.draw_CAM(pathModel, pathInputImage, True)
</code></pre>