Tobias Czempiel
commited on
Commit
•
0325572
1
Parent(s):
7ea95d6
initial commit
Browse files- .DS_Store +0 -0
- README.md +12 -0
- checkpoints/.DS_Store +0 -0
- checkpoints/ham10k_checkpoint_mobile_0.82_epoch24.pt +3 -0
- pytorch_grad_cam/.DS_Store +0 -0
- pytorch_grad_cam/__init__.py +14 -0
- pytorch_grad_cam/__pycache__/__init__.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/ablation_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/ablation_layer.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/activations_and_gradients.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/base_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/eigen_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/eigen_grad_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/fullgrad_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/grad_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/grad_cam_plusplus.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/guided_backprop.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/layer_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/score_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/__pycache__/xgrad_cam.cpython-39.pyc +0 -0
- pytorch_grad_cam/ablation_cam.py +134 -0
- pytorch_grad_cam/ablation_cam_multilayer.py +136 -0
- pytorch_grad_cam/ablation_layer.py +124 -0
- pytorch_grad_cam/activations_and_gradients.py +46 -0
- pytorch_grad_cam/base_cam.py +199 -0
- pytorch_grad_cam/eigen_cam.py +20 -0
- pytorch_grad_cam/eigen_grad_cam.py +21 -0
- pytorch_grad_cam/fullgrad_cam.py +95 -0
- pytorch_grad_cam/grad_cam.py +22 -0
- pytorch_grad_cam/grad_cam_plusplus.py +32 -0
- pytorch_grad_cam/guided_backprop.py +100 -0
- pytorch_grad_cam/layer_cam.py +36 -0
- pytorch_grad_cam/score_cam.py +63 -0
- pytorch_grad_cam/utils/__init__.py +4 -0
- pytorch_grad_cam/utils/__pycache__/__init__.cpython-39.pyc +0 -0
- pytorch_grad_cam/utils/__pycache__/find_layers.cpython-39.pyc +0 -0
- pytorch_grad_cam/utils/__pycache__/image.cpython-39.pyc +0 -0
- pytorch_grad_cam/utils/__pycache__/model_targets.cpython-39.pyc +0 -0
- pytorch_grad_cam/utils/__pycache__/reshape_transforms.cpython-39.pyc +0 -0
- pytorch_grad_cam/utils/__pycache__/svd_on_activations.cpython-39.pyc +0 -0
- pytorch_grad_cam/utils/find_layers.py +30 -0
- pytorch_grad_cam/utils/image.py +73 -0
- pytorch_grad_cam/utils/model_targets.py +61 -0
- pytorch_grad_cam/utils/reshape_transforms.py +27 -0
- pytorch_grad_cam/utils/svd_on_activations.py +19 -0
- pytorch_grad_cam/xgrad_cam.py +31 -0
- runSDSdemo.py +96 -0
.DS_Store
ADDED
Binary file (6.15 kB). View file
|
|
README.md
ADDED
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: SDSdemo
|
3 |
+
emoji: 👩⚕️
|
4 |
+
colorFrom: pink
|
5 |
+
colorTo: indigo
|
6 |
+
sdk: gradio
|
7 |
+
app_file: runSDSdemo.py
|
8 |
+
pinned: false
|
9 |
+
license: afl-3.0
|
10 |
+
---
|
11 |
+
|
12 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference
|
checkpoints/.DS_Store
ADDED
Binary file (6.15 kB). View file
|
|
checkpoints/ham10k_checkpoint_mobile_0.82_epoch24.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7c671f87198d4c5bb3a442ca9bf810dad45f755f6f7045b0b7b5186df9f767db
|
3 |
+
size 50851683
|
pytorch_grad_cam/.DS_Store
ADDED
Binary file (6.15 kB). View file
|
|
pytorch_grad_cam/__init__.py
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from pytorch_grad_cam.grad_cam import GradCAM
|
2 |
+
from pytorch_grad_cam.ablation_layer import AblationLayer, AblationLayerVit, AblationLayerFasterRCNN
|
3 |
+
from pytorch_grad_cam.ablation_cam import AblationCAM
|
4 |
+
from pytorch_grad_cam.xgrad_cam import XGradCAM
|
5 |
+
from pytorch_grad_cam.grad_cam_plusplus import GradCAMPlusPlus
|
6 |
+
from pytorch_grad_cam.score_cam import ScoreCAM
|
7 |
+
from pytorch_grad_cam.layer_cam import LayerCAM
|
8 |
+
from pytorch_grad_cam.eigen_cam import EigenCAM
|
9 |
+
from pytorch_grad_cam.eigen_grad_cam import EigenGradCAM
|
10 |
+
from pytorch_grad_cam.fullgrad_cam import FullGrad
|
11 |
+
from pytorch_grad_cam.guided_backprop import GuidedBackpropReLUModel
|
12 |
+
from pytorch_grad_cam.activations_and_gradients import ActivationsAndGradients
|
13 |
+
import pytorch_grad_cam.utils.model_targets
|
14 |
+
import pytorch_grad_cam.utils.reshape_transforms
|
pytorch_grad_cam/__pycache__/__init__.cpython-39.pyc
ADDED
Binary file (1.15 kB). View file
|
|
pytorch_grad_cam/__pycache__/ablation_cam.cpython-39.pyc
ADDED
Binary file (3.67 kB). View file
|
|
pytorch_grad_cam/__pycache__/ablation_layer.cpython-39.pyc
ADDED
Binary file (4.9 kB). View file
|
|
pytorch_grad_cam/__pycache__/activations_and_gradients.cpython-39.pyc
ADDED
Binary file (1.91 kB). View file
|
|
pytorch_grad_cam/__pycache__/base_cam.cpython-39.pyc
ADDED
Binary file (5.81 kB). View file
|
|
pytorch_grad_cam/__pycache__/eigen_cam.cpython-39.pyc
ADDED
Binary file (931 Bytes). View file
|
|
pytorch_grad_cam/__pycache__/eigen_grad_cam.cpython-39.pyc
ADDED
Binary file (952 Bytes). View file
|
|
pytorch_grad_cam/__pycache__/fullgrad_cam.cpython-39.pyc
ADDED
Binary file (3.21 kB). View file
|
|
pytorch_grad_cam/__pycache__/grad_cam.cpython-39.pyc
ADDED
Binary file (899 Bytes). View file
|
|
pytorch_grad_cam/__pycache__/grad_cam_plusplus.cpython-39.pyc
ADDED
Binary file (1.15 kB). View file
|
|
pytorch_grad_cam/__pycache__/guided_backprop.cpython-39.pyc
ADDED
Binary file (3.44 kB). View file
|
|
pytorch_grad_cam/__pycache__/layer_cam.cpython-39.pyc
ADDED
Binary file (1.08 kB). View file
|
|
pytorch_grad_cam/__pycache__/score_cam.cpython-39.pyc
ADDED
Binary file (2.1 kB). View file
|
|
pytorch_grad_cam/__pycache__/xgrad_cam.cpython-39.pyc
ADDED
Binary file (1.01 kB). View file
|
|
pytorch_grad_cam/ablation_cam.py
ADDED
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import torch
|
3 |
+
import tqdm
|
4 |
+
from typing import Callable, List
|
5 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
6 |
+
from pytorch_grad_cam.utils.find_layers import replace_layer_recursive
|
7 |
+
from pytorch_grad_cam.ablation_layer import AblationLayer
|
8 |
+
|
9 |
+
|
10 |
+
""" Implementation of AblationCAM
|
11 |
+
https://openaccess.thecvf.com/content_WACV_2020/papers/Desai_Ablation-CAM_Visual_Explanations_for_Deep_Convolutional_Network_via_Gradient-free_Localization_WACV_2020_paper.pdf
|
12 |
+
|
13 |
+
Ablate individual activations, and then measure the drop in the target score.
|
14 |
+
|
15 |
+
In the current implementation, the target layer activations is cached, so it won't be re-computed.
|
16 |
+
However layers before it, if any, will not be cached.
|
17 |
+
This means that if the target layer is a large block, for example model.featuers (in vgg), there will
|
18 |
+
be a large save in run time.
|
19 |
+
|
20 |
+
Since we have to go over many channels and ablate them, and every channel ablation requires a forward pass,
|
21 |
+
it would be nice if we could avoid doing that for channels that won't contribute anwyay, making it much faster.
|
22 |
+
The parameter ratio_channels_to_ablate controls how many channels should be ablated, using an experimental method
|
23 |
+
(to be improved). The default 1.0 value means that all channels will be ablated.
|
24 |
+
"""
|
25 |
+
|
26 |
+
|
27 |
+
class AblationCAM(BaseCAM):
|
28 |
+
def __init__(self,
|
29 |
+
model: torch.nn.Module,
|
30 |
+
target_layers: List[torch.nn.Module],
|
31 |
+
use_cuda: bool = False,
|
32 |
+
reshape_transform: Callable = None,
|
33 |
+
ablation_layer: torch.nn.Module = AblationLayer(),
|
34 |
+
batch_size: int = 32,
|
35 |
+
ratio_channels_to_ablate: float = 1.0) -> None:
|
36 |
+
|
37 |
+
super(AblationCAM, self).__init__(model,
|
38 |
+
target_layers,
|
39 |
+
use_cuda,
|
40 |
+
reshape_transform,
|
41 |
+
uses_gradients=False)
|
42 |
+
self.batch_size = batch_size
|
43 |
+
self.ablation_layer = ablation_layer
|
44 |
+
self.ratio_channels_to_ablate = ratio_channels_to_ablate
|
45 |
+
|
46 |
+
def save_activation(self, module, input, output) -> None:
|
47 |
+
""" Helper function to save the raw activations from the target layer """
|
48 |
+
self.activations = output
|
49 |
+
|
50 |
+
def assemble_ablation_scores(self,
|
51 |
+
new_scores: list,
|
52 |
+
original_score: float ,
|
53 |
+
ablated_channels: np.ndarray,
|
54 |
+
number_of_channels: int) -> np.ndarray:
|
55 |
+
""" Take the value from the channels that were ablated,
|
56 |
+
and just set the original score for the channels that were skipped """
|
57 |
+
|
58 |
+
index = 0
|
59 |
+
result = []
|
60 |
+
sorted_indices = np.argsort(ablated_channels)
|
61 |
+
ablated_channels = ablated_channels[sorted_indices]
|
62 |
+
new_scores = np.float32(new_scores)[sorted_indices]
|
63 |
+
|
64 |
+
for i in range(number_of_channels):
|
65 |
+
if index < len(ablated_channels) and ablated_channels[index] == i:
|
66 |
+
weight = new_scores[index]
|
67 |
+
index = index + 1
|
68 |
+
else:
|
69 |
+
weight = original_score
|
70 |
+
result.append(weight)
|
71 |
+
|
72 |
+
return result
|
73 |
+
|
74 |
+
def get_cam_weights(self,
|
75 |
+
input_tensor: torch.Tensor,
|
76 |
+
target_layer: torch.nn.Module,
|
77 |
+
targets: List[Callable],
|
78 |
+
activations: torch.Tensor,
|
79 |
+
grads: torch.Tensor) -> np.ndarray:
|
80 |
+
|
81 |
+
# Do a forward pass, compute the target scores, and cache the activations
|
82 |
+
handle = target_layer.register_forward_hook(self.save_activation)
|
83 |
+
with torch.no_grad():
|
84 |
+
outputs = self.model(input_tensor)
|
85 |
+
handle.remove()
|
86 |
+
original_scores = np.float32([target(output).cpu().item() for target, output in zip(targets, outputs)])
|
87 |
+
|
88 |
+
# Replace the layer with the ablation layer.
|
89 |
+
# When we finish, we will replace it back, so the original model is unchanged.
|
90 |
+
ablation_layer = self.ablation_layer
|
91 |
+
replace_layer_recursive(self.model, target_layer, ablation_layer)
|
92 |
+
|
93 |
+
number_of_channels = activations.shape[1]
|
94 |
+
weights = []
|
95 |
+
# This is a "gradient free" method, so we don't need gradients here.
|
96 |
+
with torch.no_grad():
|
97 |
+
# Loop over each of the batch images and ablate activations for it.
|
98 |
+
for batch_index, (target, tensor) in enumerate(zip(targets, input_tensor)):
|
99 |
+
new_scores = []
|
100 |
+
batch_tensor = tensor.repeat(self.batch_size, 1, 1, 1)
|
101 |
+
|
102 |
+
# Check which channels should be ablated. Normally this will be all channels,
|
103 |
+
# But we can also try to speed this up by using a low ratio_channels_to_ablate.
|
104 |
+
channels_to_ablate = ablation_layer.activations_to_be_ablated(activations[batch_index, :],
|
105 |
+
self.ratio_channels_to_ablate)
|
106 |
+
number_channels_to_ablate = len(channels_to_ablate)
|
107 |
+
|
108 |
+
for i in tqdm.tqdm(range(0, number_channels_to_ablate, self.batch_size)):
|
109 |
+
if i + self.batch_size > number_channels_to_ablate:
|
110 |
+
batch_tensor = batch_tensor[:(number_channels_to_ablate - i)]
|
111 |
+
|
112 |
+
# Change the state of the ablation layer so it ablates the next channels.
|
113 |
+
# TBD: Move this into the ablation layer forward pass.
|
114 |
+
ablation_layer.set_next_batch(input_batch_index=batch_index,
|
115 |
+
activations=self.activations,
|
116 |
+
num_channels_to_ablate=batch_tensor.size(0))
|
117 |
+
score = [target(o).cpu().item() for o in self.model(batch_tensor)]
|
118 |
+
new_scores.extend(score)
|
119 |
+
ablation_layer.indices = ablation_layer.indices[batch_tensor.size(0):]
|
120 |
+
|
121 |
+
new_scores = self.assemble_ablation_scores(new_scores,
|
122 |
+
original_scores[batch_index],
|
123 |
+
channels_to_ablate,
|
124 |
+
number_of_channels)
|
125 |
+
weights.extend(new_scores)
|
126 |
+
|
127 |
+
weights = np.float32(weights)
|
128 |
+
weights = weights.reshape(activations.shape[:2])
|
129 |
+
original_scores = original_scores[:, None]
|
130 |
+
weights = (original_scores - weights) / original_scores
|
131 |
+
|
132 |
+
# Replace the model back to the original state
|
133 |
+
replace_layer_recursive(self.model, ablation_layer, target_layer)
|
134 |
+
return weights
|
pytorch_grad_cam/ablation_cam_multilayer.py
ADDED
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import cv2
|
2 |
+
import numpy as np
|
3 |
+
import torch
|
4 |
+
import tqdm
|
5 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
6 |
+
|
7 |
+
|
8 |
+
class AblationLayer(torch.nn.Module):
|
9 |
+
def __init__(self, layer, reshape_transform, indices):
|
10 |
+
super(AblationLayer, self).__init__()
|
11 |
+
|
12 |
+
self.layer = layer
|
13 |
+
self.reshape_transform = reshape_transform
|
14 |
+
# The channels to zero out:
|
15 |
+
self.indices = indices
|
16 |
+
|
17 |
+
def forward(self, x):
|
18 |
+
self.__call__(x)
|
19 |
+
|
20 |
+
def __call__(self, x):
|
21 |
+
output = self.layer(x)
|
22 |
+
|
23 |
+
# Hack to work with ViT,
|
24 |
+
# Since the activation channels are last and not first like in CNNs
|
25 |
+
# Probably should remove it?
|
26 |
+
if self.reshape_transform is not None:
|
27 |
+
output = output.transpose(1, 2)
|
28 |
+
|
29 |
+
for i in range(output.size(0)):
|
30 |
+
|
31 |
+
# Commonly the minimum activation will be 0,
|
32 |
+
# And then it makes sense to zero it out.
|
33 |
+
# However depending on the architecture,
|
34 |
+
# If the values can be negative, we use very negative values
|
35 |
+
# to perform the ablation, deviating from the paper.
|
36 |
+
if torch.min(output) == 0:
|
37 |
+
output[i, self.indices[i], :] = 0
|
38 |
+
else:
|
39 |
+
ABLATION_VALUE = 1e5
|
40 |
+
output[i, self.indices[i], :] = torch.min(
|
41 |
+
output) - ABLATION_VALUE
|
42 |
+
|
43 |
+
if self.reshape_transform is not None:
|
44 |
+
output = output.transpose(2, 1)
|
45 |
+
|
46 |
+
return output
|
47 |
+
|
48 |
+
|
49 |
+
def replace_layer_recursive(model, old_layer, new_layer):
|
50 |
+
for name, layer in model._modules.items():
|
51 |
+
if layer == old_layer:
|
52 |
+
model._modules[name] = new_layer
|
53 |
+
return True
|
54 |
+
elif replace_layer_recursive(layer, old_layer, new_layer):
|
55 |
+
return True
|
56 |
+
return False
|
57 |
+
|
58 |
+
|
59 |
+
class AblationCAM(BaseCAM):
|
60 |
+
def __init__(self, model, target_layers, use_cuda=False,
|
61 |
+
reshape_transform=None):
|
62 |
+
super(AblationCAM, self).__init__(model, target_layers, use_cuda,
|
63 |
+
reshape_transform)
|
64 |
+
|
65 |
+
if len(target_layers) > 1:
|
66 |
+
print(
|
67 |
+
"Warning. You are usign Ablation CAM with more than 1 layers. "
|
68 |
+
"This is supported only if all layers have the same output shape")
|
69 |
+
|
70 |
+
def set_ablation_layers(self):
|
71 |
+
self.ablation_layers = []
|
72 |
+
for target_layer in self.target_layers:
|
73 |
+
ablation_layer = AblationLayer(target_layer,
|
74 |
+
self.reshape_transform, indices=[])
|
75 |
+
self.ablation_layers.append(ablation_layer)
|
76 |
+
replace_layer_recursive(self.model, target_layer, ablation_layer)
|
77 |
+
|
78 |
+
def unset_ablation_layers(self):
|
79 |
+
# replace the model back to the original state
|
80 |
+
for ablation_layer, target_layer in zip(
|
81 |
+
self.ablation_layers, self.target_layers):
|
82 |
+
replace_layer_recursive(self.model, ablation_layer, target_layer)
|
83 |
+
|
84 |
+
def set_ablation_layer_batch_indices(self, indices):
|
85 |
+
for ablation_layer in self.ablation_layers:
|
86 |
+
ablation_layer.indices = indices
|
87 |
+
|
88 |
+
def trim_ablation_layer_batch_indices(self, keep):
|
89 |
+
for ablation_layer in self.ablation_layers:
|
90 |
+
ablation_layer.indices = ablation_layer.indices[:keep]
|
91 |
+
|
92 |
+
def get_cam_weights(self,
|
93 |
+
input_tensor,
|
94 |
+
target_category,
|
95 |
+
activations,
|
96 |
+
grads):
|
97 |
+
with torch.no_grad():
|
98 |
+
outputs = self.model(input_tensor).cpu().numpy()
|
99 |
+
original_scores = []
|
100 |
+
for i in range(input_tensor.size(0)):
|
101 |
+
original_scores.append(outputs[i, target_category[i]])
|
102 |
+
original_scores = np.float32(original_scores)
|
103 |
+
|
104 |
+
self.set_ablation_layers()
|
105 |
+
|
106 |
+
if hasattr(self, "batch_size"):
|
107 |
+
BATCH_SIZE = self.batch_size
|
108 |
+
else:
|
109 |
+
BATCH_SIZE = 32
|
110 |
+
|
111 |
+
number_of_channels = activations.shape[1]
|
112 |
+
weights = []
|
113 |
+
|
114 |
+
with torch.no_grad():
|
115 |
+
# Iterate over the input batch
|
116 |
+
for tensor, category in zip(input_tensor, target_category):
|
117 |
+
batch_tensor = tensor.repeat(BATCH_SIZE, 1, 1, 1)
|
118 |
+
for i in tqdm.tqdm(range(0, number_of_channels, BATCH_SIZE)):
|
119 |
+
self.set_ablation_layer_batch_indices(
|
120 |
+
list(range(i, i + BATCH_SIZE)))
|
121 |
+
|
122 |
+
if i + BATCH_SIZE > number_of_channels:
|
123 |
+
keep = number_of_channels - i
|
124 |
+
batch_tensor = batch_tensor[:keep]
|
125 |
+
self.trim_ablation_layer_batch_indices(self, keep)
|
126 |
+
score = self.model(batch_tensor)[:, category].cpu().numpy()
|
127 |
+
weights.extend(score)
|
128 |
+
|
129 |
+
weights = np.float32(weights)
|
130 |
+
weights = weights.reshape(activations.shape[:2])
|
131 |
+
original_scores = original_scores[:, None]
|
132 |
+
weights = (original_scores - weights) / original_scores
|
133 |
+
|
134 |
+
# replace the model back to the original state
|
135 |
+
self.unset_ablation_layers()
|
136 |
+
return weights
|
pytorch_grad_cam/ablation_layer.py
ADDED
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
from collections import OrderedDict
|
3 |
+
import numpy as np
|
4 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
5 |
+
|
6 |
+
|
7 |
+
class AblationLayer(torch.nn.Module):
|
8 |
+
def __init__(self):
|
9 |
+
super(AblationLayer, self).__init__()
|
10 |
+
|
11 |
+
def objectiveness_mask_from_svd(self, activations, threshold=0.01):
|
12 |
+
""" Experimental method to get a binary mask to compare if the activation is worth ablating.
|
13 |
+
The idea is to apply the EigenCAM method by doing PCA on the activations.
|
14 |
+
Then we create a binary mask by comparing to a low threshold.
|
15 |
+
Areas that are masked out, are probably not interesting anyway.
|
16 |
+
"""
|
17 |
+
|
18 |
+
projection = get_2d_projection(activations[None, :])[0, :]
|
19 |
+
projection = np.abs(projection)
|
20 |
+
projection = projection - projection.min()
|
21 |
+
projection = projection / projection.max()
|
22 |
+
projection = projection > threshold
|
23 |
+
return projection
|
24 |
+
|
25 |
+
def activations_to_be_ablated(self, activations, ratio_channels_to_ablate=1.0):
|
26 |
+
""" Experimental method to get a binary mask to compare if the activation is worth ablating.
|
27 |
+
Create a binary CAM mask with objectiveness_mask_from_svd.
|
28 |
+
Score each Activation channel, by seeing how much of its values are inside the mask.
|
29 |
+
Then keep the top channels.
|
30 |
+
|
31 |
+
"""
|
32 |
+
if ratio_channels_to_ablate == 1.0:
|
33 |
+
self.indices = np.int32(range(activations.shape[0]))
|
34 |
+
return self.indices
|
35 |
+
|
36 |
+
projection = self.objectiveness_mask_from_svd(activations)
|
37 |
+
|
38 |
+
scores = []
|
39 |
+
for channel in activations:
|
40 |
+
normalized = np.abs(channel)
|
41 |
+
normalized = normalized - normalized.min()
|
42 |
+
normalized = normalized / np.max(normalized)
|
43 |
+
score = (projection*normalized).sum() / normalized.sum()
|
44 |
+
scores.append(score)
|
45 |
+
scores = np.float32(scores)
|
46 |
+
|
47 |
+
indices = list(np.argsort(scores))
|
48 |
+
high_score_indices = indices[::-1][: int(len(indices) * ratio_channels_to_ablate)]
|
49 |
+
low_score_indices = indices[: int(len(indices) * ratio_channels_to_ablate)]
|
50 |
+
self.indices = np.int32(high_score_indices + low_score_indices)
|
51 |
+
return self.indices
|
52 |
+
|
53 |
+
def set_next_batch(self, input_batch_index, activations, num_channels_to_ablate):
|
54 |
+
""" This creates the next batch of activations from the layer.
|
55 |
+
Just take corresponding batch member from activations, and repeat it num_channels_to_ablate times.
|
56 |
+
"""
|
57 |
+
self.activations = activations[input_batch_index, :, :, :].clone().unsqueeze(0).repeat(num_channels_to_ablate, 1, 1, 1)
|
58 |
+
|
59 |
+
def __call__(self, x):
|
60 |
+
output = self.activations
|
61 |
+
for i in range(output.size(0)):
|
62 |
+
# Commonly the minimum activation will be 0,
|
63 |
+
# And then it makes sense to zero it out.
|
64 |
+
# However depending on the architecture,
|
65 |
+
# If the values can be negative, we use very negative values
|
66 |
+
# to perform the ablation, deviating from the paper.
|
67 |
+
if torch.min(output) == 0:
|
68 |
+
output[i, self.indices[i], :] = 0
|
69 |
+
else:
|
70 |
+
ABLATION_VALUE = 1e7
|
71 |
+
output[i, self.indices[i], :] = torch.min(
|
72 |
+
output) - ABLATION_VALUE
|
73 |
+
|
74 |
+
return output
|
75 |
+
|
76 |
+
|
77 |
+
class AblationLayerVit(AblationLayer):
|
78 |
+
def __init__(self):
|
79 |
+
super(AblationLayerVit, self).__init__()
|
80 |
+
|
81 |
+
def __call__(self, x):
|
82 |
+
output = self.activations
|
83 |
+
output = output.transpose(1, 2)
|
84 |
+
for i in range(output.size(0)):
|
85 |
+
|
86 |
+
# Commonly the minimum activation will be 0,
|
87 |
+
# And then it makes sense to zero it out.
|
88 |
+
# However depending on the architecture,
|
89 |
+
# If the values can be negative, we use very negative values
|
90 |
+
# to perform the ablation, deviating from the paper.
|
91 |
+
if torch.min(output) == 0:
|
92 |
+
output[i, self.indices[i], :] = 0
|
93 |
+
else:
|
94 |
+
ABLATION_VALUE = 1e7
|
95 |
+
output[i, self.indices[i], :] = torch.min(
|
96 |
+
output) - ABLATION_VALUE
|
97 |
+
|
98 |
+
output = output.transpose(2, 1)
|
99 |
+
|
100 |
+
return output
|
101 |
+
|
102 |
+
|
103 |
+
class AblationLayerFasterRCNN(AblationLayer):
|
104 |
+
def __init__(self):
|
105 |
+
super(AblationLayerFasterRCNN, self).__init__()
|
106 |
+
|
107 |
+
def set_next_batch(self, input_batch_index, activations, num_channels_to_ablate):
|
108 |
+
""" Extract the next batch member from activations,
|
109 |
+
and repeat it num_channels_to_ablate times.
|
110 |
+
"""
|
111 |
+
self.activations = OrderedDict()
|
112 |
+
for key, value in activations.items():
|
113 |
+
fpn_activation = value[input_batch_index, :, :, :].clone().unsqueeze(0)
|
114 |
+
self.activations[key] = fpn_activation.repeat(num_channels_to_ablate, 1, 1, 1)
|
115 |
+
|
116 |
+
def __call__(self, x):
|
117 |
+
result = self.activations
|
118 |
+
layers = {0: '0', 1: '1', 2: '2', 3: '3', 4: 'pool'}
|
119 |
+
num_channels_to_ablate = result['pool'].size(0)
|
120 |
+
for i in range(num_channels_to_ablate):
|
121 |
+
pyramid_layer = int(self.indices[i]/256)
|
122 |
+
index_in_pyramid_layer = int(self.indices[i] % 256)
|
123 |
+
result[layers[pyramid_layer]][i, index_in_pyramid_layer, :, :] = -1000
|
124 |
+
return result
|
pytorch_grad_cam/activations_and_gradients.py
ADDED
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
class ActivationsAndGradients:
|
2 |
+
""" Class for extracting activations and
|
3 |
+
registering gradients from targetted intermediate layers """
|
4 |
+
|
5 |
+
def __init__(self, model, target_layers, reshape_transform):
|
6 |
+
self.model = model
|
7 |
+
self.gradients = []
|
8 |
+
self.activations = []
|
9 |
+
self.reshape_transform = reshape_transform
|
10 |
+
self.handles = []
|
11 |
+
for target_layer in target_layers:
|
12 |
+
self.handles.append(
|
13 |
+
target_layer.register_forward_hook(self.save_activation))
|
14 |
+
# Because of https://github.com/pytorch/pytorch/issues/61519,
|
15 |
+
# we don't use backward hook to record gradients.
|
16 |
+
self.handles.append(
|
17 |
+
target_layer.register_forward_hook(self.save_gradient))
|
18 |
+
|
19 |
+
def save_activation(self, module, input, output):
|
20 |
+
activation = output
|
21 |
+
|
22 |
+
if self.reshape_transform is not None:
|
23 |
+
activation = self.reshape_transform(activation)
|
24 |
+
self.activations.append(activation.cpu().detach())
|
25 |
+
|
26 |
+
def save_gradient(self, module, input, output):
|
27 |
+
if not hasattr(output, "requires_grad") or not output.requires_grad:
|
28 |
+
# You can only register hooks on tensor requires grad.
|
29 |
+
return
|
30 |
+
|
31 |
+
# Gradients are computed in reverse order
|
32 |
+
def _store_grad(grad):
|
33 |
+
if self.reshape_transform is not None:
|
34 |
+
grad = self.reshape_transform(grad)
|
35 |
+
self.gradients = [grad.cpu().detach()] + self.gradients
|
36 |
+
|
37 |
+
output.register_hook(_store_grad)
|
38 |
+
|
39 |
+
def __call__(self, x):
|
40 |
+
self.gradients = []
|
41 |
+
self.activations = []
|
42 |
+
return self.model(x)
|
43 |
+
|
44 |
+
def release(self):
|
45 |
+
for handle in self.handles:
|
46 |
+
handle.remove()
|
pytorch_grad_cam/base_cam.py
ADDED
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import torch
|
3 |
+
import ttach as tta
|
4 |
+
from typing import Callable, List, Tuple
|
5 |
+
from pytorch_grad_cam.activations_and_gradients import ActivationsAndGradients
|
6 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
7 |
+
from pytorch_grad_cam.utils.image import scale_cam_image
|
8 |
+
from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
|
9 |
+
|
10 |
+
|
11 |
+
class BaseCAM:
|
12 |
+
def __init__(self,
|
13 |
+
model: torch.nn.Module,
|
14 |
+
target_layers: List[torch.nn.Module],
|
15 |
+
use_cuda: bool = False,
|
16 |
+
reshape_transform: Callable = None,
|
17 |
+
compute_input_gradient: bool = False,
|
18 |
+
uses_gradients: bool = True) -> None:
|
19 |
+
self.model = model.eval()
|
20 |
+
self.target_layers = target_layers
|
21 |
+
self.cuda = use_cuda
|
22 |
+
if self.cuda:
|
23 |
+
self.model = model.cuda()
|
24 |
+
self.reshape_transform = reshape_transform
|
25 |
+
self.compute_input_gradient = compute_input_gradient
|
26 |
+
self.uses_gradients = uses_gradients
|
27 |
+
self.activations_and_grads = ActivationsAndGradients(
|
28 |
+
self.model, target_layers, reshape_transform)
|
29 |
+
|
30 |
+
""" Get a vector of weights for every channel in the target layer.
|
31 |
+
Methods that return weights channels,
|
32 |
+
will typically need to only implement this function. """
|
33 |
+
|
34 |
+
def get_cam_weights(self,
|
35 |
+
input_tensor: torch.Tensor,
|
36 |
+
target_layers: List[torch.nn.Module],
|
37 |
+
targets: List[torch.nn.Module],
|
38 |
+
activations: torch.Tensor,
|
39 |
+
grads: torch.Tensor) -> np.ndarray:
|
40 |
+
raise Exception("Not Implemented")
|
41 |
+
|
42 |
+
def get_cam_image(self,
|
43 |
+
input_tensor: torch.Tensor,
|
44 |
+
target_layer: torch.nn.Module,
|
45 |
+
targets: List[torch.nn.Module],
|
46 |
+
activations: torch.Tensor,
|
47 |
+
grads: torch.Tensor,
|
48 |
+
eigen_smooth: bool = False) -> np.ndarray:
|
49 |
+
|
50 |
+
weights = self.get_cam_weights(input_tensor,
|
51 |
+
target_layer,
|
52 |
+
targets,
|
53 |
+
activations,
|
54 |
+
grads)
|
55 |
+
weighted_activations = weights[:, :, None, None] * activations
|
56 |
+
if eigen_smooth:
|
57 |
+
cam = get_2d_projection(weighted_activations)
|
58 |
+
else:
|
59 |
+
cam = weighted_activations.sum(axis=1)
|
60 |
+
return cam
|
61 |
+
|
62 |
+
def forward(self,
|
63 |
+
input_tensor: torch.Tensor,
|
64 |
+
targets: List[torch.nn.Module],
|
65 |
+
eigen_smooth: bool = False) -> np.ndarray:
|
66 |
+
|
67 |
+
if self.cuda:
|
68 |
+
input_tensor = input_tensor.cuda()
|
69 |
+
|
70 |
+
if self.compute_input_gradient:
|
71 |
+
input_tensor = torch.autograd.Variable(input_tensor,
|
72 |
+
requires_grad=True)
|
73 |
+
|
74 |
+
outputs = self.activations_and_grads(input_tensor)
|
75 |
+
if targets is None:
|
76 |
+
target_categories = np.argmax(outputs.cpu().data.numpy(), axis=-1)
|
77 |
+
targets = [ClassifierOutputTarget(category) for category in target_categories]
|
78 |
+
|
79 |
+
if self.uses_gradients:
|
80 |
+
self.model.zero_grad()
|
81 |
+
loss = sum([target(output) for target, output in zip(targets, outputs)])
|
82 |
+
loss.backward(retain_graph=True)
|
83 |
+
|
84 |
+
# In most of the saliency attribution papers, the saliency is
|
85 |
+
# computed with a single target layer.
|
86 |
+
# Commonly it is the last convolutional layer.
|
87 |
+
# Here we support passing a list with multiple target layers.
|
88 |
+
# It will compute the saliency image for every image,
|
89 |
+
# and then aggregate them (with a default mean aggregation).
|
90 |
+
# This gives you more flexibility in case you just want to
|
91 |
+
# use all conv layers for example, all Batchnorm layers,
|
92 |
+
# or something else.
|
93 |
+
cam_per_layer = self.compute_cam_per_layer(input_tensor,
|
94 |
+
targets,
|
95 |
+
eigen_smooth)
|
96 |
+
return self.aggregate_multi_layers(cam_per_layer)
|
97 |
+
|
98 |
+
def get_target_width_height(self,
|
99 |
+
input_tensor: torch.Tensor) -> Tuple[int, int]:
|
100 |
+
width, height = input_tensor.size(-1), input_tensor.size(-2)
|
101 |
+
return width, height
|
102 |
+
|
103 |
+
def compute_cam_per_layer(
|
104 |
+
self,
|
105 |
+
input_tensor: torch.Tensor,
|
106 |
+
targets: List[torch.nn.Module],
|
107 |
+
eigen_smooth: bool) -> np.ndarray:
|
108 |
+
activations_list = [a.cpu().data.numpy()
|
109 |
+
for a in self.activations_and_grads.activations]
|
110 |
+
grads_list = [g.cpu().data.numpy()
|
111 |
+
for g in self.activations_and_grads.gradients]
|
112 |
+
target_size = self.get_target_width_height(input_tensor)
|
113 |
+
|
114 |
+
cam_per_target_layer = []
|
115 |
+
# Loop over the saliency image from every layer
|
116 |
+
for i in range(len(self.target_layers)):
|
117 |
+
target_layer = self.target_layers[i]
|
118 |
+
layer_activations = None
|
119 |
+
layer_grads = None
|
120 |
+
if i < len(activations_list):
|
121 |
+
layer_activations = activations_list[i]
|
122 |
+
if i < len(grads_list):
|
123 |
+
layer_grads = grads_list[i]
|
124 |
+
|
125 |
+
cam = self.get_cam_image(input_tensor,
|
126 |
+
target_layer,
|
127 |
+
targets,
|
128 |
+
layer_activations,
|
129 |
+
layer_grads,
|
130 |
+
eigen_smooth)
|
131 |
+
cam = np.maximum(cam, 0)
|
132 |
+
scaled = scale_cam_image(cam, target_size)
|
133 |
+
cam_per_target_layer.append(scaled[:, None, :])
|
134 |
+
|
135 |
+
return cam_per_target_layer
|
136 |
+
|
137 |
+
def aggregate_multi_layers(self, cam_per_target_layer: np.ndarray) -> np.ndarray:
|
138 |
+
cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1)
|
139 |
+
cam_per_target_layer = np.maximum(cam_per_target_layer, 0)
|
140 |
+
result = np.mean(cam_per_target_layer, axis=1)
|
141 |
+
return scale_cam_image(result)
|
142 |
+
|
143 |
+
def forward_augmentation_smoothing(self,
|
144 |
+
input_tensor: torch.Tensor,
|
145 |
+
targets: List[torch.nn.Module],
|
146 |
+
eigen_smooth: bool = False) -> np.ndarray:
|
147 |
+
transforms = tta.Compose(
|
148 |
+
[
|
149 |
+
tta.HorizontalFlip(),
|
150 |
+
tta.Multiply(factors=[0.9, 1, 1.1]),
|
151 |
+
]
|
152 |
+
)
|
153 |
+
cams = []
|
154 |
+
for transform in transforms:
|
155 |
+
augmented_tensor = transform.augment_image(input_tensor)
|
156 |
+
cam = self.forward(augmented_tensor,
|
157 |
+
targets,
|
158 |
+
eigen_smooth)
|
159 |
+
|
160 |
+
# The ttach library expects a tensor of size BxCxHxW
|
161 |
+
cam = cam[:, None, :, :]
|
162 |
+
cam = torch.from_numpy(cam)
|
163 |
+
cam = transform.deaugment_mask(cam)
|
164 |
+
|
165 |
+
# Back to numpy float32, HxW
|
166 |
+
cam = cam.numpy()
|
167 |
+
cam = cam[:, 0, :, :]
|
168 |
+
cams.append(cam)
|
169 |
+
|
170 |
+
cam = np.mean(np.float32(cams), axis=0)
|
171 |
+
return cam
|
172 |
+
|
173 |
+
def __call__(self,
|
174 |
+
input_tensor: torch.Tensor,
|
175 |
+
targets: List[torch.nn.Module] = None,
|
176 |
+
aug_smooth: bool = False,
|
177 |
+
eigen_smooth: bool = False) -> np.ndarray:
|
178 |
+
|
179 |
+
# Smooth the CAM result with test time augmentation
|
180 |
+
if aug_smooth is True:
|
181 |
+
return self.forward_augmentation_smoothing(
|
182 |
+
input_tensor, targets, eigen_smooth)
|
183 |
+
|
184 |
+
return self.forward(input_tensor,
|
185 |
+
targets, eigen_smooth)
|
186 |
+
|
187 |
+
def __del__(self):
|
188 |
+
self.activations_and_grads.release()
|
189 |
+
|
190 |
+
def __enter__(self):
|
191 |
+
return self
|
192 |
+
|
193 |
+
def __exit__(self, exc_type, exc_value, exc_tb):
|
194 |
+
self.activations_and_grads.release()
|
195 |
+
if isinstance(exc_value, IndexError):
|
196 |
+
# Handle IndexError here...
|
197 |
+
print(
|
198 |
+
f"An exception occurred in CAM with block: {exc_type}. Message: {exc_value}")
|
199 |
+
return True
|
pytorch_grad_cam/eigen_cam.py
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
2 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
3 |
+
|
4 |
+
# https://arxiv.org/abs/2008.00299
|
5 |
+
|
6 |
+
|
7 |
+
class EigenCAM(BaseCAM):
|
8 |
+
def __init__(self, model, target_layers, use_cuda=False,
|
9 |
+
reshape_transform=None):
|
10 |
+
super(EigenCAM, self).__init__(model, target_layers, use_cuda,
|
11 |
+
reshape_transform)
|
12 |
+
|
13 |
+
def get_cam_image(self,
|
14 |
+
input_tensor,
|
15 |
+
target_layer,
|
16 |
+
target_category,
|
17 |
+
activations,
|
18 |
+
grads,
|
19 |
+
eigen_smooth):
|
20 |
+
return get_2d_projection(activations)
|
pytorch_grad_cam/eigen_grad_cam.py
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
2 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
3 |
+
|
4 |
+
# Like Eigen CAM: https://arxiv.org/abs/2008.00299
|
5 |
+
# But multiply the activations x gradients
|
6 |
+
|
7 |
+
|
8 |
+
class EigenGradCAM(BaseCAM):
|
9 |
+
def __init__(self, model, target_layers, use_cuda=False,
|
10 |
+
reshape_transform=None):
|
11 |
+
super(EigenGradCAM, self).__init__(model, target_layers, use_cuda,
|
12 |
+
reshape_transform)
|
13 |
+
|
14 |
+
def get_cam_image(self,
|
15 |
+
input_tensor,
|
16 |
+
target_layer,
|
17 |
+
target_category,
|
18 |
+
activations,
|
19 |
+
grads,
|
20 |
+
eigen_smooth):
|
21 |
+
return get_2d_projection(grads * activations)
|
pytorch_grad_cam/fullgrad_cam.py
ADDED
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import torch
|
3 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
4 |
+
from pytorch_grad_cam.utils.find_layers import find_layer_predicate_recursive
|
5 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
6 |
+
from pytorch_grad_cam.utils.image import scale_accross_batch_and_channels, scale_cam_image
|
7 |
+
|
8 |
+
# https://arxiv.org/abs/1905.00780
|
9 |
+
|
10 |
+
|
11 |
+
class FullGrad(BaseCAM):
|
12 |
+
def __init__(self, model, target_layers, use_cuda=False,
|
13 |
+
reshape_transform=None):
|
14 |
+
if len(target_layers) > 0:
|
15 |
+
print(
|
16 |
+
"Warning: target_layers is ignored in FullGrad. All bias layers will be used instead")
|
17 |
+
|
18 |
+
def layer_with_2D_bias(layer):
|
19 |
+
bias_target_layers = [torch.nn.Conv2d, torch.nn.BatchNorm2d]
|
20 |
+
if type(layer) in bias_target_layers and layer.bias is not None:
|
21 |
+
return True
|
22 |
+
return False
|
23 |
+
target_layers = find_layer_predicate_recursive(
|
24 |
+
model, layer_with_2D_bias)
|
25 |
+
super(
|
26 |
+
FullGrad,
|
27 |
+
self).__init__(
|
28 |
+
model,
|
29 |
+
target_layers,
|
30 |
+
use_cuda,
|
31 |
+
reshape_transform,
|
32 |
+
compute_input_gradient=True)
|
33 |
+
self.bias_data = [self.get_bias_data(
|
34 |
+
layer).cpu().numpy() for layer in target_layers]
|
35 |
+
|
36 |
+
def get_bias_data(self, layer):
|
37 |
+
# Borrowed from official paper impl:
|
38 |
+
# https://github.com/idiap/fullgrad-saliency/blob/master/saliency/tensor_extractor.py#L47
|
39 |
+
if isinstance(layer, torch.nn.BatchNorm2d):
|
40 |
+
bias = - (layer.running_mean * layer.weight
|
41 |
+
/ torch.sqrt(layer.running_var + layer.eps)) + layer.bias
|
42 |
+
return bias.data
|
43 |
+
else:
|
44 |
+
return layer.bias.data
|
45 |
+
|
46 |
+
def compute_cam_per_layer(
|
47 |
+
self,
|
48 |
+
input_tensor,
|
49 |
+
target_category,
|
50 |
+
eigen_smooth):
|
51 |
+
input_grad = input_tensor.grad.data.cpu().numpy()
|
52 |
+
grads_list = [g.cpu().data.numpy() for g in
|
53 |
+
self.activations_and_grads.gradients]
|
54 |
+
cam_per_target_layer = []
|
55 |
+
target_size = self.get_target_width_height(input_tensor)
|
56 |
+
|
57 |
+
gradient_multiplied_input = input_grad * input_tensor.data.cpu().numpy()
|
58 |
+
gradient_multiplied_input = np.abs(gradient_multiplied_input)
|
59 |
+
gradient_multiplied_input = scale_accross_batch_and_channels(
|
60 |
+
gradient_multiplied_input,
|
61 |
+
target_size)
|
62 |
+
cam_per_target_layer.append(gradient_multiplied_input)
|
63 |
+
|
64 |
+
# Loop over the saliency image from every layer
|
65 |
+
assert(len(self.bias_data) == len(grads_list))
|
66 |
+
for bias, grads in zip(self.bias_data, grads_list):
|
67 |
+
bias = bias[None, :, None, None]
|
68 |
+
# In the paper they take the absolute value,
|
69 |
+
# but possibily taking only the positive gradients will work
|
70 |
+
# better.
|
71 |
+
bias_grad = np.abs(bias * grads)
|
72 |
+
result = scale_accross_batch_and_channels(
|
73 |
+
bias_grad, target_size)
|
74 |
+
result = np.sum(result, axis=1)
|
75 |
+
cam_per_target_layer.append(result[:, None, :])
|
76 |
+
cam_per_target_layer = np.concatenate(cam_per_target_layer, axis=1)
|
77 |
+
if eigen_smooth:
|
78 |
+
# Resize to a smaller image, since this method typically has a very large number of channels,
|
79 |
+
# and then consumes a lot of memory
|
80 |
+
cam_per_target_layer = scale_accross_batch_and_channels(
|
81 |
+
cam_per_target_layer, (target_size[0] // 8, target_size[1] // 8))
|
82 |
+
cam_per_target_layer = get_2d_projection(cam_per_target_layer)
|
83 |
+
cam_per_target_layer = cam_per_target_layer[:, None, :, :]
|
84 |
+
cam_per_target_layer = scale_accross_batch_and_channels(
|
85 |
+
cam_per_target_layer,
|
86 |
+
target_size)
|
87 |
+
else:
|
88 |
+
cam_per_target_layer = np.sum(
|
89 |
+
cam_per_target_layer, axis=1)[:, None, :]
|
90 |
+
|
91 |
+
return cam_per_target_layer
|
92 |
+
|
93 |
+
def aggregate_multi_layers(self, cam_per_target_layer):
|
94 |
+
result = np.sum(cam_per_target_layer, axis=1)
|
95 |
+
return scale_cam_image(result)
|
pytorch_grad_cam/grad_cam.py
ADDED
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
3 |
+
|
4 |
+
|
5 |
+
class GradCAM(BaseCAM):
|
6 |
+
def __init__(self, model, target_layers, use_cuda=False,
|
7 |
+
reshape_transform=None):
|
8 |
+
super(
|
9 |
+
GradCAM,
|
10 |
+
self).__init__(
|
11 |
+
model,
|
12 |
+
target_layers,
|
13 |
+
use_cuda,
|
14 |
+
reshape_transform)
|
15 |
+
|
16 |
+
def get_cam_weights(self,
|
17 |
+
input_tensor,
|
18 |
+
target_layer,
|
19 |
+
target_category,
|
20 |
+
activations,
|
21 |
+
grads):
|
22 |
+
return np.mean(grads, axis=(2, 3))
|
pytorch_grad_cam/grad_cam_plusplus.py
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
3 |
+
|
4 |
+
# https://arxiv.org/abs/1710.11063
|
5 |
+
|
6 |
+
|
7 |
+
class GradCAMPlusPlus(BaseCAM):
|
8 |
+
def __init__(self, model, target_layers, use_cuda=False,
|
9 |
+
reshape_transform=None):
|
10 |
+
super(GradCAMPlusPlus, self).__init__(model, target_layers, use_cuda,
|
11 |
+
reshape_transform)
|
12 |
+
|
13 |
+
def get_cam_weights(self,
|
14 |
+
input_tensor,
|
15 |
+
target_layers,
|
16 |
+
target_category,
|
17 |
+
activations,
|
18 |
+
grads):
|
19 |
+
grads_power_2 = grads**2
|
20 |
+
grads_power_3 = grads_power_2 * grads
|
21 |
+
# Equation 19 in https://arxiv.org/abs/1710.11063
|
22 |
+
sum_activations = np.sum(activations, axis=(2, 3))
|
23 |
+
eps = 0.000001
|
24 |
+
aij = grads_power_2 / (2 * grads_power_2 +
|
25 |
+
sum_activations[:, :, None, None] * grads_power_3 + eps)
|
26 |
+
# Now bring back the ReLU from eq.7 in the paper,
|
27 |
+
# And zero out aijs where the activations are 0
|
28 |
+
aij = np.where(grads != 0, aij, 0)
|
29 |
+
|
30 |
+
weights = np.maximum(grads, 0) * aij
|
31 |
+
weights = np.sum(weights, axis=(2, 3))
|
32 |
+
return weights
|
pytorch_grad_cam/guided_backprop.py
ADDED
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import torch
|
3 |
+
from torch.autograd import Function
|
4 |
+
from pytorch_grad_cam.utils.find_layers import replace_all_layer_type_recursive
|
5 |
+
|
6 |
+
|
7 |
+
class GuidedBackpropReLU(Function):
|
8 |
+
@staticmethod
|
9 |
+
def forward(self, input_img):
|
10 |
+
positive_mask = (input_img > 0).type_as(input_img)
|
11 |
+
output = torch.addcmul(
|
12 |
+
torch.zeros(
|
13 |
+
input_img.size()).type_as(input_img),
|
14 |
+
input_img,
|
15 |
+
positive_mask)
|
16 |
+
self.save_for_backward(input_img, output)
|
17 |
+
return output
|
18 |
+
|
19 |
+
@staticmethod
|
20 |
+
def backward(self, grad_output):
|
21 |
+
input_img, output = self.saved_tensors
|
22 |
+
grad_input = None
|
23 |
+
|
24 |
+
positive_mask_1 = (input_img > 0).type_as(grad_output)
|
25 |
+
positive_mask_2 = (grad_output > 0).type_as(grad_output)
|
26 |
+
grad_input = torch.addcmul(
|
27 |
+
torch.zeros(
|
28 |
+
input_img.size()).type_as(input_img),
|
29 |
+
torch.addcmul(
|
30 |
+
torch.zeros(
|
31 |
+
input_img.size()).type_as(input_img),
|
32 |
+
grad_output,
|
33 |
+
positive_mask_1),
|
34 |
+
positive_mask_2)
|
35 |
+
return grad_input
|
36 |
+
|
37 |
+
|
38 |
+
class GuidedBackpropReLUasModule(torch.nn.Module):
|
39 |
+
def __init__(self):
|
40 |
+
super(GuidedBackpropReLUasModule, self).__init__()
|
41 |
+
|
42 |
+
def forward(self, input_img):
|
43 |
+
return GuidedBackpropReLU.apply(input_img)
|
44 |
+
|
45 |
+
|
46 |
+
class GuidedBackpropReLUModel:
|
47 |
+
def __init__(self, model, use_cuda):
|
48 |
+
self.model = model
|
49 |
+
self.model.eval()
|
50 |
+
self.cuda = use_cuda
|
51 |
+
if self.cuda:
|
52 |
+
self.model = self.model.cuda()
|
53 |
+
|
54 |
+
def forward(self, input_img):
|
55 |
+
return self.model(input_img)
|
56 |
+
|
57 |
+
def recursive_replace_relu_with_guidedrelu(self, module_top):
|
58 |
+
|
59 |
+
for idx, module in module_top._modules.items():
|
60 |
+
self.recursive_replace_relu_with_guidedrelu(module)
|
61 |
+
if module.__class__.__name__ == 'ReLU':
|
62 |
+
module_top._modules[idx] = GuidedBackpropReLU.apply
|
63 |
+
print("b")
|
64 |
+
|
65 |
+
def recursive_replace_guidedrelu_with_relu(self, module_top):
|
66 |
+
try:
|
67 |
+
for idx, module in module_top._modules.items():
|
68 |
+
self.recursive_replace_guidedrelu_with_relu(module)
|
69 |
+
if module == GuidedBackpropReLU.apply:
|
70 |
+
module_top._modules[idx] = torch.nn.ReLU()
|
71 |
+
except BaseException:
|
72 |
+
pass
|
73 |
+
|
74 |
+
def __call__(self, input_img, target_category=None):
|
75 |
+
replace_all_layer_type_recursive(self.model,
|
76 |
+
torch.nn.ReLU,
|
77 |
+
GuidedBackpropReLUasModule())
|
78 |
+
|
79 |
+
if self.cuda:
|
80 |
+
input_img = input_img.cuda()
|
81 |
+
|
82 |
+
input_img = input_img.requires_grad_(True)
|
83 |
+
|
84 |
+
output = self.forward(input_img)
|
85 |
+
|
86 |
+
if target_category is None:
|
87 |
+
target_category = np.argmax(output.cpu().data.numpy())
|
88 |
+
|
89 |
+
loss = output[0, target_category]
|
90 |
+
loss.backward(retain_graph=True)
|
91 |
+
|
92 |
+
output = input_img.grad.cpu().data.numpy()
|
93 |
+
output = output[0, :, :, :]
|
94 |
+
output = output.transpose((1, 2, 0))
|
95 |
+
|
96 |
+
replace_all_layer_type_recursive(self.model,
|
97 |
+
GuidedBackpropReLUasModule,
|
98 |
+
torch.nn.ReLU())
|
99 |
+
|
100 |
+
return output
|
pytorch_grad_cam/layer_cam.py
ADDED
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
3 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
4 |
+
|
5 |
+
# https://ieeexplore.ieee.org/document/9462463
|
6 |
+
|
7 |
+
|
8 |
+
class LayerCAM(BaseCAM):
|
9 |
+
def __init__(
|
10 |
+
self,
|
11 |
+
model,
|
12 |
+
target_layers,
|
13 |
+
use_cuda=False,
|
14 |
+
reshape_transform=None):
|
15 |
+
super(
|
16 |
+
LayerCAM,
|
17 |
+
self).__init__(
|
18 |
+
model,
|
19 |
+
target_layers,
|
20 |
+
use_cuda,
|
21 |
+
reshape_transform)
|
22 |
+
|
23 |
+
def get_cam_image(self,
|
24 |
+
input_tensor,
|
25 |
+
target_layer,
|
26 |
+
target_category,
|
27 |
+
activations,
|
28 |
+
grads,
|
29 |
+
eigen_smooth):
|
30 |
+
spatial_weighted_activations = np.maximum(grads, 0) * activations
|
31 |
+
|
32 |
+
if eigen_smooth:
|
33 |
+
cam = get_2d_projection(spatial_weighted_activations)
|
34 |
+
else:
|
35 |
+
cam = spatial_weighted_activations.sum(axis=1)
|
36 |
+
return cam
|
pytorch_grad_cam/score_cam.py
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
import tqdm
|
3 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
4 |
+
|
5 |
+
|
6 |
+
class ScoreCAM(BaseCAM):
|
7 |
+
def __init__(
|
8 |
+
self,
|
9 |
+
model,
|
10 |
+
target_layers,
|
11 |
+
use_cuda=False,
|
12 |
+
reshape_transform=None):
|
13 |
+
super(ScoreCAM, self).__init__(model,
|
14 |
+
target_layers,
|
15 |
+
use_cuda,
|
16 |
+
reshape_transform=reshape_transform,
|
17 |
+
uses_gradients=False)
|
18 |
+
|
19 |
+
if len(target_layers) > 0:
|
20 |
+
print("Warning: You are using ScoreCAM with target layers, "
|
21 |
+
"however ScoreCAM will ignore them.")
|
22 |
+
|
23 |
+
def get_cam_weights(self,
|
24 |
+
input_tensor,
|
25 |
+
target_layer,
|
26 |
+
targets,
|
27 |
+
activations,
|
28 |
+
grads):
|
29 |
+
with torch.no_grad():
|
30 |
+
upsample = torch.nn.UpsamplingBilinear2d(
|
31 |
+
size=input_tensor.shape[-2:])
|
32 |
+
activation_tensor = torch.from_numpy(activations)
|
33 |
+
if self.cuda:
|
34 |
+
activation_tensor = activation_tensor.cuda()
|
35 |
+
|
36 |
+
upsampled = upsample(activation_tensor)
|
37 |
+
|
38 |
+
maxs = upsampled.view(upsampled.size(0),
|
39 |
+
upsampled.size(1), -1).max(dim=-1)[0]
|
40 |
+
mins = upsampled.view(upsampled.size(0),
|
41 |
+
upsampled.size(1), -1).min(dim=-1)[0]
|
42 |
+
|
43 |
+
maxs, mins = maxs[:, :, None, None], mins[:, :, None, None]
|
44 |
+
upsampled = (upsampled - mins) / (maxs - mins)
|
45 |
+
|
46 |
+
input_tensors = input_tensor[:, None,
|
47 |
+
:, :] * upsampled[:, :, None, :, :]
|
48 |
+
|
49 |
+
if hasattr(self, "batch_size"):
|
50 |
+
BATCH_SIZE = self.batch_size
|
51 |
+
else:
|
52 |
+
BATCH_SIZE = 16
|
53 |
+
|
54 |
+
scores = []
|
55 |
+
for target, tensor in zip(targets, input_tensors):
|
56 |
+
for i in tqdm.tqdm(range(0, tensor.size(0), BATCH_SIZE)):
|
57 |
+
batch = tensor[i: i + BATCH_SIZE, :]
|
58 |
+
outputs = [target(o).cpu().item() for o in self.model(batch)]
|
59 |
+
scores.extend(outputs)
|
60 |
+
scores = torch.Tensor(scores)
|
61 |
+
scores = scores.view(activations.shape[0], activations.shape[1])
|
62 |
+
weights = torch.nn.Softmax(dim=-1)(scores).numpy()
|
63 |
+
return weights
|
pytorch_grad_cam/utils/__init__.py
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from pytorch_grad_cam.utils.image import deprocess_image
|
2 |
+
from pytorch_grad_cam.utils.svd_on_activations import get_2d_projection
|
3 |
+
from pytorch_grad_cam.utils import model_targets
|
4 |
+
from pytorch_grad_cam.utils import reshape_transforms
|
pytorch_grad_cam/utils/__pycache__/__init__.cpython-39.pyc
ADDED
Binary file (426 Bytes). View file
|
|
pytorch_grad_cam/utils/__pycache__/find_layers.cpython-39.pyc
ADDED
Binary file (1.23 kB). View file
|
|
pytorch_grad_cam/utils/__pycache__/image.cpython-39.pyc
ADDED
Binary file (2.52 kB). View file
|
|
pytorch_grad_cam/utils/__pycache__/model_targets.cpython-39.pyc
ADDED
Binary file (2.72 kB). View file
|
|
pytorch_grad_cam/utils/__pycache__/reshape_transforms.cpython-39.pyc
ADDED
Binary file (1.06 kB). View file
|
|
pytorch_grad_cam/utils/__pycache__/svd_on_activations.cpython-39.pyc
ADDED
Binary file (685 Bytes). View file
|
|
pytorch_grad_cam/utils/find_layers.py
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
def replace_layer_recursive(model, old_layer, new_layer):
|
2 |
+
for name, layer in model._modules.items():
|
3 |
+
if layer == old_layer:
|
4 |
+
model._modules[name] = new_layer
|
5 |
+
return True
|
6 |
+
elif replace_layer_recursive(layer, old_layer, new_layer):
|
7 |
+
return True
|
8 |
+
return False
|
9 |
+
|
10 |
+
|
11 |
+
def replace_all_layer_type_recursive(model, old_layer_type, new_layer):
|
12 |
+
for name, layer in model._modules.items():
|
13 |
+
if isinstance(layer, old_layer_type):
|
14 |
+
model._modules[name] = new_layer
|
15 |
+
replace_all_layer_type_recursive(layer, old_layer_type, new_layer)
|
16 |
+
|
17 |
+
|
18 |
+
def find_layer_types_recursive(model, layer_types):
|
19 |
+
def predicate(layer):
|
20 |
+
return type(layer) in layer_types
|
21 |
+
return find_layer_predicate_recursive(model, predicate)
|
22 |
+
|
23 |
+
|
24 |
+
def find_layer_predicate_recursive(model, predicate):
|
25 |
+
result = []
|
26 |
+
for name, layer in model._modules.items():
|
27 |
+
if predicate(layer):
|
28 |
+
result.append(layer)
|
29 |
+
result.extend(find_layer_predicate_recursive(layer, predicate))
|
30 |
+
return result
|
pytorch_grad_cam/utils/image.py
ADDED
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import cv2
|
2 |
+
import numpy as np
|
3 |
+
import torch
|
4 |
+
from torchvision.transforms import Compose, Normalize, ToTensor
|
5 |
+
|
6 |
+
|
7 |
+
def preprocess_image(img: np.ndarray, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) -> torch.Tensor:
|
8 |
+
preprocessing = Compose([
|
9 |
+
ToTensor(),
|
10 |
+
Normalize(mean=mean, std=std)
|
11 |
+
])
|
12 |
+
return preprocessing(img.copy()).unsqueeze(0)
|
13 |
+
|
14 |
+
|
15 |
+
def deprocess_image(img):
|
16 |
+
""" see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 """
|
17 |
+
img = img - np.mean(img)
|
18 |
+
img = img / (np.std(img) + 1e-5)
|
19 |
+
img = img * 0.1
|
20 |
+
img = img + 0.5
|
21 |
+
img = np.clip(img, 0, 1)
|
22 |
+
return np.uint8(img * 255)
|
23 |
+
|
24 |
+
|
25 |
+
def show_cam_on_image(img: np.ndarray,
|
26 |
+
mask: np.ndarray,
|
27 |
+
use_rgb: bool = False,
|
28 |
+
colormap: int = cv2.COLORMAP_JET) -> np.ndarray:
|
29 |
+
""" This function overlays the cam mask on the image as an heatmap.
|
30 |
+
By default the heatmap is in BGR format.
|
31 |
+
|
32 |
+
:param img: The base image in RGB or BGR format.
|
33 |
+
:param mask: The cam mask.
|
34 |
+
:param use_rgb: Whether to use an RGB or BGR heatmap, this should be set to True if 'img' is in RGB format.
|
35 |
+
:param colormap: The OpenCV colormap to be used.
|
36 |
+
:returns: The default image with the cam overlay.
|
37 |
+
"""
|
38 |
+
heatmap = cv2.applyColorMap(np.uint8(255 * mask), colormap)
|
39 |
+
if use_rgb:
|
40 |
+
heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)
|
41 |
+
heatmap = np.float32(heatmap) / 255
|
42 |
+
|
43 |
+
if np.max(img) > 1:
|
44 |
+
raise Exception(
|
45 |
+
"The input image should np.float32 in the range [0, 1]")
|
46 |
+
|
47 |
+
cam = heatmap + img
|
48 |
+
cam = cam / np.max(cam)
|
49 |
+
return np.uint8(255 * cam)
|
50 |
+
|
51 |
+
def scale_cam_image(cam, target_size=None):
|
52 |
+
result = []
|
53 |
+
for img in cam:
|
54 |
+
img = img - np.min(img)
|
55 |
+
img = img / (1e-7 + np.max(img))
|
56 |
+
if target_size is not None:
|
57 |
+
img = cv2.resize(img, target_size)
|
58 |
+
result.append(img)
|
59 |
+
result = np.float32(result)
|
60 |
+
|
61 |
+
return result
|
62 |
+
|
63 |
+
def scale_accross_batch_and_channels(tensor, target_size):
|
64 |
+
batch_size, channel_size = tensor.shape[:2]
|
65 |
+
reshaped_tensor = tensor.reshape(
|
66 |
+
batch_size * channel_size, *tensor.shape[2:])
|
67 |
+
result = scale_cam_image(reshaped_tensor, target_size)
|
68 |
+
result = result.reshape(
|
69 |
+
batch_size,
|
70 |
+
channel_size,
|
71 |
+
target_size[1],
|
72 |
+
target_size[0])
|
73 |
+
return result
|
pytorch_grad_cam/utils/model_targets.py
ADDED
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import torch
|
3 |
+
import torchvision
|
4 |
+
|
5 |
+
class ClassifierOutputTarget:
|
6 |
+
def __init__(self, category):
|
7 |
+
self.category = category
|
8 |
+
def __call__(self, model_output):
|
9 |
+
if len(model_output.shape) == 1:
|
10 |
+
return model_output[self.category]
|
11 |
+
return model_output[:, self.category]
|
12 |
+
|
13 |
+
class SemanticSegmentationTarget:
|
14 |
+
""" Gets a binary spatial mask and a category,
|
15 |
+
And return the sum of the category scores,
|
16 |
+
of the pixels in the mask. """
|
17 |
+
def __init__(self, category, mask):
|
18 |
+
self.category = category
|
19 |
+
self.mask = torch.from_numpy(mask)
|
20 |
+
if torch.cuda.is_available():
|
21 |
+
self.mask = self.mask.cuda()
|
22 |
+
|
23 |
+
def __call__(self, model_output):
|
24 |
+
return (model_output[self.category, :, : ] * self.mask).sum()
|
25 |
+
|
26 |
+
|
27 |
+
class FasterRCNNBoxScoreTarget:
|
28 |
+
""" For every original detected bounding box specified in "bounding boxes",
|
29 |
+
assign a score on how the current bounding boxes match it,
|
30 |
+
1. In IOU
|
31 |
+
2. In the classification score.
|
32 |
+
If there is not a large enough overlap, or the category changed,
|
33 |
+
assign a score of 0.
|
34 |
+
|
35 |
+
The total score is the sum of all the box scores.
|
36 |
+
"""
|
37 |
+
|
38 |
+
def __init__(self, labels, bounding_boxes, iou_threshold=0.5):
|
39 |
+
self.labels = labels
|
40 |
+
self.bounding_boxes = bounding_boxes
|
41 |
+
self.iou_threshold = iou_threshold
|
42 |
+
|
43 |
+
def __call__(self, model_outputs):
|
44 |
+
output = torch.Tensor([0])
|
45 |
+
if torch.cuda.is_available():
|
46 |
+
output = output.cuda()
|
47 |
+
|
48 |
+
if len(model_outputs["boxes"]) == 0:
|
49 |
+
return output
|
50 |
+
|
51 |
+
for box, label in zip(self.bounding_boxes, self.labels):
|
52 |
+
box = torch.Tensor(box[None, :])
|
53 |
+
if torch.cuda.is_available():
|
54 |
+
box = box.cuda()
|
55 |
+
|
56 |
+
ious = torchvision.ops.box_iou(box, model_outputs["boxes"])
|
57 |
+
index = ious.argmax()
|
58 |
+
if ious[0, index] > self.iou_threshold and model_outputs["labels"][index] == label:
|
59 |
+
score = ious[0, index] + model_outputs["scores"][index]
|
60 |
+
output = output + score
|
61 |
+
return output
|
pytorch_grad_cam/utils/reshape_transforms.py
ADDED
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
|
3 |
+
def fasterrcnn_reshape_transform(x):
|
4 |
+
target_size = x['pool'].size()[-2 : ]
|
5 |
+
activations = []
|
6 |
+
for key, value in x.items():
|
7 |
+
activations.append(torch.nn.functional.interpolate(torch.abs(value), target_size, mode='bilinear'))
|
8 |
+
activations = torch.cat(activations, axis=1)
|
9 |
+
return activations
|
10 |
+
|
11 |
+
def swinT_reshape_transform(tensor, height=7, width=7):
|
12 |
+
result = tensor.reshape(tensor.size(0),
|
13 |
+
height, width, tensor.size(2))
|
14 |
+
|
15 |
+
# Bring the channels to the first dimension,
|
16 |
+
# like in CNNs.
|
17 |
+
result = result.transpose(2, 3).transpose(1, 2)
|
18 |
+
return result
|
19 |
+
|
20 |
+
def vit_reshape_transform(tensor, height=14, width=14):
|
21 |
+
result = tensor[:, 1:, :].reshape(tensor.size(0),
|
22 |
+
height, width, tensor.size(2))
|
23 |
+
|
24 |
+
# Bring the channels to the first dimension,
|
25 |
+
# like in CNNs.
|
26 |
+
result = result.transpose(2, 3).transpose(1, 2)
|
27 |
+
return result
|
pytorch_grad_cam/utils/svd_on_activations.py
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
|
3 |
+
|
4 |
+
def get_2d_projection(activation_batch):
|
5 |
+
# TBD: use pytorch batch svd implementation
|
6 |
+
activation_batch[np.isnan(activation_batch)] = 0
|
7 |
+
projections = []
|
8 |
+
for activations in activation_batch:
|
9 |
+
reshaped_activations = (activations).reshape(
|
10 |
+
activations.shape[0], -1).transpose()
|
11 |
+
# Centering before the SVD seems to be important here,
|
12 |
+
# Otherwise the image returned is negative
|
13 |
+
reshaped_activations = reshaped_activations - \
|
14 |
+
reshaped_activations.mean(axis=0)
|
15 |
+
U, S, VT = np.linalg.svd(reshaped_activations, full_matrices=True)
|
16 |
+
projection = reshaped_activations @ VT[0, :]
|
17 |
+
projection = projection.reshape(activations.shape[1:])
|
18 |
+
projections.append(projection)
|
19 |
+
return np.float32(projections)
|
pytorch_grad_cam/xgrad_cam.py
ADDED
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
from pytorch_grad_cam.base_cam import BaseCAM
|
3 |
+
|
4 |
+
|
5 |
+
class XGradCAM(BaseCAM):
|
6 |
+
def __init__(
|
7 |
+
self,
|
8 |
+
model,
|
9 |
+
target_layers,
|
10 |
+
use_cuda=False,
|
11 |
+
reshape_transform=None):
|
12 |
+
super(
|
13 |
+
XGradCAM,
|
14 |
+
self).__init__(
|
15 |
+
model,
|
16 |
+
target_layers,
|
17 |
+
use_cuda,
|
18 |
+
reshape_transform)
|
19 |
+
|
20 |
+
def get_cam_weights(self,
|
21 |
+
input_tensor,
|
22 |
+
target_layer,
|
23 |
+
target_category,
|
24 |
+
activations,
|
25 |
+
grads):
|
26 |
+
sum_activations = np.sum(activations, axis=(2, 3))
|
27 |
+
eps = 1e-7
|
28 |
+
weights = grads * activations / \
|
29 |
+
(sum_activations[:, :, None, None] + eps)
|
30 |
+
weights = weights.sum(axis=(2, 3))
|
31 |
+
return weights
|
runSDSdemo.py
ADDED
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# import pytorch related dependencies
|
2 |
+
import torch
|
3 |
+
from PIL import Image
|
4 |
+
from torch import nn
|
5 |
+
import numpy as np
|
6 |
+
import torchvision as torchvision
|
7 |
+
import torchvision.transforms as transforms
|
8 |
+
from pytorch_grad_cam import GradCAM, ScoreCAM, GradCAMPlusPlus, AblationCAM, XGradCAM, EigenCAM, FullGrad
|
9 |
+
from pytorch_grad_cam.utils.image import show_cam_on_image
|
10 |
+
import gradio as gr
|
11 |
+
|
12 |
+
# model setup
|
13 |
+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
14 |
+
classes = [ 'actinic keratoses', 'basal cell carcinoma', 'benign keratosis-like lesions',
|
15 |
+
'dermatofibroma','melanoma', 'melanocytic nevi', 'vascular lesions']
|
16 |
+
model = torchvision.models.mobilenet_v3_large(pretrained = False) # This is a very well known network but it is designed for 1000 classes and not just cats and dogs this is why we need the next line
|
17 |
+
model.classifier[3] = nn.Linear(1280, 7)
|
18 |
+
#state_dict_trained = torch.hub.load_state_dict_from_url("https://github.com/tobiascz/demotime/raw/main/checkpoints/ham10k_checkpoint_mobile_0.82_epoch24.pt", model_dir=".", map_location = device)
|
19 |
+
import os
|
20 |
+
print(os.getcwd())
|
21 |
+
state_dict_trained = torch.load('checkpoints/ham10k_checkpoint_mobile_0.82_epoch24.pt', map_location=torch.device('cpu'))
|
22 |
+
model.load_state_dict(state_dict_trained["model_state_dict"]) ## Here we load the trained weights (state_dict) in our model
|
23 |
+
model.eval() # This
|
24 |
+
|
25 |
+
# image pre-processing
|
26 |
+
norm_mean = (0.4914, 0.4822, 0.4465)
|
27 |
+
norm_std = (0.2023, 0.1994, 0.2010)
|
28 |
+
transform = transforms.Compose([ # resize image to the network input size
|
29 |
+
transforms.CenterCrop((400,400)),
|
30 |
+
transforms.ToTensor(),
|
31 |
+
transforms.Normalize(norm_mean, norm_std)
|
32 |
+
])
|
33 |
+
# convert tensot to numpy array
|
34 |
+
def tensor2npimg(tensor, mean, std):
|
35 |
+
# inverse of normalization
|
36 |
+
tensor = tensor.clone()
|
37 |
+
mean_tensor = torch.as_tensor(list(mean), dtype=tensor.dtype, device=tensor.device).view(-1,1,1)
|
38 |
+
std_tensor = torch.as_tensor(list(std), dtype=tensor.dtype, device=tensor.device).view(-1,1,1)
|
39 |
+
tensor.mul_(std_tensor).add_(mean_tensor)
|
40 |
+
# convert tensor to numpy format for plt presentation
|
41 |
+
npimg = tensor.numpy()
|
42 |
+
npimg = np.transpose(npimg,(1,2,0)) # C*H*W => H*W*C
|
43 |
+
return npimg
|
44 |
+
|
45 |
+
|
46 |
+
# draw Grad-CAM on image
|
47 |
+
# target layer could be any layer before the final attention block
|
48 |
+
# Some common choices are:
|
49 |
+
# FasterRCNN: model.backbone
|
50 |
+
# Resnet18 and 50: model.layer4[-1]
|
51 |
+
# VGG and densenet161: model.features[-1]
|
52 |
+
# mnasnet1_0: model.layers[-1]
|
53 |
+
# ViT: model.blocks[-1].norm1
|
54 |
+
# SwinT: model.layers[-1].blocks[-1].norm1
|
55 |
+
def image_grad_cam(model, input_tensor, input_float_np, target_layers):
|
56 |
+
cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False)
|
57 |
+
grayscale_cam = cam(input_tensor=input_tensor, aug_smooth=True, eigen_smooth=True)
|
58 |
+
grayscale_cam = grayscale_cam[0, :]
|
59 |
+
return show_cam_on_image(input_float_np, grayscale_cam, use_rgb=True)
|
60 |
+
|
61 |
+
|
62 |
+
# config the predict function for Gradio, input type of image is numpy.nparray
|
63 |
+
def predict(input_img):
|
64 |
+
# numpy.nparray -> PIL.Image
|
65 |
+
leasionExample = Image.fromarray(input_img.astype('uint8'), 'RGB')
|
66 |
+
# normalize the image to fit the input size of our model
|
67 |
+
leasion_tensor = transform(leasionExample)
|
68 |
+
input_float_np = tensor2npimg(leasion_tensor, norm_mean, norm_std)
|
69 |
+
leasion_tensor = leasion_tensor.unsqueeze(dim=0)
|
70 |
+
# predict
|
71 |
+
with torch.no_grad():
|
72 |
+
outputs = model(leasion_tensor)
|
73 |
+
outputs = torch.exp(outputs)
|
74 |
+
# probabilities of all classes
|
75 |
+
pred_softmax = torch.softmax(outputs, dim=1).cpu().numpy()[0]
|
76 |
+
# class with hightest probability
|
77 |
+
pred = torch.argmax(outputs, dim=1).cpu().numpy()
|
78 |
+
# diagnostic suggestions
|
79 |
+
if pred == 1 or pred == 4:
|
80 |
+
suggestion = "CHECK WITH YOUR MD!"
|
81 |
+
else:
|
82 |
+
suggestion = "Nothing to be worried about."
|
83 |
+
# grad_cam image
|
84 |
+
target_layers = model.features[-1]
|
85 |
+
output_img = image_grad_cam(model,leasion_tensor,input_float_np,target_layers)
|
86 |
+
# return label dict and suggestion
|
87 |
+
return {classes[i]: float(pred_softmax[i]) for i in range(len(classes))}, suggestion, output_img
|
88 |
+
|
89 |
+
# start gradio application
|
90 |
+
gr.Interface(
|
91 |
+
fn=predict,
|
92 |
+
inputs=gr.inputs.Image(),
|
93 |
+
outputs=[gr.outputs.Label(label="Predict Result"), gr.outputs.Textbox(type="str", label="Recommendation"), gr.outputs.Image(label="GradCAM")],
|
94 |
+
examples=[['sample1.png'],['sample2.jpg'],['sample3.jpg']],
|
95 |
+
title="Skin Lesion Classifier"
|
96 |
+
).launch()
|