ValueError: Input images must have the same dimensions.

Angela_Koh: <p>import tensorflow as tf<br>
from tensorflow.keras.layers import Input, Conv2D, Dense, Flatten, Dropout, GlobalAveragePooling2D, MaxPooling2D, GlobalMaxPooling2D, BatchNormalization, Reshape, Multiply, Add, Concatenate<br>
from tensorflow.keras.models import Model<br>
from tensorflow.keras.preprocessing import image<br>
from tensorflow.keras.preprocessing.image import ImageDataGenerator<br>
from tensorflow.keras.utils import to_categorical<br>
from tensorflow.keras.optimizers import SGD<br>
from skimage.metrics import structural_similarity as ssim<br>
from skimage.transform import resize<br>
import numpy as np<br>
import matplotlib.pyplot as plt<br>
from PIL import Image</p>
<h1>
<a name="define-the-self-attention-1" class="anchor" href="#define-the-self-attention-1"></a>Define the self attention</h1>
<p>class SelfAttention(tf.keras.layers.Layer):<br>
def <strong>init</strong>(self, num_channels):<br>
super(SelfAttention, self).<strong>init</strong>()<br>
self.query_conv = tf.keras.layers.Conv2D(num_channels // 8, kernel_size=1)<br>
self.key_conv = tf.keras.layers.Conv2D(num_channels // 8, kernel_size=1)<br>
self.value_conv = tf.keras.layers.Conv2D(num_channels, kernel_size=1)<br>
self.gamma = self.add_weight(‘gamma’, shape=[1], initializer=‘zeros’, trainable=True)</p>
<pre><code>def call(self, inputs):
    batch_size, height, width, num_channels = inputs.shape
    query = self.query_conv(inputs)
    key = self.key_conv(inputs)
    value = self.value_conv(inputs)
    energy = tf.matmul(query, tf.transpose(key, [0, 1, 3, 2]))
    attention = tf.nn.softmax(energy, axis=-1)
    context = tf.matmul(attention, value)
    out = self.gamma * context + inputs
    return out, attention
</code></pre>
<h1>
<a name="define-the-channel-attention-2" class="anchor" href="#define-the-channel-attention-2"></a>Define the channel attention</h1>
<p>class ChannelAttention(tf.keras.layers.Layer):<br>
def <strong>init</strong>(self, num_channels, reduction_ratio=8):<br>
super(ChannelAttention, self).<strong>init</strong>()<br>
self.avg_pool = tf.keras.layers.GlobalAveragePooling2D()<br>
self.max_pool = tf.keras.layers.GlobalMaxPooling2D()<br>
self.fc1 = tf.keras.layers.Dense(num_channels // reduction_ratio, activation=‘relu’)<br>
self.fc2 = tf.keras.layers.Dense(num_channels, activation=‘sigmoid’)</p>
<pre><code>def call(self, inputs):
    avg_pool = self.avg_pool(inputs)
    max_pool = self.max_pool(inputs)
    avg_fc = self.fc2(self.fc1(avg_pool))
    max_fc = self.fc2(self.fc1(max_pool))
    max_fc = tf.expand_dims(max_fc, axis=1)  # Add this line
    attention = avg_fc[:, tf.newaxis, :] * max_fc[:, :, tf.newaxis]
    out = attention * inputs
    return out, attention
</code></pre>
<h1>
<a name="define-the-spatial-attention-3" class="anchor" href="#define-the-spatial-attention-3"></a>Define the spatial attention</h1>
<p>class SpatialAttention(tf.keras.layers.Layer):<br>
def <strong>init</strong>(self, kernel_size=7):<br>
super(SpatialAttention, self).<strong>init</strong>()<br>
self.conv = tf.keras.layers.Conv2D(filters=1, kernel_size=kernel_size, padding=‘same’, activation=‘sigmoid’)</p>
<pre><code>def call(self, inputs):
    attention = self.conv(inputs)
    out = attention * inputs
    return out, attention
</code></pre>
<h1>
<a name="define-the-channel-spatial-attention-4" class="anchor" href="#define-the-channel-spatial-attention-4"></a>Define the channel-spatial attention</h1>
<p>class ChannelSpatialAttention(tf.keras.layers.Layer):<br>
def <strong>init</strong>(self, num_channels, reduction_ratio=8, kernel_size=7):<br>
super(ChannelSpatialAttention, self).<strong>init</strong>()<br>
self.channel_attention = ChannelAttention(num_channels, reduction_ratio)<br>
self.spatial_attention = SpatialAttention(kernel_size)</p>
<pre><code>def call(self, inputs):
    out, attention_channel = self.channel_attention(inputs)
    out, attention_spatial = self.spatial_attention(out)
    attention = attention_channel * attention_spatial
    out = attention * inputs
    return out, attention
</code></pre>
<h1>
<a name="define-a-function-to-get-attention-map-for-input-image-5" class="anchor" href="#define-a-function-to-get-attention-map-for-input-image-5"></a>Define a function to get attention map for input image</h1>
<p>def get_attention_map(model, image, size=(224, 224)):<br>
layer_outputs = [layer.output for layer in model.layers if ‘attention’ in layer.name]<br>
activation_model = tf.keras.models.Model(inputs=model.inputs, outputs=layer_outputs)<br>
activations = activation_model.predict(image)<br>
attention_maps = []<br>
for activation in activations:<br>
if isinstance(activation, tuple):<br>
activation = activation[0]<br>
if activation.ndim != 4:<br>
continue<br>
attention_map = tf.reduce_mean(activation, axis=-1)<br>
attention_map = tf.expand_dims(attention_map, axis=-1)<br>
attention_maps.append(attention_map)<br>
return attention_maps</p>
<h1>
<a name="load-an-image-6" class="anchor" href="#load-an-image-6"></a>Load an image</h1>
<p>img_path = ‘sample_img.jpg’<br>
image = tf.keras.preprocessing.image.load_img(img_path, target_size=(224, 224))<br>
image = tf.keras.preprocessing.image.img_to_array(image)<br>
image = tf.expand_dims(image, axis=0)</p>
<h1>
<a name="create-the-models-7" class="anchor" href="#create-the-models-7"></a>Create the models</h1>
<p>model_self_attention = tf.keras.applications.ResNet50(include_top=False, input_shape=(224, 224, 3))<br>
model_channel_attention = tf.keras.applications.ResNet50(include_top=False, input_shape=(224, 224, 3))<br>
model_spatial_attention = tf.keras.applications.ResNet50(include_top=False, input_shape=(224, 224, 3))<br>
model_channel_spatial_attention = tf.keras.applications.ResNet50(include_top=False, input_shape=(224, 224, 3))</p>
<p>for layer in model_self_attention.layers:<br>
if ‘conv5_block3_out’ in layer.name:<br>
out, _ = SelfAttention(2048)(layer.output)<br>
model_self_attention = tf.keras.models.Model(inputs=model_self_attention.inputs, outputs=out)</p>
<p>for layer in model_channel_attention.layers:<br>
if ‘conv5_block3_out’ in layer.name:<br>
out, _ = ChannelAttention(2048)(layer.output)<br>
model_channel_attention = tf.keras.models.Model(inputs=model_channel_attention.inputs, outputs=out)</p>
<p>for layer in model_spatial_attention.layers:<br>
if ‘conv5_block3_out’ in layer.name:<br>
out, _ = SpatialAttention()(layer.output)<br>
model_spatial_attention = tf.keras.models.Model(inputs=model_spatial_attention.inputs, outputs=out)</p>
<p>for layer in model_channel_spatial_attention.layers:<br>
if ‘conv5_block3_out’ in layer.name:<br>
out, _ = ChannelSpatialAttention(2048)(layer.output)<br>
model_channel_spatial_attention = tf.keras.models.Model(inputs=model_channel_spatial_attention.inputs, outputs=out)</p>
<h1>
<a name="get-the-attention-maps-8" class="anchor" href="#get-the-attention-maps-8"></a>Get the attention maps</h1>
<p>attention_maps_self_attention = get_attention_map(model_self_attention, image)<br>
attention_maps_channel_attention = get_attention_map(model_channel_attention, image)<br>
attention_maps_spatial_attention = get_attention_map(model_spatial_attention, image)<br>
attention_maps_channel_spatial_attention = get_attention_map(model_channel_spatial_attention, image)</p>
<h1>
<a name="plot-the-attention-maps-9" class="anchor" href="#plot-the-attention-maps-9"></a>Plot the attention maps</h1>
<p>plt.figure(figsize=(20, 20))<br>
plt.subplot(331)<br>
plt.imshow(tf.keras.preprocessing.image.array_to_img(image[0]))<br>
plt.title(‘Original Image’)<br>
plt.axis(‘off’)</p>
<p>attention_maps_self_attention = get_attention_map(model_self_attention, image)<br>
plt.subplot(332)<br>
plt.imshow(tf.squeeze(attention_maps_self_attention[0]))<br>
plt.title(‘Self-Attention Map’)<br>
plt.axis(‘off’)</p>
<p>attention_maps_channel_attention = get_attention_map(model_channel_attention, image)<br>
plt.subplot(333)<br>
plt.imshow(tf.squeeze(attention_maps_channel_attention[0]))<br>
plt.title(‘Channel Attention Map’)<br>
plt.axis(‘off’)</p>
<p>plt.subplot(334)<br>
plt.imshow(tf.squeeze(attention_maps_spatial_attention[0]))<br>
plt.title(‘Spatial Attention Map’)<br>
plt.axis(‘off’)</p>
<p>attention_maps_channel_spatial_attention = get_attention_map(model_channel_spatial_attention, image)<br>
plt.subplot(335)<br>
plt.imshow(tf.squeeze(attention_maps_channel_spatial_attention[0]))<br>
plt.title(‘Channel-Spatial Attention Map’)<br>
plt.axis(‘off’)</p>
<p>plt.tight_layout()<br>
plt.show()</p>
<p>print(image.shape)<br>
print(attention_maps_self_attention)</p>
<h1>
<a name="calculate-the-ssim-scores-between-the-attention-maps-and-the-original-image-10" class="anchor" href="#calculate-the-ssim-scores-between-the-attention-maps-and-the-original-image-10"></a>Calculate the SSIM scores between the attention maps and the original image</h1>
<p>ssim_score_self_attention = ssim(image[0], tf.expand_dims(attention_maps_self_attention[0], axis=-1))<br>
ssim_score_channel_attention = ssim(image[0], tf.expand_dims(attention_maps_channel_attention[0], axis=-1))<br>
ssim_score_spatial_attention = ssim(image[0], tf.expand_dims(attention_maps_spatial_attention[0], axis=-1))<br>
ssim_score_channel_spatial_attention = ssim(image[0], tf.expand_dims(attention_maps_channel_spatial_attention[0], axis=-1))</p>

ptrblck: <p>Based on your code snippet it seems you are using TensorFlow with Keras, so I would recommend posting this question in their discussion board as you will find TF experts there <img src="https://discuss.pytorch.org/images/emoji/apple/wink.png?v=12" title=":wink:" class="emoji" alt=":wink:" loading="lazy" width="20" height="20"></p>