added adversarial attacks & change to streamlit 1.19.0

.vscode/settings.json

@@ -1,3 +1,5 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ce4af7987964cdde596ea51e2af55a4f18b804fc77e090cedd3d6ae426c7e401
-size 92
+{
+    "python.analysis.extraPaths": [
+        "./Visual-Explanation-Methods-PyTorch"
+    ]
+}

README.md

@@ -1,7 +1,7 @@
 ---
 title: CNNs Interpretation Visualization
 emoji: 💡
-colorFrom: yellow
+colorFrom: blue
 colorTo: green
 sdk: streamlit
 sdk_version: 1.10.0

backend/adversarial_attack.py

@@ -0,0 +1,99 @@
+import PIL
+from PIL import Image
+import numpy as np
+from matplotlib import pylab as P
+import cv2
+
+import torch
+from torch.utils.data import TensorDataset
+from torchvision import transforms
+import torch.nn.functional as F
+
+from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
+
+from torchvex.base import ExplanationMethod
+from torchvex.utils.normalization import clamp_quantile
+
+from backend.utils import load_image, load_model
+from backend.smooth_grad import generate_smoothgrad_mask
+
+import streamlit as st
+
+IMAGENET_DEFAULT_MEAN = np.asarray(IMAGENET_DEFAULT_MEAN).reshape([1,3,1,1])
+IMAGENET_DEFAULT_STD = np.asarray(IMAGENET_DEFAULT_STD).reshape([1,3,1,1])
+
+def deprocess_image(image_inputs):
+    return (image_inputs * IMAGENET_DEFAULT_STD + IMAGENET_DEFAULT_MEAN) * 255
+
+
+def feed_forward(input_image):
+    model, feature_extractor = load_model('ConvNeXt')
+    inputs = feature_extractor(input_image, do_resize=False, return_tensors="pt")['pixel_values']
+    logits = model(inputs).logits
+    prediction_prob = F.softmax(logits, dim=-1).max()                   # prediction probability
+    # prediction class id, start from 1 to 1000 so it needs to +1 in the end
+    prediction_class = logits.argmax(-1).item()                 
+    prediction_label = model.config.id2label[prediction_class]  # prediction class label
+    return prediction_prob, prediction_class, prediction_label
+
+# FGSM attack code
+def fgsm_attack(image, epsilon, data_grad):
+    # Collect the element-wise sign of the data gradient and normalize it
+    sign_data_grad = torch.gt(data_grad, 0).type(torch.FloatTensor) * 2.0 - 1.0
+    perturbed_image = image + epsilon*sign_data_grad
+    return perturbed_image
+
+# perform attack on the model
+def perform_attack(input_image, target, epsilon):
+    model, feature_extractor = load_model("ConvNeXt")
+    # preprocess input image
+    inputs = feature_extractor(input_image, do_resize=False, return_tensors="pt")['pixel_values']
+    inputs.requires_grad = True
+    
+    # predict
+    logits = model(inputs).logits
+    prediction_prob = F.softmax(logits, dim=-1).max()
+    prediction_class = logits.argmax(-1).item()
+    prediction_label = model.config.id2label[prediction_class]
+    
+    # Calculate the loss
+    loss = F.nll_loss(logits, torch.tensor([target]))
+
+    # Zero all existing gradients
+    model.zero_grad()
+
+    # Calculate gradients of model in backward pass
+    loss.backward()
+    
+    # Collect datagrad
+    data_grad = inputs.grad.data
+    
+    # Call FGSM Attack
+    perturbed_data = fgsm_attack(inputs, epsilon, data_grad)
+    
+    # Re-classify the perturbed image
+    new_prediction = model(perturbed_data).logits
+    new_pred_prob = F.softmax(new_prediction, dim=-1).max()
+    new_pred_class = new_prediction.argmax(-1).item()
+    new_pred_label = model.config.id2label[new_pred_class]
+
+    return perturbed_data, new_pred_prob.item(), new_pred_class, new_pred_label
+
+
+def find_smallest_epsilon(input_image, target):
+    epsilons = [i*0.001 for i in range(1000)]
+
+    for epsilon in epsilons:
+        perturbed_data, new_prob, new_id, new_label = perform_attack(input_image, target, epsilon)
+        if new_id != target:
+            return perturbed_data, new_prob, new_id, new_label, epsilon
+    return None
+
+@st.cache_data
+def generate_images(image_id, epsilon=0):
+    model, feature_extractor = load_model("ConvNeXt")
+    original_image_dict = load_image(image_id)
+    image = original_image_dict['image']
+    return generate_smoothgrad_mask(
+        image, 'ConvNeXt',
+        model, feature_extractor, num_samples=10, return_mask=True)

backend/load_file.py

@@ -19,7 +19,11 @@ def load_json(filename):
         loaded_dict = json.loads(read_file.read())
     loaded_dict = OrderedDict(loaded_dict)
     for k, v in loaded_dict.items():
-        loaded_dict[k] = np.asarray(v)
+        if type(v) == list:
+            loaded_dict[k] = np.asarray(v)
+        else:
+            for k_, v_ in v.items():
+                v[k_] = np.asarray(v_)
     return loaded_dict
 
 class NumpyEncoder(json.JSONEncoder):
@@ -32,6 +36,6 @@ class NumpyEncoder(json.JSONEncoder):
 # save_pickle_to_json('data/layer_infos/resnet_layer_infos.pkl')
 # save_pickle_to_json('data/layer_infos/mobilenet_layer_infos.pkl')
 
-file = load_json('data/layer_infos/convnext_layer_infos.json')
-print(type(file))
-print(type(file['embeddings.patch_embeddings']))
+# file = load_json('data/layer_infos/convnext_layer_infos.json')
+# print(type(file))
+# print(type(file['embeddings.patch_embeddings']))

backend/maximally_activating_patches.py

@@ -4,12 +4,14 @@ import streamlit as st
 from backend.load_file import load_json
 
 
-@st.cache(allow_output_mutation=True)
+# @st.cache(allow_output_mutation=True)
+st.cache_data
 def load_activation(filename):
     activation = load_json(filename)
     return activation
 
-@st.cache(allow_output_mutation=True)
+# @st.cache(allow_output_mutation=True)
+@st.cache_data
 def load_dataset(data_index):
     with open(f'./data/preprocessed_image_net/val_data_{data_index}.pkl', 'rb') as file:
         dataset = pickle.load(file)

backend/smooth_grad.py

@@ -1,5 +1,3 @@
-import sys
-import os
 import PIL
 from PIL import Image
 import numpy as np
@@ -10,8 +8,8 @@ import torch
 from torch.utils.data import TensorDataset
 from torchvision import transforms
 
-dirpath_to_modules = './Visual-Explanation-Methods-PyTorch'
-sys.path.append(dirpath_to_modules)
+# dirpath_to_modules = './Visual-Explanation-Methods-PyTorch'
+# sys.path.append(dirpath_to_modules)
 
 from torchvex.base import ExplanationMethod
 from torchvex.utils.normalization import clamp_quantile
@@ -212,7 +210,7 @@ def fig2img(fig):
     img = Image.open(buf)
     return img
 
-def generate_smoothgrad_mask(image, model_name, model=None, feature_extractor=None, num_samples=25):
+def generate_smoothgrad_mask(image, model_name, model=None, feature_extractor=None, num_samples=25, return_mask=False):
     inputs, prediction_class = feed_forward(model_name, image, model, feature_extractor)
 
     smoothgrad_gen = SmoothGradient(
@@ -230,4 +228,8 @@ def generate_smoothgrad_mask(image, model_name, model=None, feature_extractor=No
     # ori_image = ShowImage(image)
     heat_map_image = ShowHeatMap(smoothgrad_mask)
     masked_image = ShowMaskedImage(smoothgrad_mask, image)
-    return heat_map_image, masked_image
+
+    if return_mask:
+        return heat_map_image, masked_image, smoothgrad_mask
+    else:
+        return heat_map_image, masked_image

backend/utils.py

@@ -14,12 +14,17 @@ from plotly import express as px
 from plotly.subplots import make_subplots
 from tqdm import trange
 
-@st.cache(allow_output_mutation=True)
+import torch
+from transformers import AutoFeatureExtractor, AutoModelForImageClassification
+
+# @st.cache(allow_output_mutation=True)
+@st.cache_resource
 def load_dataset(data_index):
     with open(f'./data/preprocessed_image_net/val_data_{data_index}.pkl', 'rb') as file:
         dataset = pickle.load(file)
     return dataset
 
+@st.cache_resource
 def load_dataset_dict():
     dataset_dict = {}
     progress_empty = st.empty()
@@ -33,6 +38,43 @@ def load_dataset_dict():
     text_empty.empty()
     return dataset_dict
 
+
+@st.cache_data
+def load_image(image_id):
+    dataset = load_dataset(image_id//10000)
+    image = dataset[image_id%10000]
+    return image
+
+@st.cache_data
+def load_images(image_ids):
+    images = []
+    for image_id in image_ids:
+        image = load_image(image_id)
+        images.append(image)
+    return images
+
+
+# @st.cache(allow_output_mutation=True, suppress_st_warning=True, show_spinner=False)
+@st.cache_resource
+def load_model(model_name):
+    with st.spinner(f"Loading {model_name} model! This process might take 1-2 minutes..."):
+        if model_name == 'ResNet':
+            model_file_path = 'microsoft/resnet-50'
+            feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
+            model = AutoModelForImageClassification.from_pretrained(model_file_path)
+            model.eval()
+        elif model_name == 'ConvNeXt':
+            model_file_path = 'facebook/convnext-tiny-224'
+            feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
+            model = AutoModelForImageClassification.from_pretrained(model_file_path)
+            model.eval()
+        else:
+            model = torch.hub.load('pytorch/vision:v0.10.0', 'mobilenet_v2', pretrained=True)
+            model.eval()
+            feature_extractor = None
+    return model, feature_extractor
+
+
 def make_grid(cols=None,rows=None):
     grid = [0]*rows
     for i in range(rows):

load_file.py

@@ -1,37 +0,0 @@
-import json
-import pickle
-import numpy as np
-from collections import OrderedDict
-
-def load_pickle(filename):
-    with open(filename, 'rb') as file:
-        data = pickle.load(file)
-    return data
-
-def save_pickle_to_json(filename):
-    ordered_dict = load_pickle(filename)
-    json_obj = json.dumps(ordered_dict, cls=NumpyEncoder)
-    with open(filename.replace('.pkl', '.json'), 'w') as f:
-        f.write(json_obj)
-
-def load_json(filename):
-    with open(filename, 'r') as read_file:
-        loaded_dict = json.loads(read_file.read())
-    loaded_dict = OrderedDict(loaded_dict)
-    for k, v in loaded_dict.items():
-        loaded_dict[k] = np.asarray(v)
-    return loaded_dict
-
-class NumpyEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return json.JSONEncoder.default(self, obj)
-
-# save_pickle_to_json('data/layer_infos/convnext_layer_infos.pkl')
-# save_pickle_to_json('data/layer_infos/resnet_layer_infos.pkl')
-# save_pickle_to_json('data/layer_infos/mobilenet_layer_infos.pkl')
-
-file = load_json('data/layer_infos/convnext_layer_infos.json')
-print(type(file))
-print(type(file['embeddings.patch_embeddings']))

pages/1_Maximally_activating_patches.py

@@ -28,7 +28,7 @@ def load_dot_to_graph(filename):
     return graph, dot
     
 st.title('Maximally activating image patches')
-st.write('Visualize image patches that maximize the activation of layers in three models: ConvNeXt, ResNet, MobileNet')
+st.write('Visualize image patches that maximize the activation of layers in ConvNeXt model')
 
 # st.header('ConvNeXt')
 convnext_dot_file = './data/dot_architectures/convnext_architecture.dot'
@@ -130,7 +130,7 @@ if nodes != None:
                         subplot_titles=tuple([f"#{i+1}" for i in range(top_k)]), shared_yaxes=True)
                 else:
                     top_margin = 0
-                    fig = make_subplots(rows=1, cols=num_cols)
+                    fig = make_subplots(rows=1, cols=num_cols, shared_yaxes=True)
                 for col in range(1, num_cols+1):
                     k, c = col-1, row-1
                     img_index = int(top_k_coor_max_[k, c, 3])

pages/2_SmoothGrad.py

@@ -2,7 +2,7 @@ import streamlit as st
 import pandas as pd
 import numpy as np
 import random
-from backend.utils import make_grid, load_dataset
+from backend.utils import make_grid, load_dataset, load_model, load_images
 
 from backend.smooth_grad import generate_smoothgrad_mask, ShowImage, fig2img
 from transformers import AutoFeatureExtractor, AutoModelForImageClassification
@@ -22,32 +22,34 @@ imagenet_df = pd.read_csv('./data/ImageNet_metadata.csv')
 
 # --------------------------- LOAD function -----------------------------
 
-@st.cache(allow_output_mutation=True)
-def load_images(image_ids):
-    images = []
-    for image_id in image_ids:
-        dataset = load_dataset(image_id//10000)
-        images.append(dataset[image_id%10000])
-    return images
-
-@st.cache(allow_output_mutation=True, suppress_st_warning=True, show_spinner=False)
-def load_model(model_name):
-    with st.spinner(f"Loading {model_name} model! This process might take 1-2 minutes..."):
-        if model_name == 'ResNet':
-            model_file_path = 'microsoft/resnet-50'
-            feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
-            model = AutoModelForImageClassification.from_pretrained(model_file_path)
-            model.eval()
-        elif model_name == 'ConvNeXt':
-            model_file_path = 'facebook/convnext-tiny-224'
-            feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
-            model = AutoModelForImageClassification.from_pretrained(model_file_path)
-            model.eval()
-        else:
-            model = torch.hub.load('pytorch/vision:v0.10.0', 'mobilenet_v2', pretrained=True)
-            model.eval()
-            feature_extractor = None  
-    return model, feature_extractor
+# @st.cache(allow_output_mutation=True)
+# @st.cache_data
+# def load_images(image_ids):
+#     images = []
+#     for image_id in image_ids:
+#         dataset = load_dataset(image_id//10000)
+#         images.append(dataset[image_id%10000])
+#     return images
+
+# @st.cache(allow_output_mutation=True, suppress_st_warning=True, show_spinner=False)
+# @st.cache_resource
+# def load_model(model_name):
+#     with st.spinner(f"Loading {model_name} model! This process might take 1-2 minutes..."):
+#         if model_name == 'ResNet':
+#             model_file_path = 'microsoft/resnet-50'
+#             feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
+#             model = AutoModelForImageClassification.from_pretrained(model_file_path)
+#             model.eval()
+#         elif model_name == 'ConvNeXt':
+#             model_file_path = 'facebook/convnext-tiny-224'
+#             feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
+#             model = AutoModelForImageClassification.from_pretrained(model_file_path)
+#             model.eval()
+#         else:
+#             model = torch.hub.load('pytorch/vision:v0.10.0', 'mobilenet_v2', pretrained=True)
+#             model.eval()
+#             feature_extractor = None
+#     return model, feature_extractor
 
 images = []
 image_ids = []
@@ -56,28 +58,7 @@ st.header('Input')
 with st.form('smooth_grad_form'):
     st.markdown('**Model and Input Setting**')
     selected_models = st.multiselect('Model', options=['ConvNeXt', 'ResNet', 'MobileNet'])
-    # selected_image_set = st.selectbox('Image set', ['Random set', 'User-defined set'])
     selected_image_set = st.selectbox('Image set', ['User-defined set', 'Random set'])
-
-    # if selected_image_set == 'Class set':
-    #     class_labels = imagenet_df.ClassLabel.unique().tolist()
-    #     class_labels.sort()
-    #     selected_classes = st.multiselect('Class filter', options=['All'] + class_labels)
-    #     if not ('All' in selected_classes or len(selected_classes) == 0):
-    #         imagenet_df = imagenet_df[imagenet_df['ClassLabel'].isin(selected_classes)]
-    #     no_images = st.slider('Number of images', 1, len(imagenet_df), value=10)
-    #     image_ids = random.sample(imagenet_df.index.tolist(), k=no_images)
-    
-
-    # user_defined_button = st.form_submit_button('User-defined set')
-    # random_set_button = st.form_submit_button('Random set')
-
-    # if user_defined_button:
-    #     text = st.text_area('Specific Image IDs', value='0')
-    #     image_ids = list(map(lambda x: int(x.strip()), text.split(',')))
-    # if random_set_button:
-    #     no_images = st.slider('Number of images', 1, 50, value=10)
-    #     image_ids = random.sample(list(range(50_000)), k=no_images)
     
     summit_button = st.form_submit_button('Set')
     if summit_button:
@@ -123,8 +104,11 @@ grids = make_grid(cols=2+len(selected_models)*2, rows=len(image_ids)+1)
 #     models[model_name], feature_extractors[model_name] = load_model(model_name)
 
 
-@st.cache(allow_output_mutation=True)
-def generate_images(image, model_name):
+# @st.cache(allow_output_mutation=True)
+@st.cache_data
+def generate_images(image_id, model_name):
+    j = image_ids.index(image_id)
+    image = images[j]['image']
     return generate_smoothgrad_mask(
         image, model_name,
         models[model_name], feature_extractors[model_name], num_samples=10)
@@ -139,7 +123,7 @@ with _lock:
         for i, model_name in enumerate(selected_models):
             # ori_image, heatmap_image, masked_image = generate_smoothgrad_mask(image,
             # model_name, models[model_name], feature_extractors[model_name], num_samples=10)
-            heatmap_image, masked_image = generate_images(image, model_name)
+            heatmap_image, masked_image = generate_images(image_id, model_name)
             # grids[j][1].image(ori_image)
             grids[j][i*2+2].image(heatmap_image)
             grids[j][i*2+3].image(masked_image)

pages/3_Adversarial_attack.py

@@ -0,0 +1,184 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import random
+from backend.utils import make_grid, load_dataset, load_model, load_image
+
+from backend.smooth_grad import generate_smoothgrad_mask, ShowImage, fig2img, LoadImage, ShowHeatMap, ShowMaskedImage
+from transformers import AutoFeatureExtractor, AutoModelForImageClassification
+import torch
+
+from matplotlib.backends.backend_agg import RendererAgg
+
+from backend.adversarial_attack import *
+
+_lock = RendererAgg.lock
+
+st.set_page_config(layout='wide')
+BACKGROUND_COLOR = '#bcd0e7'
+SECONDARY_COLOR = '#bce7db'
+
+
+st.title('Adversarial Attack')
+st.write('How adversarial attacks affect ConvNeXt interpretation?')
+
+imagenet_df = pd.read_csv('./data/ImageNet_metadata.csv')
+image_id = None
+
+if 'image_id' not in st.session_state:
+    st.session_state.image_id = 0
+
+# def on_change_random_input():
+#     st.session_state.image_id = st.session_state.image_id
+
+# ----------------------------- INPUT ----------------------------------
+st.header('Input')
+input_col_1, input_col_2, input_col_3 = st.columns(3)
+# --------------------------- INPUT column 1 ---------------------------
+with input_col_1:
+    with st.form('image_form'):
+        
+        # image_id = st.number_input('Image ID: ', format='%d', step=1)
+        st.write('**Choose or generate a random image**')
+        chosen_image_id_input = st.empty()
+        image_id = chosen_image_id_input.number_input('Image ID:', format='%d', step=1, value=st.session_state.image_id)
+        
+        choose_image_button = st.form_submit_button('Choose the defined image')
+        random_id = st.form_submit_button('Generate a random image')
+
+        if random_id:
+            image_id = random.randint(0, 50000)
+            st.session_state.image_id = image_id
+            chosen_image_id_input.number_input('Image ID:', format='%d', step=1, value=st.session_state.image_id)
+            
+        if choose_image_button:
+            image_id = int(image_id)
+            st.session_state.image_id = int(image_id)
+        # st.write(image_id, st.session_state.image_id)
+
+# ---------------------------- SET UP OUTPUT ------------------------------
+epsilon_container = st.empty()
+st.header('Output')
+st.subheader('Perform attack')
+
+# perform attack container
+header_col_1, header_col_2, header_col_3, header_col_4, header_col_5 = st.columns([1,1,1,1,1])
+output_col_1, output_col_2, output_col_3, output_col_4, output_col_5 = st.columns([1,1,1,1,1])
+
+# prediction error container
+error_container = st.empty()
+smoothgrad_header_container = st.empty()
+
+# smoothgrad container
+smooth_head_1, smooth_head_2, smooth_head_3, smooth_head_4, smooth_head_5 = st.columns([1,1,1,1,1])
+smoothgrad_col_1, smoothgrad_col_2, smoothgrad_col_3, smoothgrad_col_4, smoothgrad_col_5 = st.columns([1,1,1,1,1])
+
+original_image_dict = load_image(st.session_state.image_id)
+input_image = original_image_dict['image']
+input_label = original_image_dict['label']
+input_id = original_image_dict['id']
+
+# ---------------------------- DISPLAY COL 1 ROW 1 ------------------------------
+with output_col_1:
+    pred_prob, pred_class_id, pred_class_label = feed_forward(input_image)
+    # st.write(f'Class ID {input_id} - {input_label}: {pred_prob*100:.3f}% confidence')
+    st.image(input_image)
+    header_col_1.write(f'Class ID {input_id} - {input_label}: {pred_prob*100:.1f}% confidence')
+
+
+
+if pred_class_id != (input_id-1):
+    with error_container.container():
+        st.write(f'Predicted output: Class ID {pred_class_id} - {pred_class_label} {pred_prob*100:.1f}% confidence')
+        st.error('ConvNeXt misclassified the chosen image. Please choose or generate another image.',
+            icon = "🚫")
+
+# ----------------------------- INPUT column 2 & 3 ----------------------------
+with input_col_2:
+    with st.form('epsilon_form'):
+        st.write('**Set epsilon or find the smallest epsilon automatically**')
+        chosen_epsilon_input = st.empty()
+        epsilon = chosen_epsilon_input.number_input('Epsilon:', min_value=0.001, format='%.3f', step=0.001)
+
+        epsilon_button = st.form_submit_button('Choose the defined epsilon')
+        find_epsilon = st.form_submit_button('Find the smallest epsilon automatically')
+
+
+with input_col_3:
+    with st.form('iterate_epsilon_form'):
+        max_epsilon = st.number_input('Maximum value of epsilon (Optional setting)', value=0.500, format='%.3f')
+        step_epsilon = st.number_input('Step (Optional setting)', value=0.001, format='%.3f')
+        setting_button = st.form_submit_button('Set iterating mode')
+
+
+# ---------------------------- DISPLAY COL 2 ROW 1 ------------------------------
+if pred_class_id == (input_id-1) and (epsilon_button or find_epsilon or setting_button):
+    with output_col_3:
+        if epsilon_button:
+            perturbed_data, new_prob, new_id, new_label = perform_attack(input_image, input_id-1, epsilon)
+        else:
+            epsilons = [i*step_epsilon for i in range(1, 1001) if i*step_epsilon <= max_epsilon]
+            epsilon_container.progress(0, text='Checking epsilon')
+        
+            for i, e in enumerate(epsilons):
+                print(e)
+                
+                perturbed_data, new_prob, new_id, new_label = perform_attack(input_image, input_id-1, e)
+                epsilon_container.progress(i/len(epsilons), text=f'Checking epsilon={e:.3f}. Confidence={new_prob*100:.1f}%')
+                epsilon = e
+
+                if new_id != input_id - 1:
+                    epsilon_container.empty()
+                    st.balloons()
+                    break
+                if i == len(epsilons)-1:
+                    epsilon_container.error(f'FSGM failed to attack on this image at epsilon={e:.3f}. Set higher maximum value of epsilon or choose another image',
+                                            icon = "🚫")
+
+        perturbed_image = deprocess_image(perturbed_data.detach().numpy())[0].astype(np.uint8).transpose(1,2,0)
+        perturbed_amount = perturbed_image - input_image
+        header_col_3.write(f'Pertubed amount - epsilon={epsilon:.3f}')
+        st.image(ShowImage(perturbed_amount))
+    
+    with output_col_2:
+        # st.write('plus sign')
+        st.image(LoadImage('frontend/images/plus-sign.png'))
+    
+    with output_col_4:
+        # st.write('equal sign')
+        st.image(LoadImage('frontend/images/equal-sign.png'))
+
+    # ---------------------------- DISPLAY COL 5 ROW 1 ------------------------------
+    with output_col_5:
+        # st.write(f'ID {new_id+1} - {new_label}: {new_prob*100:.3f}% confidence')
+        st.image(ShowImage(perturbed_image))
+    header_col_5.write(f'Class ID {new_id+1} - {new_label}: {new_prob*100:.1f}% confidence')
+
+    # -------------------------- DISPLAY SMOOTHGRAD ---------------------------
+    smoothgrad_header_container.subheader('SmoothGrad visualization')
+
+    with smoothgrad_col_1:
+        smooth_head_1.write(f'SmoothGrad before attacked')
+        heatmap_image, masked_image, mask = generate_images(st.session_state.image_id, epsilon=0)
+        st.image(heatmap_image)
+        st.image(masked_image) 
+    with smoothgrad_col_3:
+        smooth_head_3.write('SmoothGrad after attacked')
+        heatmap_image_attacked, masked_image_attacked, attacked_mask= generate_images(st.session_state.image_id, epsilon=epsilon)
+        st.image(heatmap_image_attacked)
+        st.image(masked_image_attacked)
+    
+    with smoothgrad_col_2:
+        st.image(LoadImage('frontend/images/minus-sign-5.png'))
+
+    with smoothgrad_col_5:
+        smooth_head_5.write('SmoothGrad difference')
+        difference_mask = abs(attacked_mask-mask)
+        st.image(ShowHeatMap(difference_mask))
+        masked_image = ShowMaskedImage(difference_mask, perturbed_image)
+        st.image(masked_image)
+
+    with smoothgrad_col_4:
+        st.image(LoadImage('frontend/images/equal-sign.png'))
+
+

requirements.txt

@@ -1,18 +1,17 @@
 captum==0.5.0
-deta==1.1.0
 graphviz==0.20.1
 Markdown==3.4.1
 matplotlib==3.6.2
 numpy==1.22.3
 opencv_python_headless==4.6.0.66
 pandas==1.5.2
-Pillow==9.3.0
+Pillow==9.4.0
 plotly==5.11.0
-scipy==1.9.3
+scipy==1.10.1
 setuptools==65.5.0
-# streamlit==1.15.2
-streamlit==1.10.0
+streamlit==1.19.0
 torch==1.10.1
 torchvision==0.11.2
 tqdm==4.64.1
 transformers==4.25.1
+git+https://github.com/vlue-c/Visual-Explanation-Methods-PyTorch.git