Initial manual application publish

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+utils/resources/models/xview_model.pt filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,633 @@
+'''
+HEART Gradio Example App
+
+To run: 
+- clone the repository
+- execute: gradio examples/gradio_app.py or python examples/gradio_app.py
+- navigate to local URL e.g. http://127.0.0.1:7860
+'''
+
+import torch
+import numpy as np
+import pandas as pd
+# from carbon_theme import Carbon
+
+import gradio as gr
+import os
+
+css = """
+.input-image { margin: auto !important }
+.small-font span{
+ font-size: 0.6em;
+}
+.df-padding {
+    padding-left: 50px !important;
+    padding-right: 50px !important;
+}
+"""
+
+def basic_cifar10_model():
+    '''
+    Load an example CIFAR10 model
+    '''
+    from heart.estimators.classification.pytorch import JaticPyTorchClassifier
+    
+    labels = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
+    path = './'
+    class Model(torch.nn.Module):
+            """
+            Create model for pytorch.
+            Here the model does not use maxpooling. Needed for certification tests.
+            """
+
+            def __init__(self):
+                super(Model, self).__init__()
+
+                self.conv = torch.nn.Conv2d(
+                    in_channels=3, out_channels=16, kernel_size=(4, 4), dilation=(1, 1), padding=(0, 0), stride=(3, 3)
+                )
+
+                self.fullyconnected = torch.nn.Linear(in_features=1600, out_features=10)
+
+                self.relu = torch.nn.ReLU()
+
+                w_conv2d = np.load(
+                    os.path.join(
+                        os.path.dirname(path),
+                        "utils/resources/models",
+                        "W_CONV2D_NO_MPOOL_CIFAR10.npy",
+                    )
+                )
+                b_conv2d = np.load(
+                    os.path.join(
+                        os.path.dirname(path),
+                        "utils/resources/models",
+                        "B_CONV2D_NO_MPOOL_CIFAR10.npy",
+                    )
+                )
+                w_dense = np.load(
+                    os.path.join(
+                        os.path.dirname(path),
+                        "utils/resources/models",
+                        "W_DENSE_NO_MPOOL_CIFAR10.npy",
+                    )
+                )
+                b_dense = np.load(
+                    os.path.join(
+                        os.path.dirname(path),
+                        "utils/resources/models",
+                        "B_DENSE_NO_MPOOL_CIFAR10.npy",
+                    )
+                )
+
+                self.conv.weight = torch.nn.Parameter(torch.Tensor(w_conv2d))
+                self.conv.bias = torch.nn.Parameter(torch.Tensor(b_conv2d))
+                self.fullyconnected.weight = torch.nn.Parameter(torch.Tensor(w_dense))
+                self.fullyconnected.bias = torch.nn.Parameter(torch.Tensor(b_dense))
+
+            # pylint: disable=W0221
+            # disable pylint because of API requirements for function
+            def forward(self, x):
+                """
+                Forward function to evaluate the model
+                :param x: Input to the model
+                :return: Prediction of the model
+                """
+                x = self.conv(x)
+                x = self.relu(x)
+                x = x.reshape(-1, 1600)
+                x = self.fullyconnected(x)
+                return x
+
+    # Define the network
+    model = Model()
+
+    # Define a loss function and optimizer
+    loss_fn = torch.nn.CrossEntropyLoss(reduction="sum")
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
+
+    # Get classifier
+    jptc = JaticPyTorchClassifier(
+        model=model, loss=loss_fn, optimizer=optimizer, input_shape=(3, 32, 32), nb_classes=10, clip_values=(0, 1), labels=labels
+    )
+    return jptc
+
+def clf_evasion_evaluate(*args):
+    '''
+    Run a classification task evaluation
+    '''
+    
+    attack = args[0]
+    model_type = args[1]
+    model_path = args[2]
+    model_channels = args[3]
+    model_height = args[4]
+    model_width = args[5]
+    model_clip = args[6]
+    
+    dataset_type = args[-4]
+    dataset_path = args[-3]
+    dataset_split = args[-2]
+    image = args[-1]
+    
+    if dataset_type == "Example XView":
+        from maite import load_dataset
+        import torchvision
+        jatic_dataset = load_dataset(
+            provider="huggingface",
+            dataset_name="CDAO/xview-subset-classification",
+            task="image-classification",
+            split="test",
+        )
+        IMAGE_H, IMAGE_W = 224, 224
+        transform = torchvision.transforms.Compose(
+            [  
+                torchvision.transforms.Resize((IMAGE_H, IMAGE_W)),
+                torchvision.transforms.ToTensor(),
+            ]
+        )  
+        jatic_dataset.set_transform(lambda x: {"image": transform(x["image"]), "label": x["label"]})
+        image = {'image': [i['image'].numpy() for i in jatic_dataset],
+                'label': [i['label'] for i in jatic_dataset]}   
+    elif dataset_type=="huggingface":
+        from maite import load_dataset
+        jatic_dataset = load_dataset(
+            provider=dataset_type,
+            dataset_name=dataset_path,
+            task="image-classification",
+            split=dataset_split,
+            drop_labels=False
+        )
+        
+        image = {'image': [i['image'] for i in jatic_dataset],
+                'label': [i['label'] for i in jatic_dataset]}
+    elif dataset_type=="torchvision":
+        from maite import load_dataset
+        jatic_dataset = load_dataset(
+            provider=dataset_type,
+            dataset_name=dataset_path,
+            task="image-classification",
+            split=dataset_split,
+            root='./data/',
+            download=True
+        )
+        image = {'image': [i['image'] for i in jatic_dataset],
+                'label': [i['label'] for i in jatic_dataset]}  
+    elif dataset_type=="Example CIFAR10":
+        from maite import load_dataset
+        jatic_dataset = load_dataset(
+            provider="torchvision",
+            dataset_name="CIFAR10",
+            task="image-classification",
+            split=dataset_split,
+            root='./data/',
+            download=True
+        )
+        image = {'image': [i['image'] for i in jatic_dataset][:100],
+                'label': [i['label'] for i in jatic_dataset][:100]}  
+        
+    if model_type == "Example CIFAR10":
+        jptc = basic_cifar10_model()  
+    elif model_type == "Example XView":
+        import torchvision
+        from heart.estimators.classification.pytorch import JaticPyTorchClassifier
+        classes = {
+            0:'Building',
+            1:'Construction Site',
+            2:'Engineering Vehicle',
+            3:'Fishing Vessel',
+            4:'Oil Tanker',
+            5:'Vehicle Lot'
+        }
+        model = torchvision.models.resnet18(False)
+        num_ftrs = model.fc.in_features 
+        model.fc = torch.nn.Linear(num_ftrs, len(classes.keys())) 
+        model.load_state_dict(torch.load('./utils/resources/models/xview_model.pt'))
+        _ = model.eval()
+        jptc = JaticPyTorchClassifier(
+            model=model, loss = torch.nn.CrossEntropyLoss(), input_shape=(3, 224, 224),
+            nb_classes=len(classes), clip_values=(0, 1), labels=list(classes.values())
+        )
+    elif model_type == "torchvision":
+        from maite.interop.torchvision import TorchVisionClassifier 
+        from heart.estimators.classification.pytorch import JaticPyTorchClassifier
+        
+        clf = TorchVisionClassifier.from_pretrained(model_path)
+        loss_fn = torch.nn.CrossEntropyLoss(reduction="sum")
+        jptc = JaticPyTorchClassifier(
+            model=clf._model, loss=loss_fn, input_shape=(model_channels, model_height, model_width), 
+            nb_classes=len(clf._labels), clip_values=(0, model_clip), labels=clf._labels
+        )
+    elif model_type == "huggingface":
+        from maite.interop.huggingface import HuggingFaceImageClassifier 
+        from heart.estimators.classification.pytorch import JaticPyTorchClassifier
+        
+        clf = HuggingFaceImageClassifier.from_pretrained(model_path)
+        loss_fn = torch.nn.CrossEntropyLoss(reduction="sum")
+        jptc = JaticPyTorchClassifier(
+            model=clf._model, loss=loss_fn, input_shape=(model_channels, model_height, model_width), 
+            nb_classes=len(clf._labels), clip_values=(0, model_clip), labels=clf._labels
+        )
+    
+    if attack=="PGD":
+        from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_pytorch import ProjectedGradientDescentPyTorch
+        from heart.attacks.attack import JaticAttack
+        from heart.metrics import AccuracyPerturbationMetric
+        from torch.nn.functional import softmax
+        from maite.protocols import HasDataImage, is_typed_dict, ArrayLike
+        
+        pgd_attack = ProjectedGradientDescentPyTorch(estimator=jptc, max_iter=args[7], eps=args[8],
+                                                 eps_step=args[9], targeted=args[10]!="")
+        attack = JaticAttack(pgd_attack)
+        
+        preds = jptc(image)
+        preds = softmax(torch.from_numpy(preds.logits), dim=1)
+        labels = {}
+        for i, label in enumerate(jptc.get_labels()):
+            labels[label] = preds[0][i]
+        
+        if args[10]!="":
+            if is_typed_dict(image, HasDataImage):
+                data = {'image': image['image'], 'label': [args[10]]*len(image['image'])}
+            else:
+                data = {'image': image, 'label': [args[10]]}
+        else:
+            data = image
+        
+        x_adv = attack.run_attack(data=data)
+        adv_preds = jptc(x_adv.adversarial_examples)
+        adv_preds = softmax(torch.from_numpy(adv_preds.logits), dim=1)
+        adv_labels = {}
+        for i, label in enumerate(jptc.get_labels()):
+            adv_labels[label] = adv_preds[0][i]
+        
+        metric = AccuracyPerturbationMetric()
+        metric.update(jptc, jptc.device, image, x_adv.adversarial_examples)
+        clean_accuracy, robust_accuracy, perturbation_added = metric.compute()
+        metrics = pd.DataFrame([[clean_accuracy, robust_accuracy, perturbation_added]],
+                               columns=['clean accuracy', 'robust accuracy', 'perturbation'])
+
+        adv_imgs = [img.transpose(1,2,0) for img in x_adv.adversarial_examples]
+        if is_typed_dict(image, HasDataImage):
+            image = image['image']
+        if not isinstance(image, list):
+            image = [image]
+            
+        # in case where multiple images, use argmax to get the predicted label and add as caption
+        if dataset_type!="local":
+            temp = []
+            for i, img in enumerate(image):
+                if isinstance(img, ArrayLike):
+                    temp.append((img.transpose(1,2,0), str(jptc.get_labels()[np.argmax(preds[i])]) ))
+                else:
+                    temp.append((img, str(jptc.get_labels()[np.argmax(preds[i])]) ))
+            image = temp
+            
+            temp = []
+            for i, img in enumerate(adv_imgs):
+                temp.append((img, str(jptc.get_labels()[np.argmax(adv_preds[i])]) ))
+            adv_imgs = temp
+        
+        return [image, labels, adv_imgs, adv_labels, clean_accuracy, robust_accuracy, perturbation_added]
+
+    elif attack=="Adversarial Patch":
+        from art.attacks.evasion.adversarial_patch.adversarial_patch_pytorch import AdversarialPatchPyTorch
+        from heart.attacks.attack import JaticAttack
+        from heart.metrics import AccuracyPerturbationMetric
+        from torch.nn.functional import softmax
+        from maite.protocols import HasDataImage, is_typed_dict, ArrayLike
+        
+        batch_size = 16
+        scale_min = 0.3
+        scale_max = 1.0
+        rotation_max = 0
+        learning_rate = 5000.
+        max_iter = 2000
+        patch_shape = (3, 14, 14)
+        patch_location = (18,18)
+
+        patch_attack = AdversarialPatchPyTorch(estimator=jptc, rotation_max=rotation_max, patch_location=(args[8], args[9]),
+                            scale_min=scale_min, scale_max=scale_max, patch_type='square',
+                            learning_rate=learning_rate, max_iter=args[7], batch_size=batch_size,
+                            patch_shape=(3, args[10], args[11]), verbose=False, targeted=args[12]!="")
+        
+        attack = JaticAttack(patch_attack)
+        
+        preds = jptc(image)
+        preds = softmax(torch.from_numpy(preds.logits), dim=1)
+        labels = {}
+        for i, label in enumerate(jptc.get_labels()):
+            labels[label] = preds[0][i]
+        
+        if args[12]!="":
+            if is_typed_dict(image, HasDataImage):
+                data = {'image': image['image'], 'label': [args[12]]*len(image['image'])}
+            else:
+                data = {'image': image, 'label': [args[12]]}
+        else:
+            data = image
+        
+        attack_output = attack.run_attack(data=data)
+        adv_preds = jptc(attack_output.adversarial_examples)
+        adv_preds = softmax(torch.from_numpy(adv_preds.logits), dim=1)
+        adv_labels = {}
+        for i, label in enumerate(jptc.get_labels()):
+            adv_labels[label] = adv_preds[0][i]
+        
+        metric = AccuracyPerturbationMetric()
+        metric.update(jptc, jptc.device, image, attack_output.adversarial_examples)
+        clean_accuracy, robust_accuracy, perturbation_added = metric.compute()
+        metrics = pd.DataFrame([[clean_accuracy, robust_accuracy, perturbation_added]],
+                               columns=['clean accuracy', 'robust accuracy', 'perturbation'])
+
+        adv_imgs = [img.transpose(1,2,0) for img in attack_output.adversarial_examples]
+        if is_typed_dict(image, HasDataImage):
+            image = image['image']
+        if not isinstance(image, list):
+            image = [image]
+            
+        # in case where multiple images, use argmax to get the predicted label and add as caption
+        if dataset_type!="local":
+            temp = []
+            for i, img in enumerate(image):
+                
+                if isinstance(img, ArrayLike):
+                    temp.append((img.transpose(1,2,0), str(jptc.get_labels()[np.argmax(preds[i])]) ))
+                else:
+                    temp.append((img, str(jptc.get_labels()[np.argmax(preds[i])]) ))
+                
+            image = temp
+            
+            temp = []
+            for i, img in enumerate(adv_imgs):
+                temp.append((img, str(jptc.get_labels()[np.argmax(adv_preds[i])]) ))
+            adv_imgs = temp
+            
+        patch, patch_mask = attack_output.adversarial_patch
+        patch_image = ((patch) * patch_mask).transpose(1,2,0)
+            
+        return [image, labels, adv_imgs, adv_labels, clean_accuracy, robust_accuracy, patch_image]
+            
+def show_model_params(model_type):
+    '''
+    Show model parameters based on selected model type
+    '''
+    if model_type!="Example CIFAR10" and model_type!="Example XView":
+        return gr.Column(visible=True)
+    return gr.Column(visible=False)
+    
+def show_dataset_params(dataset_type):
+    '''
+    Show dataset parameters based on dataset type
+    '''
+    if dataset_type=="Example CIFAR10" or dataset_type=="Example XView":
+        return [gr.Column(visible=False), gr.Row(visible=False), gr.Row(visible=False)]
+    elif dataset_type=="local":
+        return [gr.Column(visible=True), gr.Row(visible=True), gr.Row(visible=False)]
+    return [gr.Column(visible=True), gr.Row(visible=False), gr.Row(visible=True)]
+  
+def pgd_show_label_output(dataset_type):
+    '''
+    Show PGD output component based on dataset type
+    '''
+    if dataset_type=="local":
+        return [gr.Label(visible=True), gr.Label(visible=True), gr.Number(visible=False), gr.Number(visible=False), gr.Number(visible=True)]
+    return [gr.Label(visible=False), gr.Label(visible=False), gr.Number(visible=True), gr.Number(visible=True), gr.Number(visible=True)]
+
+def pgd_update_epsilon(clip_values):
+    '''
+    Update max value of PGD epsilon slider based on model clip values
+    '''
+    if clip_values == 255:
+        return gr.Slider(minimum=0.0001, maximum=255, label="Epslion", value=55) 
+    return gr.Slider(minimum=0.0001, maximum=1, label="Epslion", value=0.05) 
+
+def patch_show_label_output(dataset_type):
+    '''
+    Show adversarial patch output components based on dataset type
+    '''
+    if dataset_type=="local":
+        return [gr.Label(visible=True), gr.Label(visible=True), gr.Number(visible=False), gr.Number(visible=False), gr.Number(visible=True)]
+    return [gr.Label(visible=False), gr.Label(visible=False), gr.Number(visible=True), gr.Number(visible=True), gr.Number(visible=True)]
+
+def show_target_label_dataframe(dataset_type):
+    if dataset_type == "Example CIFAR10":
+        return gr.Dataframe(visible=True), gr.Dataframe(visible=False)
+    elif dataset_type == "Example XView":
+        return gr.Dataframe(visible=False), gr.Dataframe(visible=True)
+    return gr.Dataframe(visible=False), gr.Dataframe(visible=False)
+    
+# e.g. To use a local alternative theme: carbon_theme = Carbon()
+with gr.Blocks(css=css, theme='xiaobaiyuan/theme_brief') as demo:
+    gr.Markdown("<h1>HEART Adversarial Robustness Gradio Example</h1>")
+    
+    with gr.Tab("Info"):
+        gr.Markdown('This is step 1. Using the tabs, select a task for evaluation.')
+    
+    with gr.Tab("Classification", elem_classes="task-tab"):
+        gr.Markdown("Classifying images with a set of categories.")
+        
+        # Model and Dataset Selection
+        with gr.Row():
+            # Model and Dataset type e.g. Torchvision, HuggingFace, local etc.
+            with gr.Column():
+                model_type = gr.Radio(label="Model type", choices=["Example CIFAR10", "Example XView", "torchvision"],
+                                    value="Example CIFAR10")
+                dataset_type = gr.Radio(label="Dataset", choices=["Example CIFAR10", "Example XView", "local", "torchvision", "huggingface"],
+                                    value="Example CIFAR10")
+            # Model parameters e.g. RESNET, VIT, input dimensions, clipping values etc.
+            with gr.Column(visible=False) as model_params:
+                model_path = gr.Textbox(placeholder="URL", label="Model path")
+                with gr.Row():
+                    with gr.Column():
+                        model_channels = gr.Textbox(placeholder="Integer, 3 for RGB images", label="Input Channels", value=3)
+                    with gr.Column():
+                        model_width = gr.Textbox(placeholder="Integer", label="Input Width", value=640)
+                with gr.Row():
+                    with gr.Column():
+                        model_height = gr.Textbox(placeholder="Integer", label="Input Height", value=480)
+                    with gr.Column():
+                        model_clip = gr.Radio(choices=[1, 255], label="Pixel clip", value=1)
+            # Dataset parameters e.g. Torchvision, HuggingFace, local etc. 
+            with gr.Column(visible=False) as dataset_params:
+                with gr.Row() as local_image:
+                    image = gr.Image(sources=['upload'], type="pil", height=150, width=150, elem_classes="input-image")
+                with gr.Row() as hosted_image:
+                    dataset_path = gr.Textbox(placeholder="URL", label="Dataset path")
+                    dataset_split = gr.Textbox(placeholder="test", label="Dataset split")
+            
+            model_type.change(show_model_params, model_type, model_params)
+            dataset_type.change(show_dataset_params, dataset_type, [dataset_params, local_image, hosted_image])
+        
+        # Attack Selection
+        with gr.Row():
+            
+            with gr.Tab("Info"):
+                gr.Markdown("This is step 2. Select the type of attack for evaluation.")
+                
+            with gr.Tab("White Box"):
+                gr.Markdown("White box attacks assume the attacker has __full access__ to the model.")
+                
+                with gr.Tab("Info"):
+                    gr.Markdown("This is step 3. Select the type of white-box attack to evaluate.")
+                
+                with gr.Tab("Evasion"):
+                    gr.Markdown("Evasion attacks are deployed to cause a model to incorrectly classify or detect items/objects in an image.")
+                    
+                    with gr.Tab("Info"):
+                        gr.Markdown("This is step 4. Select the type of Evasion attack to evaluate.")
+                    
+                    with gr.Tab("Projected Gradient Descent"):
+                        gr.Markdown("This attack uses PGD to identify adversarial examples.")
+                        
+                        
+                        with gr.Row():
+                            
+                            with gr.Column():
+                                attack = gr.Textbox(visible=True, value="PGD", label="Attack", interactive=False)
+                                max_iter = gr.Slider(minimum=1, maximum=5000, label="Max iterations", value=1000)
+                                eps = gr.Slider(minimum=0.0001, maximum=1, label="Epslion", value=0.05) 
+                                eps_steps = gr.Slider(minimum=0.001, maximum=1000, label="Epsilon steps", value=0.1) 
+                                targeted = gr.Textbox(placeholder="Target label (integer)", label="Target")
+                                with gr.Accordion("Target mapping", open=False):
+                                    cifar_labels = gr.Dataframe(pd.DataFrame(['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'],
+                                                                columns=['label']).rename_axis('target').reset_index(),
+                                                                visible=True, elem_classes=["small-font", "df-padding"],
+                                                                type="pandas",interactive=False)
+                                    xview_labels = gr.Dataframe(pd.DataFrame(['Building', 'Construction Site', 'Engineering Vehicle', 'Fishing Vessel', 'Oil Tanker', 
+                                                                            'Vehicle Lot'],
+                                                                columns=['label']).rename_axis('target').reset_index(), 
+                                                                visible=False, elem_classes=["small-font", "df-padding"],
+                                                                type="pandas",interactive=False)
+                                eval_btn_pgd = gr.Button("Evaluate")
+                                model_clip.change(pgd_update_epsilon, model_clip, eps)
+                                dataset_type.change(show_target_label_dataframe, dataset_type, [cifar_labels, xview_labels])
+                                
+                            # Evaluation Output. Visualisations of success/failures of running evaluation attacks.
+                            with gr.Column():
+                                with gr.Row():
+                                    with gr.Column():
+                                        original_gallery = gr.Gallery(label="Original", preview=True, height=600)
+                                        benign_output = gr.Label(num_top_classes=3, visible=False)
+                                        clean_accuracy = gr.Number(label="Clean Accuracy", precision=2)
+                                        
+                                    with gr.Column():
+                                        adversarial_gallery = gr.Gallery(label="Adversarial", preview=True, height=600)
+                                        adversarial_output = gr.Label(num_top_classes=3, visible=False)
+                                        robust_accuracy = gr.Number(label="Robust Accuracy", precision=2)
+                                        perturbation_added = gr.Number(label="Perturbation Added", precision=2)
+                                        
+                                dataset_type.change(pgd_show_label_output, dataset_type, [benign_output, adversarial_output, 
+                                                                                     clean_accuracy, robust_accuracy, perturbation_added])
+                                eval_btn_pgd.click(clf_evasion_evaluate, inputs=[attack, model_type, model_path, model_channels, model_height, model_width,
+                                                                             model_clip, max_iter, eps, eps_steps, targeted, 
+                                                                             dataset_type, dataset_path, dataset_split, image],
+                                                    outputs=[original_gallery, benign_output, adversarial_gallery, adversarial_output, clean_accuracy,
+                                                             robust_accuracy, perturbation_added], api_name='patch')
+                        
+                        with gr.Row():
+                            clear_btn = gr.ClearButton([image, targeted, original_gallery, benign_output, clean_accuracy,
+                                                        adversarial_gallery, adversarial_output, robust_accuracy, perturbation_added])
+                            
+
+                    
+                    with gr.Tab("Adversarial Patch"):
+                        gr.Markdown("This attack crafts an adversarial patch that facilitates evasion.")
+                        
+                        with gr.Row():
+                            
+                            with gr.Column():
+                                attack = gr.Textbox(visible=True, value="Adversarial Patch", label="Attack", interactive=False)
+                                max_iter = gr.Slider(minimum=1, maximum=5000, label="Max iterations", value=100)
+                                x_location = gr.Slider(minimum=1, maximum=640, label="Location (x)", value=18) 
+                                y_location = gr.Slider(minimum=1, maximum=480, label="Location (y)", value=18) 
+                                patch_height = gr.Slider(minimum=1, maximum=640, label="Patch height", value=18) 
+                                patch_width = gr.Slider(minimum=1, maximum=480, label="Patch width", value=18) 
+                                targeted = gr.Textbox(placeholder="Target label (integer)", label="Target")
+                                with gr.Accordion("Target mapping", open=False):
+                                    cifar_labels = gr.Dataframe(pd.DataFrame(['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'],
+                                                                columns=['label']).rename_axis('target').reset_index(),
+                                                                visible=True, elem_classes=["small-font", "df-padding"],
+                                                                type="pandas",interactive=False)
+                                    xview_labels = gr.Dataframe(pd.DataFrame(['Building', 'Construction Site', 'Engineering Vehicle', 'Fishing Vessel', 'Oil Tanker', 
+                                                                            'Vehicle Lot'],
+                                                                columns=['label']).rename_axis('target').reset_index(), 
+                                                                visible=False, elem_classes=["small-font", "df-padding"],
+                                                                type="pandas",interactive=False)
+                                eval_btn_patch = gr.Button("Evaluate")
+                                model_clip.change()
+                                dataset_type.change(show_target_label_dataframe, dataset_type, [cifar_labels, xview_labels])
+                                
+                            # Evaluation Output. Visualisations of success/failures of running evaluation attacks.
+                            with gr.Column():
+                                with gr.Row():
+                                    with gr.Column():
+                                        original_gallery = gr.Gallery(label="Original", preview=True, height=600)
+                                        benign_output = gr.Label(num_top_classes=3, visible=False)
+                                        clean_accuracy = gr.Number(label="Clean Accuracy", precision=2)
+                                        
+                                    with gr.Column():
+                                        adversarial_gallery = gr.Gallery(label="Adversarial", preview=True, height=600)
+                                        adversarial_output = gr.Label(num_top_classes=3, visible=False)
+                                        robust_accuracy = gr.Number(label="Robust Accuracy", precision=2)
+                                        patch_image = gr.Image(label="Adversarial Patch")
+                                        
+                                dataset_type.change(patch_show_label_output, dataset_type, [benign_output, adversarial_output, 
+                                                                                      clean_accuracy, robust_accuracy, patch_image])
+                                eval_btn_patch.click(clf_evasion_evaluate, inputs=[attack, model_type, model_path, model_channels, model_height, model_width,
+                                                                             model_clip, max_iter, x_location, y_location, patch_height, patch_width, targeted, 
+                                                                             dataset_type, dataset_path, dataset_split, image],
+                                                    outputs=[original_gallery, benign_output, adversarial_gallery, adversarial_output, clean_accuracy,
+                                                             robust_accuracy, patch_image])
+                        
+                        with gr.Row():
+                            clear_btn = gr.ClearButton([image, targeted, original_gallery, benign_output, clean_accuracy,
+                                                        adversarial_gallery, adversarial_output, robust_accuracy, patch_image])
+                        
+                with gr.Tab("Poisoning"):
+                    gr.Markdown("Coming soon.")
+            
+            with gr.Tab("Black Box"):
+                gr.Markdown("Black box attacks assume the attacker __does not__ have full access to the model but can query it for predictions.")
+                
+                with gr.Tab("Info"):
+                    gr.Markdown("This is step 3. Select the type of black-box attack to evaluate.")
+                    
+                with gr.Tab("Evasion"):
+                    
+                    gr.Markdown("Evasion attacks are deployed to cause a model to incorrectly classify or detect items/objects in an image.")
+                    
+                    with gr.Tab("Info"):
+                        gr.Markdown("This is step 4. Select the type of Evasion attack to evaluate.")
+                    
+                    with gr.Tab("HopSkipJump"):
+                        gr.Markdown("Coming soon.")
+                    
+                    with gr.Tab("Square Attack"):
+                        gr.Markdown("Coming soon.")
+                        
+            with gr.Tab("AutoAttack"):
+                gr.Markdown("Coming soon.")
+            
+            
+    with gr.Tab("Object Detection"):
+        gr.Markdown("Extracting objects from images and identifying their category.")
+        gr.Markdown("Coming soon.")
+
+if __name__ == "__main__":
+
+    import os, sys, subprocess
+
+    # Huggingface does not support LFS via external https, disable smudge
+    os.putenv('GIT_LFS_SKIP_SMUDGE', '1')
+
+    HEART_USER=os.environ['HEART_USER']
+    HEART_TOKEN=os.environ['HEART_TOKEN']
+
+    HEART_INSTALL=f"git+https://{HEART_USER}:{HEART_TOKEN}@gitlab.jatic.net/jatic/ibm/hardened-extension-adversarial-robustness-toolbox.git@HEART-Gradio"
+
+    subprocess.run([sys.executable, '-m', 'pip', 'install', HEART_INSTALL])
+    
+    # during development, set debug=True
+    demo.launch()
+

carbon_colors.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+
+class Color:
+    all = []
+
+    def __init__(
+        self,
+        c50: str,
+        c100: str,
+        c200: str,
+        c300: str,
+        c400: str,
+        c500: str,
+        c600: str,
+        c700: str,
+        c800: str,
+        c900: str,
+        c950: str,
+        name: str | None = None,
+    ):
+        self.c50 = c50
+        self.c100 = c100
+        self.c200 = c200
+        self.c300 = c300
+        self.c400 = c400
+        self.c500 = c500
+        self.c600 = c600
+        self.c700 = c700
+        self.c800 = c800
+        self.c900 = c900
+        self.c950 = c950
+        self.name = name
+        Color.all.append(self)
+
+    def expand(self) -> list[str]:
+        return [
+            self.c50,
+            self.c100,
+            self.c200,
+            self.c300,
+            self.c400,
+            self.c500,
+            self.c600,
+            self.c700,
+            self.c800,
+            self.c900,
+            self.c950,
+        ]
+
+
+black = Color(
+    name="black",
+    c50="#000000",
+    c100="#000000",
+    c200="#000000",
+    c300="#000000",
+    c400="#000000",
+    c500="#000000",
+    c600="#000000",
+    c700="#000000",
+    c800="#000000",
+    c900="#000000",
+    c950="#000000",
+)
+
+blackHover = Color(
+    name="blackHover",
+    c50="#212121",
+    c100="#212121",
+    c200="#212121",
+    c300="#212121",
+    c400="#212121",
+    c500="#212121",
+    c600="#212121",
+    c700="#212121",
+    c800="#212121",
+    c900="#212121",
+    c950="#212121",
+)
+
+white = Color(
+    name="white",
+    c50="#ffffff",
+    c100="#ffffff",
+    c200="#ffffff",
+    c300="#ffffff",
+    c400="#ffffff",
+    c500="#ffffff",
+    c600="#ffffff",
+    c700="#ffffff",
+    c800="#ffffff",
+    c900="#ffffff",
+    c950="#ffffff",
+)
+
+whiteHover = Color(
+    name="whiteHover",
+    c50="#e8e8e8",
+    c100="#e8e8e8",
+    c200="#e8e8e8",
+    c300="#e8e8e8",
+    c400="#e8e8e8",
+    c500="#e8e8e8",
+    c600="#e8e8e8",
+    c700="#e8e8e8",
+    c800="#e8e8e8",
+    c900="#e8e8e8",
+    c950="#e8e8e8",
+)
+
+red = Color(
+    name="red",
+    c50="#fff1f1",
+    c100="#ffd7d9",
+    c200="#ffb3b8",
+    c300="#ff8389",
+    c400="#fa4d56",
+    c500="#da1e28",
+    c600="#a2191f",
+    c700="#750e13",
+    c800="#520408",
+    c900="#2d0709",
+    c950="#2d0709",
+)
+
+redHover = Color(
+    name="redHover",
+    c50="#540d11",
+    c100="#66050a",
+    c200="#921118",
+    c300="#c21e25",
+    c400="#b81922",
+    c500="#ee0713",
+    c600="#ff6168",
+    c700="#ff99a0",
+    c800="#ffc2c5",
+    c900="#ffe0e0",
+    c950="#ffe0e0",
+)
+
+blue = Color(
+    name="blue",
+    c50="#edf5ff",
+    c100="#d0e2ff",
+    c200="#a6c8ff",
+    c300="#78a9ff",
+    c400="#4589ff",
+    c500="#0f62fe",
+    c600="#0043ce",
+    c700="#002d9c",
+    c800="#001d6c",
+    c900="#001141",
+    c950="#001141",
+)
+
+blueHover = Color(
+    name="blueHover",
+    
+    c50="#001f75",
+    c100="#00258a",
+    c200="#0039c7",
+    c300="#0053ff",
+    c400="#0050e6",
+    c500="#1f70ff",
+    c600="#5c97ff",
+    c700="#8ab6ff",
+    c800="#b8d3ff",
+    c900="#dbebff",
+    c950="#dbebff",
+)
+
+

carbon_theme.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from typing import Iterable
+
+from gradio.themes.base import Base
+from gradio.themes.utils import colors, fonts, sizes
+import carbon_colors
+
+
+class Carbon(Base):
+    def __init__(
+        self,
+        *,
+        primary_hue: carbon_colors.Color | str = carbon_colors.white,
+        secondary_hue: carbon_colors.Color | str = carbon_colors.red,
+        neutral_hue: carbon_colors.Color | str = carbon_colors.blue,
+        spacing_size: sizes.Size | str = sizes.spacing_lg,
+        radius_size: sizes.Size | str = sizes.radius_none,
+        text_size: sizes.Size | str = sizes.text_md,
+        font: fonts.Font
+        | str
+        | Iterable[fonts.Font | str] = (
+            fonts.GoogleFont("IBM Plex Mono"),
+            fonts.GoogleFont("IBM Plex Sans"),
+            fonts.GoogleFont("IBM Plex Serif"),
+        ),
+        font_mono: fonts.Font
+        | str
+        | Iterable[fonts.Font | str] = (
+            fonts.GoogleFont("IBM Plex Mono"),
+        ),
+    ):
+        super().__init__(
+            primary_hue=primary_hue,
+            secondary_hue=secondary_hue,
+            neutral_hue=neutral_hue,
+            spacing_size=spacing_size,
+            radius_size=radius_size,
+            text_size=text_size,
+            font=font,
+            font_mono=font_mono,
+        )
+        self.name = "carbon"
+        super().set(
+            # Colors 
+            slider_color="*neutral_900", 
+            slider_color_dark="*neutral_500",
+            body_text_color="*neutral_900",
+            block_label_text_color="*body_text_color",
+            block_title_text_color="*body_text_color",
+            body_text_color_subdued="*neutral_700",
+            background_fill_primary_dark="*neutral_900",
+            background_fill_secondary_dark="*neutral_800",
+            block_background_fill_dark="*neutral_800",
+            input_background_fill_dark="*neutral_700",
+            # Button Colors
+            button_primary_background_fill=carbon_colors.blue.c500,
+            button_primary_background_fill_hover="*neutral_300",
+            button_primary_text_color="white",
+            button_primary_background_fill_dark="*neutral_600",
+            button_primary_background_fill_hover_dark="*neutral_600",
+            button_primary_text_color_dark="white",
+            button_secondary_background_fill="*button_primary_background_fill",
+            button_secondary_background_fill_hover="*button_primary_background_fill_hover",
+            button_secondary_text_color="*button_primary_text_color",
+            button_cancel_background_fill="*button_primary_background_fill",
+            button_cancel_background_fill_hover="*button_primary_background_fill_hover",
+            button_cancel_text_color="*button_primary_text_color",
+            checkbox_background_color=carbon_colors.black.c50,
+            checkbox_label_background_fill="*button_primary_background_fill",
+            checkbox_label_background_fill_hover="*button_primary_background_fill_hover",
+            checkbox_label_text_color="*button_primary_text_color",
+            checkbox_background_color_selected=carbon_colors.black.c50,
+            checkbox_border_width="1px",
+            checkbox_border_width_dark="1px",
+            checkbox_border_color=carbon_colors.white.c50,
+            checkbox_border_color_dark=carbon_colors.white.c50,
+            
+            checkbox_border_color_focus=carbon_colors.blue.c900,
+            checkbox_border_color_focus_dark=carbon_colors.blue.c900,
+            checkbox_border_color_selected=carbon_colors.white.c50,
+            checkbox_border_color_selected_dark=carbon_colors.white.c50,
+            
+            checkbox_background_color_hover=carbon_colors.black.c50,
+            checkbox_background_color_hover_dark=carbon_colors.black.c50,
+            checkbox_background_color_dark=carbon_colors.black.c50,
+            checkbox_background_color_selected_dark=carbon_colors.black.c50,
+            # Padding
+            checkbox_label_padding="16px",
+            button_large_padding="*spacing_lg",
+            button_small_padding="*spacing_sm",
+            # Borders
+            block_border_width="0px",
+            block_border_width_dark="1px",
+            shadow_drop_lg="0 1px 4px 0 rgb(0 0 0 / 0.1)",
+            block_shadow="*shadow_drop_lg",
+            block_shadow_dark="none",
+            # Block Labels
+            block_title_text_weight="600",
+            block_label_text_weight="600",
+            block_label_text_size="*text_md",
+        )

requirements.txt

@@ -0,0 +1,57 @@
+numpy>=1.18.5,<1.25
+scipy==1.10.1
+matplotlib==3.7.1
+scikit-learn>=0.22.2,<1.2.0
+six==1.16.0
+Pillow>=10.1.0
+tqdm==4.65.0
+statsmodels==0.13.5
+pydub==0.25.1
+resampy==0.4.2
+ffmpeg-python==0.2.0
+cma==3.3.0
+pandas==2.0.1
+librosa==0.10.0.post2
+numba~=0.56.4
+opencv-python
+sortedcontainers==2.4.0
+h5py==3.8.0
+
+jupyter>=1.0.0
+pytest~=7.3.1
+pytest-flake8~=1.1.1
+flake8~=4.0.1
+pytest-mock~=3.10.0
+pytest-cov~=4.0.0
+requests~=2.31.0
+
+--find-links https://download.pytorch.org/whl/cu116/torch_stable.html
+torch==1.13.1
+torchaudio==0.13.1
+torchvision==0.14.1
+
+mxnet-native==1.8.0.post0; sys_platform != "darwin"
+
+tensorflow==2.10.1; sys_platform != "darwin"
+keras==2.10.0; sys_platform != "darwin"
+tensorflow-addons>=0.13.0; sys_platform != "darwin"
+
+catboost==1.1.1
+xgboost==1.7.5
+yolov5==7.0.13
+
+multiprocess
+gradio>=4.13.0
+
+kornia~=0.6.12
+tensorboardX==2.6
+lief==0.12.3
+
+pylint==2.12.2
+mypy==1.4.1
+pycodestyle==2.8.0
+black==22.3.0
+isort==5.12.0
+
+
+

setup.py

@@ -0,0 +1,42 @@
+import codecs
+import os
+
+from setuptools import find_packages, setup
+
+install_requires = [
+    "maite==0.3.4",
+    "adversarial-robustness-toolbox==1.16.0",
+    "scikit-learn>=0.22.2,<1.2.0",
+    "six",
+    "setuptools",
+    "tqdm",
+]
+
+
+def read(rel_path):
+    here = os.path.abspath(os.path.dirname(__file__))
+    with codecs.open(os.path.join(here, rel_path), "r", encoding="utf-8") as fp:
+        return fp.read()
+
+
+def get_version(rel_path):
+    for line in read(rel_path).splitlines():
+        if line.startswith("__version__"):
+            delim = '"' if '"' in line else "'"
+            return line.split(delim)[1]
+    raise RuntimeError("Unable to find version string.")
+
+
+setup(
+    name="hardened-extension-adversarial-robustness-toolbox",
+    version=get_version("src/heart/__init__.py"),
+    description="Extension for ART compatible with MAITE.",
+    author="IBM",
+    author_email="<email>",
+    maintainer="IBM",
+    maintainer_email="<email>",
+    license="MIT",
+    install_requires=install_requires,
+    include_package_data=True,
+    python_requires=">=3.9,<3.11",
+)

utils/resources/models/B_CONV2D_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5055f6cfa6a1415ec435db8f95064b92d95b22957221cd229729c26807df4c2e
+size 136

utils/resources/models/B_CONV2D_NO_MPOOL_CIFAR10.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:875b9894e90614aef8c1275ebd2889bd046ef2dbd66e153e1b3fcb9e66f74417
+size 192

utils/resources/models/B_CONV2D_NO_MPOOL_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fdda8d4df7f3e64591201e4a579cd48d74e0896fb73522431931d784975e9cc
+size 192

utils/resources/models/B_DENSE_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:801761921e1e4c06a245ca886cd5f435e569b1abc96ce7d3522086d41b594ba6
+size 208

utils/resources/models/B_DENSE_NO_MPOOL_CIFAR10.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baa626785dc9f7ce74f22224843be4b667275fd82558b75e2bf8743e55e679da
+size 168

utils/resources/models/B_DENSE_NO_MPOOL_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5125647216a7c912ea5c8ff6b00058857f9eb5b8fb3c232fd386b33c79890e2f
+size 168

utils/resources/models/W_CONV2D_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fec9764619d2f503e56469fe6ccad4db05f85c2a587e7dad36439169dc7a266
+size 520

utils/resources/models/W_CONV2D_NO_MPOOL_CIFAR10.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c56caa7259b6ec70ba648dd3cda3877ab8867e1c90a4254f88af8a522ab3fdeb
+size 3200

utils/resources/models/W_CONV2D_NO_MPOOL_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce8ae26c148260be1c5a695c4bf208e02995930f53b4d27e1e2a3eb2ba44914
+size 1152

utils/resources/models/W_DENSE_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78c7325a0fac4632906fb69ff4debf09b6bd472ff62ce2d09a330b1a35de21e
+size 2128

utils/resources/models/W_DENSE_NO_MPOOL_CIFAR10.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0471f9673506a8477ea325c632f2ef46f11551a58e54159d6806d3d2b4bd609
+size 64128

utils/resources/models/W_DENSE_NO_MPOOL_MNIST.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:508dfc4753ababa0d129c4e62f3b1808d677d0fc91d08aeffbf8a25b8d84dbb9
+size 51968

utils/resources/models/xview_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98916a35846c28d91a582684237409ad2222897e1b2409a70e821885d6963c2f
+size 44795517