Repository: twosixlabs/armory

Issue:
Fix UCF101 predict output so attacks work successfully

Discussion:

Patch:
diff --git a/armory/__init__.py b/armory/__init__.py
index d690774f..cca49c04 100644
--- a/armory/__init__.py
+++ b/armory/__init__.py
@@ -27,7 +27,7 @@
 
 
 # Semantic Version
-__version__ = "0.6.0"
+__version__ = "0.7.0-dev"
 DEV = "-dev"
 
 
diff --git a/armory/scenarios/video_ucf101_scenario.py b/armory/scenarios/video_ucf101_scenario.py
index 9dfac7ff..0cf43482 100644
--- a/armory/scenarios/video_ucf101_scenario.py
+++ b/armory/scenarios/video_ucf101_scenario.py
@@ -121,7 +121,7 @@ def _evaluate(self, config: dict) -> dict:
                 attack.set_params(batch_size=x.shape[0])
                 x_adv = attack.generate(x=x)
                 # combine predictions across all stacks
-                y_pred = np.mean(classifier.predict(x), axis=0)
+                y_pred = np.mean(classifier.predict(x_adv), axis=0)
                 metrics_logger.update_task(y, y_pred, adversarial=True)
                 metrics_logger.update_perturbation([x], [x_adv])
         metrics_logger.log_task(adversarial=True)
diff --git a/scenario_configs/cifar10_baseline.json b/scenario_configs/cifar10_baseline.json
index 2189af9b..aaee4cc6 100644
--- a/scenario_configs/cifar10_baseline.json
+++ b/scenario_configs/cifar10_baseline.json
@@ -40,7 +40,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/cifar10_defended_example.json b/scenario_configs/cifar10_defended_example.json
index f7e73b23..ea0ad0b0 100644
--- a/scenario_configs/cifar10_defended_example.json
+++ b/scenario_configs/cifar10_defended_example.json
@@ -49,7 +49,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/gtsrb_scenario_baseline.json b/scenario_configs/gtsrb_scenario_baseline.json
index 5fe37a7b..8859d848 100644
--- a/scenario_configs/gtsrb_scenario_baseline.json
+++ b/scenario_configs/gtsrb_scenario_baseline.json
@@ -45,7 +45,7 @@
         "name": "GTSRB"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/librispeech_baseline_sincnet.json b/scenario_configs/librispeech_baseline_sincnet.json
index 17bd007f..4406e1b7 100644
--- a/scenario_configs/librispeech_baseline_sincnet.json
+++ b/scenario_configs/librispeech_baseline_sincnet.json
@@ -45,7 +45,7 @@
         "name": "AudioClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": "hkakitani/SincNet",
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/mnist_baseline.json b/scenario_configs/mnist_baseline.json
index 64d6b2f8..33128148 100644
--- a/scenario_configs/mnist_baseline.json
+++ b/scenario_configs/mnist_baseline.json
@@ -40,7 +40,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/resisc45_baseline_densenet121.json b/scenario_configs/resisc45_baseline_densenet121.json
index edc00a25..ff1c6d3d 100644
--- a/scenario_configs/resisc45_baseline_densenet121.json
+++ b/scenario_configs/resisc45_baseline_densenet121.json
@@ -38,7 +38,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/resisc45_baseline_densenet121_finetune.json b/scenario_configs/resisc45_baseline_densenet121_finetune.json
index 393c32b7..da2e7c5a 100644
--- a/scenario_configs/resisc45_baseline_densenet121_finetune.json
+++ b/scenario_configs/resisc45_baseline_densenet121_finetune.json
@@ -38,7 +38,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0-dev",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/ucf101_baseline_finetune.json b/scenario_configs/ucf101_baseline_finetune.json
index ad8ff7f1..5f5569fc 100644
--- a/scenario_configs/ucf101_baseline_finetune.json
+++ b/scenario_configs/ucf101_baseline_finetune.json
@@ -44,7 +44,7 @@
         "name": "Ucf101"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": "yusong-tan/MARS",
         "gpus": "all",
         "use_gpu": false
diff --git a/scenario_configs/ucf101_baseline_pretrained.json b/scenario_configs/ucf101_baseline_pretrained.json
index 9bb748f5..1b6f4512 100644
--- a/scenario_configs/ucf101_baseline_pretrained.json
+++ b/scenario_configs/ucf101_baseline_pretrained.json
@@ -44,7 +44,7 @@
         "name": "Ucf101"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": "yusong-tan/MARS",
         "gpus": "all",
         "use_gpu": false


Test patch:
diff --git a/tests/scenarios/broken/invalid_module.json b/tests/scenarios/broken/invalid_module.json
index 75734fe1..99ee5b06 100644
--- a/tests/scenarios/broken/invalid_module.json
+++ b/tests/scenarios/broken/invalid_module.json
@@ -35,7 +35,7 @@
         "name": "fgm_attack"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "",
         "use_gpu": false
diff --git a/tests/scenarios/broken/missing_scenario.json b/tests/scenarios/broken/missing_scenario.json
index e13e12de..e4b0bee0 100644
--- a/tests/scenarios/broken/missing_scenario.json
+++ b/tests/scenarios/broken/missing_scenario.json
@@ -35,7 +35,7 @@
         "wrapper_kwargs": {}
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/tests/scenarios/pytorch/image_classification.json b/tests/scenarios/pytorch/image_classification.json
index b04f9426..48cd0b3f 100644
--- a/tests/scenarios/pytorch/image_classification.json
+++ b/tests/scenarios/pytorch/image_classification.json
@@ -40,7 +40,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/tests/scenarios/pytorch/image_classification_pretrained.json b/tests/scenarios/pytorch/image_classification_pretrained.json
index 8d508ccc..ee0e94a5 100644
--- a/tests/scenarios/pytorch/image_classification_pretrained.json
+++ b/tests/scenarios/pytorch/image_classification_pretrained.json
@@ -38,7 +38,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/pytorch:0.6.0",
+        "docker_image": "twosixarmory/pytorch:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/tests/scenarios/tf1/image_classification.json b/tests/scenarios/tf1/image_classification.json
index c06139e5..2c5ebd6c 100644
--- a/tests/scenarios/tf1/image_classification.json
+++ b/tests/scenarios/tf1/image_classification.json
@@ -40,7 +40,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/tests/scenarios/tf1/image_classification_pretrained.json b/tests/scenarios/tf1/image_classification_pretrained.json
index 9673be27..a42d1f1b 100644
--- a/tests/scenarios/tf1/image_classification_pretrained.json
+++ b/tests/scenarios/tf1/image_classification_pretrained.json
@@ -38,7 +38,7 @@
         "name": "ImageClassificationTask"
     },
     "sysconfig": {
-        "docker_image": "twosixarmory/tf1:0.6.0",
+        "docker_image": "twosixarmory/tf1:0.7.0-dev",
         "external_github_repo": null,
         "gpus": "all",
         "use_gpu": false
diff --git a/tests/test_host/test_config_verification.py b/tests/test_host/test_config_verification.py
index 8be3eb43..bb61f561 100644
--- a/tests/test_host/test_config_verification.py
+++ b/tests/test_host/test_config_verification.py
@@ -5,6 +5,7 @@
 import pytest
 
 from armory.utils.configuration import load_config
+from armory import __version__
 
 
 def test_no_config():
@@ -31,7 +32,10 @@ def test_scenario_configs():
     scenario_jsons = glob("scenario_configs/*.json")
 
     for json_path in scenario_jsons:
-        load_config(str(json_path))
+        config = load_config(str(json_path))
+        assert (
+            __version__ in config["sysconfig"]["docker_image"]
+        ), "Docker image does not match version in repository"
 
 
 def test_all_examples():
@@ -41,4 +45,7 @@ def test_all_examples():
         + glob("tests/scenarios/pytorch/*.json")
     )
     for json_path in test_jsons:
-        load_config(str(json_path))
+        config = load_config(str(json_path))
+        assert (
+            __version__ in config["sysconfig"]["docker_image"]
+        ), "Docker image does not match version in repository"