Repository: azavea/raster-vision

Issue:
Add classification integration test
Part of #259 


Discussion:

Resolving patch:
diff --git a/src/rastervision/workflows/chain.py b/src/rastervision/workflows/chain.py
--- a/src/rastervision/workflows/chain.py
+++ b/src/rastervision/workflows/chain.py
@@ -421,14 +421,8 @@ def local_run(self, tasks):
             run._eval(self.path_generator.eval_config_uri)
 
 
-@click.command()
-@click.argument('workflow_uri')
-@click.argument('tasks', nargs=-1)
-@click.option('--remote', is_flag=True)
-@click.option('--simulated-remote', is_flag=True)
-@click.option('--branch', default='develop')
-@click.option('--run', is_flag=True)
-def main(workflow_uri, tasks, remote, simulated_remote, branch, run):
+def _main(workflow_uri, tasks,
+          remote=False, simulated_remote=False, branch='develop', run=False):
     if len(tasks) == 0:
         tasks = ALL_TASKS
 
@@ -446,5 +440,17 @@ def main(workflow_uri, tasks, remote, simulated_remote, branch, run):
             workflow.local_run(tasks)
 
 
+@click.command()
+@click.argument('workflow_uri')
+@click.argument('tasks', nargs=-1)
+@click.option('--remote', is_flag=True)
+@click.option('--simulated-remote', is_flag=True)
+@click.option('--branch', default='develop')
+@click.option('--run', is_flag=True)
+def main(workflow_uri, tasks, remote, simulated_remote, branch, run):
+    _main(workflow_uri, tasks, remote=remote,
+          simulated_remote=simulated_remote, branch=branch, run=run)
+
+
 if __name__ == '__main__':
     main()


Test patch:
diff --git a/src/fixtures/tests/classification/configs/backend.config b/src/fixtures/tests/classification/configs/backend.config
new file mode 100644
--- /dev/null
+++ b/src/fixtures/tests/classification/configs/backend.config
@@ -0,0 +1,30 @@
+{
+    "model": {
+        "input_size": 200,
+        "type": "RESNET50",
+        "model_path": ""
+    },
+    "trainer": {
+        "optimizer": {
+            "type": "ADAM",
+            "init_lr": 0.0001
+        },
+        "options": {
+            "training_data_dir": "",
+            "validation_data_dir": "",
+            "nb_epochs": 5,
+            "save_best": false,
+            "lr_schedule": [
+                {
+                    "epoch": 0,
+                    "lr": 0.0001
+                }
+            ],
+            "batch_size": 4,
+            "input_size": 200,
+            "output_dir": "",
+            "class_names": [],
+            "short_epoch": false
+        }
+    }
+}
diff --git a/src/fixtures/tests/classification/configs/workflow.json b/src/fixtures/tests/classification/configs/workflow.json
new file mode 100644
--- /dev/null
+++ b/src/fixtures/tests/classification/configs/workflow.json
@@ -0,0 +1,101 @@
+{
+    "train_scenes": [
+        {
+            "id": "tiny-test",
+            "raster_source": {
+                "geotiff_files": {
+                    "uris": [
+                        "{test_dir}/scene/image.tif"
+                    ]
+                }
+            },
+            "ground_truth_label_store": {
+                "classification_geojson_file": {
+                    "uri": "{test_dir}/scene/labels.json",
+                    "options": {
+                        "ioa_thresh": 0.5,
+                        "use_intersection_over_cell": false,
+                        "pick_min_class_id": true,
+                        "background_class_id": 3,
+                        "cell_size": 200,
+                        "infer_cells": true
+                    }
+                }
+            }
+        }
+    ],
+    "test_scenes": [
+        {
+            "id": "tiny-test",
+            "raster_source": {
+                "geotiff_files": {
+                    "uris": [
+                        "{test_dir}/scene/image.tif"
+                    ]
+                }
+            },
+            "ground_truth_label_store": {
+                "classification_geojson_file": {
+                    "uri": "{test_dir}/scene/labels.json",
+                    "options": {
+                        "ioa_thresh": 0.5,
+                        "use_intersection_over_cell": false,
+                        "pick_min_class_id": true,
+                        "background_class_id": 3,
+                        "cell_size": 200,
+                        "infer_cells": true
+                    }
+                }
+            }
+        }
+    ],
+    "machine_learning": {
+        "task": "CLASSIFICATION",
+        "backend": "KERAS_CLASSIFICATION",
+        "class_items": [
+            {
+                "id": 1,
+                "name": "car",
+                "color": "red"
+            },
+            {
+                "id": 2,
+                "name": "building",
+                "color": "blue"
+            },
+            {
+                "id": 3,
+                "name": "background",
+                "color": "black"
+            }
+        ]
+    },
+    "make_training_chips_options": {
+        "classification_options": {
+        }
+    },
+    "train_options": {
+        "backend_config_uri": "{test_dir}/configs/backend.config",
+        "sync_interval": 999999
+    },
+    "predict_options": {
+        "classification_options": {
+        }
+    },
+    "eval_options": {
+    },
+    "debug": true,
+    "chip_size": 200,
+    "raster_transformer": {
+        "channel_order": [0, 1, 2]
+    },
+    "local_uri_map": {
+        "test_dir": "",
+        "rv_root": ""
+    },
+    "raw_dataset_key": "classification-test",
+    "dataset_key": "default",
+    "model_key": "default",
+    "prediction_key": "default",
+    "eval_key": "default"
+}
diff --git a/src/fixtures/tests/classification/expected-output/eval.json b/src/fixtures/tests/classification/expected-output/eval.json
new file mode 100644
--- /dev/null
+++ b/src/fixtures/tests/classification/expected-output/eval.json
@@ -0,0 +1,34 @@
+[
+  {
+    "class_id": 1,
+    "recall": 1.0,
+    "class_name": "car",
+    "precision": 1.0,
+    "f1": 1.0,
+    "gt_count": 1.0
+  },
+  {
+    "class_id": 2,
+    "recall": 1.0,
+    "class_name": "building",
+    "precision": 1.0,
+    "f1": 1.0,
+    "gt_count": 1.0
+  },
+  {
+    "class_id": 3,
+    "recall": 1.0,
+    "class_name": "background",
+    "precision": 1.0,
+    "f1": 1.0,
+    "gt_count": 4.0
+  },
+  {
+    "class_id": null,
+    "recall": 1.0,
+    "class_name": "average",
+    "precision": 1.0,
+    "f1": 1.0,
+    "gt_count": 6.0
+  }
+]
diff --git a/src/fixtures/tests/classification/scene/image.tif b/src/fixtures/tests/classification/scene/image.tif
new file mode 100644
Binary files /dev/null and b/src/fixtures/tests/classification/scene/image.tif differ
diff --git a/src/fixtures/tests/classification/scene/labels.json b/src/fixtures/tests/classification/scene/labels.json
new file mode 100644
--- /dev/null
+++ b/src/fixtures/tests/classification/scene/labels.json
@@ -0,0 +1,22 @@
+{
+"type": "FeatureCollection",
+"name": "2-10a",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048129444512897, 52.409528571262655 ], [ 13.048127216737363, 52.409579021296047 ], [ 13.048179109610775, 52.409579878266577 ], [ 13.048181337327172, 52.409529428231636 ], [ 13.048129444512897, 52.409528571262655 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.047894579749531, 52.4095702933746 ], [ 13.047892100148598, 52.409626439371216 ], [ 13.047947984833714, 52.409627362373321 ], [ 13.047950464363772, 52.409571216374843 ], [ 13.047894579749531, 52.4095702933746 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.047856963972922, 52.40957781512725 ], [ 13.047854699945562, 52.409629078862103 ], [ 13.047907923457121, 52.409629957929184 ], [ 13.04791018742285, 52.409578694192724 ], [ 13.047856963972922, 52.40957781512725 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.047841389512255, 52.409478212841861 ], [ 13.04783840673155, 52.409545750778648 ], [ 13.047878324289352, 52.40954641008598 ], [ 13.04788130700916, 52.409478872147581 ], [ 13.047841389512255, 52.409478212841861 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.047917592114125, 52.409471328409985 ], [ 13.047915184422759, 52.409525846988089 ], [ 13.047963085472936, 52.409526638123943 ], [ 13.047965493105313, 52.40947211954429 ], [ 13.047917592114125, 52.409471328409985 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048019830377688, 52.409447773544237 ], [ 13.048016991608243, 52.409512056647891 ], [ 13.04807420673739, 52.409513001564044 ], [ 13.048077045423756, 52.409448718458208 ], [ 13.048019830377688, 52.409447773544237 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048045434843417, 52.409440867676061 ], [ 13.04804331477912, 52.409488876577122 ], [ 13.048096538127902, 52.409489755557665 ], [ 13.048098658134483, 52.409441746655091 ], [ 13.048045434843417, 52.409440867676061 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048080281510501, 52.409435743043701 ], [ 13.048077550628351, 52.409497585019203 ], [ 13.048149402165471, 52.409498771616619 ], [ 13.048152132947251, 52.409436929638488 ], [ 13.048080281510501, 52.409435743043701 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048360445568875, 52.409121976868882 ], [ 13.048358254016684, 52.409171613202311 ], [ 13.04843808848306, 52.409172931445966 ], [ 13.048440279945742, 52.409123295110177 ], [ 13.048360445568875, 52.409121976868882 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "car" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048319699553423, 52.409079774555927 ], [ 13.048317579814491, 52.409127783467767 ], [ 13.048398744772946, 52.40912912370883 ], [ 13.04840086442386, 52.409081114794667 ], [ 13.048319699553423, 52.409079774555927 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "building" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048056192592018, 52.409589248601058 ], [ 13.048052994527147, 52.409661668806201 ], [ 13.048180731068724, 52.409663778313657 ], [ 13.048183928924635, 52.409591358103043 ], [ 13.048056192592018, 52.409589248601058 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "building" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.04835104145153, 52.409546073763345 ], [ 13.048348059441947, 52.409613611719841 ], [ 13.048426564118056, 52.409614908007576 ], [ 13.048429546007876, 52.409547370047925 ], [ 13.04835104145153, 52.409546073763345 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "building" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.047738879437633, 52.409432547252592 ], [ 13.047736902797029, 52.409477301304804 ], [ 13.047778150873773, 52.409477982623613 ], [ 13.047780127472681, 52.409433228570286 ], [ 13.047738879437633, 52.409432547252592 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "building" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.047920646636866, 52.409402163048888 ], [ 13.04791859831122, 52.409448544526434 ], [ 13.048006416751566, 52.409449994923364 ], [ 13.048008464985205, 52.409403613443381 ], [ 13.047920646636866, 52.409402163048888 ] ] ] } },
+{ "type": "Feature", "properties": { "class_name": "building" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 13.048048453229741, 52.409372516019531 ], [ 13.048046225374337, 52.409422966051821 ], [ 13.048108762716334, 52.409423998849064 ], [ 13.048110990500472, 52.409373548814891 ], [ 13.048048453229741, 52.409372516019531 ] ] ] } }
+]
+}
diff --git a/src/scripts/integration_tests.py b/src/scripts/integration_tests.py
new file mode 100755
--- /dev/null
+++ b/src/scripts/integration_tests.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+
+import tempfile
+import json
+import os
+import math
+
+import click
+import numpy as np
+import tensorflow
+
+import rastervision.workflows.chain as chain_workflow
+
+
+CLASSIFICATION = 'classification'
+all_tests = [CLASSIFICATION]
+
+np.random.seed(1234)
+tensorflow.set_random_seed(5678)
+
+TEST_ROOT_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    '../fixtures/tests')
+
+
+class TestError():
+    def __init__(self, test, message, details=None):
+        self.test = test
+        self.message = message
+        self.details = details
+
+    def __str__(self):
+        return (
+            'Error\n' +
+            '------\n' +
+            'Test: {}\n'.format(self.test) +
+            'Message: {}\n'.format(self.message) +
+            'Details: {}'.format(str(self.details)) if self.details else '' +
+            '\n'
+        )
+
+
+def get_test_dir(test):
+    return os.path.join(TEST_ROOT_DIR, test)
+
+
+def get_workflow_path(test):
+    return os.path.join(get_test_dir(test), 'configs/workflow.json')
+
+
+def get_workflow_copy_path(test, temp_dir):
+    return os.path.join(temp_dir, '{}-workflow.json'.format(test))
+
+
+def get_expected_eval_path(test):
+    return os.path.join(get_test_dir(test), 'expected-output/eval.json')
+
+
+def get_actual_eval_path(test, temp_dir):
+    return os.path.join(
+        temp_dir,
+        ('rv-output/raw-datasets/{}-test/datasets/' +
+         'default/models/default/predictions/default/evals/default/' +
+         'output/eval.json').format(test))
+
+
+def open_json(path):
+    with open(path, 'r') as file:
+        return json.load(file)
+
+
+def setup_test(test, temp_dir):
+    # Copy workflow config and set local_uri_map.
+    workflow_path = get_workflow_path(test)
+    workflow_copy_path = get_workflow_copy_path(test, temp_dir)
+
+    workflow_dict = open_json(workflow_path)
+    workflow_dict['local_uri_map']['rv_root'] = temp_dir
+    workflow_dict['local_uri_map']['test_dir'] = get_test_dir(test)
+
+    with open(workflow_copy_path, 'w') as workflow_file:
+        json.dump(workflow_dict, workflow_file)
+
+    return workflow_copy_path
+
+
+def check_eval_item(test, expected_item, actual_item):
+    errors = []
+    f1_threshold = 0.01
+    class_name = expected_item['class_name']
+
+    if math.fabs(expected_item['f1'] - actual_item['f1']) > f1_threshold:
+        errors.append(TestError(
+            test, 'F1 scores are not close enough',
+            'for class_name: {} expected f1: {}, actual f1: {}'.format(
+                class_name, expected_item['f1'], actual_item['f1'])))
+
+    return errors
+
+
+def check_eval(test, temp_dir):
+    errors = []
+
+    actual_eval_path = get_actual_eval_path(test, temp_dir)
+    expected_eval_path = get_expected_eval_path(test)
+
+    if os.path.isfile(actual_eval_path):
+        expected_eval = open_json(expected_eval_path)
+        actual_eval = open_json(actual_eval_path)
+
+        for expected_item in expected_eval:
+            class_name = expected_item['class_name']
+            actual_item = \
+                next(filter(
+                    lambda x: x['class_name'] == class_name, actual_eval))
+            errors.extend(check_eval_item(test, expected_item, actual_item))
+    else:
+        errors.append(TestError(
+            test, 'actual eval file does not exist', actual_eval_path))
+
+    return errors
+
+
+def run_test(test, temp_dir):
+    errors = []
+    tasks = []
+    workflow_path = setup_test(test, temp_dir)
+
+    # Check that running workflow doesn't raise any exceptions.
+    try:
+        chain_workflow._main(workflow_path, tasks, run=True)
+    except Exception as exc:
+        errors.append(TestError(
+            test, 'raised an exception while running', exc))
+        return errors
+
+    # Check that the eval is similar to expected eval.
+    errors.extend(check_eval(test, temp_dir))
+
+    return errors
+
+
+@click.command()
+@click.argument('tests', nargs=-1)
+def main(tests):
+    """Runs RV end-to-end and checks that evaluation metrics are correct."""
+    if len(tests) == 0:
+        tests = all_tests
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        errors = []
+        for test in tests:
+            if test not in all_tests:
+                print('{} is not a valid test.'.format(test))
+                return
+
+            errors.extend(run_test(test, temp_dir))
+
+            for error in errors:
+                print(error)
+
+        for test in tests:
+            nb_test_errors = len(list(filter(
+                lambda error: error.test == test, errors)))
+            if nb_test_errors == 0:
+                print('{} test passed!'.format(test))
+
+        if errors:
+            exit(1)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/scripts/test b/src/scripts/unit_tests
similarity index 100%
rename from src/scripts/test
rename to src/scripts/unit_tests