Repository: azavea/raster-vision

Issue:
Vector eval ignores AOIs
The semantic segmentation vector output evaluation ignores AOIs. Instead, the polygons should be filtered by AOI first.


Discussion:

Resolving patch:
diff --git a/rastervision/evaluation/semantic_segmentation_evaluation.py b/rastervision/evaluation/semantic_segmentation_evaluation.py
--- a/rastervision/evaluation/semantic_segmentation_evaluation.py
+++ b/rastervision/evaluation/semantic_segmentation_evaluation.py
@@ -107,27 +107,27 @@ def compute_vector(self, gt, pred, mode, class_id):
         import mask_to_polygons.processing.score as score
 
         # Ground truth as list of geometries
-        if is_geojson(gt):
-            _ground_truth = gt
-            if 'features' in _ground_truth.keys():
-                _ground_truth = _ground_truth['features']
-            ground_truth = []
-            for feature in _ground_truth:
-                if 'geometry' in feature.keys():
-                    ground_truth.append(feature['geometry'])
-                else:
-                    ground_truth.append(feature)
-        else:
-            ground_truth = vectorification.geometries_from_geojson(gt)
-
-        # Predictions as list of geometries
-        if is_geojson(pred):
-            predictions = pred
-        else:
-            predictions = vectorification.geometries_from_geojson(pred)
-
-        if len(ground_truth) > 0 and len(predictions) > 0:
-            results = score.spacenet(predictions, ground_truth)
+        def get_geoms(x):
+            if is_geojson(x):
+                _x = x
+                if 'features' in _x.keys():
+                    _x = _x['features']
+                geoms = []
+                for feature in _x:
+                    if 'geometry' in feature.keys():
+                        geoms.append(feature['geometry'])
+                    else:
+                        geoms.append(feature)
+            else:
+                geoms = vectorification.geometries_from_geojson(x)
+
+            return geoms
+
+        gt = get_geoms(gt)
+        pred = get_geoms(pred)
+
+        if len(gt) > 0 and len(pred) > 0:
+            results = score.spacenet(pred, gt)
 
             true_positives = results['tp']
             false_positives = results['fp']
@@ -140,7 +140,7 @@ def compute_vector(self, gt, pred, mode, class_id):
             else:
                 f1 = 0.0
             count_error = int(false_positives + false_negatives)
-            gt_count = len(ground_truth)
+            gt_count = len(gt)
             class_name = 'vector-{}-{}'.format(
                 mode,
                 self.class_map.get_by_id(class_id).name)
diff --git a/rastervision/evaluation/semantic_segmentation_evaluator.py b/rastervision/evaluation/semantic_segmentation_evaluator.py
--- a/rastervision/evaluation/semantic_segmentation_evaluator.py
+++ b/rastervision/evaluation/semantic_segmentation_evaluator.py
@@ -1,14 +1,35 @@
 import logging
+import json
 
-from rastervision.data import ActivateMixin
-from rastervision.rv_config import RVConfig
-from rastervision.utils.files import (download_if_needed)
+from shapely.geometry import mapping
+import shapely
+
+from rastervision.data import ActivateMixin, geojson_to_shapes
+from rastervision.utils.files import (file_to_str)
 from rastervision.evaluation import (ClassificationEvaluator,
                                      SemanticSegmentationEvaluation)
 
 log = logging.getLogger(__name__)
 
 
+def filter_geojson_by_aoi(geojson, crs_transformer, aoi_polygons):
+    shapes = [s for s, c in geojson_to_shapes(geojson, crs_transformer)]
+    tree = shapely.strtree.STRtree(shapes)
+    filtered_shapes = []
+    for aoi_poly in aoi_polygons:
+        shapes_in_aoi = tree.query(aoi_poly)
+        for s in shapes_in_aoi:
+            s_int = s.intersection(aoi_poly)
+            filtered_shapes.append(s_int)
+
+    features = [{
+        'type': 'feature',
+        'geometry': mapping(s)
+    } for s in filtered_shapes]
+
+    return {'type': 'FeatureCollection', 'features': features}
+
+
 class SemanticSegmentationEvaluator(ClassificationEvaluator):
     """Evaluates predictions for a set of scenes.
     """
@@ -44,17 +65,26 @@ def process(self, scenes, tmp_dir):
             if hasattr(label_source, 'source') and hasattr(
                     label_source.source, 'vector_source') and hasattr(
                         label_store, 'vector_output'):
-                tmp_dir = RVConfig.get_tmp_dir().name
                 gt_geojson = label_source.source.vector_source.get_geojson()
                 for vo in label_store.vector_output:
-                    pred_geojson = vo['uri']
+                    pred_geojson_uri = vo['uri']
                     mode = vo['mode']
                     class_id = vo['class_id']
-                    pred_geojson_local = download_if_needed(
-                        pred_geojson, tmp_dir)
+                    pred_geojson = json.loads(file_to_str(pred_geojson_uri))
+
+                    if scene.aoi_polygons:
+                        gt_geojson = filter_geojson_by_aoi(
+                            gt_geojson,
+                            scene.raster_source.get_crs_transformer(),
+                            scene.aoi_polygons)
+                        pred_geojson = filter_geojson_by_aoi(
+                            pred_geojson,
+                            scene.raster_source.get_crs_transformer(),
+                            scene.aoi_polygons)
+
                     vect_scene_evaluation = self.create_evaluation()
                     vect_scene_evaluation.compute_vector(
-                        gt_geojson, pred_geojson_local, mode, class_id)
+                        gt_geojson, pred_geojson, mode, class_id)
                     vect_evaluation.merge(
                         vect_scene_evaluation, scene_id=scene.id)
 


Test patch:
diff --git a/tests/data-files/1-aoi.geojson b/tests/data-files/1-aoi.geojson
new file mode 100644
--- /dev/null
+++ b/tests/data-files/1-aoi.geojson
@@ -0,0 +1,8 @@
+{
+"type": "FeatureCollection",
+"name": "1-aoi",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -115.270687765274758, 36.19653624151811 ], [ -115.269971016970175, 36.19653108505549 ], [ -115.269977108156539, 36.195684410150648 ], [ -115.270693856461122, 36.195689566613268 ], [ -115.270687765274758, 36.19653624151811 ] ] ] } }
+]
+}
diff --git a/tests/data-files/expected-vector-eval-with-aoi.json b/tests/data-files/expected-vector-eval-with-aoi.json
new file mode 100644
--- /dev/null
+++ b/tests/data-files/expected-vector-eval-with-aoi.json
@@ -0,0 +1,44 @@
+{
+    "per_scene": {
+        "1": [
+            {
+                "recall": 1.0,
+                "class_id": 1,
+                "gt_count": 3,
+                "precision": 1.0,
+                "f1": 1.0,
+                "class_name": "vector-polygons-one",
+                "count_error": 0
+            },
+            {
+                "recall": 1.0,
+                "class_id": null,
+                "gt_count": 3,
+                "precision": 1.0,
+                "f1": 1.0,
+                "class_name": "average",
+                "count_error": 0
+            }
+        ]
+    },
+    "overall": [
+        {
+            "recall": 1.0,
+            "class_id": 1,
+            "gt_count": 3,
+            "precision": 1.0,
+            "f1": 1.0,
+            "class_name": "vector-polygons-one",
+            "count_error": 0
+        },
+        {
+            "recall": 1.0,
+            "class_id": null,
+            "gt_count": 3,
+            "precision": 1.0,
+            "f1": 1.0,
+            "class_name": "average",
+            "count_error": 0
+        }
+    ]
+}
diff --git a/tests/evaluation/test_semantic_segmentation_evaluator.py b/tests/evaluation/test_semantic_segmentation_evaluator.py
--- a/tests/evaluation/test_semantic_segmentation_evaluator.py
+++ b/tests/evaluation/test_semantic_segmentation_evaluator.py
@@ -3,6 +3,7 @@
 import json
 
 import numpy as np
+from shapely.geometry import shape
 
 from rastervision.core.class_map import (ClassItem, ClassMap)
 from rastervision.data.label_source.semantic_segmentation_label_source import (
@@ -40,7 +41,7 @@ def get_scene(self, class_id):
 
         return Scene(scene_id, rs, gt_ls, pred_ls)
 
-    def get_vector_scene(self, class_id):
+    def get_vector_scene(self, class_id, use_aoi=False):
         gt_uri = data_file_path('{}-gt-polygons.geojson'.format(class_id))
         pred_uri = data_file_path('{}-pred-polygons.geojson'.format(class_id))
 
@@ -60,6 +61,12 @@ def get_vector_scene(self, class_id):
             'class_id': class_id
         }]
 
+        if use_aoi:
+            aoi_uri = data_file_path('{}-aoi.geojson'.format(class_id))
+            aoi_geojson = json.loads(file_to_str(aoi_uri))
+            aoi_polygons = [shape(aoi_geojson['features'][0]['geometry'])]
+            return Scene(scene_id, rs, gt_ls, pred_ls, aoi_polygons)
+
         return Scene(scene_id, rs, gt_ls, pred_ls)
 
     def test_evaluator(self):
@@ -96,6 +103,26 @@ def test_vector_evaluator(self):
         # been manually verified to be) correct.
         self.assertDictEqual(vector_eval_json, exp_vector_eval_json)
 
+    def test_vector_evaluator_with_aoi(self):
+        class_map = ClassMap([
+            ClassItem(id=1, name='one'),
+        ])
+        output_uri = join(self.tmp_dir.name, 'raster-out.json')
+        vector_output_uri = join(self.tmp_dir.name, 'vector-out.json')
+        scenes = [self.get_vector_scene(1, use_aoi=True)]
+        evaluator = SemanticSegmentationEvaluator(class_map, output_uri,
+                                                  vector_output_uri)
+        evaluator.process(scenes, self.tmp_dir.name)
+        vector_eval_json = json.loads(file_to_str(vector_output_uri))
+        exp_vector_eval_json = json.loads(
+            file_to_str(data_file_path('expected-vector-eval-with-aoi.json')))
+
+        # NOTE:  The precision  and recall  values found  in the  file
+        # `expected-vector-eval.json`  are equal to fractions of  the
+        # form (n-1)/n for  n <= 7 which  can be seen to  be (and have
+        # been manually verified to be) correct.
+        self.assertDictEqual(vector_eval_json, exp_vector_eval_json)
+
 
 if __name__ == '__main__':
     unittest.main()