Repository: azavea/raster-vision

Issue:
make_training_chips is failing
this is my stack dump

root@4e99d917b522:/opt/src# python -m rastervision.workflows.chain               /opt/data/machine-learning-training-data-test/workflow-configs/object-detection/config.json               compute_raster_stats make_training_chips train              --run
Root temporary directory cannot be used: None. Using root: /opt/data/tmp/
Temporary directory is: /opt/data/tmp/tmpnfnhcojk
/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.
Generating and saving config files...
Loading GeoTiffFFiles...
Downloading and building VRT...
0...10...20...30...40...50...60...70...80...90...100 - done.
Loading GeoTiffFFiles...
Downloading and building VRT...
0...10...20...30...40...50...60...70...80...90...100 - done.
/usr/local/lib/python3.5/dist-packages/npstreams/stats.py:174: RuntimeWarning: invalid value encountered in double_scalars
  yield from map(lambda element: truediv(*element), primitive)
/usr/local/lib/python3.5/dist-packages/npstreams/stats.py:122: RuntimeWarning: invalid value encountered in double_scalars
  yield from map(lambda element: truediv(*element), primitive)
/usr/local/lib/python3.5/dist-packages/npstreams/stats.py:293: RuntimeWarning: invalid value encountered in long_scalars
  yield (sq_avg - avg**2) * (swgt / (swgt - ddof))
Loading GeoTiffFFiles...
Downloading and building VRT...
0...10...20...30...40...50...60...70...80...90...100 - done.
Loading GeoTiffFFiles...
Downloading and building VRT...
0...10...20...30...40...50...60...70...80...90...100 - done.
Making train chips for scene: 2-10Traceback (most recent call last):
  File "/usr/lib/python3.5/runpy.py", line 184, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.5/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/opt/src/rastervision/workflows/chain.py", line 489, in <module>
    main()
  File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/opt/src/rastervision/workflows/chain.py", line 485, in main
    run=run)
  File "/opt/src/rastervision/workflows/chain.py", line 468, in _main
    workflow.local_run(tasks)
  File "/opt/src/rastervision/workflows/chain.py", line 436, in local_run
    self.path_generator.make_training_chips_config_uri)
  File "/opt/src/rastervision/run.py", line 22, in _make_training_chips
    command.run()
  File "/opt/src/rastervision/commands/make_training_chips.py", line 13, in run
    self.validation_scenes, self.options)
  File "/opt/src/rastervision/core/ml_task.py", line 140, in make_training_chips
    processed_training_results = _process_scenes(train_scenes, TRAIN)
  File "/opt/src/rastervision/core/ml_task.py", line 137, in _process_scenes
    return [_process_scene(scene, type_) for scene in scenes]
  File "/opt/src/rastervision/core/ml_task.py", line 137, in <listcomp>
    return [_process_scene(scene, type_) for scene in scenes]
  File "/opt/src/rastervision/core/ml_task.py", line 121, in _process_scene
    windows = self.get_train_windows(scene, options)
  File "/opt/src/rastervision/ml_tasks/object_detection.py", line 110, in get_train_windows
    options)
  File "/opt/src/rastervision/ml_tasks/object_detection.py", line 73, in make_pos_windows
    return _make_chip_pos_windows(image_extent, label_store, options)
  File "/opt/src/rastervision/ml_tasks/object_detection.py", line 41, in _make_chip_pos_windows
    window = box.make_random_square_container(chip_size)
  File "/opt/src/rastervision/core/box.py", line 101, in make_random_square_container
    rand_y = random.randint(lb, ub)
  File "/usr/lib/python3.5/random.py", line 218, in randint
    return self.randrange(a, b+1)
  File "/usr/lib/python3.5/random.py", line 196, in randrange
    raise ValueError("empty range for randrange() (%d,%d, %d)" % (istart, istop, width))
ValueError: empty range for randrange() (2710,2515, -195)
root@4e99d917b522:/opt/src# exit
exit


Discussion:
I'm not 100% sure without having some test data, but I *think* I know what's causing this. The error message is saying that the lower bound is greater than the upper bound at https://github.com/azavea/raster-vision/blob/develop/src/rastervision/core/box.py#L101. This can only occur if the chip size is smaller than the box for an object. So you must have a big object that doesn't fit inside a chip. We need to guard against this condition and raise an exception if it occurs. We also need to handle this exception by skipping creating a chip for that object. We should also emit a warning that this is happening, and suggest that the chip size be increased.
ahhh makes sense. in our data we had some cases where the object was larger than the chip for sure. 

Resolving patch:
diff --git a/src/rastervision/core/box.py b/src/rastervision/core/box.py
--- a/src/rastervision/core/box.py
+++ b/src/rastervision/core/box.py
@@ -5,6 +5,10 @@
 from shapely.geometry import box as ShapelyBox
 
 
+class BoxSizeError(ValueError):
+    pass
+
+
 class Box():
     """A multi-purpose box (ie. rectangle)."""
 
@@ -96,6 +100,12 @@ def make_random_square_container(self, size):
         Args:
             size: the width and height of the new Box
         """
+        if size < self.get_width():
+            raise BoxSizeError('size of random container cannot be < width')
+
+        if size < self.get_height():
+            raise BoxSizeError('size of random container cannot be < height')
+
         lb = self.ymin - (size - self.get_height())
         ub = self.ymin
         rand_y = random.randint(lb, ub)
@@ -113,10 +123,10 @@ def make_random_square(self, size):
             size: the height and width of the new Box
         """
         if size >= self.get_width():
-            raise ValueError('size of random square cannot be >= width')
+            raise BoxSizeError('size of random square cannot be >= width')
 
         if size >= self.get_height():
-            raise ValueError('size of random square cannot be >= height')
+            raise BoxSizeError('size of random square cannot be >= height')
 
         lb = self.ymin
         ub = self.ymax - size
diff --git a/src/rastervision/ml_tasks/object_detection.py b/src/rastervision/ml_tasks/object_detection.py
--- a/src/rastervision/ml_tasks/object_detection.py
+++ b/src/rastervision/ml_tasks/object_detection.py
@@ -7,6 +7,7 @@
     ObjectDetectionEvaluation)
 from rastervision.labels.object_detection_labels import ObjectDetectionLabels
 from rastervision.utils.misc import save_img
+from rastervision.core.box import BoxSizeError
 
 
 def save_debug_image(im, labels, class_map, output_path):
@@ -38,16 +39,20 @@ def _make_chip_pos_windows(image_extent, label_store, options):
     # in a window, then it is skipped.
     for box in boxes:
         if box.tuple_format() not in done_boxes:
-            window = box.make_random_square_container(chip_size)
-            pos_windows.append(window)
-
-            # Get boxes that lie completely within window
-            window_boxes = label_store.get_labels(window=window)
-            window_boxes = ObjectDetectionLabels.get_overlapping(
-                window_boxes, window, ioa_thresh=1.0)
-            window_boxes = window_boxes.get_boxes()
-            window_boxes = [box.tuple_format() for box in window_boxes]
-            done_boxes.update(window_boxes)
+            try:
+                window = box.make_random_square_container(chip_size)
+                pos_windows.append(window)
+
+                # Get boxes that lie completely within window
+                window_boxes = label_store.get_labels(window=window)
+                window_boxes = ObjectDetectionLabels.get_overlapping(
+                    window_boxes, window, ioa_thresh=1.0)
+                window_boxes = window_boxes.get_boxes()
+                window_boxes = [box.tuple_format() for box in window_boxes]
+                done_boxes.update(window_boxes)
+            except BoxSizeError:
+                print(('\nSkipping box {} because chip_size is set too small' +
+                       ' for it.').format(box))
 
     return pos_windows
 


Test patch:
diff --git a/src/rastervision/core/box_test.py b/src/rastervision/core/box_test.py
--- a/src/rastervision/core/box_test.py
+++ b/src/rastervision/core/box_test.py
@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from rastervision.core.box import Box
+from rastervision.core.box import Box, BoxSizeError
 from shapely.geometry import box as ShapelyBox
 
 np.random.seed(1)
@@ -64,6 +64,11 @@ def test_make_random_square_container(self):
             self.assertTrue(container.get_shapely().contains(
                 self.box.get_shapely()))
 
+    def test_make_random_square_container_too_big(self):
+        size = 1
+        with self.assertRaises(BoxSizeError):
+            self.box.make_random_square_container(size)
+
     def test_make_random_square(self):
         window = Box(5, 5, 15, 15)
         size = 5