Repository: azavea/raster-vision

Issue:
Batch submit jobs InvalidSignatureException
Sometimes when submitting a Batch job in the VM, I get the following InvalidSignatureException error. The workaround is to reload the VM and try again.
```
root@59542c61e46b:/opt/src# ./scripts/batch_submit.py lf/pytorch     "python -m rv.classification.run train \
        s3://raster-vision-od/configs/training/cl/merced.json \
        s3://raster-vision-od/training-data/cl/merced-train.zip \
        s3://raster-vision-od/training-data/cl/merced-test.zip \
        s3://raster-vision-od/trained-models/cl/merced"     --attempts 1
Traceback (most recent call last):
  File "./scripts/batch_submit.py", line 52, in <module>
    batch_submit()
  File "/opt/conda/lib/python3.5/site-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/opt/conda/lib/python3.5/site-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/opt/conda/lib/python3.5/site-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/opt/conda/lib/python3.5/site-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "./scripts/batch_submit.py", line 44, in batch_submit
    'attempts': attempts
  File "/opt/conda/lib/python3.5/site-packages/botocore/client.py", line 312, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/opt/conda/lib/python3.5/site-packages/botocore/client.py", line 605, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (InvalidSignatureException) when calling the SubmitJob operation: Signature expired: 20171214T213244Z is now earlier than 20171214T213657Z (20171214T214157Z - 5 min.)
```


Discussion:
This may be occurring if your VM clock is getting out of sync. Do you have a sense for if this occurs after you wake your workstation up from sleep?
I'm not sure if it happens then. I just tried submitting a job after waking it up, and it worked fine.
This problem should go away once we stop using Vagrant in https://github.com/azavea/raster-vision/issues/222

Resolving patch:
diff --git a/deployment/batch_amis.py b/deployment/batch_amis.py
--- a/deployment/batch_amis.py
+++ b/deployment/batch_amis.py
@@ -30,7 +30,7 @@ def main():
                             help='AWS region to use for launching an EC2 '
                                  'instance')
 
-    build_amis.add_argument('--source-ami-id', default=None,
+    build_amis.add_argument('--source-ami', default=None,
                             help='ID of the base ami to use for launching the '
                                  'EC2 instance.')
     build_amis.set_defaults(func=create_ami)
diff --git a/src/rastervision/evaluation_items/object_detection_evaluation_item.py b/src/rastervision/evaluation_items/object_detection_evaluation_item.py
--- a/src/rastervision/evaluation_items/object_detection_evaluation_item.py
+++ b/src/rastervision/evaluation_items/object_detection_evaluation_item.py
@@ -12,11 +12,12 @@ def __init__(self, precision, recall, f1, count_error, gt_count=None,
     def merge(self, other):
         super().merge(other)
 
-        total_gt_count = self.gt_count + other.gt_count
-        self_ratio = self.gt_count / total_gt_count
-        other_ratio = other.gt_count / total_gt_count
+        if other.gt_count > 0:
+            total_gt_count = self.gt_count + other.gt_count
+            self_ratio = self.gt_count / total_gt_count
+            other_ratio = other.gt_count / total_gt_count
 
-        def avg(self_val, other_val):
-            return self_ratio * self_val + other_ratio * other_val
+            def avg(self_val, other_val):
+                return self_ratio * self_val + other_ratio * other_val
 
-        self.count_error = avg(self.count_error, other.count_error)
+            self.count_error = avg(self.count_error, other.count_error)
diff --git a/src/rastervision/evaluations/object_detection_evaluation.py b/src/rastervision/evaluations/object_detection_evaluation.py
--- a/src/rastervision/evaluations/object_detection_evaluation.py
+++ b/src/rastervision/evaluations/object_detection_evaluation.py
@@ -1,5 +1,7 @@
 import json
 
+import numpy as np
+
 from object_detection.utils import object_detection_evaluation
 
 from rastervision.core.evaluation import Evaluation
@@ -30,22 +32,33 @@ def parse_od_eval(od_eval, class_map):
     class_to_eval_item = {}
     for class_id in range(1, len(class_map) + 1):
         class_name = class_map.get_by_id(class_id).name
+        gt_count = int(od_eval.num_gt_instances_per_class[class_id - 1])
 
-        precisions = eval_result.precisions[class_id - 1]
-        recalls = eval_result.recalls[class_id - 1]
-        # Get precision and recall assuming all predicted boxes are used.
-        precision = float(precisions[-1])
-        recall = float(recalls[-1])
-        f1 = (2 * precision * recall) / (precision + recall)
+        # If there are predictions for this class.
+        if len(eval_result.precisions[class_id - 1]) > 0:
+            precisions = eval_result.precisions[class_id - 1]
+            recalls = eval_result.recalls[class_id - 1]
+            # Get precision and recall assuming all predicted boxes are used.
+            precision = float(precisions[-1])
+            recall = float(recalls[-1])
+            f1 = 0.
+            if precision + recall != 0.0:
+                f1 = (2 * precision * recall) / (precision + recall)
 
-        gt_count = int(od_eval.num_gt_instances_per_class[class_id - 1])
-        pred_count = len(recalls)
-        count_error = pred_count - gt_count
-        norm_count_error = count_error / gt_count
+            pred_count = len(recalls)
+            count_error = pred_count - gt_count
+            norm_count_error = None
+            if gt_count > 0:
+                norm_count_error = count_error / gt_count
+
+            eval_item = ObjectDetectionEvaluationItem(
+                precision, recall, f1, norm_count_error, gt_count=gt_count,
+                class_id=class_id, class_name=class_name)
+        else:
+            eval_item = ObjectDetectionEvaluationItem(
+                0, 0, 0, 1.0, gt_count=gt_count, class_id=class_id,
+                class_name=class_name)
 
-        eval_item = ObjectDetectionEvaluationItem(
-            precision, recall, f1, norm_count_error, gt_count=gt_count,
-            class_id=class_id, class_name=class_name)
         class_to_eval_item[class_id] = eval_item
 
     return class_to_eval_item


Test patch: