Repository: sql-machine-learning/elasticdl

Issue:
Evaluation minibatches may be calculated repeatedly
Currently, each task contains multiple minibatches. In the evaluation phase, workers calculate each minibatch in turn. Every time a minibatch calculation is completed, the worker immediately passes the calculated evaluation metrics to the master. When the worker completes all the minibatches in a task, the worker tells the master the results of the entire task.
If a worker A is killed in the process of calculating a task, the master will find another worker B to calculate the failed task, but it will not clear the metrics of the minibatch already calculated in the task. So some minibatches are calculated both by worker A and worker B, which will lead to errors in the final evaluation metrics.


Discussion:
Interesting. I think workers should report results per task as the task is the minimum unit watched by the master.
> Interesting. I think workers should report results per task as the task is the minimum unit watched by the master.

Agree. In evaluation phase, worker should report results per task. 

Resolving patch:
diff --git a/elasticdl/python/common/constants.py b/elasticdl/python/common/constants.py
--- a/elasticdl/python/common/constants.py
+++ b/elasticdl/python/common/constants.py
@@ -37,3 +37,4 @@ class Redis(object):
 
 class MetricsDictKey(object):
     MODEL_OUTPUT = "output"
+    LABEL = "label"
diff --git a/elasticdl/python/worker/worker.py b/elasticdl/python/worker/worker.py
--- a/elasticdl/python/worker/worker.py
+++ b/elasticdl/python/worker/worker.py
@@ -1,5 +1,6 @@
 import traceback
 
+import numpy as np
 import tensorflow as tf
 
 from elasticdl.proto import elasticdl_pb2, elasticdl_pb2_grpc
@@ -114,6 +115,7 @@ def __init__(
         if self._get_model_steps > 1:
             self._opt = self._opt_fn()
             self._non_embed_grads = None
+        self._evaluation_result = {}
 
     # TODO: Multiple tests are currently using this function to initialize
     # self._model, where the initialization should be done via constructor.
@@ -267,11 +269,11 @@ def report_evaluation_metrics(self, model_outputs, labels):
         from rpc call.
         """
         req = elasticdl_pb2.ReportEvaluationMetricsRequest()
-        if not isinstance(model_outputs, dict):
-            model_outputs = {MetricsDictKey.MODEL_OUTPUT: model_outputs}
         for name, output in model_outputs.items():
-            req.model_outputs[name].CopyFrom(ndarray_to_tensor(output.numpy()))
-        req.labels.CopyFrom(ndarray_to_tensor(labels.numpy()))
+            output = np.concatenate(output)
+            req.model_outputs[name].CopyFrom(ndarray_to_tensor(output))
+        labels = np.concatenate(labels)
+        req.labels.CopyFrom(ndarray_to_tensor(labels))
         req.model_version = self._model_version
         res = self._stub.ReportEvaluationMetrics(req)
         return res.accepted, res.model_version
@@ -380,10 +382,26 @@ def _run_training_task(self, features, labels):
         self._reset_embedding()
         return accepted, min_model_version, loss
 
+    def _collect_evaluation_result(self, outputs, labels):
+        key = MetricsDictKey.MODEL_OUTPUT
+        if key not in self._evaluation_result:
+            outputs = {k: [v.numpy()] for k, v in outputs.items()}
+            self._evaluation_result[key] = outputs
+        else:
+            for k, v in outputs.items():
+                self._evaluation_result[key][k].append(v.numpy())
+        key = MetricsDictKey.LABEL
+        if key not in self._evaluation_result:
+            self._evaluation_result[key] = [labels.numpy()]
+        else:
+            self._evaluation_result[key].append(labels.numpy())
+
     def _run_evaluation_task(self, features, labels):
         outputs = self.forward_process(features)
-        accepted, _ = self.report_evaluation_metrics(outputs, labels)
-        return accepted
+        if not isinstance(outputs, dict):
+            outputs = {MetricsDictKey.MODEL_OUTPUT: outputs}
+        self._collect_evaluation_result(outputs, labels)
+        return True
 
     def _run_prediction_task(self, features):
         predictions = self.forward_process(features)
@@ -465,7 +483,14 @@ def _process_eval_task_if_needed(self):
                 err_msg = data_err_msg
                 break
         del eval_dataset
+        accepted, _ = self.report_evaluation_metrics(
+            self._evaluation_result[MetricsDictKey.MODEL_OUTPUT],
+            self._evaluation_result[MetricsDictKey.LABEL],
+        )
+        if not accepted:
+            raise RuntimeError("Report evaluation metric failed!")
         self.report_task_result(task_id, err_msg)
+        self._evaluation_result = {}
         return True
 
     def _process_minibatch_and_report(


Test patch: