Repository: sql-machine-learning/elasticdl

Issue:
Add tests for evaluation metrics reporting service
We previously added service to report evaluation metrics in #510. However, there isn't any test coverage for the added code yet. We should add some tests to cover `MasterServicer.ReportEvaluationMetrics`.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/master/servicer.py b/elasticdl/python/elasticdl/master/servicer.py
--- a/elasticdl/python/elasticdl/master/servicer.py
+++ b/elasticdl/python/elasticdl/master/servicer.py
@@ -77,6 +77,10 @@ def GetModel(self, request, _):
                 res.param[k].CopyFrom(ndarray_to_tensor(v.numpy()))
         return res
 
+    def _update_model_version(self):
+        assert self._lock.locked()
+        self._version += 1
+
     def _update_model(self):
         assert self._lock.locked()
         grad_var = []
@@ -84,7 +88,7 @@ def _update_model(self):
             self._gradient_sum[k] = self._gradient_sum[k] / self._grad_to_wait
             grad_var.append((self._gradient_sum[k], self._model[k]))
         self._opt.apply_gradients(grad_var)
-        self._version += 1
+        self._update_model_version()
         self._gradient_sum.clear()
         self._grad_n = 0
 
@@ -167,7 +171,7 @@ def ReportEvaluationMetrics(self, request, _):
                 arr = tensor_to_ndarray(v)
                 self._evaluation_metrics[k] = arr
 
-            self._update_model()
+            self._update_model_version()
         res.accepted = True
         res.model_version = self._version
         return res


Test patch:
diff --git a/elasticdl/python/tests/servicer_test.py b/elasticdl/python/tests/servicer_test.py
--- a/elasticdl/python/tests/servicer_test.py
+++ b/elasticdl/python/tests/servicer_test.py
@@ -181,6 +181,46 @@ def makeGrad():
             master._model["y"].numpy(),
         )
 
+    def testReportEvaluationMetrics(self):
+        def makeEvaluationMetrics():
+            """ Make a ReportEvaluationMetricsRequest compatible with model"""
+            req = elasticdl_pb2.ReportEvaluationMetricsRequest()
+            req.evaluation_metrics['mse'].CopyFrom(
+                ndarray_to_tensor(np.array([100, 200], dtype=np.float32))
+            )
+            req.model_version = 1
+            return req
+
+        master = MasterServicer(
+            3,
+            3,
+            tf.train.GradientDescentOptimizer(0.1),
+            None,
+        )
+        master._version = 1
+
+        # Report a future version, should raise exception
+        req = makeEvaluationMetrics()
+        req.model_version = 2
+        self.assertRaisesRegex(
+            ValueError,
+            'Model version %s not available yet, current version: %s' % (
+                req.model_version, master._version
+            ), master.ReportEvaluationMetrics, req, None)
+
+        # Report an old version, should not be accepted
+        req = makeEvaluationMetrics()
+        req.model_version = 0
+        res = master.ReportEvaluationMetrics(req, None)
+        self.assertFalse(res.accepted)
+        self.assertEqual(1, res.model_version)
+
+        # Report a current version, should be accepted, and a new version is created
+        req = makeEvaluationMetrics()
+        res = master.ReportEvaluationMetrics(req, None)
+        self.assertTrue(res.accepted)
+        self.assertEqual(2, res.model_version)
+
     def testReportTaskResult(self):
         task_q = _TaskQueue(
             {"shard_1": 10, "shard_2": 9}, record_per_task=3, num_epoch=2