Repository: sql-machine-learning/elasticdl

Issue:
[master]Use different RPC for reporting gradient and task result.



Discussion:

Resolving patch:
diff --git a/elasticdl/master/servicer.py b/elasticdl/master/servicer.py
--- a/elasticdl/master/servicer.py
+++ b/elasticdl/master/servicer.py
@@ -66,7 +66,7 @@ def _update_model(self):
         self._gradient_sum.clear()
         self._grad_n = 0
 
-    def ReportTaskResult(self, request, context):
+    def ReportGradient(self, request, context):
         if request.model_version > self._version:
             err_msg = "Model version %d out of range, current version: %d" % (
                 request.model_version,
@@ -75,7 +75,7 @@ def ReportTaskResult(self, request, context):
             self.logger.warning(err_msg)
             raise ValueError(err_msg)
 
-        res = master_pb2.ReportTaskResultReply()
+        res = master_pb2.ReportGradientReply()
         if request.model_version < self._version:
             self.logger.warning(
                 "Task result for outdated version %d dropped",
@@ -85,12 +85,6 @@ def ReportTaskResult(self, request, context):
             res.model_version = self._version
             return res
 
-        if request.err_message:
-            self.logger.warning("Worker error: %s" % request.err_message)
-            res.accepted = False
-            res.model_version = self._version
-            return res
-
         # TODO: Update task queue with task_id
         with self._lock:
             tmp = {}


Test patch:
diff --git a/elasticdl/master/servicer_test.py b/elasticdl/master/servicer_test.py
--- a/elasticdl/master/servicer_test.py
+++ b/elasticdl/master/servicer_test.py
@@ -59,10 +59,10 @@ def testGetModel(self):
         req.min_version = 2
         self.assertRaises(ValueError, master.GetModel, req, None)
 
-    def testReportTaskResult(self):
+    def testReportGradient(self):
         def makeGrad():
             """ Make a ReportTaskResultRequest compatible with model"""
-            req = master_pb2.ReportTaskResultRequest()
+            req = master_pb2.ReportGradientRequest()
             req.gradient["x"].CopyFrom(
                 ndarray_to_tensor(np.array([0.1], dtype=np.float32))
             )
@@ -82,19 +82,12 @@ def makeGrad():
         # Report a future version, should raise exception
         req = makeGrad()
         req.model_version = 2
-        self.assertRaises(ValueError, master.ReportTaskResult, req, None)
+        self.assertRaises(ValueError, master.ReportGradient, req, None)
 
         # Report an old version, should not be accepted
         req = makeGrad()
         req.model_version = 0
-        res = master.ReportTaskResult(req, None)
-        self.assertFalse(res.accepted)
-        self.assertEqual(1, res.model_version)
-
-        # Report a current version, but with error, should not be accepted.
-        req = makeGrad()
-        req.err_message = "worker error"
-        res = master.ReportTaskResult(req, None)
+        res = master.ReportGradient(req, None)
         self.assertFalse(res.accepted)
         self.assertEqual(1, res.model_version)
 
@@ -103,25 +96,25 @@ def makeGrad():
         req.gradient["z"].CopyFrom(
             ndarray_to_tensor(np.array([0.1], dtype=np.float32))
         )
-        self.assertRaises(ValueError, master.ReportTaskResult, req, None)
+        self.assertRaises(ValueError, master.ReportGradient, req, None)
 
         # Report an incompatible gradient, should raise.
         req = makeGrad()
         req.gradient["y"].CopyFrom(
             ndarray_to_tensor(np.array([0.1], dtype=np.float32))
         )
-        self.assertRaises(ValueError, master.ReportTaskResult, req, None)
+        self.assertRaises(ValueError, master.ReportGradient, req, None)
 
-        # Report a current version without error, should be accepted.
+        # Report a current version, should be accepted.
         req = makeGrad()
-        res = master.ReportTaskResult(req, None)
+        res = master.ReportGradient(req, None)
         self.assertTrue(res.accepted)
         self.assertEqual(1, res.model_version)
 
         # Report a current version with part of gradients, should be accepted.
         req = makeGrad()
         del req.gradient["y"]
-        res = master.ReportTaskResult(req, None)
+        res = master.ReportGradient(req, None)
         self.assertTrue(res.accepted)
         self.assertEqual(1, res.model_version)
         # Gradient should be accumulated.
@@ -133,10 +126,10 @@ def makeGrad():
         )
         self.assertEqual(2, master._grad_n)
 
-        # Report a current version without error, should be accepted, and a new
-        # version created
+        # Report a current version, should be accepted, and a new version
+        # created
         req = makeGrad()
-        res = master.ReportTaskResult(req, None)
+        res = master.ReportGradient(req, None)
         self.assertTrue(res.accepted)
         self.assertEqual(2, res.model_version)
         self.assertFalse(master._gradient_sum)
@@ -151,3 +144,7 @@ def makeGrad():
             np.array([11.998, 12.996], dtype=np.float32),
             master._model["y"].numpy(),
         )
+
+    def testReportTaskResult(self):
+        # TODO: add tests and verify task queue
+        pass