Repository: sql-machine-learning/elasticdl

Issue:
Run evaluation every n many steps
Currently, we support evaluation based on training time. We also need to support the evaluation based on model update steps.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/evaluation_service.py b/elasticdl/python/master/evaluation_service.py
--- a/elasticdl/python/master/evaluation_service.py
+++ b/elasticdl/python/master/evaluation_service.py
@@ -81,7 +81,7 @@ def run(self):
             time_now = time.time()
             if self._wait_enough_time(time_now, previous_round_start_secs):
                 # Time is up, add an evaluation task
-                self._eval_service.add_evaluation_task()
+                self._eval_service.add_evaluation_task(is_time_based_eval=True)
                 previous_round_start_secs = time_now
             time.sleep(5)
 
@@ -96,6 +96,7 @@ def __init__(
         task_d,
         start_delay_secs,
         throttle_secs,
+        eval_steps,
     ):
         self._logger = logging.getLogger(__name__)
         self._checkpoint_service = checkpoint_service
@@ -107,6 +108,7 @@ def __init__(
             self, start_delay_secs, throttle_secs
         )
         self._time_based_eval = throttle_secs > 0
+        self._eval_steps = eval_steps
         self._eval_checkpoint_versions = []
         self._last_eval_checkpoint_version = -1
 
@@ -121,18 +123,19 @@ def stop(self):
     def set_master_servicer(self, master_servicer):
         self._master_servicer = master_servicer
 
-    def add_evaluation_task(self, model_version=-1):
+    def add_evaluation_task(self, is_time_based_eval, master_locking=True):
         """
-        Add evaluation task with model_version.
-        If model_version<0, use current model version.
+        Add evaluation task with current model_version.
         """
-        if model_version < 0:
-            model_version = self._master_servicer.get_model_version()
+        # Do not create time-based eval after all tasks are done
+        if is_time_based_eval and self._task_d.finished():
+            return
+        model_version = self._master_servicer.get_model_version()
         if model_version == self._last_eval_checkpoint_version:
             return
 
         checkpoint_version = self._master_servicer._save_checkpoint(
-            locking=True, is_eval_checkpoint=True
+            locking=master_locking, is_eval_checkpoint=True
         )
         with self._lock:
             self._eval_checkpoint_versions.append(checkpoint_version)
@@ -150,6 +153,13 @@ def try_to_create_new_job(self):
                 return True
         return False
 
+    def add_evaluation_task_if_needed(self, master_locking):
+        model_version = self._master_servicer.get_model_version()
+        if self._eval_steps and model_version % self._eval_steps == 0:
+            self.add_evaluation_task(
+                is_time_based_eval=False, master_locking=master_locking
+            )
+
     def report_evaluation_metrics(
         self, evaluation_version, evaluation_metrics
     ):
diff --git a/elasticdl/python/master/main.py b/elasticdl/python/master/main.py
--- a/elasticdl/python/master/main.py
+++ b/elasticdl/python/master/main.py
@@ -114,8 +114,10 @@ def main():
     evaluation_service = None
     if evaluation_while_training:
         logger.info(
-            "Starting evaluation service with throttle seconds %d",
+            "Starting evaluation service with throttle seconds %d ",
+            " and evaluation steps %d",
             args.evaluation_throttle_secs,
+            args.evaluation_steps,
         )
         evaluation_service = EvaluationService(
             checkpoint_service,
@@ -123,6 +125,7 @@ def main():
             task_d,
             args.evaluation_start_delay_secs,
             args.evaluation_throttle_secs,
+            args.evaluation_steps,
         )
         evaluation_service.start()
         task_d.set_evaluation_service(evaluation_service)
diff --git a/elasticdl/python/master/servicer.py b/elasticdl/python/master/servicer.py
--- a/elasticdl/python/master/servicer.py
+++ b/elasticdl/python/master/servicer.py
@@ -160,8 +160,17 @@ def _save_checkpoint(self, locking, is_eval_checkpoint):
             self._lock.release()
         return checkpoint_version
 
+    def _update_evaluation(self):
+        if self._evaluation_service:
+            self._evaluation_service.add_evaluation_task_if_needed(
+                master_locking=False
+            )
+
     def _update_checkpoint(self):
-        if self._checkpoint_service.need_to_checkpoint(self._version):
+        if (
+            self._checkpoint_service
+            and self._checkpoint_service.need_to_checkpoint(self._version)
+        ):
             try:
                 self._logger.info(
                     "Saving checkpoint for model version %d" % self._version
@@ -236,6 +245,7 @@ def ReportGradient(self, request, _):
             self._grad_n += 1
             if self._grad_n >= self._grad_to_wait:
                 self._update_model()
+                self._update_evaluation()
                 self._update_checkpoint()
 
         res.accepted = True


Test patch:
diff --git a/elasticdl/python/tests/evaluation_service_test.py b/elasticdl/python/tests/evaluation_service_test.py
--- a/elasticdl/python/tests/evaluation_service_test.py
+++ b/elasticdl/python/tests/evaluation_service_test.py
@@ -87,7 +87,7 @@ def testEvaluationService(self):
 
             # Evaluation metrics will not be accepted if no evaluation ongoing
             evaluation_service = EvaluationService(
-                checkpoint_service, None, task_d, 10, 20
+                checkpoint_service, None, task_d, 10, 20, 0
             )
             evaluation_metrics = {
                 "mse": ndarray_to_tensor(
diff --git a/elasticdl/scripts/client_test.sh b/elasticdl/scripts/client_test.sh
--- a/elasticdl/scripts/client_test.sh
+++ b/elasticdl/scripts/client_test.sh
@@ -13,6 +13,7 @@ python -m elasticdl.python.client.client train \
   --records_per_task=100 \
   --num_workers=2 \
   --checkpoint_steps=10 \
+  --evaluation_steps=15 \
   --grads_to_wait=2 \
   --job_name=test-mnist \
   --log_level=INFO \