Repository: sql-machine-learning/elasticdl

Issue:
Implement high-level API elasticdl.evaluate
We also need `elasticdl.evaluate`, but should be able to share most implementation code with `elasticdl.predict`.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/api.py b/elasticdl/python/elasticdl/api.py
--- a/elasticdl/python/elasticdl/api.py
+++ b/elasticdl/python/elasticdl/api.py
@@ -8,77 +8,58 @@
 MODEL_ROOT_PATH = "/model_zoo"
 
 
-def train(
-    job_name,
-    namespace,
-    model_def,
-    master_resource_request,
-    master_resource_limit,
-    num_workers,
-    worker_resource_request,
-    worker_resource_limit,
-    master_pod_priority,
-    image_base,
-    docker_image_prefix,
-    extra_pypi_index,
-    tensorboard_log_dir,
-    image_pull_policy,
-    restart_policy,
-    volume_name,
-    mount_path,
-    records_per_task,
-    num_epochs,
-    grads_to_wait,
-    minibatch_size,
-    training_data_dir,
-    evaluation_data_dir,
-):
+def train(args):
     image_name = build_and_push_docker_image(
-        model_zoo=model_def,
-        base_image=image_base,
-        docker_image_prefix=docker_image_prefix,
-        extra_pypi=extra_pypi_index,
+        model_zoo=args.model_def,
+        base_image=args.image_base,
+        docker_image_prefix=args.docker_image_prefix,
+        extra_pypi=args.extra_pypi_index,
     )
     container_args = [
         "-m",
         "elasticdl.python.master.main",
         "--job_name",
-        job_name,
+        args.job_name,
         "--worker_image",
         image_name,
         "--model_def",
-        _model_def_in_docker(model_def),
+        _model_def_in_docker(args.model_def),
         "--num_workers",
-        str(num_workers),
+        str(args.num_workers),
         "--worker_resource_request",
-        worker_resource_request,
+        args.worker_resource_request,
         "--worker_resource_limit",
-        worker_resource_limit,
+        args.worker_resource_limit,
         "--namespace",
-        namespace,
+        args.namespace,
         "--tensorboard_log_dir",
-        tensorboard_log_dir,
+        args.tensorboard_log_dir,
         "--records_per_task",
-        str(records_per_task),
+        str(args.records_per_task),
         "--num_epochs",
-        str(num_epochs),
+        str(args.num_epochs),
         "--grads_to_wait",
-        str(grads_to_wait),
+        str(args.grads_to_wait),
         "--minibatch_size",
-        str(minibatch_size),
+        str(args.minibatch_size),
         "--training_data_dir",
-        training_data_dir,
+        args.training_data_dir,
         "--evaluation_data_dir",
-        evaluation_data_dir,
+        args.evaluation_data_dir,
     ]
-    container_args.extend(["--image_pull_policy", image_pull_policy])
-    container_args.extend(["--restart_policy", restart_policy])
+    container_args.extend(["--image_pull_policy", args.image_pull_policy])
+    container_args.extend(["--restart_policy", args.restart_policy])
 
-    if all([volume_name, mount_path]):
+    if all([args.volume_name, args.mount_path]):
         container_args.extend(
-            ["--mount_path", mount_path, "--volume_name", volume_name]
+            [
+                "--mount_path",
+                args.mount_path,
+                "--volume_name",
+                args.volume_name,
+            ]
         )
-    elif any([volume_name, mount_path]):
+    elif any([args.volume_name, args.mount_path]):
         raise ValueError(
             "Not both of the parameters volume_name and "
             "mount_path are provided."
@@ -86,25 +67,98 @@ def train(
 
     k8s.Client(
         image_name=image_name,
-        namespace=namespace,
-        job_name=job_name,
+        namespace=args.namespace,
+        job_name=args.job_name,
         event_callback=None,
     ).create_master(
-        resource_requests=master_resource_request,
-        resource_limits=master_resource_limit,
+        resource_requests=args.master_resource_request,
+        resource_limits=args.master_resource_limit,
         args=container_args,
-        pod_priority=master_pod_priority,
-        image_pull_policy=image_pull_policy,
-        restart_policy=restart_policy,
-        volume_name=volume_name,
-        mount_path=mount_path,
+        pod_priority=args.master_pod_priority,
+        image_pull_policy=args.image_pull_policy,
+        restart_policy=args.restart_policy,
+        volume_name=args.volume_name,
+        mount_path=args.mount_path,
     )
     # TODO: print dashboard url after launching the master pod
 
 
-def evaluate(args, argv):
-    # TODO implement distributed evaluation.
-    raise NotImplementedError()
+def evaluate(args):
+    image_name = build_and_push_docker_image(
+        model_zoo=args.model_def,
+        base_image=args.image_base,
+        docker_image_prefix=args.docker_image_prefix,
+        extra_pypi=args.extra_pypi_index,
+    )
+    container_args = [
+        "-m",
+        "elasticdl.python.master.main",
+        "--job_name",
+        args.job_name,
+        "--worker_image",
+        image_name,
+        "--model_def",
+        _model_def_in_docker(args.model_def),
+        "--num_workers",
+        str(args.num_workers),
+        "--worker_resource_request",
+        args.worker_resource_request,
+        "--worker_resource_limit",
+        args.worker_resource_limit,
+        "--namespace",
+        args.namespace,
+        "--tensorboard_log_dir",
+        args.tensorboard_log_dir,
+        "--records_per_task",
+        str(args.records_per_task),
+        "--num_epochs",
+        str(args.num_epochs),
+        "--minibatch_size",
+        str(args.minibatch_size),
+        "--evaluation_data_dir",
+        args.evaluation_data_dir,
+        "--evaluation_steps",
+        args.evaluation_steps,
+        "--evaluation_start_delay_secs",
+        args.evaluation_start_delay_secs,
+        "--evaluation_throttle_secs",
+        args.evaluation_throttle_secs,
+        "--checkpoint_filename_for_init",
+        args.checkpoint_filename_for_init,
+    ]
+    container_args.extend(["--image_pull_policy", args.image_pull_policy])
+    container_args.extend(["--restart_policy", args.restart_policy])
+
+    if all([args.volume_name, args.mount_path]):
+        container_args.extend(
+            [
+                "--mount_path",
+                args.mount_path,
+                "--volume_name",
+                args.volume_name,
+            ]
+        )
+    elif any([args.volume_name, args.mount_path]):
+        raise ValueError(
+            "Not both of the parameters volume_name and "
+            "mount_path are provided."
+        )
+
+    k8s.Client(
+        image_name=image_name,
+        namespace=args.namespace,
+        job_name=args.job_name,
+        event_callback=None,
+    ).create_master(
+        resource_requests=args.master_resource_request,
+        resource_limits=args.master_resource_limit,
+        args=container_args,
+        pod_priority=args.master_pod_priority,
+        image_pull_policy=args.image_pull_policy,
+        restart_policy=args.restart_policy,
+        volume_name=args.volume_name,
+        mount_path=args.mount_path,
+    )
 
 
 def _model_def_in_docker(model_def):
diff --git a/elasticdl/python/elasticdl/client.py b/elasticdl/python/elasticdl/client.py
--- a/elasticdl/python/elasticdl/client.py
+++ b/elasticdl/python/elasticdl/client.py
@@ -13,6 +13,7 @@ def main():
 """
     )
     subparsers = parser.add_subparsers()
+
     train_parser = subparsers.add_parser("train", help="elasticdl.py train -h")
     train_parser.set_defaults(func=train)
     _add_train_params(train_parser)
@@ -23,32 +24,8 @@ def main():
     evaluate_parser.set_defaults(func=evaluate)
     _add_evaluate_params(evaluate_parser)
 
-    args, argv = train_parser.parse_known_args()
-    args.func(
-        args.job_name,
-        args.namespace,
-        args.model_def,
-        args.master_resource_request,
-        args.master_resource_limit,
-        args.num_workers,
-        args.worker_resource_request,
-        args.worker_resource_limit,
-        args.master_pod_priority,
-        args.image_base,
-        args.docker_image_prefix,
-        args.extra_pypi_index,
-        args.tensorboard_log_dir,
-        args.image_pull_policy,
-        args.restart_policy,
-        args.volume_name,
-        args.mount_path,
-        args.records_per_task,
-        args.num_epochs,
-        args.grads_to_wait,
-        args.minibatch_size,
-        args.training_data_dir,
-        args.evaluation_data_dir,
-    )
+    args, argv = parser.parse_known_args()
+    args.func(args)
 
 
 def _add_train_params(parser):
@@ -165,8 +142,132 @@ def _add_train_params(parser):
 
 
 def _add_evaluate_params(parser):
-    # TODO add parameters for evaluation parser..
-    pass
+    parser.add_argument(
+        "--model_def",
+        help="The directory that contains user-defined model files "
+        "or a specific model file",
+        required=True,
+    )
+    parser.add_argument(
+        "--docker_image_prefix",
+        default="",
+        help="The prefix for generated Docker images, if set, the image is "
+        "also pushed to the registry",
+    )
+    parser.add_argument("--image_base", help="Base Docker image.")
+    parser.add_argument("--job_name", help="ElasticDL job name", required=True)
+    parser.add_argument(
+        "--master_resource_request",
+        default="cpu=0.1,memory=1024Mi",
+        type=str,
+        help="The minimal resource required by master, "
+        "e.g. cpu=0.1,memory=1024Mi,disk=1024Mi,gpu=1",
+    )
+    parser.add_argument(
+        "--master_resource_limit",
+        type=str,
+        help="The maximal resource required by master, "
+        "e.g. cpu=0.1,memory=1024Mi,disk=1024Mi,gpu=1, "
+        "default to master_resource_request",
+    )
+    parser.add_argument(
+        "--num_workers", type=int, help="Number of workers", default=0
+    )
+    parser.add_argument(
+        "--worker_resource_request",
+        default="cpu=1,memory=4096Mi",
+        type=str,
+        help="The minimal resource required by worker, "
+        "e.g. cpu=1,memory=1024Mi,disk=1024Mi,gpu=1",
+    )
+    parser.add_argument(
+        "--worker_resource_limit",
+        type=str,
+        help="The maximal resource required by worker, "
+        "e.g. cpu=1,memory=1024Mi,disk=1024Mi,gpu=1,"
+        "default to worker_resource_request",
+    )
+    parser.add_argument(
+        "--master_pod_priority", help="The requested priority of master pod"
+    )
+    parser.add_argument(
+        "--volume_name", help="The volume name of network file system"
+    )
+    parser.add_argument(
+        "--mount_path", help="The mount path in the Docker container"
+    )
+    parser.add_argument(
+        "--image_pull_policy",
+        default="Always",
+        help="The image pull policy of master and worker",
+    )
+    parser.add_argument(
+        "--restart_policy",
+        default="Never",
+        help="The pod restart policy when pod crashed",
+    )
+    parser.add_argument(
+        "--extra_pypi_index", help="The extra python package repository"
+    )
+    parser.add_argument(
+        "--namespace",
+        default="default",
+        type=str,
+        help="The name of the Kubernetes namespace where ElasticDL "
+        "pods will be created",
+    )
+    parser.add_argument(
+        "--tensorboard_log_dir",
+        default="",
+        type=str,
+        help="Directory where TensorBoard will look to find "
+        "TensorFlow event files that it can display. "
+        "TensorBoard will recursively walk the directory "
+        "structure rooted at log dir, looking for .*tfevents.* "
+        "files. You may also pass a comma separated list of log "
+        "directories, and TensorBoard will watch each "
+        "directory.",
+    )
+    parser.add_argument("--records_per_task", type=int, required=True)
+    parser.add_argument("--num_epochs", type=int, required=True)
+    parser.add_argument(
+        "--minibatch_size",
+        type=int,
+        help="Minibatch size used by workers",
+        required=True,
+    )
+    parser.add_argument(
+        "--evaluation_data_dir",
+        help="Evaluation data directory. Files should be in RecordIO format",
+        default="",
+    )
+    parser.add_argument(
+        "--evaluation_steps",
+        type=int,
+        help="Evaluate the model every this many steps."
+        "If 0, step-based evaluation is disabled",
+        default=0,
+    )
+    parser.add_argument(
+        "--evaluation_start_delay_secs",
+        type=int,
+        help="Start time-based evaluation only after waiting for "
+        "this many seconds",
+        default=100,
+    )
+    parser.add_argument(
+        "--evaluation_throttle_secs",
+        type=int,
+        help="Do not re-evaluate unless the last evaluation was started "
+        "at least this many seconds ago."
+        "If 0, time-based evaluation is disabled",
+        default=0,
+    )
+    parser.add_argument(
+        "--checkpoint_filename_for_init",
+        help="The checkpoint file to initialize the training model",
+        required=True,
+    )
 
 
 if __name__ == "__main__":
diff --git a/elasticdl/python/master/args.py b/elasticdl/python/master/args.py
--- a/elasticdl/python/master/args.py
+++ b/elasticdl/python/master/args.py
@@ -73,8 +73,8 @@ def parse_args():
     parser.add_argument(
         "--grads_to_wait",
         type=_pos_int,
+        default=2,
         help="Number of gradients to wait before updating model",
-        required=True,
     )
     parser.add_argument(
         "--minibatch_size",


Test patch: