Repository: sql-machine-learning/elasticdl

Issue:
Change use_async and lr_staleness_modulation to boolean arguments and pass them
The arguments:
+ `use_async`
+ `lr_staleness_modulation`

are arguments with default value `False`. They are not boolean arguments.

And also pass them from client to master and worker


Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/args.py b/elasticdl/python/common/args.py
--- a/elasticdl/python/common/args.py
+++ b/elasticdl/python/common/args.py
@@ -1,4 +1,5 @@
 import argparse
+from itertools import chain
 
 from elasticdl.python.common.log_util import default_logger as logger
 
@@ -85,6 +86,17 @@ def parse_envs(arg):
     return envs
 
 
+def add_bool_param(parser, name, default, help):
+    parser.add_argument(
+        name,  # should be in "--foo" format
+        nargs="?",
+        const=not default,
+        default=default,
+        type=lambda x: x.lower() in ["true", "yes", "t", "y"],
+        help=help,
+    )
+
+
 def add_common_params(parser):
     """Common arguments for training/prediction/evaluation"""
     add_common_args_between_master_and_worker(parser)
@@ -262,13 +274,16 @@ def add_train_params(parser):
         default="",
         help="The path to save the final trained model",
     )
-    parser.add_argument(
-        "--use_async",
+
+    add_bool_param(
+        parser=parser,
+        name="--use_async",
         default=False,
         help="True for asynchronous SGD, False for synchronous SGD",
     )
-    parser.add_argument(
-        "--lr_staleness_modulation",
+    add_bool_param(
+        parser=parser,
+        name="--lr_staleness_modulation",
         default=False,
         help="If True, master will modulate the learning rate with staleness "
         "in asynchronous SGD",
@@ -473,3 +488,27 @@ def parse_worker_args(worker_args=None):
     print_args(args, groups=ALL_ARGS_GROUPS)
     logger.warning("Unknown arguments: %s", _)
     return args
+
+
+def build_arguments_from_parsed_result(args, filter_args=None):
+    """Resconstruct arguments from parsed result
+    Args:
+        args: result from `parser.parse_args()`
+    Returns:
+        list of string: ready for parser to parse,
+        such as ["--foo", "3", "--bar", False]
+    """
+    items = vars(args).items()
+    if filter_args:
+        items = filter(lambda item: item[0] not in filter_args, items)
+
+    def _str_ignore_none(s):
+        if s is None:
+            return s
+        return str(s)
+
+    arguments = map(_str_ignore_none, chain(*items))
+    arguments = [
+        "--" + k if i % 2 == 0 else k for i, k in enumerate(arguments)
+    ]
+    return arguments
diff --git a/elasticdl/python/elasticdl/api.py b/elasticdl/python/elasticdl/api.py
--- a/elasticdl/python/elasticdl/api.py
+++ b/elasticdl/python/elasticdl/api.py
@@ -1,7 +1,10 @@
 import os
 
 from elasticdl.python.common import k8s_client as k8s
-from elasticdl.python.common.args import parse_envs
+from elasticdl.python.common.args import (
+    build_arguments_from_parsed_result,
+    parse_envs,
+)
 from elasticdl.python.common.log_util import default_logger as logger
 from elasticdl.python.elasticdl.image_builder import (
     build_and_push_docker_image,
@@ -23,70 +26,18 @@ def train(args):
     container_args = [
         "-m",
         "elasticdl.python.master.main",
-        "--job_name",
-        args.job_name,
         "--worker_image",
         image_name,
         "--model_zoo",
         _model_zoo_in_docker(args.model_zoo),
         "--cluster_spec",
         _cluster_spec_def_in_docker(args.cluster_spec),
-        "--num_workers",
-        str(args.num_workers),
-        "--master_resource_request",
-        args.master_resource_request,
-        "--master_resource_limit",
-        args.master_resource_limit,
-        "--worker_resource_request",
-        args.worker_resource_request,
-        "--worker_resource_limit",
-        args.worker_resource_limit,
-        "--envs",
-        args.envs,
-        "--namespace",
-        args.namespace,
-        "--tensorboard_log_dir",
-        args.tensorboard_log_dir,
-        "--records_per_task",
-        str(args.records_per_task),
-        "--num_epochs",
-        str(args.num_epochs),
-        "--grads_to_wait",
-        str(args.grads_to_wait),
-        "--minibatch_size",
-        str(args.minibatch_size),
-        "--training_data_dir",
-        args.training_data_dir,
-        "--evaluation_data_dir",
-        args.evaluation_data_dir,
-        "--checkpoint_steps",
-        str(args.checkpoint_steps),
-        "--checkpoint_dir",
-        args.checkpoint_dir,
-        "--keep_checkpoint_max",
-        str(args.keep_checkpoint_max),
-        "--evaluation_steps",
-        str(args.evaluation_steps),
-        "--evaluation_start_delay_secs",
-        str(args.evaluation_start_delay_secs),
-        "--evaluation_throttle_secs",
-        str(args.evaluation_throttle_secs),
-        "--dataset_fn",
-        args.dataset_fn,
-        "--loss",
-        args.loss,
-        "--optimizer",
-        args.optimizer,
-        "--eval_metrics_fn",
-        args.eval_metrics_fn,
-        "--model_def",
-        args.model_def,
-        "--model_params",
-        args.model_params,
     ]
-    container_args.extend(["--image_pull_policy", args.image_pull_policy])
-    container_args.extend(["--restart_policy", args.restart_policy])
-    container_args.extend(["--volume", args.volume])
+    container_args.extend(
+        build_arguments_from_parsed_result(
+            args, filter_args=["model_zoo", "cluster_spec", "worker_image"]
+        )
+    )
 
     _submit_job(image_name, args, container_args)
     # TODO: print dashboard url after launching the master pod
@@ -106,44 +57,18 @@ def evaluate(args):
     container_args = [
         "-m",
         "elasticdl.python.master.main",
-        "--job_name",
-        args.job_name,
         "--worker_image",
         image_name,
         "--model_zoo",
         _model_zoo_in_docker(args.model_zoo),
         "--cluster_spec",
         _cluster_spec_def_in_docker(args.cluster_spec),
-        "--num_workers",
-        str(args.num_workers),
-        "--worker_resource_request",
-        args.worker_resource_request,
-        "--worker_resource_limit",
-        args.worker_resource_limit,
-        "--envs",
-        args.envs,
-        "--namespace",
-        args.namespace,
-        "--records_per_task",
-        str(args.records_per_task),
-        "--minibatch_size",
-        str(args.minibatch_size),
-        "--evaluation_data_dir",
-        args.evaluation_data_dir,
-        "--checkpoint_filename_for_init",
-        args.checkpoint_filename_for_init,
-        "--dataset_fn",
-        args.dataset_fn,
-        "--eval_metrics_fn",
-        args.eval_metrics_fn,
-        "--model_def",
-        args.model_def,
-        "--model_params",
-        args.model_params,
     ]
-    container_args.extend(["--image_pull_policy", args.image_pull_policy])
-    container_args.extend(["--restart_policy", args.restart_policy])
-    container_args.extend(["--volume", args.volume])
+    container_args.extend(
+        build_arguments_from_parsed_result(
+            args, filter_args=["model_zoo", "cluster_spec", "worker_image"]
+        )
+    )
 
     _submit_job(image_name, args, container_args)
 
@@ -162,42 +87,18 @@ def predict(args):
     container_args = [
         "-m",
         "elasticdl.python.master.main",
-        "--job_name",
-        args.job_name,
         "--worker_image",
         image_name,
         "--model_zoo",
         _model_zoo_in_docker(args.model_zoo),
         "--cluster_spec",
         _cluster_spec_def_in_docker(args.cluster_spec),
-        "--num_workers",
-        str(args.num_workers),
-        "--worker_resource_request",
-        args.worker_resource_request,
-        "--worker_resource_limit",
-        args.worker_resource_limit,
-        "--envs",
-        args.envs,
-        "--namespace",
-        args.namespace,
-        "--records_per_task",
-        str(args.records_per_task),
-        "--minibatch_size",
-        str(args.minibatch_size),
-        "--prediction_data_dir",
-        args.prediction_data_dir,
-        "--checkpoint_filename_for_init",
-        args.checkpoint_filename_for_init,
-        "--dataset_fn",
-        args.dataset_fn,
-        "--model_def",
-        args.model_def,
-        "--model_params",
-        args.model_params,
     ]
-    container_args.extend(["--image_pull_policy", args.image_pull_policy])
-    container_args.extend(["--restart_policy", args.restart_policy])
-    container_args.extend(["--volume", args.volume])
+    container_args.extend(
+        build_arguments_from_parsed_result(
+            args, filter_args=["model_zoo", "cluster_spec", "worker_image"]
+        )
+    )
 
     _submit_job(image_name, args, container_args)
 
diff --git a/elasticdl/python/master/embedding_service.py b/elasticdl/python/master/embedding_service.py
--- a/elasticdl/python/master/embedding_service.py
+++ b/elasticdl/python/master/embedding_service.py
@@ -164,7 +164,7 @@ def _parse_embedding_service_args():
             "can be unavailable",
         )
 
-        args = parser.parse_args()
+        args, _ = parser.parse_known_args()
 
         return args
 


Test patch:
diff --git a/elasticdl/python/tests/args_test.py b/elasticdl/python/tests/args_test.py
new file mode 100644
--- /dev/null
+++ b/elasticdl/python/tests/args_test.py
@@ -0,0 +1,36 @@
+import argparse
+import unittest
+
+from elasticdl.python.common.args import (
+    add_bool_param,
+    build_arguments_from_parsed_result,
+)
+
+
+class ArgsTest(unittest.TestCase):
+    def setUp(self):
+        self._parser = argparse.ArgumentParser()
+        self._parser.add_argument("--foo", default=3, type=int)
+        add_bool_param(self._parser, "--bar", False, "")
+
+    def test_build_arguments_from_parsed_result(self):
+        args = ["--foo", "4", "--bar"]
+        results = self._parser.parse_args(args=args)
+        original_arguments = build_arguments_from_parsed_result(results)
+        value = "\t".join(sorted(original_arguments))
+        target = "\t".join(sorted(["--foo", "4", "--bar", "True"]))
+        self.assertEqual(value, target)
+
+        original_arguments = build_arguments_from_parsed_result(
+            results, filter_args=["foo"]
+        )
+        value = "\t".join(sorted(original_arguments))
+        target = "\t".join(sorted(["--bar", "True"]))
+        self.assertEqual(value, target)
+
+        args = ["--foo", "4"]
+        results = self._parser.parse_args(args=args)
+        original_arguments = build_arguments_from_parsed_result(results)
+        value = "\t".join(sorted(original_arguments))
+        target = "\t".join(sorted(["--bar", "False", "--foo", "4"]))
+        self.assertEqual(value, target)