internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: sql-machine-learning/elasticdl

Issue:
Set the max_worker of grpc server in PS according to num_workers.
The threads may be wasted if the number of threads in grpc server is bigger than the number of workers. So we should set the `max_worker` according to the number of workers.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/master.py b/elasticdl/python/master/master.py
--- a/elasticdl/python/master/master.py
+++ b/elasticdl/python/master/master.py
@@ -379,6 +379,8 @@ def _create_instance_manager(self, args):
                 str(args.num_ps_pods),
                 "--checkpoint_dir_for_init",
                 str(args.checkpoint_dir_for_init),
+                "--num_workers",
+                str(args.num_workers),
             ]
 
             env_dict = parse_envs(args.envs)
diff --git a/elasticdl/python/ps/parameter_server.py b/elasticdl/python/ps/parameter_server.py
--- a/elasticdl/python/ps/parameter_server.py
+++ b/elasticdl/python/ps/parameter_server.py
@@ -34,6 +34,7 @@ def __init__(self, args):
         self._set_lr_scheduler(model_module, args.learning_rate_scheduler)
         self.ps_id = args.ps_id
         self.num_ps_pods = args.num_ps_pods
+        self.num_workers = args.num_workers
         # Create Parameters instance
         self.parameters = Parameters()
         if args.master_addr is None:
@@ -94,8 +95,10 @@ def _init_checkpoint_saver(self, args):
             )
 
     def prepare(self):
+        max_workers = min(self.num_workers, 64)
+        self.logger.info("The max threads in PS servers is %d" % max_workers)
         server = grpc.server(
-            futures.ThreadPoolExecutor(max_workers=64),
+            futures.ThreadPoolExecutor(max_workers=max_workers),
             options=[
                 ("grpc.max_send_message_length", GRPC.MAX_SEND_MESSAGE_LENGTH),
                 (


Test patch:
diff --git a/elasticdl/python/tests/test_utils.py b/elasticdl/python/tests/test_utils.py
--- a/elasticdl/python/tests/test_utils.py
+++ b/elasticdl/python/tests/test_utils.py
@@ -55,6 +55,7 @@ def __init__(
         keep_checkpoint_max=0,
         ps_id=0,
         num_ps_pods=1,
+        num_workers=2,
         checkpoint_dir_for_init=None,
     ):
         self.grads_to_wait = grads_to_wait
@@ -75,6 +76,7 @@ def __init__(
         self.keep_checkpoint_max = keep_checkpoint_max
         self.ps_id = ps_id
         self.num_ps_pods = num_ps_pods
+        self.num_workers = num_workers
         self.checkpoint_dir_for_init = checkpoint_dir_for_init
Repository: sql-machine-learning/elasticdl

Issue:
The memory grows fast when the num_minibatches_per_task in args is big using odps table.
A task may contain a larger number of records. If we directly download all records in the task from ODPS, huge memory is needed which may cause a memory leak or OOM. So, we need to split the records in the task to small chunks and download the small chunk to yield each time.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/data/odps_io.py b/elasticdl/python/data/odps_io.py
--- a/elasticdl/python/data/odps_io.py
+++ b/elasticdl/python/data/odps_io.py
@@ -226,17 +226,14 @@ def read_batch(self, start, end, columns=None, max_retries=3):
             columns = self._odps_table.schema.names
         while retry_count < max_retries:
             try:
-                batch_record = []
                 with self._odps_table.open_reader(
-                    partition=self._partition, reopen=True
+                    partition=self._partition, reopen=False
                 ) as reader:
                     for record in reader.read(
                         start=start, count=end - start, columns=columns
                     ):
-                        batch_record.append(
-                            [str(record[column]) for column in columns]
-                        )
-                return batch_record
+                        yield [str(record[column]) for column in columns]
+                    return
             except Exception as e:
                 if retry_count >= max_retries:
                     raise Exception("Exceeded maximum number of retries")
diff --git a/elasticdl/python/data/reader/odps_reader.py b/elasticdl/python/data/reader/odps_reader.py
--- a/elasticdl/python/data/reader/odps_reader.py
+++ b/elasticdl/python/data/reader/odps_reader.py
@@ -19,16 +19,11 @@ def read_records(self, task):
         reader = self._get_reader(
             table_name=self._get_odps_table_name(task.shard_name)
         )
-        if self._metadata.column_names is None:
-            columns = self._kwargs.get("columns")
-            self._metadata.column_names = (
-                reader._odps_table.schema.names if columns is None else columns
-            )
-        records = reader.read_batch(
+
+        for record in reader.read_batch(
             start=task.start, end=task.end, columns=self._metadata.column_names
-        )
-        for batch in records:
-            yield batch
+        ):
+            yield record
 
     def create_shards(self):
         check_required_kwargs(["table", "records_per_task"], self._kwargs)


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Parse arguments(flag) in ps/server.go



Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/args.py b/elasticdl/python/common/args.py
--- a/elasticdl/python/common/args.py
+++ b/elasticdl/python/common/args.py
@@ -333,7 +333,7 @@ def add_train_params(parser):
         parser=parser,
         name="--lr_staleness_modulation",
         default=False,
-        help="If True, master will modulate the learning rate with staleness "
+        help="If True, PS will modulate the learning rate with staleness "
         "in asynchronous SGD",
     )
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Need to start GRPC server using pod's IP instead of localhost
Go PS needs to find pod IP to use.



Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/k8s_client.py b/elasticdl/python/common/k8s_client.py
--- a/elasticdl/python/common/k8s_client.py
+++ b/elasticdl/python/common/k8s_client.py
@@ -32,6 +32,20 @@ def get_ps_pod_name(job_name, ps_id):
     return "elasticdl-%s-ps-%s" % (job_name, str(ps_id))
 
 
+def append_pod_ip_to_env(env):
+    pod_ip_var = V1EnvVar(
+        name="MY_POD_IP",
+        value_from=V1EnvVarSource(
+            field_ref=V1ObjectFieldSelector(field_path="status.podIP")
+        ),
+    )
+    if env:
+        env.append(pod_ip_var)
+    else:
+        env = [pod_ip_var]
+    return env
+
+
 class Client(object):
     def __init__(
         self,
@@ -285,17 +299,11 @@ def dump_master_yaml(self, **kargs):
             yaml.safe_dump(pod_dict, f)
 
     def _create_master_pod_obj(self, **kargs):
-        env = [
-            V1EnvVar(
-                name="MY_POD_IP",
-                value_from=V1EnvVarSource(
-                    field_ref=V1ObjectFieldSelector(field_path="status.podIP")
-                ),
-            )
-        ]
+        env = []
         if "envs" in kargs:
             for key in kargs["envs"]:
                 env.append(V1EnvVar(name=key, value=kargs["envs"][key]))
+        env = append_pod_ip_to_env(env)
 
         pod = self._create_pod(
             pod_name=self.get_master_pod_name(),
@@ -324,6 +332,7 @@ def _create_ps_worker_pod(self, pod_name, type_key, index_key, **kargs):
         # for the ps or worker pod.
         master_pod = self.get_master_pod()
         env = kargs["envs"] if "envs" in kargs else None
+        env = append_pod_ip_to_env(env)
         pod = self._create_pod(
             pod_name=pod_name,
             job_name=self.job_name,


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Worker occasionally crashes when reports  evaluation task result.
The error log:
```
status = StatusCode.UNKNOWN
details = "Exception calling application: 'NoneType' object has no attribute 'complete_task'"
debug_error_string = "{"created":"@1582833503.778925101","description":"Error received from peer ipv4:11.253.195.11:50001","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Exception calling application: 'NoneType' object has no attribute 'complete_task'","grpc_status":2}"
```



Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/evaluation_service.py b/elasticdl/python/master/evaluation_service.py
--- a/elasticdl/python/master/evaluation_service.py
+++ b/elasticdl/python/master/evaluation_service.py
@@ -194,6 +194,8 @@ def report_evaluation_metrics(self, model_outputs, labels):
             )
 
     def complete_task(self):
+        if self._eval_job is None:
+            return
         self._eval_job.complete_task()
         if self._eval_job.finished():
             evaluation_metrics = (


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Worker hangs when connects to PS.
The worker will wait the grpc channel of PS is ready using `channel_ready_future`. We should set timeout and retry.
![image](https://user-images.githubusercontent.com/18071380/76311338-b4f58200-630b-11ea-927f-d7e87206f015.png)



Discussion:

Resolving patch:
diff --git a/elasticdl/python/worker/main.py b/elasticdl/python/worker/main.py
--- a/elasticdl/python/worker/main.py
+++ b/elasticdl/python/worker/main.py
@@ -5,6 +5,9 @@
 from elasticdl.python.common.grpc_utils import build_channel
 from elasticdl.python.worker.worker import Worker
 
+CONNECT_PS_MAX_RETRIES = 3
+CONNECT_PS_TIMEOUT = 60
+
 
 def main():
     args = parse_worker_args()
@@ -23,13 +26,29 @@ def main():
             # addr is in the form as "ps-pod-name.namespace.svc:port"
             channel = build_channel(addr)
 
-            # Wait the channel is ready by a Future object.
-            grpc.channel_ready_future(channel).result()
-            logger.info(
-                "grpc channel %s to connect pod %s is ready"
-                % (addr, addr.split(".")[0])
-            )
-            ps_channels.append(channel)
+            succeeded = False
+            for i in range(CONNECT_PS_MAX_RETRIES):
+                try:
+                    grpc.channel_ready_future(channel).result(
+                        timeout=CONNECT_PS_TIMEOUT
+                    )
+                    logger.info(
+                        "grpc channel %s to connect pod %s is ready"
+                        % (addr, addr.split(".")[0])
+                    )
+                    ps_channels.append(channel)
+                    succeeded = True
+                    break
+                except grpc.FutureTimeoutError:
+                    logger.warning(
+                        "Failed to connect pod %s with %d retry"
+                        % (addr.split(".")[0], i)
+                    )
+            if not succeeded:
+                raise TimeoutError(
+                    "Time out to connect pod %s with 3 retries"
+                    % addr.split(".")[0]
+                )
 
     worker = Worker(
         args,


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Exception calling application: dictionary changed size during iteration
The exception occasionally happens when the worker reports gradients to ps.
![image](https://user-images.githubusercontent.com/18071380/76288063-d3925380-62e0-11ea-869f-64cccb4e7d37.png)



Discussion:

Resolving patch:
diff --git a/elasticdl/python/ps/embedding_table.py b/elasticdl/python/ps/embedding_table.py
--- a/elasticdl/python/ps/embedding_table.py
+++ b/elasticdl/python/ps/embedding_table.py
@@ -1,3 +1,5 @@
+import threading
+
 import numpy as np
 import tensorflow as tf
 
@@ -44,6 +46,7 @@ def __init__(self, name, dim=None, initializer=None, is_slot=False):
             )
         self.is_slot = is_slot
         self.embedding_vectors = {}
+        self._lock = threading.Lock()
 
     def get(self, indices):
         if len(indices) == 0:
@@ -52,8 +55,9 @@ def get(self, indices):
         for i in indices:
             value = self.embedding_vectors.get(i, None)
             if value is None:
-                value = self.initializer(shape=(self.dim,)).numpy()
-                self.embedding_vectors[i] = value
+                with self._lock:
+                    value = self.initializer(shape=(self.dim,)).numpy()
+                    self.embedding_vectors[i] = value
             values.append(value)
         return np.stack(values)
 
@@ -69,9 +73,10 @@ def clear(self):
     def to_indexed_slices(self):
         indices = []
         embedding_vectors = []
-        for id, embedding_vector in self.embedding_vectors.items():
-            indices.append(id)
-            embedding_vectors.append(embedding_vector)
+        with self._lock:
+            for id, embedding_vector in self.embedding_vectors.items():
+                indices.append(id)
+                embedding_vectors.append(embedding_vector)
         return tf.IndexedSlices(
             values=np.array(embedding_vectors), indices=np.array(indices)
         )


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
worker pod crashes when get task
```
2020-03-12 22:03:32.300671: W tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Cancelled: Operation was cancelled
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/elasticdl/python/worker/main.py", line 44, in <module>
    main()
  File "/elasticdl/python/worker/main.py", line 40, in main
    worker.run()
  File "/elasticdl/python/worker/worker.py", line 1144, in run
    self._train_and_evaluate()
  File "/elasticdl/python/worker/worker.py", line 1074, in _train_and_evaluate
    self._minibatch_size, err_msg
  File "/elasticdl/python/worker/task_data_service.py", line 86, in report_record_done
    task = self._pending_tasks[0]
IndexError: deque index out of range
```


Discussion:
It seems that timeout mechanisim does not worker too. I will debug this.
I describe the pod:

```
Status:       Failed
State:          Terminated
      Reason:       Error
      Exit Code:    1
      Started:      Thu, 12 Mar 2020 17:07:32 +0800
      Finished:     Fri, 13 Mar 2020 06:04:06 +0800
```
The exit code is 1.


And the master pod log:

```
   for event in stream:
[2020-03-13 01:58:50,535] [INFO] [k8s_instance_manager.py:233:_event_cb] Got event ADDED, phase Running for pod: elasticdl-test-edl-dban-qitian910-worker-1
[2020-03-13 01:58:50,538] [INFO] [k8s_instance_manager.py:233:_event_cb] Got event ADDED, phase Failed for pod: elasticdl-test-edl-dban-qitian910-worker-6
[2020-03-13 01:58:50,540] [INFO] [k8s_instance_manager.py:233:_event_cb] Got event ADDED, phase Running for pod: elasticdl-test-edl-dban-qitian910-ps-2
```

The master find the failed pod.

It seems that we only handle exit_code 137

https://github.com/sql-machine-learning/elasticdl/blob/d002f4a49127a16a38955340bc2711606913f7c0/elasticdl/python/master/k8s_instance_manager.py#L251-L259

So this failed worker pod is not handled.

Resolving patch:
diff --git a/elasticdl/python/worker/task_data_service.py b/elasticdl/python/worker/task_data_service.py
--- a/elasticdl/python/worker/task_data_service.py
+++ b/elasticdl/python/worker/task_data_service.py
@@ -83,6 +83,9 @@ def report_record_done(self, count, err_msg=""):
         if err_msg:
             self._failed_record_count += count
 
+        # TODO(qijun) This is a workaround for #1829
+        if not self._pending_tasks:
+            return False
         task = self._pending_tasks[0]
         total_record_num = task.end - task.start
         if self._reported_record_count >= total_record_num:


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Decouple elasticdl_preprocessing from elasticdl main package.
elasticdl_preprocessing only depends on TensorFlow. It can be used in elasticdl model training and vanilla tensorflow training.


Discussion:
Use a dependent repo for `elasticdl_preprocessing`?
> elasticdl_preprocessing

We can use another setup.py for elasticdl_preprocessing at first.

Resolving patch:
diff --git a/elasticdl_preprocessing/setup.py b/elasticdl_preprocessing/setup.py
new file mode 100644
--- /dev/null
+++ b/elasticdl_preprocessing/setup.py
@@ -0,0 +1,26 @@
+from setuptools import setup
+
+with open("elasticdl_preprocessing/requirements.txt") as f:
+    required_deps = f.read().splitlines()
+
+extras = {}
+with open("elasticdl_preprocessing/requirements-dev.txt") as f:
+    extras["develop"] = f.read().splitlines()
+
+setup(
+    name="elasticdl_preprocessing",
+    version="0.1.0",
+    description="A Kubernetes-native Deep Learning Framework",
+    long_description="This is an extension of the native Keras Preprocessing"
+    " Layers and Feature Column API from TensorFlow. We can develop our model"
+    " using the native high-level API from TensorFlow and our library."
+    "We can train this model using native TensorFlow or ElasticDL.",
+    long_description_content_type="text/markdown",
+    author="Ant Financial",
+    url="https://elasticdl.org",
+    install_requires=required_deps,
+    extras_require=extras,
+    python_requires=">=3.5",
+    packages=["elasticdl_preprocessing"],
+    package_data={"": ["requirements.txt"]},
+)


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Don't package datasets into Docker images
https://github.com/sql-machine-learning/elasticdl/blob/develop/docs/tutorials/elasticdl_local.md

Instead, bind mount MNIST dataset on local filesystem to minikube containers.


Discussion:
Update the following parts to avoid package the data into docker image. Use volume mount instead.
- Update the elasticdl_local tutorial
- Update the integration testing in travis CI

Resolving patch:
diff --git a/scripts/gen_mnist_checkpoint.py b/scripts/travis/gen_mnist_checkpoint.py
similarity index 100%
rename from scripts/gen_mnist_checkpoint.py
rename to scripts/travis/gen_mnist_checkpoint.py


Test patch:
diff --git a/scripts/client_test.sh b/scripts/client_test.sh
--- a/scripts/client_test.sh
+++ b/scripts/client_test.sh
@@ -17,10 +17,10 @@
 JOB_TYPE=$1
 PS_NUM=$2
 WORKER_NUM=$3
+DATA_PATH=$4
 
 MNIST_CKPT_DIR=/model_zoo/test_data/mnist_ckpt/
 
-
 if [[ "$JOB_TYPE" == "train" ]]; then
     elasticdl train \
       --image_name=elasticdl:ci \
@@ -47,8 +47,8 @@ if [[ "$JOB_TYPE" == "train" ]]; then
       --job_name=test-train \
       --log_level=INFO \
       --image_pull_policy=Never \
-      --output=/saved_model/model_output \
-      --volume="host_path=${PWD},mount_path=/saved_model"
+      --output=/data/saved_model/model_output \
+      --volume="host_path=${DATA_PATH},mount_path=/data"
 elif [[ "$JOB_TYPE" == "evaluate" ]]; then
     elasticdl evaluate \
       --image_name=elasticdl:ci \
@@ -71,7 +71,8 @@ elif [[ "$JOB_TYPE" == "evaluate" ]]; then
       --tensorboard_log_dir=/tmp/tensorboard-log \
       --job_name=test-evaluate \
       --log_level=INFO \
-      --image_pull_policy=Never
+      --image_pull_policy=Never \
+      --volume="host_path=${DATA_PATH},mount_path=/data"
 elif [[ "$JOB_TYPE" == "predict" ]]; then
     elasticdl predict \
       --image_name=elasticdl:ci \
@@ -91,7 +92,8 @@ elif [[ "$JOB_TYPE" == "predict" ]]; then
       --num_ps_pods="$PS_NUM" \
       --job_name=test-predict \
       --log_level=INFO \
-      --image_pull_policy=Never
+      --image_pull_policy=Never \
+      --volume="host_path=${DATA_PATH},mount_path=/data"
 elif [[ "$JOB_TYPE" == "odps" ]]; then
     elasticdl train \
       --image_name=elasticdl:ci \
Repository: sql-machine-learning/elasticdl

Issue:
Make the path of `elasticdl_ps` compatible with various Python versions and OS in the Dockerfile. 



Discussion:

Resolving patch:
diff --git a/elasticdl_client/api.py b/elasticdl_client/api.py
--- a/elasticdl_client/api.py
+++ b/elasticdl_client/api.py
@@ -53,7 +53,9 @@ def init_zoo(args):
  --extra-index-url={{ EXTRA_PYPI_INDEX }}
 
 RUN pip install elasticdl --extra-index-url={{ EXTRA_PYPI_INDEX }}
-ENV PATH /usr/local/lib/python3.6/site-packages/elasticdl/go/bin:$PATH
+RUN /bin/bash -c\
+ 'PYTHON_PKG_PATH=$(pip3 show elasticdl | grep "Location:" | cut -d " " -f2);\
+ echo "PATH=${PYTHON_PKG_PATH}/elasticdl/go/bin:$PATH" >> /root/.bashrc'
 
 COPY . /model_zoo
 RUN pip install -r /model_zoo/requirements.txt\


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Set the model zoo path in the `elasticdl zoo init`.



Discussion:

Resolving patch:
diff --git a/elasticdl_client/api.py b/elasticdl_client/api.py
--- a/elasticdl_client/api.py
+++ b/elasticdl_client/api.py
@@ -57,7 +57,7 @@ def init_zoo(args):
  'PYTHON_PKG_PATH=$(pip3 show elasticdl | grep "Location:" | cut -d " " -f2);\
  echo "PATH=${PYTHON_PKG_PATH}/elasticdl/go/bin:$PATH" >> /root/.bashrc'
 
-COPY . /model_zoo
+COPY {{MODEL_ZOO_PATH}} /model_zoo
 RUN pip install -r /model_zoo/requirements.txt\
  --extra-index-url={{ EXTRA_PYPI_INDEX }}
 
@@ -70,6 +70,7 @@ def init_zoo(args):
         BASE_IMAGE=args.base_image,
         EXTRA_PYPI_INDEX=args.extra_pypi_index,
         CLUSTER_SPEC_NAME=cluster_spec_name,
+        MODEL_ZOO_PATH=args.model_zoo,
     )
 
     with open("./Dockerfile", mode="w") as f:
diff --git a/elasticdl_client/common/args.py b/elasticdl_client/common/args.py
--- a/elasticdl_client/common/args.py
+++ b/elasticdl_client/common/args.py
@@ -38,6 +38,13 @@ def add_zoo_init_params(parser):
         "the file path can be accessed by ElasticDL client.",
         default="",
     )
+    parser.add_argument(
+        "--model_zoo",
+        help="The directory that contains user-defined model files "
+        "or a specific model file.",
+        required=False,
+        default=".",
+    )
 
 
 def add_zoo_build_params(parser):


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Allreduce worker should not require ps_addrs
main.py: error: argument --ps_addrs: expected one argument


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/k8s_instance_manager.py b/elasticdl/python/master/k8s_instance_manager.py
--- a/elasticdl/python/master/k8s_instance_manager.py
+++ b/elasticdl/python/master/k8s_instance_manager.py
@@ -147,11 +147,14 @@ def _start_worker(self, worker_id):
         logger.info("Starting worker: %d" % worker_id)
         bash_command = self._worker_args[1]
         bash_command += " --worker_id {}".format(worker_id)
-        bash_command += " --ps_addrs {}".format(self._ps_addrs)
+        if self._ps_addrs:
+            bash_command += " --ps_addrs {}".format(self._ps_addrs)
         if self._log_file_path:
             bash_command += BashCommandTemplate.REDIRECTION.format(
                 self._log_file_path
             )
+        for extra_arg in self._worker_args[2:]:
+            bash_command += " {}".format(extra_arg)
         worker_args = [self._worker_args[0], bash_command]
         with self._lock:
             pod = self._k8s_client.create_worker(


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Elasticdl client crashes with invalid args
```
$ elasticdl -v
Traceback (most recent call last):
  File "/usr/local/bin/elasticdl", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.7/site-packages/elasticdl_client/main.py", line 97, in main
    args, _ = parser.parse_known_args()
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/argparse.py", line 1787, in parse_known_args
    namespace, args = self._parse_known_args(args, namespace)
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/argparse.py", line 2022, in _parse_known_args
    ', '.join(required_actions))
TypeError: sequence item 0: expected str instance, NoneType found
```


Discussion:

Resolving patch:
diff --git a/elasticdl_client/main.py b/elasticdl_client/main.py
--- a/elasticdl_client/main.py
+++ b/elasticdl_client/main.py
@@ -94,7 +94,12 @@ def main():
         parser.print_help(sys.stderr)
         sys.exit(1)
 
-    args, _ = parser.parse_known_args()
+    try:
+        args, _ = parser.parse_known_args()
+    except TypeError:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
     args.func(args)
 
 


Test patch: