internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: sql-machine-learning/elasticdl

Issue:
Python version support
I run the unit tests in a local environment, with Python 3.7.4

https://github.com/wangkuiyi/elasticdl/blob/424a9b45999fe77eb671d7cfe86d7e22130f263b/elasticdl/python/common/odps_recordio_conversion_utils.py#L52-L56

I find that it throws an exception. This is an interface change from Python 3.6 to Python 3.7
Please refer to https://docs.python.org/3/library/collections.html#collections.namedtuple

```
Changed in version 3.1: Added support for rename.

Changed in version 3.6: The verbose and rename parameters became keyword-only arguments.

Changed in version 3.6: Added the module parameter.

Changed in version 3.7: Remove the verbose parameter and the _source attribute.

Changed in version 3.7: Added the defaults parameter and the _field_defaults attribute.
```

Here is just a record.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/odps_recordio_conversion_utils.py b/elasticdl/python/common/odps_recordio_conversion_utils.py
--- a/elasticdl/python/common/odps_recordio_conversion_utils.py
+++ b/elasticdl/python/common/odps_recordio_conversion_utils.py
@@ -50,9 +50,7 @@ def _find_features_indices(
 ):
     """Finds the indices for different types of features."""
     FeatureIndices = namedtuple(
-        "FeatureIndices",
-        ["int_features", "float_features", "bytes_features"],
-        verbose=False,
+        "FeatureIndices", ["int_features", "float_features", "bytes_features"]
     )
     int_features_indices = [features_list.index(key) for key in int_features]
     float_features_indices = [
@@ -110,9 +108,7 @@ def _find_feature_indices_from_record(record):
     """Find the indices of different feature types."""
     feature_types = [type(value) for value in record]
     FeatureIndices = namedtuple(
-        "FeatureIndices",
-        ["int_features", "float_features", "bytes_features"],
-        verbose=False,
+        "FeatureIndices", ["int_features", "float_features", "bytes_features"]
     )
     return FeatureIndices(
         [i for i, x in enumerate(feature_types) if x == int],
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,6 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-
-with open('elasticdl/requirements.txt') as f:
+with open("elasticdl/requirements.txt") as f:
     requirements = f.read().splitlines()
 
 setup(


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
When async-SGD, ReportGradient does not accumulate gradients, but apply_gradients directly
According to [design doc](https://github.com/sql-machine-learning/elasticdl/blob/develop/elasticdl/doc/async_sgd_design.md#change-in-ps).

No need to accumulate gradients in ReportGradient in server.py. ReportGradient calls _update_model directly.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/servicer.py b/elasticdl/python/master/servicer.py
--- a/elasticdl/python/master/servicer.py
+++ b/elasticdl/python/master/servicer.py
@@ -185,7 +185,10 @@ def _update_model(self):
         # (grad, var) pairs excluding keras Embedding layer and
         # ElasticDL Embedding layer
         for k in self._gradient_sum:
-            self._gradient_sum[k] = self._gradient_sum[k] / self._grad_to_wait
+            if not self._use_async:
+                self._gradient_sum[k] = (
+                    self._gradient_sum[k] / self._grad_to_wait
+                )
             grad_var.append((self._gradient_sum[k], self._model[k]))
 
         # (grad, var) pair of Keras Embedding layer
@@ -402,13 +405,13 @@ def ReportGradient(self, request, _):
 
             # other grads
             for k, v in tmp.items():
-                if k in self._gradient_sum:
+                if not self._use_async and k in self._gradient_sum:
                     self._gradient_sum[k] = self._gradient_sum[k] + v
                 else:
                     self._gradient_sum[k] = v
 
             self._grad_n += 1
-            if self._grad_n >= self._grad_to_wait:
+            if self._use_async or self._grad_n >= self._grad_to_wait:
                 self._update_model()
                 self._update_evaluation()
                 self._update_checkpoint()


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Do not check model_version_valid in ReportGradient if async.



Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/servicer.py b/elasticdl/python/master/servicer.py
--- a/elasticdl/python/master/servicer.py
+++ b/elasticdl/python/master/servicer.py
@@ -310,7 +310,7 @@ def ReportVariable(self, request, _):
         return empty_pb2.Empty()
 
     def ReportGradient(self, request, _):
-        model_version_valid = self._validate_model_version(
+        model_version_valid = self._use_async or self._validate_model_version(
             request.model_version
         )
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Remove manually create arguments in master/worker
In ElasticDL's master and worker. We manually create the arguments collection for them. It is a little hacky as we are adding more and more arguments.
After #1210 is merged, we can also update these two places where we manually add arguments.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/main.py b/elasticdl/python/master/main.py
--- a/elasticdl/python/master/main.py
+++ b/elasticdl/python/master/main.py
@@ -6,7 +6,11 @@
 from kubernetes.client import V1EnvVar
 
 from elasticdl.proto import elasticdl_pb2_grpc
-from elasticdl.python.common.args import parse_envs, parse_master_args
+from elasticdl.python.common.args import (
+    build_arguments_from_parsed_result,
+    parse_envs,
+    parse_master_args,
+)
 from elasticdl.python.common.constants import (
     GRPC,
     JobType,
@@ -227,31 +231,14 @@ def main():
         worker_args = [
             "-m",
             "elasticdl.python.worker.main",
-            "--model_zoo",
-            args.model_zoo,
             "--master_addr",
             master_addr,
-            "--log_level",
-            args.log_level,
-            "--dataset_fn",
-            args.dataset_fn,
-            "--loss",
-            args.loss,
-            "--optimizer",
-            args.optimizer,
-            "--eval_metrics_fn",
-            args.eval_metrics_fn,
-            "--model_def",
-            args.model_def,
             "--job_type",
             job_type,
-            "--minibatch_size",
-            str(args.minibatch_size),
             "--embedding_service_endpoint",
             str(embedding_service_endpoint),
-            "--get_model_steps",
-            str(args.get_model_steps),
         ]
+        worker_args.extend(build_arguments_from_parsed_result(args))
 
         env_dict = parse_envs(args.envs)
         env = []


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Only keep necessary dependencies in requirements.txt
Currently, we have one single `requirements.txt` file which contains some unnecessary dependencies used for testing only such as `pytest`, `pytest-cov`, and `mock`. They should not be part of `requirements.txt` and should not be distributed with `elasticdl` Python package.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,11 @@
 from setuptools import find_packages, setup
 
 with open("elasticdl/requirements.txt") as f:
-    requirements = f.read().splitlines()
+    required_deps = f.read().splitlines()
+
+extras = {}
+with open("elasticdl/requirements-dev.txt") as f:
+    extras["develop"] = f.read().splitlines()
 
 setup(
     name="elasticdl",
@@ -13,7 +17,8 @@
     long_description_content_type="text/markdown",
     author="Ant Financial",
     url="https://elasticdl.org",
-    install_requires=requirements,
+    install_requires=required_deps,
+    extras_require=extras,
     packages=find_packages(exclude=["*test*"]),
     package_data={"": ["proto/elasticdl.proto", "docker/*", "Makefile"]},
     entry_points={


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Warnings show up when pip installing the requirements
```
Step 9/12 : RUN pip install -r /requirements.txt --extra-index-url=${EXTRA_PYPI_INDEX}
 ---> Running in 06f0db0a8f37
Looking in indexes: https://pypi.org/simple, 
Collecting grpcio-tools (from -r /requirements.txt (line 1))
  WARNING: Url 'grpcio-tools/' is ignored. It is either a non-existing path or lacks a specific scheme.
  Downloading https://files.pythonhosted.org/packages/5b/39/7a8e49822b7618e347d15108c8499ce9b6f07c488c6635403d9ff72d396d/grpcio_tools-1.24.1-cp36-cp36m-manylinux1_x86_64.whl (2.3MB)
Collecting kubernetes (from -r /requirements.txt (line 2))
  WARNING: Url 'kubernetes/' is ignored. It is either a non-existing path or lacks a specific scheme.
  Downloading https://files.pythonhosted.org/packages/6e/fc/2cab119f679648b348b8940de0dd744a1f0ee99c690aa2ef6072f050816c/kubernetes-10.0.1-py2.py3-none-any.whl (1.5MB)
Collecting docker (from -r /requirements.txt (line 3))
  WARNING: Url 'docker/' is ignored. It is either a non-existing path or lacks a specific scheme.
  Downloading https://files.pythonhosted.org/packages/cc/ca/699d4754a932787ef353a157ada74efd1ceb6d1fc0bfb7989ae1e7b33111/docker-4.1.0-py2.py3-none-any.whl (139kB)
```

Example build log with the above warnings can be found [here](https://travis-ci.org/sql-machine-learning/elasticdl/jobs/599159646).

This might indicate that `--extra-index-url` is not being passed in correctly to `pip install` and we should probably investigate this.


Discussion:
This should be due to `${EXTRA_PYPI_INDEX}` is none, causing the command becomes `pip install -r /requirements.txt --extra-index-url=none`

Resolving patch:
diff --git a/elasticdl/python/common/args.py b/elasticdl/python/common/args.py
--- a/elasticdl/python/common/args.py
+++ b/elasticdl/python/common/args.py
@@ -164,7 +164,9 @@ def add_common_params(parser):
         "comma is supported in value field",
     )
     parser.add_argument(
-        "--extra_pypi_index", help="The extra python package repository"
+        "--extra_pypi_index",
+        default="https://pypi.org/simple",
+        help="The extra URLs of Python package repository indexes",
     )
     parser.add_argument(
         "--namespace",


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Master uses legacy PS only when `num_ps_pods==0`
When `num_ps_pods == 0`, the master uses the legacy PS to create `MasterServicer`, `embedding_service`, `evaluation_service`, `checkpoint_service`.

For `num_ps_pods > 0 > 0`, the master will use the new PS design and create PS pods.

The old `MasterServicer` and `embedding_service` will be implemented in PS pods. 
`evaluation_service` and `checkpoint_service` will be redesigned in other projects.

Note that the legacy PS codes will be removed after we have implemented the new PS.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/master.py b/elasticdl/python/master/master.py
--- a/elasticdl/python/master/master.py
+++ b/elasticdl/python/master/master.py
@@ -68,6 +68,7 @@ class Master(object):
     def __init__(self, args):
         self.logger = get_logger("master", level=args.log_level.upper())
         self.checkpoint_output_path = args.output
+        self.num_ps_pods = args.num_ps_pods
 
         # Master addr
         master_ip = os.getenv("MY_POD_IP", "localhost")
@@ -104,21 +105,33 @@ def __init__(self, args):
         )
         self.optimizer = self.model_module[args.optimizer]()
 
-        # Initialize checkpoint service
-        self.checkpoint_service = self._create_checkpoint_service(args)
+        # TODO: checkpoint_service, evaluation_service and embedding_service
+        #       will be redesigned after distributed PS is implemented
+        if self.num_ps_pods:
+            self.checkpoint_service = None
+            self.evaluation_service = None
+            self.embedding_service_endpoint = None
+            self.embedding_dims = None
+        else:
+            # Initialize checkpoint service
+            self.checkpoint_service = self._create_checkpoint_service(args)
 
-        # Initialize evaluation service
-        self.evaluation_service = self._create_evaluation_service(args)
+            # Initialize evaluation service
+            self.evaluation_service = self._create_evaluation_service(args)
 
-        # Initialize embedding service
-        (
-            self.embedding_service_endpoint,
-            self.embedding_dims,
-        ) = self._create_embedding_service(args)
+            # Initialize embedding service
+            (
+                self.embedding_service_endpoint,
+                self.embedding_dims,
+            ) = self._create_embedding_service(args)
 
         # Initialize master service
         self.master_servicer, self.server = self._create_master_service(args)
 
+        if self.num_ps_pods:
+            # TODO: create ps pod manager for distributed PS
+            pass
+
         # Initialize worker manager
         self.worker_manager = self._create_worker_manager(args)
 
@@ -137,6 +150,10 @@ def prepare(self):
         self.server.start()
         self.logger.info("Master RPC server started")
 
+        if self.num_ps_pods:
+            # TODO: start ps pods
+            pass
+
         # Start the worker manager if requested
         if self.worker_manager:
             self.worker_manager.update_status(WorkerManagerStatus.PENDING)


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
No module named 'elasticdl.python.elasticdl.layers' on master
```
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/local/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/elasticdl/python/master/main.py", line 28, in <module>
    from elasticdl.python.elasticdl.layers.embedding import Embedding
ModuleNotFoundError: No module named 'elasticdl.python.elasticdl.layers'
```

Seems `layers` directory is not installed to `/usr/local/lib/python3.7/site-packages/elasticdl-develop-py3.7.egg/elasticdl/python/elasticdl` after running `python setup.py install`

Steps to reproduce:

1. In a Python Docker container, clone ElasticDL and run `python setup.py install`
1. remove the cloned source
1. execute a demo job by: `elasticdl train ...`


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/__init__.py b/elasticdl/python/elasticdl/__init__.py
--- a/elasticdl/python/elasticdl/__init__.py
+++ b/elasticdl/python/elasticdl/__init__.py
@@ -0,0 +1 @@
+from elasticdl.python.elasticdl import layers  # noqa: F401


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Use async grpc call to parallel worker-ps communication



Discussion:

Resolving patch:
diff --git a/elasticdl/python/worker/worker.py b/elasticdl/python/worker/worker.py
--- a/elasticdl/python/worker/worker.py
+++ b/elasticdl/python/worker/worker.py
@@ -203,16 +203,21 @@ def get_model_from_master(self, version, method):
 
     def get_model_from_ps(self, version, method):
         model_version = -1
+        variable_future_and_id_pairs = []
+        req = empty_pb2.Empty()
         for ps_id, stub in enumerate(self._ps_stubs):
             if ps_id not in self._ps_vars:
                 continue
-            req = empty_pb2.Empty()
-            res = stub.pull_variable(req)
+            # async grpc call
+            var_future = stub.pull_variable.future(req)
+            variable_future_and_id_pairs.append((var_future, ps_id))
 
+        for var_future, ps_id in variable_future_and_id_pairs:
+            res = var_future.result()
             if not res.model_init_status:
                 # push variable to ps for initialization
                 self.report_variable_to_ps(ps_id)
-                res = stub.pull_variable(req)
+                res = self._ps_stubs[ps_id].pull_variable(req)
                 if not res.model_init_status:
                     # TODO: support PS fault-tolerance
                     raise RuntimeError(
@@ -237,11 +242,15 @@ def pull_embedding_vector(self, layer_name, embedding_ids):
 
         embeddings = []
         index = []
+        pb_future_and_id_pairs = []
         for ps_id, embedding_ids in ps_ids.items():
             req = elasticdl_pb2.PullEmbeddingVectorRequest()
             req.name = layer_name
             req.ids.extend(embedding_ids)
-            pb = self._ps_stubs[ps_id].pull_embedding_vector(req)
+            pb_future = self._ps_stubs[ps_id].pull_embedding_vector.future(req)
+            pb_future_and_id_pairs.append((pb_future, ps_id))
+        for pb_future, ps_id in pb_future_and_id_pairs:
+            pb = pb_future.result()
             embeddings.append(tensor_pb_to_ndarray(pb))
             index.extend(ps_ids_index[ps_id])
         embeddings = np.concatenate(embeddings)
@@ -441,11 +450,15 @@ def report_gradient_to_ps(self, grads):
                         req.gradients, values=gv, indices=gi, name=layer.name
                     )
 
-        # TODO: call `push_gradient` in parallel
+        report_futures = []
         for ps_id in range(len(self._ps_stubs)):
             req = reqs[ps_id]
             req.model_version = self._model_version
-            res = self._ps_stubs[ps_id].push_gradient(req)
+            report_future = self._ps_stubs[ps_id].push_gradient.future(req)
+            report_futures.append(report_future)
+
+        for report_future in report_futures:
+            res = report_future.result()
         # TODO: choose the last response temporarily
         return res.accepted, res.model_version
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
The worker should wait all channels are ready.
If the worker has started to train and the grpc channels to connect PS are not ready, a connection error will be raised.

```
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.UNAVAILABLE
	details = "failed to connect to all addresses"
	debug_error_string = "{"created":"@1574320322.398282300","description":"Failed to pick subchannel","file":"src/core/ext/filters/client_channel/client_channel.cc","file_line":3876,"referenced_errors":[{"created":"@1574320322.398280426","description":"failed to connect to all addresses","file":"src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc","file_line":395,"grpc_status":14}]}"
```


Discussion:

Resolving patch:
diff --git a/elasticdl/python/worker/main.py b/elasticdl/python/worker/main.py
--- a/elasticdl/python/worker/main.py
+++ b/elasticdl/python/worker/main.py
@@ -53,6 +53,13 @@ def main():
                     ),
                 ],
             )
+
+            # Wait the channel is ready by a Future object.
+            grpc.channel_ready_future(channel).result()
+            logger.info(
+                "grpc channel %s to connect pod %s is ready"
+                % (addr, pod.metadata.name)
+            )
             ps_channels.append(channel)
 
     worker = Worker(args, channel=master_channel, ps_channels=ps_channels)


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Master needs to relaunch a failed ps/worker pod for OOM workaround
TensorFlow 2.0 has memory leak issues in the eager mode. A ps/worker pod may fail due to OOM after running for a while.

To work around it, the master can delete the failed pod so it can be relaunched.

We can remove this workaround after all memory leak issues are resolved.



Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/k8s_instance_manager.py b/elasticdl/python/master/k8s_instance_manager.py
--- a/elasticdl/python/master/k8s_instance_manager.py
+++ b/elasticdl/python/master/k8s_instance_manager.py
@@ -73,6 +73,8 @@ def __init__(
         self._ps_pod_name_to_id = {}
         self._relaunch_deleted_live_ps = True
 
+        self._failed_pods = []
+
         self._k8s_client = k8s.Client(event_callback=self._event_cb, **kwargs)
         self._ps_addrs = self._get_addrs(
             self._num_ps, self._k8s_client.get_ps_service_address
@@ -218,10 +220,18 @@ def _event_cb(self, event):
         worker_id = -1
         ps_id = -1
         with self._lock:
+            if pod_name in self._failed_pods:
+                return
             if pod_name in self._worker_pod_name_to_id:
                 worker_id = self._worker_pod_name_to_id.get(pod_name)
                 self._worker_pods_phase[worker_id] = (pod_name, phase)
-                if evt_type == "DELETED":
+                # Workaround for memory leak issues in tf eager mode.
+                # A pod may fail due to OOM from tf eager mode memory leak.
+                failed_pod = False
+                if evt_type == "MODIFIED" and phase == "Failed":
+                    self._failed_pods.append(pod_name)
+                    failed_pod = True
+                if evt_type == "DELETED" or failed_pod:
                     del self._worker_pods_phase[worker_id]
                     del self._worker_pod_name_to_id[pod_name]
                     self._task_d.recover_tasks(worker_id)
@@ -235,7 +245,13 @@ def _event_cb(self, event):
             elif pod_name in self._ps_pod_name_to_id:
                 ps_id = self._ps_pod_name_to_id.get(pod_name)
                 self._ps_pods_phase[ps_id] = (pod_name, phase)
-                if evt_type == "DELETED":
+                # Workaround for memory leak issues in tf eager mode.
+                # A pod may fail due to OOM from tf eager mode memory leak.
+                failed_pod = False
+                if evt_type == "MODIFIED" and phase == "Failed":
+                    self._failed_pods.append(pod_name)
+                    failed_pod = True
+                if evt_type == "DELETED" or failed_pod:
                     del self._ps_pods_phase[ps_id]
                     del self._ps_pod_name_to_id[pod_name]
                     relaunch_ps = self._relaunch_deleted_live_ps


Test patch: