internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: sql-machine-learning/elasticdl

Issue:
non-trainable variables update in training
Some keras layers have non-trainable variables, such as BatchNorm and RNN. These variables are not updated by gradients, but by update ops.

We need to run the update ops to update these internal states of the layers.




Discussion:
BatchNorm has two non-trainable variables:
batch_normalization_v1/moving_mean:0
batch_normalization_v1/moving_variance:0

Resolving patch:
diff --git a/elasticdl/examples/mnist/mnist.py b/elasticdl/examples/mnist/mnist.py
--- a/elasticdl/examples/mnist/mnist.py
+++ b/elasticdl/examples/mnist/mnist.py
@@ -25,13 +25,14 @@ def __init__(self, channel_last=True):
         self._flatten = tf.keras.layers.Flatten()
         self._dense = tf.keras.layers.Dense(10)
 
-    def call(self, inputs):
+    def call(self, inputs, training=False):
         x = self._reshape(inputs)
         x = self._conv1(x)
         x = self._conv2(x)
-        x = self._batch_norm(x)
+        x = self._batch_norm(x, training=training)
         x = self._maxpooling(x)
-        x = self._dropout(x)
+        if training:
+            x = self._dropout(x, training=training)
         x = self._flatten(x)
         x = self._dense(x)
         return x
diff --git a/elasticdl/worker/worker.py b/elasticdl/worker/worker.py
--- a/elasticdl/worker/worker.py
+++ b/elasticdl/worker/worker.py
@@ -114,7 +114,7 @@ def distributed_train(self):
                                     inputs.append(batch_input_data[input_name])
                                 if len(inputs) == 1:
                                     inputs = inputs[0]
-                                outputs = self._model.call(inputs)
+                                outputs = self._model.call(inputs, training=True)
                                 loss = self._model.loss(outputs, batch_input_data)
 
                                 # TODO:  Add regularization loss if any,
@@ -164,7 +164,7 @@ def local_train(self, file_list, batch_size, epoch=1, kwargs=None):
                                 inputs.append(data[input_name])
                             if len(inputs) == 1:
                                 inputs = inputs[0]
-                            outputs = self._model.call(inputs)
+                            outputs = self._model.call(inputs, training=True)
                             loss = self._model.loss(outputs, data)
 
                             # Add regularization loss if any.


Test patch:
diff --git a/elasticdl/worker/worker_test.py b/elasticdl/worker/worker_test.py
--- a/elasticdl/worker/worker_test.py
+++ b/elasticdl/worker/worker_test.py
@@ -21,7 +21,7 @@ def __init__(self):
         super(TestModel, self).__init__(name='test_model')
         self.dense = tf.keras.layers.Dense(1)
 
-    def call(self, inputs):
+    def call(self, inputs, training=False):
         return self.dense(inputs)
 
     @staticmethod
Repository: sql-machine-learning/elasticdl

Issue:
build local docker image using docker python api
for the edl client to build docker image locally, we need a wrapper for docker python sdk:  https://pypi.org/project/docker firstly.


Discussion:

Resolving patch:
diff --git a/elasticdl/client/client.py b/elasticdl/client/client.py
--- a/elasticdl/client/client.py
+++ b/elasticdl/client/client.py
@@ -4,6 +4,8 @@
 import time
 import getpass
 from string import Template
+import docker
+
 
 def run(model_class, train_data_dir=None, 
         num_epoch=1, minibatch_size=10, 
@@ -31,7 +33,9 @@ def _build_docker_image(m_path, m_file, m_file_in_docker, timestamp):
 
     with open(new_dfile, 'a') as df:
         df.write("COPY " + m_file + " " + m_file_in_docker)
-    val = os.system('docker build -t elasticdl:dev_' + str(timestamp) + ' -f Dockerfile .')
+    client = docker.APIClient(base_url='unix://var/run/docker.sock') 
+    for line in client.build(dockerfile='Dockerfile', path='.', tag='elasticdl:dev_' + str(timestamp)):
+        print(str(line, encoding = "utf-8"))
 
     # TODO: upload docker image to docker hub.
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
submit master pod using k8s python client instead of kubectl 
use k8s python client to submit master pod instead of using the command below
`os.system('kubectl run ...')`


Discussion:

Resolving patch:
diff --git a/elasticdl/client/client.py b/elasticdl/client/client.py
--- a/elasticdl/client/client.py
+++ b/elasticdl/client/client.py
@@ -5,6 +5,9 @@
 import getpass
 from string import Template
 import docker
+import yaml
+from kubernetes.client.apis import core_v1_api
+from kubernetes import config
 
 
 def run(model_class, train_data_dir=None, 
@@ -14,11 +17,11 @@ def run(model_class, train_data_dir=None,
     m_file_in_docker = "/model/" + m_file 
     timestamp = int(round(time.time() * 1000))
     _build_docker_image(m_path, m_file, m_file_in_docker, timestamp)
-    yaml_file = _generate_yaml(m_file_in_docker, model_class.__name__, train_data_dir=train_data_dir, 
+    yaml_content = _generate_yaml(m_file_in_docker, model_class.__name__, train_data_dir=train_data_dir, 
             num_epoch=num_epoch, minibatch_size=minibatch_size, 
             record_per_task=record_per_task, num_worker=num_worker, 
             grads_to_wait=grads_to_wait, timestamp=timestamp)
-    _submit(yaml_file)
+    _submit(yaml_content)
 
 def _getModelFile():
     m_file = inspect.currentframe().f_back.f_back.f_code.co_filename
@@ -75,14 +78,15 @@ def _generate_yaml(m_file, m_class,
     restartPolicy: Never
   """
   t = Template(YAML_TEMPLATE)
-  yaml_file = 'job_desc.yaml'
-  with open(yaml_file, "w") as yaml:
-      yaml.write(t.substitute(m_file=m_file, m_class=m_class, 
-          train_data_dir=train_data_dir, 
-          timestamp=timestamp, num_worker=num_worker, num_epoch=num_epoch,
-          minibatch_size=minibatch_size, record_per_task=record_per_task,
-          user=getpass.getuser(), grads_to_wait=grads_to_wait))
-  return yaml_file
+  return t.substitute(m_file=m_file, m_class=m_class, 
+             train_data_dir=train_data_dir, 
+             timestamp=timestamp, num_worker=num_worker, num_epoch=num_epoch,
+             minibatch_size=minibatch_size, record_per_task=record_per_task,
+             user=getpass.getuser(), grads_to_wait=grads_to_wait)
 
-def _submit(yaml_file):
-    os.system('kubectl create -f ' + yaml_file)
+def _submit(yaml_content):
+    config.load_kube_config()
+    pod_desc = yaml.safe_load(yaml_content)
+    api = core_v1_api.CoreV1Api()
+    resp = api.create_namespaced_pod(body=pod_desc, namespace='default')
+    print("Pod created. status='%s'" % str(resp.status))


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Failed generating cifar10 dataset when building dev image



Discussion:

Resolving patch:
diff --git a/elasticdl/recordio_ds_gen/cifar10/gen_data.py b/elasticdl/recordio_ds_gen/cifar10/gen_data.py
--- a/elasticdl/recordio_ds_gen/cifar10/gen_data.py
+++ b/elasticdl/recordio_ds_gen/cifar10/gen_data.py
@@ -11,7 +11,7 @@
 from recordio import File
 from tensorflow.python.keras import backend
 from tensorflow.python.keras.datasets import cifar10
-from elasticdl.recordio_ds_gen.mnist import record
+from elasticdl.recordio_ds_gen.cifar10 import record
 
 # TODO: This function can be shared with MNIST dataset
 def gen(file_dir, data, label, *, chunk_size, record_per_file):
diff --git a/elasticdl/recordio_ds_gen/cifar10/show_data.py b/elasticdl/recordio_ds_gen/cifar10/show_data.py
--- a/elasticdl/recordio_ds_gen/cifar10/show_data.py
+++ b/elasticdl/recordio_ds_gen/cifar10/show_data.py
@@ -1,5 +1,5 @@
 from recordio import File
-from elasticdl.recordio_ds_gen.mnist import record
+from elasticdl.recordio_ds_gen.cifar10 import record
 import sys
 import argparse
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Fix TFExample codec



Discussion:

Resolving patch:
diff --git a/record_codec/__init__.py b/record_codec/__init__.py
--- a/record_codec/__init__.py
+++ b/record_codec/__init__.py
@@ -0,0 +1,3 @@
+from record_codec.tf_example_codec import TFExampleCodec
+
+__all__=[TFExampleCodec]
diff --git a/record_codec/tf_example_codec.py b/record_codec/tf_example_codec.py
--- a/record_codec/tf_example_codec.py
+++ b/record_codec/tf_example_codec.py
@@ -4,10 +4,8 @@
 
 class TFExampleCodec(object):
     def __init__(self, feature_columns):
-        self._f_name2type = {}
         self._example_spec = tf.feature_column.make_parse_example_spec(feature_columns)
-        for f_col in feature_columns:
-            self._f_name2type[f_col.key] = f_col.dtype
+        self._f_name2type = {f_col.key: f_col.dtype for f_col in feature_columns}
 
     def encode(self, example):
         f_dict = {}


Test patch:
diff --git a/record_codec/tf_example_codec_test.py b/record_codec/tf_example_codec_test.py
--- a/record_codec/tf_example_codec_test.py
+++ b/record_codec/tf_example_codec_test.py
@@ -4,7 +4,7 @@
 import tensorflow as tf
 import numpy as np
 from recordio import File
-from elasticdl.record_codec.tf_example_codec import TFExampleCodec
+from record_codec import TFExampleCodec
 
 
 class TestTFExampleCodec(unittest.TestCase):
Repository: sql-machine-learning/elasticdl

Issue:
Better check for codec names
currently, codec name argument is not checked. A typo would result in worker interpreting encoded data.


Discussion:

Resolving patch:
diff --git a/elasticdl/master/main.py b/elasticdl/master/main.py
--- a/elasticdl/master/main.py
+++ b/elasticdl/master/main.py
@@ -65,6 +65,7 @@ def _parse_args():
     parser.add_argument(
         "--codec-type",
         default=None,
+        choices=["tf_example"],
         help="Type of codec(tf_example or None)",
     )
     return parser.parse_args()


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
docker stdout log formatting
Currently the docker output log is too messy. for example:
```
{"stream":"\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\r21970944/26421880 [=======================\u003e......] - ETA: 40s"}

{"stream":"\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\r21987328/26421880 [=======================\u003e......] - ETA: 40s"}

{"stream":"\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\u0008\r22003712/26421880 [=======================\u003e......] - ETA: 40s"}
```


Discussion:

Resolving patch:
diff --git a/elasticdl/client/client.py b/elasticdl/client/client.py
--- a/elasticdl/client/client.py
+++ b/elasticdl/client/client.py
@@ -28,11 +28,15 @@ def _build_docker_image(
         df.write(DOCKER_TEMPLATE.format(image_base, m_file, _m_file_in_docker(m_file)))
 
     client = docker.APIClient(base_url="unix://var/run/docker.sock")
+    print("===== Building Docker Image =====")
     for line in client.build(
-        dockerfile=df.name, path=".", rm=True, tag=image_name
+        dockerfile=df.name, path=".", rm=True, tag=image_name, decode=True
     ):
-        print(str(line, encoding="utf-8"))
-
+        text = line.get("stream", None)
+        if text:
+            sys.stdout.write(text)
+            sys.stdout.flush()
+    print("===== Docker Image Built =====")
     if repository != None:
         for line in client.push(image_name, stream=True, decode=True):
             print(line)


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
When worker/master image creation failed, client should fail instead of trying to launch master.



Discussion:

Resolving patch:
diff --git a/elasticdl/client/client.py b/elasticdl/client/client.py
--- a/elasticdl/client/client.py
+++ b/elasticdl/client/client.py
@@ -32,12 +32,14 @@ def _build_docker_image(
     for line in client.build(
         dockerfile=df.name, path=".", rm=True, tag=image_name, decode=True
     ):
+        if "error" in line:
+            raise RuntimeError("Docker image build failure: " % line["error"])
         text = line.get("stream", None)
         if text:
             sys.stdout.write(text)
             sys.stdout.flush()
     print("===== Docker Image Built =====")
-    if repository != None:
+    if repository is not None:
         for line in client.push(image_name, stream=True, decode=True):
             print(line)
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Make master exist when there are no tasks left.
Currently, master exists when there are no tasks left AND all workers are gone. It might left hanging if a worker got preempted.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/master/main.py b/elasticdl/python/elasticdl/master/main.py
--- a/elasticdl/python/elasticdl/master/main.py
+++ b/elasticdl/python/elasticdl/master/main.py
@@ -163,13 +163,7 @@ def main():
     if args.num_worker:
         # TODO: worker_manager.remove_workers supports synchronized call
         worker_manager.remove_workers()
-        # wait for worker pod to be deleted
-        max_check_num = 10
-        for _ in range(max_check_num):
-            time.sleep(3)
-            counters = worker_manager.get_counters()
-            if not counters:
-                break
+
     server.stop(0)
 
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Fix logging issue
Currently, the logging is not working as intended. E.g. the formatting was ignored for WARNING and INFO log are not printed out.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/master/main.py b/elasticdl/python/elasticdl/master/main.py
--- a/elasticdl/python/elasticdl/master/main.py
+++ b/elasticdl/python/elasticdl/master/main.py
@@ -157,7 +157,9 @@ def main():
             "--master_addr",
             master_addr,
             "--codec_type",
-            args.codec_type
+            args.codec_type,
+            "--log_level",
+            args.log_level
         ]
 
         worker_manager = WorkerManager(


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Add CLI entry-point to `Worker.distributed_evaluate()`
Part of https://github.com/wangkuiyi/elasticdl/issues/384. Add main CLI entry-point to `Worker.distributed_evaluate()` that will be used in `WorkerManager`.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/worker/main.py b/elasticdl/python/elasticdl/worker/main.py
--- a/elasticdl/python/elasticdl/worker/main.py
+++ b/elasticdl/python/elasticdl/worker/main.py
@@ -33,6 +33,18 @@ def _parse_args():
         default='WARNING',
         help="Set the logging level",
     )
+    parser.add_argument(
+        "--task_type",
+        default="training",
+        choices=["training", "evaluation"],
+        help="Type of worker task (training or evaluation). Default is training task.",
+    )
+    parser.add_argument(
+        "--evaluate_steps",
+        default=None,
+        help="Evaluate the model by this many number of steps where the model is evaluated on one "
+             "batch of samples for each step. By default, evaluation will continue until reaching the end of input.",
+    )
 
     return parser.parse_args()
 
@@ -55,7 +67,10 @@ def main():
         codec_type=args.codec_type,
     )
 
-    worker.distributed_train()
+    if args.task_type == "training":
+        worker.distributed_train()
+    else:
+        worker.distributed_evaluate(steps=args.evaluate_steps)
 
 
 if __name__ == "__main__":
diff --git a/elasticdl/python/elasticdl/worker/worker.py b/elasticdl/python/elasticdl/worker/worker.py
--- a/elasticdl/python/elasticdl/worker/worker.py
+++ b/elasticdl/python/elasticdl/worker/worker.py
@@ -248,7 +248,7 @@ def distributed_evaluate(self, steps=None):
                     evaluation_metrics_collection = defaultdict(list)
                     while True:
                         current_step += 1
-                        if steps and current_step > steps:
+                        if steps and current_step > int(steps):
                             break
                         record_buf = self._get_batch(reader, batch_size, self._codec.decode)
                         if not record_buf:


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
Create owner references among master and worker pods
Currently we have the following two ways to delete master + worker pods:
* Delete each pod one by one
* Delete all pods related to this elasticdl run via `elasticdl_job_name` label `kubectl delete pod -l elasticdl_job_name=test-job-1559292773-93`

It would be much easier if users could just delete master pod and then worker pods can be delete automatically. This would be possible if there are owner references among master and worker pods.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/master/k8s_client.py b/elasticdl/python/elasticdl/master/k8s_client.py
--- a/elasticdl/python/elasticdl/master/k8s_client.py
+++ b/elasticdl/python/elasticdl/master/k8s_client.py
@@ -103,6 +103,16 @@ def _create_worker_pod(
         if priority is not None:
             spec.priority_class_name = priority
 
+        # Find that master pod that will be used as the owner reference
+        # for this worker pod.
+        pods = self._v1.list_namespaced_pod(
+            namespace=self._ns,
+            label_selector="elasticdl_job_name=" + self._job_name
+        ).items
+        master_pod = [pod for pod in pods if (
+                pod.metadata.name == "elasticdl-master-" + self._job_name
+        )][0]
+
         pod = client.V1Pod(
             spec=spec,
             metadata=client.V1ObjectMeta(
@@ -111,6 +121,17 @@ def _create_worker_pod(
                     "app": "elasticdl",
                     "elasticdl_job_name": self._job_name,
                 },
+                # TODO: Add tests for this once we've done refactoring on
+                # k8s client code and the constant strings
+                owner_references=[
+                    client.V1OwnerReference(
+                        api_version="v1",
+                        block_owner_deletion=True,
+                        kind="Pod",
+                        name=master_pod.metadata.name,
+                        uid=master_pod.metadata.uid,
+                    ),
+                ],
             ),
         )
         return pod


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
precommit flake8 check is missing files
elasticdl/python/data/recordio_gen/sample_pyspark_recordio_gen/spark_gen_recordio.py is missed by the checker.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/data/codec/__init__.py b/elasticdl/python/data/codec/__init__.py
--- a/elasticdl/python/data/codec/__init__.py
+++ b/elasticdl/python/data/codec/__init__.py
@@ -1,4 +1,4 @@
 from elasticdl.python.data.codec.bytes_codec import BytesCodec
 from elasticdl.python.data.codec.tf_example_codec import TFExampleCodec
 
-__all__ = [BytesCodec, TFExampleCodec]
+__all__ = ["BytesCodec", "TFExampleCodec"]
diff --git a/elasticdl/python/data/recordio_gen/sample_pyspark_recordio_gen/spark_gen_recordio.py b/elasticdl/python/data/recordio_gen/sample_pyspark_recordio_gen/spark_gen_recordio.py
--- a/elasticdl/python/data/recordio_gen/sample_pyspark_recordio_gen/spark_gen_recordio.py
+++ b/elasticdl/python/data/recordio_gen/sample_pyspark_recordio_gen/spark_gen_recordio.py
@@ -6,8 +6,9 @@
 
 import numpy as np
 from elasticdl.python.elasticdl.common.model_helper import load_user_model
-from elasticdl.python.data.recordio_gen.convert_numpy_to_recordio import \
-    convert_numpy_to_recordio
+from elasticdl.python.data.recordio_gen.convert_numpy_to_recordio import (
+    convert_numpy_to_recordio,
+)
 
 
 def process_data(
@@ -36,6 +37,7 @@ def _process_data(file_list):
             str(ctx.partitionId()),
         )
         return file_list
+
     return _process_data
 
 
@@ -50,9 +52,7 @@ def main():
         required=True,
     )
     parser.add_argument(
-        "--output_dir",
-        help="Directory of output RecordIO data",
-        required=True,
+        "--output_dir", help="Directory of output RecordIO data", required=True
     )
     parser.add_argument(
         "--model_file",
@@ -81,12 +81,12 @@ def main():
     file_list = []
     for dir_name, subdir_list, files in os.walk(args.training_data_dir):
         for fname in files:
-            if not fname.startswith('.'):
+            if not fname.startswith("."):
                 file_list.append(os.path.join(dir_name, fname))
 
     # Load user-defined model
     model_module = load_user_model(args.model_file)
-    
+
     if not os.path.exists(args.output_dir):
         os.makedirs(args.output_dir)
 


Test patch:
Repository: sql-machine-learning/elasticdl

Issue:
keep track of the correct min_model_version value
`min_model_version` should be initialized at the beginning of `_handle_task`, and updated with minibatch training.

fix #659 


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/worker/worker.py b/elasticdl/python/elasticdl/worker/worker.py
--- a/elasticdl/python/elasticdl/worker/worker.py
+++ b/elasticdl/python/elasticdl/worker/worker.py
@@ -166,6 +166,7 @@ def _run_evaluation_task(self, features, labels):
         return self.report_evaluation_metrics(evaluation_metrics)
 
     def _handle_task(self, task):
+        min_model_version = task.model_version
         with closing(
             recordio.Scanner(
                 task.shard_file_name, task.start, task.end - task.start
@@ -177,10 +178,11 @@ def _handle_task(self, task):
                 )
                 if not record_buf:
                     break
-                self._process_minibatch(task, record_buf)
+                min_model_version = self._process_minibatch(
+                    task, record_buf, min_model_version
+                )
 
-    def _process_minibatch(self, task, record_buf):
-        min_model_version = task.model_version
+    def _process_minibatch(self, task, record_buf, min_model_version):
         for _ in range(self._max_minibatch_retry_num):
             # TODO: optimize the logic to avoid unnecessary
             #       get_model call.
@@ -206,6 +208,7 @@ def _process_minibatch(self, task, record_buf):
             # TODO: stop the worker if it fails to make any
             #       progress for some time.
             raise RuntimeError("Worker got stuck")
+        return min_model_version
 
     def run(self):
         """


Test patch: