Repository: sql-machine-learning/elasticdl

Issue:
worker_test failure
```
python -m unittest elasticdl/worker/*_test.py
/usr/local/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
2019-05-21 13:57:07.262725: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
WARNING:tensorflow:From /usr/local/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:642: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
must be str, not NoneType
FLoss is  3.090042
Loss is  1.4608976
Loss is  0.913306
Loss is  0.5969497
Loss is  0.66515267
Loss is  0.3935135
Loss is  0.37774342
Loss is  0.289928
.
======================================================================
FAIL: test_distributed_train (elasticdl.worker.worker_test.WorkerTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/l.zou/git/elasticdl/elasticdl/worker/worker_test.py", line 96, in test_distributed_train
    self.assertTrue(res)
AssertionError: False is not true

----------------------------------------------------------------------
Ran 2 tests in 0.165s

```


Discussion:

Resolving patch:
diff --git a/elasticdl/worker/worker.py b/elasticdl/worker/worker.py
--- a/elasticdl/worker/worker.py
+++ b/elasticdl/worker/worker.py
@@ -30,15 +30,21 @@ def __init__(self,
             channel: grpc channel
             max_retrain_num: max number of a minibatch retrain as its gradients are not accepted by master
         """
-
         model_module = load_user_model(model_file)
         self._model = model_module.model
         self._feature_columns = model_module.feature_columns()
-        self._all_columns = self._feature_columns + model_module.label_columns()
         build_model(self._model, self._feature_columns)
         self._input_fn = model_module.input_fn 
         self._opt_fn = model_module.optimizer
         self._loss = model_module.loss
+        all_columns = self._feature_columns + model_module.label_columns()
+        if codec_type == "tf_example":
+            self._codec = TFExampleCodec(all_columns)
+        elif codec_type == "bytes":
+            self._codec = BytesCodec(all_columns)
+        else:
+            raise ValueError("invalid codec_type: " + codec_type)
+
 
         if channel is None:
             self._stub = None
@@ -93,12 +99,6 @@ def distributed_train(self):
         """
         Distributed training.
         """
-        if self._codec_type == "tf_example":
-            codec = TFExampleCodec(self._all_columns)
-        elif self._codec_type == "bytes":
-            codec = BytesCodec(self._all_columns)
-        else:
-            raise ValueError("invalid codec_type: " + self._codec_type)
         while True:
             task = self.get_task()
             if not task.shard_file_name:
@@ -107,7 +107,7 @@ def distributed_train(self):
             batch_size = task.minibatch_size
             err_msg = ""
             try:
-                with recordio.File(task.shard_file_name, "r", decoder=codec.decode) as rdio_r:
+                with recordio.File(task.shard_file_name, "r", decoder=self._codec.decode) as rdio_r:
                     reader = rdio_r.get_reader(task.start, task.end)
                     min_model_version = task.model_version
                     while True:
@@ -164,7 +164,7 @@ def local_train(self, file_list, batch_size, epoch=1, kwargs=None):
         optimizer = self._opt_fn()
         for _ in range(epoch):
             for f in file_list:
-                with recordio.File(f, "r") as rdio_r:
+                with recordio.File(f, "r", decoder=self._codec.decode) as rdio_r:
                     reader = rdio_r.get_reader(0, rdio_r.count())
                     while True:
                         record_buf = list(


Test patch:
diff --git a/elasticdl/worker/test_module.py b/elasticdl/worker/test_module.py
--- a/elasticdl/worker/test_module.py
+++ b/elasticdl/worker/test_module.py
@@ -8,8 +8,6 @@
 outputs = Dense(1)(inputs)
 model = Model(inputs, outputs)
 
-input_names = ['x']
-
 def loss(outputs, labels):
     return tf.reduce_mean(tf.square(outputs - labels)) 
 
@@ -19,16 +17,15 @@ def feature_columns():
 
 def label_columns():
     return [tf.feature_column.numeric_column(key="y",
-        dtype=tf.dtypes.int64, shape=[1])]
+        dtype=tf.dtypes.float32, shape=[1])]
 
 def input_fn(records):
     x_list = []
     y_list = []
     # deserialize
     for r in records:
-        parsed = np.frombuffer(r, dtype='float32')
-        x_list.append([parsed[0]])
-        y_list.append([parsed[1]])
+        x_list.append([r["x"]])
+        y_list.append([r["y"]])
     # batching
     batch_size = len(x_list)
     xs = np.concatenate(x_list, axis=0)
diff --git a/elasticdl/worker/worker_test.py b/elasticdl/worker/worker_test.py
--- a/elasticdl/worker/worker_test.py
+++ b/elasticdl/worker/worker_test.py
@@ -10,31 +10,42 @@
 import numpy as np
 import recordio
 
+from elasticdl.common.model_helper import load_user_model
 from elasticdl.master.task_queue import _TaskQueue
 from elasticdl.master.servicer import MasterServicer
 from google.protobuf import empty_pb2
 from elasticdl.proto import master_pb2_grpc
 from elasticdl.proto import master_pb2
 from elasticdl.worker.worker import Worker
+from edl_data.codec import BytesCodec
+from edl_data.codec import TFExampleCodec
 
 _module_file = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "test_module.py")
 
-def create_recordio_file(size):
+m = load_user_model(_module_file)
+columns = m.feature_columns() + m.label_columns()
+
+def create_recordio_file(size, codec_type):
+    codec = None
+    if codec_type == "bytes":
+        codec = BytesCodec(columns)
+    elif codec_type == "tf_example":
+        codec = TFExampleCodec(columns)
+
     temp_file = tempfile.NamedTemporaryFile(delete=False)
-    with recordio.File(temp_file.name, 'w', max_chunk_size=size) as f:
+    with recordio.File(temp_file.name, 'w', max_chunk_size=size, encoder=codec.encode) as f:
         for _ in range(size):
             x = np.random.rand((1)).astype(np.float32)
             y = 2 * x + 1
-            data = np.concatenate((x, y), axis=None).tobytes()
-            f.write(data)
+            f.write([("x",x), ("y",y)])
     return temp_file.name
 
 class WorkerTest(unittest.TestCase):
-    def test_local_train(self):
-        worker = Worker(_module_file)
-        filename = create_recordio_file(128)
+    def local_train(self, codec_type):
+        worker = Worker(_module_file, codec_type=codec_type)
+        filename = create_recordio_file(128, codec_type)
         batch_size = 32
         epoch = 2
         try:
@@ -44,8 +55,14 @@ def test_local_train(self):
             print(ex)
             res = False
         self.assertTrue(res)
+    
+    def test_local_train_bytes(self):
+        self.local_train("bytes")
+
+    def test_local_train_tf_example(self):
+        self.local_train("tf_example")
 
-    def test_distributed_train(self):
+    def distributed_train(self, codec_type):
         """
         Run Worker.distributed_train with a local master.
         grpc calls are mocked by local master call.
@@ -67,9 +84,9 @@ def mock_ReportTaskResult(req):
             return master.ReportTaskResult(req, None)
 
         channel = grpc.insecure_channel('localhost:9999')
-        worker = Worker(_module_file, channel)
+        worker = Worker(_module_file, channel, codec_type=codec_type)
 
-        filename = create_recordio_file(128)
+        filename = create_recordio_file(128, codec_type)
         task_q = _TaskQueue(
             {filename: 128}, record_per_task=64, num_epoch=1
         )
@@ -97,3 +114,9 @@ def mock_ReportTaskResult(req):
         task = mock_GetTask(empty_pb2.Empty())
         # No more task.
         self.assertTrue(not task.shard_file_name)
+
+    def test_distributed_train_bytes(self):
+        self.distributed_train("bytes")
+
+    def test_distributed_train_tf_example(self):
+        self.distributed_train("tf_example")