Repository: sql-machine-learning/elasticdl

Issue:
Mnist example exposed a bug?
Using the mnist example in :
https://github.com/wangkuiyi/elasticdl/pull/304

mnist example running with master/worker fails .
Start master:
```
docker run --net=host --rm -it elasticdl:dev \
    python -m master.main \
        --model-file=examples/mnist/mnist.py \
        --model-class=MnistModel \
        --train_data_dir=/data/mnist/train \
        --record_per_task=100 \
        --num_epoch=2 \
        --grads_to_wait=2 \
        --minibatch_size=10
```
Start worker:
```
docker run --net=host --rm -it elasticdl:dev \
    python -m worker.main \
        --model-file=examples/mnist/mnist.py \
        --model-class=MnistModel \
        --master_addr=localhost:50001
```

Result:
```
WARNING:master:Worker reported error: Tensor PB has no dim defined
```

Debug shows that GetModel failed, its return value is not valid. Need further debug.




Discussion:

Resolving patch:
diff --git a/elasticdl/master/main.py b/elasticdl/master/main.py
--- a/elasticdl/master/main.py
+++ b/elasticdl/master/main.py
@@ -13,7 +13,7 @@
 from proto import master_pb2_grpc
 from .servicer import MasterServicer
 from .task_queue import _TaskQueue
-from common.model_helper import load_user_model 
+from common.model_helper import load_user_model
 
 
 def _make_task_queue(data_dir, record_per_task, num_epoch):
@@ -31,12 +31,12 @@ def _parse_args():
         "--model-file",
         help="Full file path of user defined neural model",
         required=True,
-    ) 
+    )
     parser.add_argument(
         "--model-class",
         help="The model class name defined in model file",
         required=True,
-    ) 
+    )
     parser.add_argument(
         "--train_data_dir",
         help="Training data directory. Files should be in RecordIO format",
@@ -68,18 +68,17 @@ def main():
     model_cls = load_user_model(args.model_file, args.model_class)
     model_inst = model_cls()
     model_inst.build(model_inst.input_shapes())
-    optimizer = model_cls.optimizer() 
-    
-    servicer = MasterServicer(
-        logger, args.grads_to_wait, args.minibatch_size, optimizer, task_q
-    )
-    for variable in model_inst.trainable_variables:
-        servicer.set_model_var(variable.name, variable.numpy())
+    optimizer = model_cls.optimizer()
 
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=64))
     master_pb2_grpc.add_MasterServicer_to_server(
         MasterServicer(
-            logger, args.grads_to_wait, args.minibatch_size, optimizer, task_q
+            logger,
+            args.grads_to_wait,
+            args.minibatch_size,
+            optimizer,
+            task_q,
+            init_var=model_inst.trainable_variables,
         ),
         server,
     )
diff --git a/elasticdl/master/servicer.py b/elasticdl/master/servicer.py
--- a/elasticdl/master/servicer.py
+++ b/elasticdl/master/servicer.py
@@ -2,6 +2,7 @@
 import numpy as np
 
 import tensorflow as tf
+
 assert tf.executing_eagerly()
 
 from google.protobuf import empty_pb2
@@ -14,13 +15,21 @@
 class MasterServicer(master_pb2_grpc.MasterServicer):
     """Master service implementation"""
 
-    def __init__(self, logger, grads_to_wait, minibatch_size, optimizer, task_q):
+    def __init__(
+        self,
+        logger,
+        grads_to_wait,
+        minibatch_size,
+        optimizer,
+        task_q,
+        *,
+        init_var=[]
+    ):
         # TODO: group params together into a single object.
         self.logger = logger
         self._opt = optimizer
         self._task_q = task_q
         self._lock = threading.Lock()
-        # TODO: random initialization
         # A <string, tf.ResourceVariable> map. We use tf.ResourceVariable
         # instead ndarray to avoid copying and conversion when calling
         # optimizer's apply_gradients() function.
@@ -30,19 +39,21 @@ def __init__(self, logger, grads_to_wait, minibatch_size, optimizer, task_q):
         self._grad_to_wait = grads_to_wait
         self._grad_n = 0
         self._minibatch_size = minibatch_size
+        for var in init_var:
+            self.set_model_var(var.name, var.numpy())
 
     def set_model_var(self, name, value):
         """Add or set model variable. Value should be a float32 ndarray"""
         if value.dtype != np.float32:
             raise ValueError("Value should be a float32 numpy array")
-        self._model[name] = tf.Variable(value, name=MasterServicer.var_name_encode(name))
-
+        self._model[name] = tf.Variable(
+            value, name=MasterServicer.var_name_encode(name)
+        )
 
     @staticmethod
     def var_name_encode(name):
         return name.replace(":", "-")
 
-
     def GetTask(self, request, context):
         res = master_pb2.Task()
         res.model_version = self._version
@@ -55,7 +66,6 @@ def GetTask(self, request, context):
             res.end = task.end
         return res
 
-
     def GetModel(self, request, context):
         if request.min_version > self._version:
             err_msg = (
@@ -133,7 +143,9 @@ def ReportGradient(self, request, context):
 
     def ReportTaskResult(self, request, context):
         if request.err_message:
-            self.logger.warning("Worker reported error: " + request.err_message)
+            self.logger.warning(
+                "Worker reported error: " + request.err_message
+            )
             self._task_q.report(request.task_id, False)
         else:
             self._task_q.report(request.task_id, True)


Test patch: