Repository: sql-machine-learning/elasticdl

Issue:
Model size is limited by grpc limit (4M)
If the model size is larger than the grpc limit (4M), `GetModel` will fail.

```
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.RESOURCE_EXHAUSTED
	details = "Received message larger than max (8672945 vs. 4194304)"
	debug_error_string = "{"created":"@1559802279.541465331","description":"Received message larger than max (8672945 vs. 4194304)","file":"src/core/ext/filters/message_size/message_size_filter.cc","file_line":174,"grpc_status":8}"
>
Traceback (most recent call last):
  File "/elasticdl/python/elasticdl/worker/worker.py", line 159, in run
    max(self._model_version, min_model_version))
  File "/elasticdl/python/elasticdl/worker/worker.py", line 80, in get_model
    model = self._stub.GetModel(req)
  File "/usr/local/lib/python3.5/dist-packages/grpc/_channel.py", line 562, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/usr/local/lib/python3.5/dist-packages/grpc/_channel.py", line 466, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.RESOURCE_EXHAUSTED
	details = "Received message larger than max (8672945 vs. 4194304)"
	debug_error_string = "{"created":"@1559802279.755033272","description":"Received message larger than max (8672945 vs. 4194304)","file":"src/core/ext/filters/message_size/message_size_filter.cc","file_line":174,"grpc_status":8}"
```


Discussion:
Thank you for catching this bug! Should we switch to gRPC streaming? @zou000 
According to https://github.com/grpc/grpc/issues/7882#issuecomment-243188908, it seems that a quick-and-hacky solution is to call `ServerBuilder::SetMaxMessageSize()` at the gRPC server side to set the limit to a bigger number, e.g., 20MB.

Resolving patch:
diff --git a/elasticdl/python/elasticdl/common/constants.py b/elasticdl/python/elasticdl/common/constants.py
new file mode 100644
--- /dev/null
+++ b/elasticdl/python/elasticdl/common/constants.py
@@ -0,0 +1,5 @@
+class GRPC(object):
+    # gRPC limits the size of message by default to 4MB.
+    # It's too small to send model parameters.
+    MAX_SEND_MESSAGE_LENGTH = 256 * 1024 * 1024
+    MAX_RECEIVE_MESSAGE_LENGTH = 256 * 1024 * 1024
diff --git a/elasticdl/python/elasticdl/master/main.py b/elasticdl/python/elasticdl/master/main.py
--- a/elasticdl/python/elasticdl/master/main.py
+++ b/elasticdl/python/elasticdl/master/main.py
@@ -19,6 +19,7 @@
 from elasticdl.python.elasticdl.master.task_queue import _TaskQueue
 from elasticdl.python.elasticdl.master.k8s_worker_manager import WorkerManager
 from elasticdl.python.elasticdl.common.model_helper import load_module
+from elasticdl.python.elasticdl.common.constants import GRPC
 
 
 def _make_task_queue(
@@ -173,7 +174,7 @@ def _parse_args():
     parser.add_argument(
         "--image_pull_policy",
         default="Always",
-        help="Image pull policy of master and workers"
+        help="Image pull policy of master and workers",
     )
     parser.add_argument(
         "--restart_policy",
@@ -184,8 +185,7 @@ def _parse_args():
         "--namespace",
         default="default",
         type=str,
-        help="The name of the Kubernetes namespace where ElasticDL "
-             "pods will be created",
+        help="The Kubernetes namespace where ElasticDL jobs run",
     )
     return parser.parse_args()
 
@@ -237,7 +237,19 @@ def main():
         task_q.set_evaluation_service(evaluation_service)
 
     # The master service
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=64))
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=64),
+        options=[
+            (
+                "grpc.max_send_message_length",
+                GRPC.MAX_SEND_MESSAGE_LENGTH,
+            ),
+            (
+                "grpc.max_receive_message_length",
+                GRPC.MAX_RECEIVE_MESSAGE_LENGTH,
+            ),
+        ],
+    )
     master_servicer = MasterServicer(
         args.grads_to_wait,
         args.minibatch_size,
diff --git a/elasticdl/python/elasticdl/worker/main.py b/elasticdl/python/elasticdl/worker/main.py
--- a/elasticdl/python/elasticdl/worker/main.py
+++ b/elasticdl/python/elasticdl/worker/main.py
@@ -3,6 +3,7 @@
 import logging
 
 from elasticdl.python.elasticdl.worker.worker import Worker  # noqa
+from elasticdl.python.elasticdl.common.constants import GRPC
 
 
 def _parse_args():
@@ -34,7 +35,19 @@ def _parse_args():
 
 def main():
     args = _parse_args()
-    channel = grpc.insecure_channel(args.master_addr)
+    channel = grpc.insecure_channel(
+        args.master_addr,
+        options=[
+            (
+                "grpc.max_send_message_length",
+                GRPC.MAX_SEND_MESSAGE_LENGTH,
+            ),
+            (
+                "grpc.max_receive_message_length",
+                GRPC.MAX_RECEIVE_MESSAGE_LENGTH,
+            ),
+        ],
+    )
 
     # Initialize logger
     logging.basicConfig(


Test patch: