Repository: mars-project/mars

Issue:
[BUG] Mars worker failed to start when --phy-mem is specified
**Describe the bug**
A lot of BrokenPipeErrors raised when ``--phy-mem`` arg is specified.

**To Reproduce**
Run ``python -m mars.worker -a localhost -p 12003 -s localhost:12001 --phy-mem 90%`` with Python 2.7. Code branch v0.1



Discussion:

Resolving patch:
diff --git a/mars/compat/__init__.py b/mars/compat/__init__.py
--- a/mars/compat/__init__.py
+++ b/mars/compat/__init__.py
@@ -17,6 +17,7 @@
 import itertools
 import platform
 import os
+import socket
 from unicodedata import east_asian_width
 
 from ..lib import six
@@ -177,9 +178,9 @@ def np_getbuffer(n):
         return np.getbuffer(n)
 
     TimeoutError = type('TimeoutError', (Exception, ), {})
-    BrokenPipeError = type('BrokenPipeError', (Exception, ), {})
-    ConnectionRefusedError = type('ConnectionRefusedError', (Exception, ), {})
-    ConnectionResetError = type('ConnectionResetError', (Exception,), {})
+    BrokenPipeError = type('BrokenPipeError', (socket.error, ), {})
+    ConnectionRefusedError = type('ConnectionRefusedError', (socket.error, ), {})
+    ConnectionResetError = type('ConnectionResetError', (socket.error,), {})
 
 
     def accumulate(iterable, func=lambda a, b: a + b):
diff --git a/mars/worker/__main__.py b/mars/worker/__main__.py
--- a/mars/worker/__main__.py
+++ b/mars/worker/__main__.py
@@ -34,6 +34,7 @@ class WorkerApplication(BaseApplication, WorkerService):
     def __init__(self):
         BaseApplication.__init__(self)
         WorkerService.__init__(self)
+        self._total_mem = None
 
     def config_args(self, parser):
         parser.add_argument('--cpu-procs', help='number of processes used for cpu')
@@ -61,15 +62,15 @@ def create_pool(self, *args, **kwargs):
         options.worker.io_process_count = int(self.args.io_procs or '1')
 
         if self.args.phy_mem:
-            mem_total = self._calc_size_limit(self.args.phy_mem, mem_stats.total)
+            self._total_mem = self._calc_size_limit(self.args.phy_mem, mem_stats.total)
         else:
-            mem_total = mem_stats.total
+            self._total_mem = mem_stats.total
 
         options.worker.physical_memory_limit_hard = self._calc_size_limit(
-            options.worker.physical_memory_limit_hard, mem_total
+            options.worker.physical_memory_limit_hard, self._total_mem
         )
         options.worker.physical_memory_limit_soft = self._calc_size_limit(
-            options.worker.physical_memory_limit_soft, mem_total
+            options.worker.physical_memory_limit_soft, self._total_mem
         )
         options.worker.cache_memory_limit = self.args.cache_mem
         options.worker.disk_limit = self.args.disk
@@ -92,8 +93,8 @@ def create_pool(self, *args, **kwargs):
         return super(WorkerApplication, self).create_pool(*args, **kwargs)
 
     def start_service(self):
-        super(WorkerApplication, self).start(self.endpoint, self.args.schedulers,
-                                             self.pool, ignore_avail_mem=self.args.ignore_avail_mem)
+        super(WorkerApplication, self).start(self.endpoint, self.args.schedulers, self.pool,
+                                             total_mem=self._total_mem, ignore_avail_mem=self.args.ignore_avail_mem)
 
     def handle_process_down(self, proc_indices):
         logger.debug('Process %r halt. Trying to recover.', proc_indices)
diff --git a/mars/worker/service.py b/mars/worker/service.py
--- a/mars/worker/service.py
+++ b/mars/worker/service.py
@@ -72,16 +72,17 @@ def start_plasma(self, mem_limit, one_mapped_file=False):
         options.worker.plasma_socket, _ = self._plasma_store.__enter__()
 
     @classmethod
-    def _calc_soft_memory_limit(cls):
+    def _calc_soft_memory_limit(cls, total_mem=None):
         """
         Calc memory limit for MemQuotaActor given configured percentage. Cache size and memory
         already been used will be excluded from calculated size.
         """
         mem_status = resource.virtual_memory()
+        total_mem = total_mem or mem_status.total
 
         phy_soft_limit = options.worker.physical_memory_limit_soft
         cache_limit = options.worker.cache_memory_limit
-        actual_used = mem_status.total - mem_status.available
+        actual_used = total_mem - mem_status.available
         quota_soft_limit = phy_soft_limit - cache_limit - actual_used
 
         if quota_soft_limit < 512 * 1024 ** 2:
@@ -91,7 +92,7 @@ def _calc_soft_memory_limit(cls):
         cls.service_logger.info('Setting soft limit to %s.', readable_size(quota_soft_limit))
         return quota_soft_limit
 
-    def start(self, endpoint, schedulers, pool, ignore_avail_mem=False):
+    def start(self, endpoint, schedulers, pool, total_mem=None, ignore_avail_mem=False):
         if schedulers:
             if isinstance(schedulers, six.string_types):
                 schedulers = [schedulers]
@@ -120,7 +121,7 @@ def start(self, endpoint, schedulers, pool, ignore_avail_mem=False):
                 QuotaActor, options.worker.physical_memory_limit_soft, uid=MemQuotaActor.default_name())
         else:
             self._mem_quota_ref = pool.create_actor(
-                MemQuotaActor, self._calc_soft_memory_limit(),
+                MemQuotaActor, self._calc_soft_memory_limit(total_mem),
                 options.worker.physical_memory_limit_hard, uid=MemQuotaActor.default_name())
 
         # create ChunkHolderActor


Test patch: