Repository: mars-project/mars

Issue:
[BUG] Worker should be registered only when all components started
**Describe the bug**
Currently workers are registered when ``StatusActor`` starts. Workers should only be registered when all components are initialized.



Discussion:

Resolving patch:
diff --git a/mars/worker/__main__.py b/mars/worker/__main__.py
--- a/mars/worker/__main__.py
+++ b/mars/worker/__main__.py
@@ -39,6 +39,7 @@ def config_args(self, parser):
         parser.add_argument('--phy-mem', help='physical memory size limit')
         parser.add_argument('--ignore-avail-mem', action='store_true', help='ignore available memory')
         parser.add_argument('--cache-mem', help='cache memory size limit')
+        parser.add_argument('--min-mem', help='minimal free memory required to start worker')
         parser.add_argument('--spill-dir', help='spill directory')
 
     def validate_arguments(self):
@@ -58,6 +59,7 @@ def create_pool(self, *args, **kwargs):
             total_mem=self.args.phy_mem,
             cache_mem_limit=self.args.cache_mem,
             ignore_avail_mem=self.args.ignore_avail_mem,
+            min_mem_size=self.args.min_mem,
         )
         # start plasma
         self._service.start_plasma()
diff --git a/mars/worker/service.py b/mars/worker/service.py
--- a/mars/worker/service.py
+++ b/mars/worker/service.py
@@ -79,9 +79,10 @@ def __init__(self, **kwargs):
 
         self._total_mem = kwargs.pop('total_mem', None)
         self._cache_mem_limit = kwargs.pop('cache_mem_limit', None)
-        self._soft_mem_limit = kwargs.pop('soft_mem_limit', '75%')
-        self._hard_mem_limit = kwargs.pop('hard_mem_limit', '90%')
-        self._ignore_avail_mem = kwargs.pop('ignore_avail_mem', False)
+        self._soft_mem_limit = kwargs.pop('soft_mem_limit', None) or '75%'
+        self._hard_mem_limit = kwargs.pop('hard_mem_limit', None) or '90%'
+        self._ignore_avail_mem = kwargs.pop('ignore_avail_mem', None) or False
+        self._min_mem_size = kwargs.pop('min_mem_size', None) or 128 * 1024 ** 2
 
         self._soft_quota_limit = self._soft_mem_limit
 
@@ -113,6 +114,7 @@ def _calc_size_limit(limit_str, total_size):
         else:
             self._total_mem = mem_stats.total
 
+        self._min_mem_size = _calc_size_limit(self._min_mem_size, self._total_mem)
         self._hard_mem_limit = _calc_size_limit(self._hard_mem_limit, self._total_mem)
 
         self._cache_mem_limit = _calc_size_limit(self._cache_mem_limit, self._total_mem)
@@ -125,7 +127,7 @@ def _calc_size_limit(limit_str, total_size):
             self._soft_quota_limit = self._soft_mem_limit
         else:
             self._soft_quota_limit = self._soft_mem_limit - self._cache_mem_limit - actual_used
-            if self._soft_quota_limit < 512 * 1024 ** 2:
+            if self._soft_quota_limit < self._min_mem_size:
                 raise MemoryError('Memory not enough. soft_limit=%s, cache_limit=%s, used=%s' %
                                   tuple(readable_size(k) for k in (
                                       self._soft_mem_limit, self._cache_mem_limit, actual_used)))
@@ -228,6 +230,9 @@ def start(self, endpoint, pool, distributed=True, schedulers=None, process_start
                 actor = actor_holder.create_actor(SpillActor, uid=uid)
                 self._spill_actors.append(actor)
 
+        # worker can be registered when everything is ready
+        self._status_ref.enable_status_upload(_tell=True)
+
     def handle_process_down(self, pool, proc_indices):
         logger.debug('Process %r halt. Trying to recover.', proc_indices)
         for pid in proc_indices:
diff --git a/mars/worker/status.py b/mars/worker/status.py
--- a/mars/worker/status.py
+++ b/mars/worker/status.py
@@ -32,6 +32,8 @@ def __init__(self, endpoint):
         super(StatusReporterActor, self).__init__()
         self._endpoint = endpoint
 
+        self._upload_status = False
+
         self._status_ref = None
         self._resource_ref = None
 
@@ -43,12 +45,18 @@ def post_create(self):
         self._resource_ref = self.get_actor_ref(ResourceActor.default_name())
         self.ref().collect_status(_tell=True)
 
+    def enable_status_upload(self):
+        self._upload_status = True
+
     def collect_status(self):
         """
         Collect worker status and write to kvstore
         """
         meta_dict = dict()
         try:
+            if not self._upload_status:
+                return
+
             cpu_percent = resource.cpu_percent()
             disk_io = resource.disk_io_usage()
             net_io = resource.net_io_usage()
@@ -167,6 +175,9 @@ def post_create(self):
     def pre_destroy(self):
         self.ctx.destroy_actor(self._reporter_ref)
 
+    def enable_status_upload(self):
+        self._reporter_ref.enable_status_upload(_tell=True)
+
     def get_stats(self, items=None):
         if not items:
             return self._stats


Test patch:
diff --git a/mars/worker/tests/test_status.py b/mars/worker/tests/test_status.py
--- a/mars/worker/tests/test_status.py
+++ b/mars/worker/tests/test_status.py
@@ -40,6 +40,7 @@ def testStatus(self):
                                   uid=ChunkHolderActor.default_name())
                 status_ref = pool.create_actor(StatusActor, pool_address,
                                                uid=StatusActor.default_name())
+                status_ref.enable_status_upload()
 
                 status_ref.update_slots(dict(cpu=4))
                 status_ref.update_stats(dict(min_est_finish_time=10))