Repository: mars-project/mars

Issue:
Blocklist and subtask reassignment related with scale-in of auto-scaling service
This issue is mainly for tracking the progress of developing functions related with scale-in of auto-scaling service, such as **blocklist**, **subtask reassignment**. Comments and suggestions are greatly appreciated.


Discussion:

Resolving patch:
diff --git a/mars/services/scheduling/supervisor/assigner.py b/mars/services/scheduling/supervisor/assigner.py
--- a/mars/services/scheduling/supervisor/assigner.py
+++ b/mars/services/scheduling/supervisor/assigner.py
@@ -15,7 +15,7 @@
 import asyncio
 import random
 from collections import defaultdict
-from typing import List
+from typing import List, Dict, Tuple
 
 from .... import oscar as mo
 from ....core.operand import Fetch, FetchShuffle
@@ -60,7 +60,19 @@ async def assign_subtasks(self, subtasks: List[Subtask]):
         selected_bands = dict()
         for subtask in subtasks:
             if subtask.expect_bands:
-                selected_bands[subtask.subtask_id] = subtask.expect_bands
+                if all(expect_band in self._bands
+                       for expect_band in subtask.expect_bands):
+                    # pass if all expected bands are available
+                    selected_bands[subtask.subtask_id] = subtask.expect_bands
+                else:
+                    # exclude expected but unready bands
+                    expect_available_bands = [expect_band
+                                              for expect_band in subtask.expect_bands
+                                              if expect_band in self._bands]
+                    # fill in if all expected bands are unready
+                    if not expect_available_bands:
+                        expect_available_bands = [self.reassign_band()]
+                    selected_bands[subtask.subtask_id] = expect_available_bands
                 continue
             for indep_chunk in subtask.chunk_graph.iter_indep():
                 if isinstance(indep_chunk.op, Fetch):
@@ -69,7 +81,7 @@ async def assign_subtasks(self, subtasks: List[Subtask]):
                     if not self._bands:
                         self._bands = list(await self._cluster_api.get_all_bands(
                             NodeRole.WORKER))
-                    selected_bands[subtask.subtask_id] = [random.choice(self._bands)]
+                    selected_bands[subtask.subtask_id] = [self.reassign_band()]
                     break
 
         fields = ['store_size', 'bands']
@@ -90,6 +102,8 @@ async def assign_subtasks(self, subtasks: List[Subtask]):
                         continue
                     meta = inp_metas[inp.key]
                     for band in meta['bands']:
+                        if band not in self._bands:
+                            band = self.reassign_band()
                         band_sizes[band] += meta['store_size']
                 bands = []
                 max_size = -1
@@ -101,3 +115,53 @@ async def assign_subtasks(self, subtasks: List[Subtask]):
                         bands.append(band)
             assigns.append(random.choice(bands))
         return assigns
+
+    def reassign_band(self):
+        assert self._bands
+        return random.choice(self._bands)
+
+    async def reassign_subtasks(self, band_num_queued_subtasks: Dict[Tuple, int]) -> Dict[Tuple, int]:
+        num_used_bands = len(band_num_queued_subtasks.keys())
+        if num_used_bands == 1:
+            [(band, length)] = band_num_queued_subtasks.items()
+            if length == 0:
+                return {band: 0}
+            # no need to balance when there's only one band initially
+            if len(self._bands) == 1 and band == self._bands[0]:
+                return {band: 0}
+        # unready bands recorded in band_num_queued_subtasks, some of them may hold 0 subtasks
+        unready_bands = list(set(band_num_queued_subtasks.keys()) - set(self._bands))
+        # ready bands not recorded in band_num_queued_subtasks, all of them hold 0 subtasks
+        new_ready_bands = list(set(self._bands) - set(band_num_queued_subtasks.keys()))
+        # when there are new ready bands, make all bands hold same amount of subtasks
+        # when there are no new ready bands now, move out subtasks left on them
+        if not new_ready_bands and unready_bands:
+            band_num_queued_subtasks = {k: band_num_queued_subtasks[k] for k in unready_bands}
+        # approximate total of subtasks moving to each ready band
+        num_all_subtasks = sum(band_num_queued_subtasks.values())
+        mean = int(num_all_subtasks / len(self._bands))
+        # all_bands (namely) includes:
+        # a. ready bands recorded in band_num_queued_subtasks
+        # b. ready bands not recorded in band_num_queued_subtasks
+        # c. unready bands recorded in band_num_queued_subtasks
+        # a. + b. = self._bands, a. + c. = bands in band_num_queued_subtasks
+        all_bands = list(set(self._bands) | set(band_num_queued_subtasks.keys()))
+        # calculate the differential steps of moving subtasks
+        # move < 0 means subtasks should move out and vice versa
+        # unready bands no longer hold subtasks
+        # assuming bands not recorded in band_num_queued_subtasks hold 0 subtasks
+        move_queued_subtasks = {}
+        for band in all_bands:
+            if band in self._bands:
+                move_queued_subtasks[band] = mean - band_num_queued_subtasks.get(band, 0)
+            else:
+                move_queued_subtasks[band] = -band_num_queued_subtasks.get(band, 0)
+        # ensure the balance of moving in and out
+        total_move = sum(move_queued_subtasks.values())
+        # int() is going to be closer to zero, so `mean` is no more than actual mean value
+        # total_move = mean * len(self._bands) - num_all_subtasks
+        #            <= actual_mean * len(self._bands) - num_all_subtasks = 0
+        assert total_move <= 0
+        if total_move != 0:
+            move_queued_subtasks[self.reassign_band()] -= total_move
+        return dict(sorted(move_queued_subtasks.items(), key=lambda item: item[1]))
diff --git a/mars/services/scheduling/supervisor/globalslot.py b/mars/services/scheduling/supervisor/globalslot.py
--- a/mars/services/scheduling/supervisor/globalslot.py
+++ b/mars/services/scheduling/supervisor/globalslot.py
@@ -57,13 +57,15 @@ async def apply_subtask_slots(self, band: Tuple, session_id: str,
             self._band_total_slots = await self._cluster_api.get_all_bands()
 
         idx = 0
-        total_slots = self._band_total_slots[band]
-        for stid, slots in zip(subtask_ids, subtask_slots):
-            if self._band_used_slots[band] + slots > total_slots:
-                break
-            self._band_stid_slots[band][(session_id, stid)] = slots
-            self._band_used_slots[band] += slots
-            idx += 1
+        # only ready bands will pass
+        if band in self._band_total_slots:
+            total_slots = self._band_total_slots[band]
+            for stid, slots in zip(subtask_ids, subtask_slots):
+                if self._band_used_slots[band] + slots > total_slots:
+                    break
+                self._band_stid_slots[band][(session_id, stid)] = slots
+                self._band_used_slots[band] += slots
+                idx += 1
         if idx == 0:
             logger.debug('No slots available, status: %r, request: %r',
                          self._band_used_slots, subtask_slots)
diff --git a/mars/services/scheduling/supervisor/queueing.py b/mars/services/scheduling/supervisor/queueing.py
--- a/mars/services/scheduling/supervisor/queueing.py
+++ b/mars/services/scheduling/supervisor/queueing.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import copy
 import heapq
 import logging
 from collections import defaultdict
@@ -74,7 +75,11 @@ async def __post_create__(self):
 
         async def watch_bands():
             async for bands in self._cluster_api.watch_all_bands():
-                self._band_slot_nums = bands
+                # confirm ready bands indeed changed
+                if bands != self._band_slot_nums:
+                    self._band_slot_nums = copy.deepcopy(bands)
+                    if self._band_queues:
+                        await self.balance_queued_subtasks()
 
         self._band_watch_task = asyncio.create_task(watch_bands())
 
@@ -186,3 +191,23 @@ def remove_queued_subtasks(self, subtask_ids: List[str]):
         for stid in subtask_ids:
             self._stid_to_bands.pop(stid, None)
             self._stid_to_items.pop(stid, None)
+
+    async def balance_queued_subtasks(self):
+        # record length of band queues
+        band_num_queued_subtasks = {band: len(queue) for band, queue in self._band_queues.items()}
+        move_queued_subtasks = await self._assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+        items = []
+        # rewrite band queues according to feedbacks from assigner
+        for band, move in move_queued_subtasks.items():
+            task_queue = self._band_queues[band]
+            assert move + len(task_queue) >= 0
+            for _ in range(abs(move)):
+                if move < 0:
+                    # TODO: pop item of low priority
+                    item = heapq.heappop(task_queue)
+                    self._stid_to_bands[item.subtask.subtask_id].remove(band)
+                    items.append(item)
+                elif move > 0:
+                    item = items.pop()
+                    self._stid_to_bands[item.subtask.subtask_id].append(band)
+                    heapq.heappush(task_queue, item)


Test patch:
diff --git a/mars/services/scheduling/supervisor/tests/test_assigner.py b/mars/services/scheduling/supervisor/tests/test_assigner.py
--- a/mars/services/scheduling/supervisor/tests/test_assigner.py
+++ b/mars/services/scheduling/supervisor/tests/test_assigner.py
@@ -13,11 +13,16 @@
 # limitations under the License.
 
 import numpy as np
+import asyncio
 import pytest
 
 import mars.oscar as mo
 from mars.core import ChunkGraph
-from mars.services.cluster import MockClusterAPI
+from mars.services.cluster import ClusterAPI
+from mars.services.cluster.core import NodeRole, NodeStatus
+from mars.services.cluster.locator import SupervisorLocatorActor
+from mars.services.cluster.uploader import NodeInfoUploaderActor
+from mars.services.cluster.supervisor.node_info import NodeInfoCollectorActor
 from mars.services.meta import MockMetaAPI
 from mars.services.session import MockSessionAPI
 from mars.services.scheduling.supervisor import AssignerActor
@@ -26,27 +31,81 @@
 from mars.tensor.arithmetic import TensorTreeAdd
 
 
+class MockNodeInfoCollectorActor(NodeInfoCollectorActor):
+    def __init__(self, timeout=None, check_interval=None):
+        super().__init__(timeout=timeout, check_interval=check_interval)
+        self.ready_nodes = {('address0', 'numa-0'): 2,
+                            ('address1', 'numa-0'): 2,
+                            ('address2', 'numa-0'): 2,
+                            ('address3', 'numa-0'): 2}
+
+    async def update_node_info(self, address, role, env=None,
+                               resource=None, detail=None, status=None):
+        if 'address' in address and status == NodeStatus.STOPPING:
+            del self.ready_nodes[(address, 'numa-0')]
+        await super().update_node_info(address, role, env,
+                                       resource, detail, status)
+
+    def get_all_bands(self, role=None, statuses=None):
+        if statuses == {NodeStatus.READY}:
+            return self.ready_nodes
+        else:
+            return {('address0', 'numa-0'): 2,
+                    ('address1', 'numa-0'): 2,
+                    ('address2', 'numa-0'): 2,
+                    ('address3', 'numa-0'): 2}
+
+
+class FakeClusterAPI(ClusterAPI):
+    @classmethod
+    async def create(cls, address: str, **kw):
+        dones, _ = await asyncio.wait([
+            mo.create_actor(SupervisorLocatorActor, 'fixed', address,
+                            uid=SupervisorLocatorActor.default_uid(),
+                            address=address),
+            mo.create_actor(MockNodeInfoCollectorActor,
+                            uid=NodeInfoCollectorActor.default_uid(),
+                            address=address),
+            mo.create_actor(NodeInfoUploaderActor, NodeRole.WORKER,
+                            interval=kw.get('upload_interval'),
+                            band_to_slots=kw.get('band_to_slots'),
+                            use_gpu=kw.get('use_gpu', False),
+                            uid=NodeInfoUploaderActor.default_uid(),
+                            address=address),
+        ])
+
+        for task in dones:
+            try:
+                task.result()
+            except mo.ActorAlreadyExist:  # pragma: no cover
+                pass
+
+        api = await super().create(address=address)
+        await api.mark_node_ready()
+        return api
+
+
 @pytest.fixture
 async def actor_pool():
     pool = await mo.create_actor_pool('127.0.0.1', n_process=0)
 
     async with pool:
         session_id = 'test_session'
-        await MockClusterAPI.create(pool.external_address)
+        cluster_api = await FakeClusterAPI.create(pool.external_address)
         await MockSessionAPI.create(pool.external_address, session_id=session_id)
         meta_api = await MockMetaAPI.create(session_id, pool.external_address)
         assigner_ref = await mo.create_actor(
             AssignerActor, session_id, uid=AssignerActor.gen_uid(session_id),
             address=pool.external_address)
 
-        yield pool, session_id, assigner_ref, meta_api
+        yield pool, session_id, assigner_ref, cluster_api, meta_api
 
         await mo.destroy_actor(assigner_ref)
 
 
 @pytest.mark.asyncio
 async def test_assigner(actor_pool):
-    pool, session_id, assigner_ref, meta_api = actor_pool
+    pool, session_id, assigner_ref, cluster_api, meta_api = actor_pool
 
     input1 = TensorFetch(key='a', source_key='a', dtype=np.dtype(int)).new_chunk([])
     input2 = TensorFetch(key='b', source_key='b', dtype=np.dtype(int)).new_chunk([])
@@ -70,6 +129,75 @@ async def test_assigner(actor_pool):
     await meta_api.set_chunk_meta(input3, memory_size=400, store_size=400,
                                   bands=[('address2', 'numa-0')])
 
+    await cluster_api.set_node_status(
+        node='address1', role=NodeRole.WORKER, status=NodeStatus.STOPPING)
+    await cluster_api.set_node_status(
+        node='address3', role=NodeRole.WORKER, status=NodeStatus.STOPPING)
+
     subtask = Subtask('test_task', session_id, chunk_graph=chunk_graph)
     [result] = await assigner_ref.assign_subtasks([subtask])
-    assert result in (('address1', 'numa-0'), ('address2', 'numa-0'))
+    assert result in (('address0', 'numa-0'), ('address2', 'numa-0'))
+
+    subtask.expect_bands = [('address0', 'numa-0')]
+    [result] = await assigner_ref.assign_subtasks([subtask])
+    assert result == ('address0', 'numa-0')
+
+    subtask.expect_bands = [('address0', 'numa-0'), ('address1', 'numa-0')]
+    [result] = await assigner_ref.assign_subtasks([subtask])
+    assert result == ('address0', 'numa-0')
+
+    subtask.expect_bands = [('address1', 'numa-0')]
+    [result] = await assigner_ref.assign_subtasks([subtask])
+    assert result in (('address0', 'numa-0'), ('address2', 'numa-0'))
+
+
+@pytest.mark.asyncio
+async def test_reassign_subtasks(actor_pool):
+    pool, session_id, assigner_ref, cluster_api, meta_api = actor_pool
+
+    # ('address0', 'numa-0'), ('address1', 'numa-0'), ('address2', 'numa-0') are ready
+    await cluster_api.set_node_status(
+        node='address3', role=NodeRole.WORKER, status=NodeStatus.STOPPING)
+
+    band_num_queued_subtasks = {('address0', 'numa-0'): 3, ('address1', 'numa-0'): 4}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks in \
+        ({('address1', 'numa-0'): -1, ('address0', 'numa-0'): -1, ('address2', 'numa-0'): 2},
+         {('address1', 'numa-0'): -2, ('address0', 'numa-0'): 0,  ('address2', 'numa-0'): 2},
+         {('address1', 'numa-0'): -2, ('address0', 'numa-0'): -1, ('address2', 'numa-0'): 3})
+
+    # ('address0', 'numa-0'), ('address2', 'numa-0') are ready
+    await cluster_api.set_node_status(
+        node='address1', role=NodeRole.WORKER, status=NodeStatus.STOPPING)
+
+    band_num_queued_subtasks = {('address0', 'numa-0'): 9, ('address1', 'numa-0'): 7,
+                                ('address2', 'numa-0'): 0}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks in \
+        ({('address1', 'numa-0'): -7, ('address0', 'numa-0'): 3, ('address2', 'numa-0'): 4},
+         {('address1', 'numa-0'): -7, ('address0', 'numa-0'): 4, ('address2', 'numa-0'): 3})
+
+    band_num_queued_subtasks = {('address0', 'numa-0'): 9, ('address1', 'numa-0'): 7}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks == \
+        {('address1', 'numa-0'): -7, ('address0', 'numa-0'): -1, ('address2', 'numa-0'): 8}
+
+    band_num_queued_subtasks = {('address1', 'numa-0'): 8}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks == \
+        {('address1', 'numa-0'): -8, ('address0', 'numa-0'): 4, ('address2', 'numa-0'): 4}
+
+    band_num_queued_subtasks = {('address1', 'numa-0'): 0}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks == {('address1', 'numa-0'): 0}
+
+    # only ('address0', 'numa-0') is ready, i.e. there's only one band initially
+    await cluster_api.set_node_status(
+        node='address2', role=NodeRole.WORKER, status=NodeStatus.STOPPING)
+    band_num_queued_subtasks = {('address0', 'numa-0'): 8}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks == {('address0', 'numa-0'): 0}
+
+    band_num_queued_subtasks = {('address1', 'numa-0'): 8}
+    move_queued_subtasks = await assigner_ref.reassign_subtasks(band_num_queued_subtasks)
+    assert move_queued_subtasks == {('address1', 'numa-0'): -8, ('address0', 'numa-0'): 8}
diff --git a/mars/services/scheduling/supervisor/tests/test_queue_balance.py b/mars/services/scheduling/supervisor/tests/test_queue_balance.py
new file mode 100644
--- /dev/null
+++ b/mars/services/scheduling/supervisor/tests/test_queue_balance.py
@@ -0,0 +1,181 @@
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import pytest
+from typing import Tuple, List
+
+import mars.oscar as mo
+from mars.services.cluster import ClusterAPI
+from mars.services.cluster.core import NodeRole, NodeStatus
+from mars.services.cluster.locator import SupervisorLocatorActor
+from mars.services.cluster.uploader import NodeInfoUploaderActor
+from mars.services.cluster.supervisor.node_info import NodeInfoCollectorActor
+from mars.services.scheduling.supervisor import AssignerActor, \
+    SubtaskManagerActor, SubtaskQueueingActor, GlobalSlotManagerActor
+from mars.services.subtask import Subtask
+from mars.utils import extensible
+
+
+class MockNodeInfoCollectorActor(NodeInfoCollectorActor):
+    def __init__(self, timeout=None, check_interval=None):
+        super().__init__(timeout=timeout, check_interval=check_interval)
+        self.ready_nodes = {('address0', 'numa-0'): 2,
+                            ('address1', 'numa-0'): 2,
+                            ('address2', 'numa-0'): 2}
+
+    async def update_node_info(self, address, role, env=None,
+                               resource=None, detail=None, status=None):
+        if 'address' in address and status == NodeStatus.STOPPING:
+            del self.ready_nodes[(address, 'numa-0')]
+        await super().update_node_info(address, role, env,
+                                       resource, detail, status)
+
+    def get_all_bands(self, role=None, statuses=None):
+        if statuses == {NodeStatus.READY}:
+            return self.ready_nodes
+        else:
+            return {('address0', 'numa-0'): 2,
+                    ('address1', 'numa-0'): 2,
+                    ('address2', 'numa-0'): 2}
+
+
+class FakeClusterAPI(ClusterAPI):
+    @classmethod
+    async def create(cls, address: str, **kw):
+        dones, _ = await asyncio.wait([
+            mo.create_actor(SupervisorLocatorActor, 'fixed', address,
+                            uid=SupervisorLocatorActor.default_uid(),
+                            address=address),
+            mo.create_actor(MockNodeInfoCollectorActor,
+                            uid=NodeInfoCollectorActor.default_uid(),
+                            address=address),
+            mo.create_actor(NodeInfoUploaderActor, NodeRole.WORKER,
+                            interval=kw.get('upload_interval'),
+                            band_to_slots=kw.get('band_to_slots'),
+                            use_gpu=kw.get('use_gpu', False),
+                            uid=NodeInfoUploaderActor.default_uid(),
+                            address=address),
+        ])
+
+        for task in dones:
+            try:
+                task.result()
+            except mo.ActorAlreadyExist:  # pragma: no cover
+                pass
+
+        api = await super().create(address=address)
+        await api.mark_node_ready()
+        return api
+
+
+class MockSlotsActor(mo.Actor):
+    def apply_subtask_slots(self, band: Tuple, session_id: str,
+                            subtask_ids: List[str], subtask_slots: List[int]):
+        return subtask_ids
+
+
+class MockAssignerActor(mo.Actor):
+    def assign_subtasks(self, subtasks: List[Subtask]):
+        return [subtask.expect_bands[0] for subtask in subtasks]
+
+    def reassign_subtasks(self, band_num_queued_subtasks):
+        if len(band_num_queued_subtasks.keys()) == 1:
+            [(band, _)] = band_num_queued_subtasks.items()
+            return {band: 0}
+        return {('address1', 'numa-0'): -8, ('address0', 'numa-0'): 0,
+                ('address2', 'numa-0'): 8}
+
+
+class MockSubtaskManagerActor(mo.Actor):
+    def __init__(self):
+        self._subtask_ids, self._bands = [], []
+
+    @extensible
+    def submit_subtask_to_band(self, subtask_id: str, band: Tuple):
+        self._subtask_ids.append(subtask_id)
+        self._bands.append(band)
+
+    def dump_data(self):
+        return self._subtask_ids, self._bands
+
+
+@pytest.fixture
+async def actor_pool():
+    pool = await mo.create_actor_pool('127.0.0.1', n_process=0)
+
+    async with pool:
+        session_id = 'test_session'
+        cluster_api = await FakeClusterAPI.create(pool.external_address)
+
+        # create assigner actor
+        await mo.create_actor(MockAssignerActor,
+                              uid=AssignerActor.gen_uid(session_id),
+                              address=pool.external_address)
+        # create queueing actor
+        manager_ref = await mo.create_actor(MockSubtaskManagerActor,
+                                            uid=SubtaskManagerActor.gen_uid(session_id),
+                                            address=pool.external_address)
+        # create slots actor
+        slots_ref = await mo.create_actor(MockSlotsActor,
+                                          uid=GlobalSlotManagerActor.default_uid(),
+                                          address=pool.external_address)
+        # create queueing actor
+        queueing_ref = await mo.create_actor(SubtaskQueueingActor,
+                                             session_id, 1,
+                                             uid=SubtaskQueueingActor.gen_uid(session_id),
+                                             address=pool.external_address)
+
+        yield pool, session_id, cluster_api, queueing_ref, slots_ref, manager_ref
+
+        await mo.destroy_actor(queueing_ref)
+
+
+async def _queue_subtasks(num_subtasks, expect_bands, queueing_ref):
+    if not num_subtasks:
+        return
+    subtasks = [Subtask(expect_bands[0] + '-' + str(i)) for i in range(num_subtasks)]
+    for subtask in subtasks:
+        subtask.expect_bands = [expect_bands]
+    priorities = [(i,) for i in range(num_subtasks)]
+
+    await queueing_ref.add_subtasks(subtasks, priorities)
+
+
+@pytest.mark.asyncio
+async def test_subtask_queueing(actor_pool):
+    _pool, session_id, cluster_api, queueing_ref, slots_ref, manager_ref = actor_pool
+    nums_subtasks = [9, 8, 1]
+    expects_bands = [('address0', 'numa-0'), ('address1', 'numa-0'),
+                     ('address2', 'numa-0')]
+    for num_subtasks, expect_bands in zip(nums_subtasks, expects_bands):
+        await _queue_subtasks(num_subtasks, expect_bands, queueing_ref)
+
+    await cluster_api.set_node_status(
+        node='address1', role=NodeRole.WORKER, status=NodeStatus.STOPPING)
+
+    # 9 subtasks on ('address0', 'numa-0')
+    await queueing_ref.submit_subtasks(band=('address0', 'numa-0'), limit=10)
+    commited_subtask_ids, _commited_bands = await manager_ref.dump_data()
+    assert len(commited_subtask_ids) == 9
+
+    # 0 subtasks on ('address1', 'numa-0')
+    await queueing_ref.submit_subtasks(band=('address1', 'numa-0'), limit=10)
+    commited_subtask_ids, _commited_bands = await manager_ref.dump_data()
+    assert len(commited_subtask_ids) == 9
+
+    # 9 subtasks on ('address2', 'numa-0')
+    await queueing_ref.submit_subtasks(band=('address2', 'numa-0'), limit=10)
+    commited_subtask_ids, _commited_bands = await manager_ref.dump_data()
+    assert len(commited_subtask_ids) == 18