Repository: buildbot/buildbot

Issue:
Insubstantiation notifier left after the refactor
Which blocks the `BuildMaster.stopService`, see https://github.com/buildbot/buildbot/blob/master/master/buildbot/worker/latent.py#L495


Discussion:

Resolving patch:
diff --git a/master/buildbot/worker/latent.py b/master/buildbot/worker/latent.py
--- a/master/buildbot/worker/latent.py
+++ b/master/buildbot/worker/latent.py
@@ -61,6 +61,10 @@ class States(enum.Enum):
     # When in this state self.substantiation_build is not None.
     INSUBSTANTIATING_SUBSTANTIATING = 5
 
+    # This state represents a worker that is shut down. Effectively, it's NOT_SUBSTANTIATED
+    # plus that we will abort if anyone tries to substantiate it.
+    SHUT_DOWN = 6
+
 
 @implementer(ILatentWorker)
 class AbstractLatentWorker(AbstractWorker):
@@ -148,7 +152,9 @@ class AbstractLatentWorker(AbstractWorker):
         then either of:
             INSUBSTANTIATING_SUBSTANTIATING -> SUBSTANTIATING
             INSUBSTANTIATING -> NOT_SUBSTANTIATED
-        """
+
+    stopService():
+        NOT_SUBSTANTIATED -> SHUT_DOWN
     '''
 
     def checkConfig(self, name, password,
@@ -209,6 +215,9 @@ def substantiated(self):
     def substantiate(self, wfb, build):
         log.msg("substantiating worker %s" % (wfb,))
 
+        if self.state == States.SHUT_DOWN:
+            return defer.succeed(False)
+
         if self.state == States.SUBSTANTIATED and self.conn is not None:
             self._setBuildWaitTimer()
             return defer.succeed(True)
@@ -416,7 +425,7 @@ def insubstantiate(self, fast=False, force_substantiation_build=None):
             assert self.state not in [States.INSUBSTANTIATING,
                                       States.INSUBSTANTIATING_SUBSTANTIATING]
 
-            if self.state == States.NOT_SUBSTANTIATED:
+            if self.state in [States.NOT_SUBSTANTIATED, States.SHUT_DOWN]:
                 return
 
             prev_state = self.state
@@ -460,13 +469,6 @@ def insubstantiate(self, fast=False, force_substantiation_build=None):
 
     @defer.inlineCallbacks
     def _soft_disconnect(self, fast=False, stopping_service=False):
-        if self.building:
-            # If there are builds running or about to start on this worker, don't disconnect.
-            # soft_disconnect happens during master reconfiguration or shutdown. It's not a good
-            # reason to kill these builds. We effectively slow down reconfig until all workers
-            # that have been unconfigured finish their builds.
-            return
-
         # a negative build_wait_timeout means the worker should never be shut
         # down, so just disconnect.
         if not stopping_service and self.build_wait_timeout < 0:
@@ -489,14 +491,26 @@ def disconnect(self):
 
     @defer.inlineCallbacks
     def stopService(self):
-        # the worker might be insubstantiating from buildWaitTimeout
-        if self.state in [States.INSUBSTANTIATING,
-                          States.INSUBSTANTIATING_SUBSTANTIATING]:
-            yield self._insubstantiation_notifier.wait()
+        # stops the service. Waits for any pending substantiations, insubstantiations or builds
+        # that are running or about to start to complete.
+        while self.state not in [States.NOT_SUBSTANTIATED, States.SHUT_DOWN]:
+            if self.state in [States.INSUBSTANTIATING,
+                              States.INSUBSTANTIATING_SUBSTANTIATING,
+                              States.SUBSTANTIATING,
+                              States.SUBSTANTIATING_STARTING]:
+                self._log_start_stop_locked('stopService')
+                yield self._start_stop_lock.acquire()
+                self._start_stop_lock.release()
+
+            if self.conn is not None or self.state in [States.SUBSTANTIATED,
+                                                       States.SUBSTANTIATING_STARTING]:
+                yield self._soft_disconnect(stopping_service=True)
+
+        # prevent any race conditions with any future builds that are in the process of
+        # being started.
+        if self.state == States.NOT_SUBSTANTIATED:
+            self.state = States.SHUT_DOWN
 
-        if self.conn is not None or self.state in [States.SUBSTANTIATING,
-                                                   States.SUBSTANTIATED]:
-            yield self._soft_disconnect(stopping_service=True)
         self._clearBuildWaitTimer()
         res = yield super().stopService()
         return res


Test patch:
diff --git a/master/buildbot/test/integration/test_worker_latent.py b/master/buildbot/test/integration/test_worker_latent.py
--- a/master/buildbot/test/integration/test_worker_latent.py
+++ b/master/buildbot/test/integration/test_worker_latent.py
@@ -13,12 +13,14 @@
 #
 # Copyright Buildbot Team Members
 
+from parameterized import parameterized
 
 from twisted.internet import defer
 from twisted.python.failure import Failure
 from twisted.spread import pb
 
 from buildbot.config import BuilderConfig
+from buildbot.config import MasterConfig
 from buildbot.interfaces import LatentWorkerCannotSubstantiate
 from buildbot.interfaces import LatentWorkerFailedToSubstantiate
 from buildbot.interfaces import LatentWorkerSubstantiatiationCancelled
@@ -131,6 +133,15 @@ def create_single_worker_two_builder_config(self, controller_kwargs=None):
 
         return controller, master, builder_ids
 
+    def reconfig_workers_remove_all(self, master):
+        # returns a deferred that fires when the worker has been successfully removed
+        config_dict = {
+            'workers': [],
+            'multiMaster': True
+        }
+        config = MasterConfig.loadFromDict(config_dict, '<dict>')
+        return master.workers.reconfigServiceWithBuildbotConfig(config)
+
     @defer.inlineCallbacks
     def test_latent_workers_start_in_parallel(self):
         """
@@ -460,6 +471,62 @@ def test_insubstantiation_during_substantiation_refuses_substantiation(self):
 
         yield self.assertBuildResults(1, RETRY)
 
+    @defer.inlineCallbacks
+    def test_stopservice_during_insubstantiation_completes(self):
+        """
+        When stopService is called and a worker is insubstantiating, we should wait for this
+        process to complete.
+        """
+        controller, master, builder_id = yield self.create_single_worker_config(
+            controller_kwargs=dict(build_wait_timeout=1))
+
+        # Substantiate worker via a build
+        yield self.createBuildrequest(master, [builder_id])
+        yield controller.start_instance(True)
+
+        yield self.assertBuildResults(1, SUCCESS)
+        self.assertTrue(controller.started)
+
+        # Wait until worker starts insubstantiation and then shutdown worker
+        self.reactor.advance(1)
+        self.assertTrue(controller.stopping)
+
+        d = self.reconfig_workers_remove_all(master)
+        self.assertFalse(d.called)
+        yield controller.stop_instance(True)
+        yield d
+
+    @parameterized.expand([
+        ('with_substantiation_failure', False, False),
+        ('without_worker_connecting', True, False),
+        ('with_worker_connecting', True, True),
+    ])
+    @defer.inlineCallbacks
+    def test_stopservice_during_substantiation_completes(self, name, subst_success,
+                                                         worker_connects):
+        """
+        When stopService is called and a worker is substantiating, we should wait for this
+        process to complete.
+        """
+        controller, master, builder_id = yield self.create_single_worker_config(
+            controller_kwargs=dict(build_wait_timeout=1))
+        controller.auto_connect_worker = worker_connects
+
+        # Substantiate worker via a build
+        yield self.createBuildrequest(master, [builder_id])
+        self.assertTrue(controller.starting)
+
+        d = self.reconfig_workers_remove_all(master)
+        self.assertFalse(d.called)
+
+        yield controller.start_instance(subst_success)
+
+        # we should stop the instance immediately after it substantiates regardless of the result
+        self.assertTrue(controller.stopping)
+
+        yield controller.stop_instance(True)
+        yield d
+
     @defer.inlineCallbacks
     def test_substantiation_cancelled_by_insubstantiation_when_waiting_for_insubstantiation(self):
         """