Repository: buildbot/buildbot

Issue:
Master can't shutdown when worker gets missing without TCP connection being severed
Reproduction steps:

 - Create sample master config with a single worker.
 - `buildbot start path/master`
 - `buildbot-worker start path/worker`
 - `kill -SIGSTOP $(cat path/worker/twistd.pid)`
 - start build
 - `buildbot stop path/master`

The last step will say: `never saw process go away`


Discussion:
Sounds similar to #2914.

Resolving patch:
diff --git a/master/buildbot/process/botmaster.py b/master/buildbot/process/botmaster.py
--- a/master/buildbot/process/botmaster.py
+++ b/master/buildbot/process/botmaster.py
@@ -97,6 +97,11 @@ def cleanShutdown(self, quickMode=False, stopReactor=True, _reactor=reactor):
             dl = []
             for builder in self.builders.values():
                 for build in builder.building:
+                    # build may be waiting for ping to worker to succeed which
+                    # may never happen if the connection to worker was broken
+                    # without TCP connection being severed
+                    build.workerforbuilder.abortPingIfAny()
+
                     dl.append(build.waitUntilFinished())
             if not dl:
                 log.msg("No running jobs, starting shutdown immediately")
diff --git a/master/buildbot/process/workerforbuilder.py b/master/buildbot/process/workerforbuilder.py
--- a/master/buildbot/process/workerforbuilder.py
+++ b/master/buildbot/process/workerforbuilder.py
@@ -132,6 +132,11 @@ def ping(self, status=None):
 
         return d
 
+    def abortPingIfAny(self):
+        watchers, self.ping_watchers = self.ping_watchers, []
+        for d in watchers:
+            d.errback(PingException('aborted ping'))
+
     def _pong(self, res):
         watchers, self.ping_watchers = self.ping_watchers, []
         for d in watchers:


Test patch:
diff --git a/master/buildbot/test/fake/reactor.py b/master/buildbot/test/fake/reactor.py
--- a/master/buildbot/test/fake/reactor.py
+++ b/master/buildbot/test/fake/reactor.py
@@ -147,10 +147,9 @@ def callLater(self, when, what, *a, **kw):
         return Clock.callLater(self, when, what, *a, **kw)
 
     def stop(self):
-        # first fire pending calls
-        while self.getDelayedCalls():
-            last = sorted(self.getDelayedCalls(), key=lambda a: a.getTime())[-1]
-            self.advance(last.getTime() - self.seconds())
+        # first fire pending calls until the current time. Note that the real
+        # reactor only advances until the current time in the case of shutdown.
+        self.advance(0)
 
         # then, fire the shutdown event
         self.fireSystemEvent('shutdown')
diff --git a/master/buildbot/test/fake/worker.py b/master/buildbot/test/fake/worker.py
--- a/master/buildbot/test/fake/worker.py
+++ b/master/buildbot/test/fake/worker.py
@@ -17,9 +17,17 @@
 import os
 
 from twisted.internet import defer
+from twisted.python.filepath import FilePath
+from twisted.trial.unittest import SkipTest
 
 from buildbot.process import properties
 from buildbot.test.fake import fakeprotocol
+from buildbot.worker import Worker
+
+try:
+    from buildbot_worker.bot import LocalWorker as RemoteWorker
+except ImportError:
+    RemoteWorker = None
 
 
 class FakeWorker(object):
@@ -64,3 +72,76 @@ def putInQuarantine(self):
 
     def resetQuarantine(self):
         pass
+
+
+class WorkerController:
+
+    """
+    A controller for a ``Worker``.
+
+    https://glyph.twistedmatrix.com/2015/05/separate-your-fakes-and-your-inspectors.html
+    """
+
+    def __init__(self, case, name, build_wait_timeout=600,
+                 worker_class=None, **kwargs):
+        if worker_class is None:
+            worker_class = Worker
+
+        self.case = case
+        self.build_wait_timeout = build_wait_timeout
+        self.worker = worker_class(name, self, **kwargs)
+        self.remote_worker = None
+
+    def connect_worker(self):
+        if self.remote_worker is not None:
+            return
+        if RemoteWorker is None:
+            raise SkipTest("buildbot-worker package is not installed")
+        workdir = FilePath(self.case.mktemp())
+        workdir.createDirectory()
+        self.remote_worker = RemoteWorker(self.worker.name, workdir.path, False)
+        self.remote_worker.setServiceParent(self.worker)
+
+    def disconnect_worker(self):
+        if self.remote_worker is None:
+            return
+        self.worker.conn, conn = None, self.worker.conn
+        # LocalWorker does actually disconnect, so we must force disconnection
+        # via detached
+        conn.notifyDisconnected()
+        ret = self.remote_worker.disownServiceParent()
+        self.remote_worker = None
+        return ret
+
+    def sever_connection(self):
+        # stubs the worker connection so that it appears that the TCP connection
+        # has been severed in a way that no response is ever received, but
+        # messages don't fail immediately.
+        def remotePrint(message):
+            return defer.Deferred()
+        self.worker.conn.remotePrint = remotePrint
+
+        def remoteGetWorkerInfo():
+            return defer.Deferred()
+        self.worker.conn.remoteGetWorkerInfo = remoteGetWorkerInfo
+
+        def remoteSetBuilderList(builders):
+            return defer.Deferred()
+        self.worker.conn.remoteSetBuilderList = remoteSetBuilderList
+
+        def remoteStartCommand(remoteCommand, builderName, commandId,
+                               commandName, args):
+            return defer.Deferred()
+        self.worker.conn.remoteStartCommand = remoteStartCommand
+
+        def remoteShutdown():
+            return defer.Deferred()
+        self.worker.conn.remoteShutdown = remoteShutdown
+
+        def remoteStartBuild(builderName):
+            return defer.Deferred()
+        self.worker.conn.remoteStartBuild = remoteStartBuild
+
+        def remoteInterruptCommand(builderName, commandId, why):
+            return defer.Deferred()
+        self.worker.conn.remoteInterruptCommand = remoteInterruptCommand
diff --git a/master/buildbot/test/integration/test_locks.py b/master/buildbot/test/integration/test_locks.py
--- a/master/buildbot/test/integration/test_locks.py
+++ b/master/buildbot/test/integration/test_locks.py
@@ -15,35 +15,26 @@
 
 
 from twisted.internet import defer
-from twisted.python import log
 from twisted.python.filepath import FilePath
 from twisted.trial.unittest import TestCase
 
 from buildbot.config import BuilderConfig
 from buildbot.plugins import util
-from buildbot.process.buildstep import BuildStep
 from buildbot.process.factory import BuildFactory
 from buildbot.process.results import SUCCESS
 from buildbot.test.fake.step import BuildStepController
 from buildbot.test.util.integration import getMaster
+from buildbot.test.util.misc import DebugIntegrationLogsMixin
 from buildbot.test.util.misc import TestReactorMixin
 from buildbot.util.eventual import flushEventualQueue
 from buildbot.worker.local import LocalWorker
 
 
-class Tests(TestCase, TestReactorMixin):
+class Tests(TestCase, TestReactorMixin, DebugIntegrationLogsMixin):
 
     def setUp(self):
         self.setUpTestReactor()
-
-        # to ease debugging we display the error logs in the test log
-        origAddCompleteLog = BuildStep.addCompleteLog
-
-        def addCompleteLog(self, name, _log):
-            if name.endswith("err.text"):
-                log.msg("got error log!", name, _log)
-            return origAddCompleteLog(self, name, _log)
-        self.patch(BuildStep, "addCompleteLog", addCompleteLog)
+        self.setupDebugIntegrationLogs()
 
     def tearDown(self):
         self.assertFalse(self.master.running, "master is still running!")
diff --git a/master/buildbot/test/integration/test_process_botmaster.py b/master/buildbot/test/integration/test_process_botmaster.py
new file mode 100644
--- /dev/null
+++ b/master/buildbot/test/integration/test_process_botmaster.py
@@ -0,0 +1,97 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from buildbot.config import BuilderConfig
+from buildbot.process.factory import BuildFactory
+from buildbot.process.workerforbuilder import PingException
+from buildbot.test.fake.worker import WorkerController
+from buildbot.test.util.integration import getMaster
+from buildbot.test.util.misc import DebugIntegrationLogsMixin
+from buildbot.test.util.misc import TestReactorMixin
+
+
+class Tests(TestCase, TestReactorMixin, DebugIntegrationLogsMixin):
+
+    def setUp(self):
+        self.setUpTestReactor()
+        self.setupDebugIntegrationLogs()
+        self.timeout = 5
+
+    def tearDown(self):
+        self.assertFalse(self.master.running, "master is still running!")
+
+    @defer.inlineCallbacks
+    def getMaster(self, config_dict):
+        self.master = master = yield getMaster(self, self.reactor, config_dict)
+        return master
+
+    @defer.inlineCallbacks
+    def createBuildrequest(self, master, builder_ids, properties=None):
+        properties = properties.asDict() if properties is not None else None
+        ret = yield master.data.updates.addBuildset(
+            waited_for=False,
+            builderids=builder_ids,
+            sourcestamps=[
+                {'codebase': '',
+                 'repository': '',
+                 'branch': None,
+                 'revision': None,
+                 'project': ''},
+            ],
+            properties=properties,
+        )
+        return ret
+
+    @defer.inlineCallbacks
+    def do_terminates_ping_on_shutdown(self, quick_mode):
+        """
+        During shutdown we want to terminate any outstanding pings.
+        """
+        controller = WorkerController(self, 'local')
+
+        config_dict = {
+            'builders': [
+                BuilderConfig(name="testy",
+                              workernames=['local'],
+                              factory=BuildFactory()),
+            ],
+            'workers': [controller.worker],
+            'protocols': {'null': {}},
+            'multiMaster': True,
+        }
+        master = yield self.getMaster(config_dict)
+        builder_id = yield master.data.updates.findBuilderId('testy')
+
+        controller.connect_worker()
+        controller.sever_connection()
+        yield self.createBuildrequest(master, [builder_id])
+
+        # give time for any delayed actions to complete
+        self.reactor.advance(1)
+
+        yield master.botmaster.cleanShutdown(quickMode=quick_mode,
+                                             _reactor=self.reactor,
+                                             stopReactor=False)
+        self.flushLoggedErrors(PingException)
+
+    def test_terminates_ping_on_shutdown_quick_mode(self):
+        return self.do_terminates_ping_on_shutdown(quick_mode=True)
+
+    def test_terminates_ping_on_shutdown_slow_mode(self):
+        return self.do_terminates_ping_on_shutdown(quick_mode=False)
diff --git a/master/buildbot/test/integration/test_worker_latent.py b/master/buildbot/test/integration/test_worker_latent.py
--- a/master/buildbot/test/integration/test_worker_latent.py
+++ b/master/buildbot/test/integration/test_worker_latent.py
@@ -14,10 +14,7 @@
 # Copyright Buildbot Team Members
 
 
-import os
-
 from twisted.internet import defer
-from twisted.python import log
 from twisted.python.failure import Failure
 from twisted.trial.unittest import SynchronousTestCase
 
@@ -25,7 +22,6 @@
 from buildbot.interfaces import LatentWorkerCannotSubstantiate
 from buildbot.interfaces import LatentWorkerFailedToSubstantiate
 from buildbot.interfaces import LatentWorkerSubstantiatiationCancelled
-from buildbot.process.buildstep import BuildStep
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Interpolate
 from buildbot.process.properties import Properties
@@ -36,8 +32,8 @@
 from buildbot.test.fake.latent import LatentController
 from buildbot.test.fake.step import BuildStepController
 from buildbot.test.util.integration import getMaster
+from buildbot.test.util.misc import DebugIntegrationLogsMixin
 from buildbot.test.util.misc import TestReactorMixin
-from buildbot.test.util.misc import enable_trace
 
 
 class TestException(Exception):
@@ -47,23 +43,11 @@ class TestException(Exception):
     """
 
 
-class Tests(SynchronousTestCase, TestReactorMixin):
+class Tests(SynchronousTestCase, TestReactorMixin, DebugIntegrationLogsMixin):
 
     def setUp(self):
         self.setUpTestReactor()
-
-        # to ease debugging we display the error logs in the test log
-        origAddCompleteLog = BuildStep.addCompleteLog
-
-        def addCompleteLog(self, name, _log):
-            if name.endswith("err.text"):
-                log.msg("got error log!", name, _log)
-            return origAddCompleteLog(self, name, _log)
-        self.patch(BuildStep, "addCompleteLog", addCompleteLog)
-
-        if 'BBTRACE' in os.environ:
-            enable_trace(self, ["twisted", "worker_transition.py", "util/tu", "util/path",
-                                "log.py", "/mq/", "/db/", "buildbot/data/", "fake/reactor.py"])
+        self.setupDebugIntegrationLogs()
 
     def tearDown(self):
         # Flush the errors logged by the master stop cancelling the builds.
diff --git a/master/buildbot/test/util/misc.py b/master/buildbot/test/util/misc.py
--- a/master/buildbot/test/util/misc.py
+++ b/master/buildbot/test/util/misc.py
@@ -18,10 +18,12 @@
 import os
 import sys
 
+from twisted.python import log
 from twisted.python import threadpool
 from twisted.python.compat import NativeStringIO
 
 import buildbot
+from buildbot.process.buildstep import BuildStep
 from buildbot.test.fake.reactor import NonThreadPool
 from buildbot.test.fake.reactor import TestReactor
 from buildbot.util.eventual import _setReactor
@@ -74,9 +76,12 @@ class TestReactorMixin(object):
     def setUpTestReactor(self):
         self.patch(threadpool, 'ThreadPool', NonThreadPool)
         self.reactor = TestReactor()
-        self.addCleanup(self.reactor.stop)
         _setReactor(self.reactor)
+        # During shutdown sequence we must first stop the reactor and only then
+        # set unset the reactor used for eventually() because any callbacks
+        # that are run during reactor.stop() may use eventually() themselves.
         self.addCleanup(_setReactor, None)
+        self.addCleanup(self.reactor.stop)
 
 
 def encodeExecutableAndArgs(executable, args, encoding="utf-8"):
@@ -100,8 +105,10 @@ def enable_trace(case, trace_exclusions=None, f=sys.stdout):
     """This function can be called to enable tracing of the execution
     """
     if trace_exclusions is None:
-        trace_exclusions = ["twisted", "worker_transition.py", "util/tu",
-                            "log.py", "/mq/", "/db/", "buildbot/data/", "fake/reactor.py"]
+        trace_exclusions = [
+            "twisted", "worker_transition.py", "util/tu", "util/path",
+            "log.py", "/mq/", "/db/", "buildbot/data/", "fake/reactor.py"
+        ]
 
     bbbase = os.path.dirname(buildbot.__file__)
     state = {'indent': 0}
@@ -119,3 +126,19 @@ def tracefunc(frame, event, arg):
 
     sys.settrace(tracefunc)
     case.addCleanup(sys.settrace, lambda _a, _b, _c: None)
+
+
+class DebugIntegrationLogsMixin:
+
+    def setupDebugIntegrationLogs(self):
+        # to ease debugging we display the error logs in the test log
+        origAddCompleteLog = BuildStep.addCompleteLog
+
+        def addCompleteLog(self, name, _log):
+            if name.endswith("err.text"):
+                log.msg("got error log!", name, _log)
+            return origAddCompleteLog(self, name, _log)
+        self.patch(BuildStep, "addCompleteLog", addCompleteLog)
+
+        if 'BBTRACE' in os.environ:
+            enable_trace(self)