Repository: buildbot/buildbot

Issue:
buildbot.test.integration.test_worker_workerside.TestWorkerConnection.test_pb_keepalive is flaky
Examples: https://buildbot.buildbot.net/#/builders/8/builds/1856/steps/8/logs/stdio,  https://buildbot.buildbot.net/#/builders/23/builds/1149/steps/8/logs/stdio. Note the different failure reasons.


Discussion:
This passed on my local setup
The point is that the test fails only on some tests, only in certain conditions which are for some reason more prevalent on the CI environment.
In the end I was able to reproduce various issues with the test by running many instances of it in parallel while limiting the total cpu usage to a single core so that the tests are preempted many times and hopefully interleave the deferreds differently. Looks like the keepalive may be called more than once for some reason.
Ok, pretty cool!

Resolving patch:
diff --git a/master/buildbot/worker/base.py b/master/buildbot/worker/base.py
--- a/master/buildbot/worker/base.py
+++ b/master/buildbot/worker/base.py
@@ -669,8 +669,9 @@ def stopQuarantineTimer(self):
 
 class Worker(AbstractWorker):
 
+    @defer.inlineCallbacks
     def detached(self):
-        super().detached()
+        yield super().detached()
         self.botmaster.workerLost(self)
         self.startMissingTimer()
 
diff --git a/worker/buildbot_worker/pb.py b/worker/buildbot_worker/pb.py
--- a/worker/buildbot_worker/pb.py
+++ b/worker/buildbot_worker/pb.py
@@ -75,6 +75,7 @@ class BotFactory(AutoLoginPBFactory):
     def __init__(self, buildmaster_host, port, keepaliveInterval, maxDelay):
         AutoLoginPBFactory.__init__(self)
         self.keepaliveInterval = keepaliveInterval
+        self.keepalive_lock = defer.DeferredLock()
 
     def gotPerspective(self, perspective):
         log.msg("Connected to buildmaster; worker is ready")
@@ -96,24 +97,27 @@ def startTimers(self):
         assert self.keepaliveInterval
         assert not self.keepaliveTimer
 
+        @defer.inlineCallbacks
         def doKeepalive():
             self.keepaliveTimer = None
             self.startTimers()
 
+            yield self.keepalive_lock.acquire()
+            self.currentKeepaliveWaiter = defer.Deferred()
+
             # Send the keepalive request.  If an error occurs
             # was already dropped, so just log and ignore.
             log.msg("sending app-level keepalive")
-            d = self.perspective.callRemote("keepalive")
-            self.currentKeepaliveWaiter = defer.Deferred()
-
-            def keepaliveReplied(details):
+            try:
+                details = yield self.perspective.callRemote("keepalive")
                 log.msg("Master replied to keepalive, everything's fine")
                 self.currentKeepaliveWaiter.callback(details)
                 self.currentKeepaliveWaiter = None
-                return details
+            except Exception as e:
+                log.err(e, "error sending keepalive")
+            finally:
+                self.keepalive_lock.release()
 
-            d.addCallback(keepaliveReplied)
-            d.addErrback(log.err, "error sending keepalive")
         self.keepaliveTimer = self._reactor.callLater(self.keepaliveInterval,
                                                       doKeepalive)
 


Test patch:
diff --git a/master/buildbot/test/integration/test_worker_workerside.py b/master/buildbot/test/integration/test_worker_workerside.py
--- a/master/buildbot/test/integration/test_worker_workerside.py
+++ b/master/buildbot/test/integration/test_worker_workerside.py
@@ -72,8 +72,9 @@ def attached(self, conn):
         self.detach_d = defer.Deferred()
         return super().attached(conn)
 
+    @defer.inlineCallbacks
     def detached(self):
-        super().detached()
+        yield super().detached()
         self.detach_d, d = None, self.detach_d
         d.callback(None)
 
@@ -218,9 +219,9 @@ def addMasterSideWorker(self,
     def workerSideDisconnect(self, worker):
         """Disconnect from the worker side
 
-        This seems a good way to simulate a broken connection
+        This seems a good way to simulate a broken connection. Returns a Deferred
         """
-        worker.bf.disconnect()
+        return worker.bf.disconnect()
 
     def addWorker(self, connection_string_tpl=r"tcp:host=127.0.0.1:port={port}",
                   password="pw", name="testworker", keepalive=None):
@@ -232,7 +233,7 @@ def addWorker(self, connection_string_tpl=r"tcp:host=127.0.0.1:port={port}",
 
     @defer.inlineCallbacks
     def test_connect_disconnect(self):
-        self.addMasterSideWorker()
+        yield self.addMasterSideWorker()
 
         def could_not_connect():
             self.fail("Worker never got connected to master")
@@ -249,7 +250,7 @@ def could_not_connect():
 
     @defer.inlineCallbacks
     def test_reconnect_network(self):
-        self.addMasterSideWorker()
+        yield self.addMasterSideWorker()
 
         def could_not_connect():
             self.fail("Worker did not reconnect in time to master")
@@ -261,7 +262,7 @@ def could_not_connect():
         self.assertTrue('bldr' in worker.bot.builders)
 
         timeout = reactor.callLater(10, could_not_connect)
-        self.workerSideDisconnect(worker)
+        yield self.workerSideDisconnect(worker)
         yield worker.tests_connected
 
         timeout.cancel()
@@ -275,7 +276,7 @@ def test_applicative_reconnection(self):
         The worker starts with a password that the master does not accept
         at first, and then the master gets reconfigured to accept it.
         """
-        self.addMasterSideWorker()
+        yield self.addMasterSideWorker()
         worker = self.addWorker(password="pw2")
         yield worker.startService()
         why, broker = yield worker.tests_login_failed
@@ -297,6 +298,7 @@ def could_not_connect():
 
         timeout.cancel()
         self.assertTrue('bldr' in worker.bot.builders)
+
         yield worker.stopService()
         yield worker.tests_disconnected
 
@@ -311,24 +313,24 @@ def perspective_keepalive(Connection_self):
             waiter = worker.keepalive_waiter
             if waiter is not None:
                 waiter.callback(time.time())
+                worker.keepalive_waiter = None
+
         from buildbot.worker.protocols.pb import Connection
         self.patch(Connection, 'perspective_keepalive', perspective_keepalive)
 
-        self.addMasterSideWorker()
+        yield self.addMasterSideWorker()
         # short keepalive to make the test bearable to run
         worker = self.addWorker(keepalive=0.1)
-        worker.keepalive_waiter = defer.Deferred()
+        waiter = worker.keepalive_waiter = defer.Deferred()
 
         yield worker.startService()
         yield worker.tests_connected
-        first = yield worker.keepalive_waiter
+        first = yield waiter
         yield worker.bf.currentKeepaliveWaiter
 
-        worker.keepalive_waiter = defer.Deferred()
+        waiter = worker.keepalive_waiter = defer.Deferred()
 
-        second = yield worker.keepalive_waiter
-        # avoid errors if a third gets fired
-        worker.keepalive_waiter = None
+        second = yield waiter
         yield worker.bf.currentKeepaliveWaiter
 
         self.assertGreater(second, first)