Repository: buildbot/buildbot

Issue:
Regression: maxdelay.bit_length change causes workers to not reconnect
See
https://github.com/buildbot/buildbot/blame/3fcf45acd84c1e3e6b3a6c77eda5d1e4dd0f9b3e/worker/buildbot_worker/pb.py#L195 -- this changed the line from passing an int to passing a callable, which cases the following exception

```
2018-01-05T21:18:39+0000 [Uninitialized] Unhandled Error
	Traceback (most recent call last):
	  File "/usr/local/lib/python3.6/site-packages/twisted/application/app.py", line 311, in runReactorWithLogging
	    reactor.run()
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/base.py", line 1243, in run
	    self.mainLoop()
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/base.py", line 1255, in mainLoop
	    self.doIteration(t)
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/epollreactor.py", line 235, in doPoll
	    log.callWithLogger(selectable, _drdw, selectable, fd, event)
	--- <exception caught here> ---
	  File "/usr/local/lib/python3.6/site-packages/twisted/python/log.py", line 103, in callWithLogger
	    return callWithContext({"system": lp}, func, *args, **kw)
	  File "/usr/local/lib/python3.6/site-packages/twisted/python/log.py", line 86, in callWithContext
	    return context.call({ILogContext: newCtx}, func, *args, **kw)
	  File "/usr/local/lib/python3.6/site-packages/twisted/python/context.py", line 122, in callWithContext
	    return self.currentContext().callWithContext(ctx, func, *args, **kw)
	  File "/usr/local/lib/python3.6/site-packages/twisted/python/context.py", line 85, in callWithContext
	    return func(*args,**kw)
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/posixbase.py", line 627, in _doReadOrWrite
	    self._disconnectSelectable(selectable, why, inRead)
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/posixbase.py", line 258, in _disconnectSelectable
	    selectable.connectionLost(failure.Failure(why))
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/tcp.py", line 473, in connectionLost
	    self.failIfNotConnected(error.ConnectError(string=reason))
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/tcp.py", line 450, in failIfNotConnected
	    self.connector.connectionFailed(failure.Failure(err))
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/base.py", line 1128, in connectionFailed
	    self.factory.clientConnectionFailed(self, reason)
	  File "/usr/local/lib/python3.6/site-packages/buildbot_worker/util/_hangcheck.py", line 115, in clientConnectionFailed
	    self._wrapped_factory.clientConnectionFailed(connector, reason)
	  File "/usr/local/lib/python3.6/site-packages/buildbot_worker/pb.py", line 120, in clientConnectionFailed
	    connector, reason)
	  File "/usr/local/lib/python3.6/site-packages/buildbot_worker/pbutil.py", line 64, in clientConnectionFailed
	    self.retry()
	  File "/usr/local/lib/python3.6/site-packages/buildbot_worker/pb.py", line 86, in retry
	    ReconnectingPBClientFactory.retry(self, connector=connector)
	  File "/usr/local/lib/python3.6/site-packages/twisted/internet/protocol.py", line 427, in retry
	    self.delay = min(self.delay * self.factor, self.maxDelay)
	builtins.TypeError: '<' not supported between instances of 'builtin_function_or_method' and 'float'
```

Relevant buildbot tac

```

import os

from buildbot_worker.bot import Worker
from twisted.application import service

basedir = '/buildbot-worker'

# note: this line is matched against to check that this is a worker
# directory; do not edit it.
application = service.Application('buildbot-worker')
buildmaster_host = 'localhost'
port = 2020
workername = 'worker'
passwd = 'pass'
keepalive = 600
umask = None
maxdelay = 300
numcpus = None
allow_shutdown = None
maxretries = None

s = Worker(buildmaster_host, port, workername, passwd, basedir,
           keepalive, umask=umask, maxdelay=maxdelay,
           numcpus=numcpus, allow_shutdown=allow_shutdown,
           maxRetries=maxretries)
s.setServiceParent(application)
```


Discussion:
Thanks for the report.

 I have no idea why this ``".bit_length"`` sneaked in. I fear this is IDE autocomplete :/
I don't understand how the e2e tests didn't catch that.

Resolving patch:
diff --git a/worker/buildbot_worker/pb.py b/worker/buildbot_worker/pb.py
--- a/worker/buildbot_worker/pb.py
+++ b/worker/buildbot_worker/pb.py
@@ -83,7 +83,10 @@ def __init__(self, buildmaster_host, port, keepaliveInterval, maxDelay,
 
     def retry(self, connector=None):
         ReconnectingPBClientFactory.retry(self, connector=connector)
-        if self.retries > self.maxRetries and self.maxRetriesCallback:
+        log.msg("Retry attempt {}/{}".format(
+            self.retries, self.maxRetries if self.maxRetries is not None else "inf"))
+        if self.maxRetries is not None and self.retries > self.maxRetries and self.maxRetriesCallback:
+            log.msg("Giving up retrying!")
             self.maxRetriesCallback()
 
     def startedConnecting(self, connector):
@@ -192,7 +195,7 @@ def __init__(self, buildmaster_host, port, name, passwd, basedir,
             self.shutdown_mtime = 0
 
         self.allow_shutdown = allow_shutdown
-        bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay.bit_length,
+        bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay,
             maxRetries=self.maxRetries, maxRetriesCallback=self.gracefulShutdown)
         bf.startLogin(
             credentials.UsernamePassword(name, passwd), client=self.bot)


Test patch:
diff --git a/master/buildbot/test/integration/interop/test_worker_reconnect.py b/master/buildbot/test/integration/interop/test_worker_reconnect.py
new file mode 100644
--- /dev/null
+++ b/master/buildbot/test/integration/interop/test_worker_reconnect.py
@@ -0,0 +1,70 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+from twisted.internet import defer
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.test.util.integration import RunMasterBase
+
+
+class DisconnectingStep(BuildStep):
+    disconnection_list = []
+
+    def run(self):
+        self.disconnection_list.append(self)
+        if len(self.disconnection_list) < 2:
+            self.worker.disconnect()
+        return SUCCESS
+
+
+class WorkerReconnect(RunMasterBase):
+    """integration test for testing worker disconnection and reconnection"""
+    proto = "pb"
+
+    @defer.inlineCallbacks
+    def test_eventually_reconnect(self):
+        DisconnectingStep.disconnection_list = []
+        yield self.setupConfig(masterConfig())
+        build = yield self.doForceBuild()
+        self.assertEqual(build['buildid'], 2)
+        self.assertEqual(len(DisconnectingStep.disconnection_list), 2)
+
+
+# master configuration
+def masterConfig():
+    c = {}
+    from buildbot.config import BuilderConfig
+    from buildbot.process.factory import BuildFactory
+    from buildbot.plugins import schedulers
+
+    c['schedulers'] = [
+        schedulers.AnyBranchScheduler(
+            name="sched",
+            builderNames=["testy"]),
+        schedulers.ForceScheduler(
+            name="force",
+            builderNames=["testy"])]
+
+    f = BuildFactory()
+    f.addStep(DisconnectingStep())
+    c['builders'] = [
+        BuilderConfig(name="testy",
+                      workernames=["local1"],
+                      factory=f)]
+    return c