Repository: buildbot/buildbot

Issue:
workers should have exponential backoff for reconnecting to master in case of unauthorized login
In case of  unauthorized login, worker keep retrying to connect to master every 2 or 3 seconds. In case of multiple such workers, such worker connection floods the server. 

Worker should have exponential backoff for reconnecting to master in case of unauthorized login.

Sample logs:
```
2018-06-21 17:06:00-0700 [-] Connecting to localhost:9989
2018-06-21 17:06:00-0700 [HangCheckProtocol,client] ReconnectingPBClientFactory.failedToGetPerspective
2018-06-21 17:06:00-0700 [HangCheckProtocol,client] unauthorized login; check worker name and password
2018-06-21 17:06:00-0700 [HangCheckProtocol,client] Lost connection to localhost:9989
2018-06-21 17:06:00-0700 [HangCheckProtocol,client] <twisted.internet.tcp.Connector instance at 0x10ff2b200> will retry in 2 seconds
2018-06-21 17:06:00-0700 [HangCheckProtocol,client] Retry attempt 1/inf
2018-06-21 17:06:00-0700 [-] Stopping factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>

2018-06-21 17:06:03-0700 [-] Starting factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>
2018-06-21 17:06:03-0700 [-] Connecting to localhost:9989
2018-06-21 17:06:03-0700 [HangCheckProtocol,client] ReconnectingPBClientFactory.failedToGetPerspective
2018-06-21 17:06:03-0700 [HangCheckProtocol,client] unauthorized login; check worker name and password
2018-06-21 17:06:03-0700 [HangCheckProtocol,client] Lost connection to localhost:9989
2018-06-21 17:06:03-0700 [HangCheckProtocol,client] <twisted.internet.tcp.Connector instance at 0x10ff2b200> will retry in 2 seconds
2018-06-21 17:06:03-0700 [HangCheckProtocol,client] Retry attempt 1/inf
2018-06-21 17:06:03-0700 [-] Stopping factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>

2018-06-21 17:06:06-0700 [-] Starting factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>
2018-06-21 17:06:06-0700 [-] Connecting to localhost:9989
2018-06-21 17:06:06-0700 [HangCheckProtocol,client] ReconnectingPBClientFactory.failedToGetPerspective
2018-06-21 17:06:06-0700 [HangCheckProtocol,client] unauthorized login; check worker name and password
2018-06-21 17:06:06-0700 [HangCheckProtocol,client] Lost connection to localhost:9989
2018-06-21 17:06:06-0700 [HangCheckProtocol,client] <twisted.internet.tcp.Connector instance at 0x10ff2b200> will retry in 3 seconds
2018-06-21 17:06:06-0700 [HangCheckProtocol,client] Retry attempt 1/inf
2018-06-21 17:06:06-0700 [-] Stopping factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>

2018-06-21 17:06:09-0700 [-] Starting factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>
2018-06-21 17:06:09-0700 [-] Connecting to localhost:9989
2018-06-21 17:06:09-0700 [HangCheckProtocol,client] ReconnectingPBClientFactory.failedToGetPerspective
2018-06-21 17:06:09-0700 [HangCheckProtocol,client] unauthorized login; check worker name and password
2018-06-21 17:06:09-0700 [HangCheckProtocol,client] Lost connection to localhost:9989
2018-06-21 17:06:09-0700 [HangCheckProtocol,client] <twisted.internet.tcp.Connector instance at 0x10ff2b200> will retry in 2 seconds
2018-06-21 17:06:09-0700 [HangCheckProtocol,client] Retry attempt 1/inf
2018-06-21 17:06:09-0700 [-] Stopping factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>

2018-06-21 17:06:12-0700 [-] Starting factory <buildbot_worker.pb.BotFactory instance at 0x10ff2b098>
2018-06-21 17:06:12-0700 [-] Connecting to localhost:9989
2018-06-21 17:06:12-0700 [HangCheckProtocol,client] ReconnectingPBClientFactory.failedToGetPerspective
2018-06-21 17:06:12-0700 [HangCheckProtocol,client] unauthorized login; check worker name and password
2018-06-21 17:06:12-0700 [HangCheckProtocol,client] Lost connection to localhost:9989
2018-06-21 17:06:12-0700 [HangCheckProtocol,client] <twisted.internet.tcp.Connector instance at 0x10ff2b200> will retry in 2 seconds
2018-06-21 17:06:12-0700 [HangCheckProtocol,client] Retry attempt 1/inf
```



Discussion:
I agree the retry timer should increase in case of bad login. feel free to send a PR

Resolving patch:
diff --git a/worker/buildbot_worker/pb.py b/worker/buildbot_worker/pb.py
--- a/worker/buildbot_worker/pb.py
+++ b/worker/buildbot_worker/pb.py
@@ -481,8 +481,8 @@ class BotFactory(AutoLoginPBFactory):
 
     _reactor = reactor
 
-    def __init__(self, buildmaster_host, port, keepaliveInterval, maxDelay):
-        AutoLoginPBFactory.__init__(self)
+    def __init__(self, buildmaster_host, port, keepaliveInterval, maxDelay, retryPolicy=None):
+        AutoLoginPBFactory.__init__(self, retryPolicy=retryPolicy)
         self.keepaliveInterval = keepaliveInterval
         self.keepalive_lock = defer.DeferredLock()
         self._shutting_down = False
@@ -621,8 +621,15 @@ def __init__(self, buildmaster_host, port, name, passwd, basedir,
 
         self.allow_shutdown = allow_shutdown
 
+        def policy(attempt):
+            if maxRetries and attempt >= maxRetries:
+                reactor.stop()
+            return backoffPolicy()(attempt)
+
         if protocol == 'pb':
-            bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay)
+            bf = self.bf = BotFactory(
+                buildmaster_host, port, keepalive, maxdelay, retryPolicy=policy
+            )
             bf.startLogin(credentials.UsernamePassword(name, passwd), client=self.bot)
         elif protocol == 'msgpack_experimental_v7':
             if connection_string is None:
@@ -670,10 +677,6 @@ def get_connection_string(host, port):
                 connection_string = get_connection_string(buildmaster_host, port)
             endpoint = clientFromString(reactor, connection_string)
 
-        def policy(attempt):
-            if maxRetries and attempt >= maxRetries:
-                reactor.stop()
-            return backoffPolicy()(attempt)
         pb_service = ClientService(endpoint, bf, retryPolicy=policy)
         self.addService(pb_service)
 
diff --git a/worker/buildbot_worker/pbutil.py b/worker/buildbot_worker/pbutil.py
--- a/worker/buildbot_worker/pbutil.py
+++ b/worker/buildbot_worker/pbutil.py
@@ -21,8 +21,11 @@
 from __future__ import print_function
 from future.utils import iteritems
 
+from twisted.application.internet import backoffPolicy
 from twisted.cred import error
+from twisted.internet import defer
 from twisted.internet import reactor
+from twisted.internet import task
 from twisted.python import log
 from twisted.spread import pb
 from twisted.spread.pb import PBClientFactory
@@ -51,9 +54,24 @@ class AutoLoginPBFactory(PBClientFactory):
     invoked.
     """
 
-    def clientConnectionMade(self, broker):
+    def __init__(  # pylint: disable=wrong-spelling-in-docstring
+        self, retryPolicy=None, **kwargs
+    ):
+        """
+        @param retryPolicy: A policy configuring how long L{AutoLoginPBFactory} will
+            wait between attempts to connect to C{endpoint}.
+        @type retryPolicy: callable taking (the number of failed connection
+            attempts made in a row (L{int})) and returning the number of
+            seconds to wait before making another attempt.
+        """
+        PBClientFactory.__init__(self, **kwargs)
+        self._timeoutForAttempt = backoffPolicy() if retryPolicy is None else retryPolicy
+        self._failedAttempts = 0
+        self._login_d = None
+
+    def clientConnectionMade(self, broker, retryPolicy=None):
         PBClientFactory.clientConnectionMade(self, broker)
-        self.doLogin(self._root, broker)
+        self._login_d = self.doLogin(self._root, broker)
         self.gotRootObject(self._root)
 
     def login(self, *args):
@@ -70,6 +88,11 @@ def doLogin(self, root, broker):
                        errbackArgs=(broker,))
         return d
 
+    def stopFactory(self):
+        if self._login_d:
+            self._login_d.cancel()
+        PBClientFactory.stopFactory(self)
+
     # methods to override
 
     def gotPerspective(self, perspective):
@@ -81,6 +104,7 @@ def gotRootObject(self, root):
         is now available. This method will be called each time the connection
         is established and the object reference is retrieved."""
 
+    @defer.inlineCallbacks
     def failedToGetPerspective(self, why, broker):
         """The login process failed, most likely because of an authorization
         failure (bad password), but it is also possible that we lost the new
@@ -97,7 +121,22 @@ def failedToGetPerspective(self, why, broker):
         else:
             log.err(why, 'While trying to connect:')
             reactor.stop()
-            return
+            defer.returnValue(None)
+
+        self._failedAttempts += 1
+        delay = self._timeoutForAttempt(self._failedAttempts)
+        log.msg(
+            "Scheduling retry {attempt} to getPerspective in {delay} seconds.".format(
+                attempt=self._failedAttempts,
+                delay=delay,
+            )
+        )
+
+        # Delay the retry according to the backoff policy
+        try:
+            yield task.deferLater(reactor, delay, lambda: None)
+        except defer.CancelledError:
+            pass
 
         # lose the current connection, which will trigger a retry
         broker.transport.loseConnection()


Test patch: