Repository: buildbot/buildbot

Issue:
Quarantine timer unpausing manually paused worker
When I pause a worker during a build and an exception happens on that build, the worker becomes unpaused. I suspect it is the quarantine timer causing this to happen. 

If this is the case, is there a way to prevent the quarantine timer unpausing a manually paused worker or a way to increase the initial delay before the worker is unpaused?


Discussion:

Resolving patch:
diff --git a/master/buildbot/worker/base.py b/master/buildbot/worker/base.py
--- a/master/buildbot/worker/base.py
+++ b/master/buildbot/worker/base.py
@@ -620,9 +620,9 @@ def canStartBuild(self):
         this worker will not start.
         """
 
-        # If we're waiting to shutdown gracefully or paused, then we shouldn't
+        # If we're waiting to shutdown gracefully, paused or quarantined then we shouldn't
         # accept any new jobs.
-        if self._graceful or self._paused:
+        if self._graceful or self._paused or self.quarantine_timer:
             return False
 
         if self.max_builds:
@@ -668,6 +668,7 @@ def pause(self):
     def unpause(self):
         """Restart running new builds on the worker."""
         self._paused = False
+        self.stopQuarantineTimer()
         self.botmaster.maybeStartBuildsForWorker(self.name)
         self.updateState()
 
@@ -681,7 +682,6 @@ def putInQuarantine(self):
         if self.quarantine_timer:  # already in quarantine
             return
 
-        self.pause()
         self.quarantine_timer = self.master.reactor.callLater(
             self.quarantine_timeout, self.exitQuarantine)
         log.msg("{} has been put in quarantine for {}s".format(
@@ -693,14 +693,14 @@ def putInQuarantine(self):
             self.quarantine_timeout = self.quarantine_max_timeout
 
     def exitQuarantine(self):
+        log.msg("{} has left quarantine".format(self.name))
         self.quarantine_timer = None
-        self.unpause()
+        self.botmaster.maybeStartBuildsForWorker(self.name)
 
     def stopQuarantineTimer(self):
         if self.quarantine_timer is not None:
             self.quarantine_timer.cancel()
-            self.quarantine_timer = None
-            self.unpause()
+            self.exitQuarantine()
 
 
 class Worker(AbstractWorker):


Test patch:
diff --git a/master/buildbot/test/unit/worker/test_base.py b/master/buildbot/test/unit/worker/test_base.py
--- a/master/buildbot/test/unit/worker/test_base.py
+++ b/master/buildbot/test/unit/worker/test_base.py
@@ -678,11 +678,204 @@ def test_worker_actions_kill(self):
     def test_worker_actions_pause(self):
         worker = yield self.createWorker(attached=False)
         yield worker.startService()
-        worker.controlWorker(("worker", 1, "pause"), {'reason': "none"})
+        self.assertTrue(worker.canStartBuild())
+
+        worker.controlWorker(("worker", 1, "pause"), {"reason": "none"})
         self.assertEqual(worker._paused, True)
+        self.assertFalse(worker.canStartBuild())
 
-        worker.controlWorker(("worker", 1, "unpause"), {'reason': "none"})
+        worker.controlWorker(("worker", 1, "unpause"), {"reason": "none"})
         self.assertEqual(worker._paused, False)
+        self.assertTrue(worker.canStartBuild())
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_doesnt_affect_pause(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+        self.assertFalse(worker._paused)
+
+        # put worker into quarantine.
+        # Check canStartBuild() is False, and paused state is not changed
+        worker.putInQuarantine()
+        self.assertFalse(worker._paused)
+        self.assertFalse(worker.canStartBuild())
+        self.assertIsNotNone(worker.quarantine_timer)
+
+        # human manually pauses the worker
+        worker.controlWorker(("worker", 1, "pause"), {"reason": "none"})
+        self.assertTrue(worker._paused)
+        self.assertFalse(worker.canStartBuild())
+
+        # simulate wait for quarantine to end
+        # Check canStartBuild() is still False, and paused state is not changed
+        self.master.reactor.advance(10)
+        self.assertTrue(worker._paused)
+        self.assertFalse(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_unpausing_exits_quarantine(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # put worker into quarantine whilst unpaused.
+        worker.putInQuarantine()
+        self.assertFalse(worker._paused)
+        self.assertFalse(worker.canStartBuild())
+
+        # pause and unpause the worker
+        worker.controlWorker(("worker", 1, "pause"), {"reason": "none"})
+        self.assertFalse(worker.canStartBuild())
+        worker.controlWorker(("worker", 1, "unpause"), {"reason": "none"})
+        self.assertTrue(worker.canStartBuild())
+
+        # put worker into quarantine whilst paused.
+        worker.controlWorker(("worker", 1, "pause"), {"reason": "none"})
+        worker.putInQuarantine()
+        self.assertTrue(worker._paused)
+        self.assertFalse(worker.canStartBuild())
+
+        # unpause worker should start the build
+        worker.controlWorker(("worker", 1, "unpause"), {"reason": "none"})
+        self.assertFalse(worker._paused)
+        self.assertTrue(worker.canStartBuild())
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_unpausing_doesnt_reset_timeout(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # pump up the quarantine wait time
+        for quarantine_wait in (10, 20, 40, 80):
+            worker.putInQuarantine()
+            self.assertFalse(worker.canStartBuild())
+            self.assertIsNotNone(worker.quarantine_timer)
+            self.master.reactor.advance(quarantine_wait)
+            self.assertTrue(worker.canStartBuild())
+            self.assertIsNone(worker.quarantine_timer)
+
+        # put worker into quarantine (160s)
+        worker.putInQuarantine()
+        self.assertFalse(worker._paused)
+        self.assertFalse(worker.canStartBuild())
+
+        # pause and unpause the worker to exit quarantine
+        worker.controlWorker(("worker", 1, "pause"), {"reason": "none"})
+        self.assertFalse(worker.canStartBuild())
+        worker.controlWorker(("worker", 1, "unpause"), {"reason": "none"})
+        self.assertFalse(worker._paused)
+        self.assertTrue(worker.canStartBuild())
+
+        # next build fails. check timeout is 320s
+        worker.putInQuarantine()
+        self.master.reactor.advance(319)
+        self.assertFalse(worker.canStartBuild())
+        self.assertIsNotNone(worker.quarantine_timer)
+        self.master.reactor.advance(1)
+        self.assertIsNone(worker.quarantine_timer)
+        self.assertTrue(worker.canStartBuild())
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_wait_times(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        for quarantine_wait in (10, 20, 40, 80, 160, 320, 640, 1280, 2560, 3600, 3600):
+            # put worker into quarantine
+            worker.putInQuarantine()
+            self.assertFalse(worker.canStartBuild())
+            self.assertIsNotNone(worker.quarantine_timer)
+
+            # simulate wait just before quarantine ends
+            self.master.reactor.advance(quarantine_wait - 1)
+            self.assertFalse(worker.canStartBuild())
+            self.assertIsNotNone(worker.quarantine_timer)
+
+            # simulate wait to just after quarantine ends
+            self.master.reactor.advance(1)
+            self.assertTrue(worker.canStartBuild())
+            self.assertIsNone(worker.quarantine_timer)
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_reset(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # pump up the quarantine wait time
+        for quarantine_wait in (10, 20, 40, 80):
+            worker.putInQuarantine()
+            self.assertFalse(worker.canStartBuild())
+            self.assertIsNotNone(worker.quarantine_timer)
+            self.master.reactor.advance(quarantine_wait)
+            self.assertTrue(worker.canStartBuild())
+            self.assertIsNone(worker.quarantine_timer)
+
+        # Now get a successful build
+        worker.resetQuarantine()
+
+        # the workers quarantine period should reset back to 10
+        worker.putInQuarantine()
+        self.master.reactor.advance(10)
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_whilst_quarantined(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # put worker in quarantine
+        worker.putInQuarantine()
+        self.assertFalse(worker.canStartBuild())
+        self.assertIsNotNone(worker.quarantine_timer)
+
+        # simulate wait for half the time, and put in quarantine again
+        self.master.reactor.advance(5)
+        worker.putInQuarantine()
+        self.assertFalse(worker.canStartBuild())
+        self.assertIsNotNone(worker.quarantine_timer)
+
+        # simulate wait for another 5 seconds, and we should leave quarantine
+        self.master.reactor.advance(5)
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # simulate wait for yet another 5 seconds, and ensure nothing changes
+        self.master.reactor.advance(5)
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+    @defer.inlineCallbacks
+    def test_worker_quarantine_stop_timer(self):
+        worker = yield self.createWorker(attached=False)
+        yield worker.startService()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # Call stopQuarantineTimer whilst not quarantined
+        worker.stopQuarantineTimer()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
+
+        # Call stopQuarantineTimer whilst quarantined
+        worker.putInQuarantine()
+        self.assertFalse(worker.canStartBuild())
+        self.assertIsNotNone(worker.quarantine_timer)
+        worker.stopQuarantineTimer()
+        self.assertTrue(worker.canStartBuild())
+        self.assertIsNone(worker.quarantine_timer)
 
 
 class TestAbstractLatentWorker(TestReactorMixin, unittest.TestCase):