Repository: buildbot/buildbot

Issue:
Lock on step not waiting until the lock is free
While I was doing some testing around locks, I noticed a situation where the lock seems to release early. I have put what I believe is a minimal `master.cfg` below that reproduces the problem for me.

The issue I found can be replicated by triggering the builder 3 times. The one triggered first will run and the other two will wait for the lock. But if one of the other two builds is cancelled while they are still waiting for the lock, the third build will run, even though the first build should still have the lock.

I had a quick look at the code for the locks, and I think what is happening is that the `releaseLocks()` function (and more specifically, the `lock.isOwner()` function) only uses the step config to check if it is the owner, not which build the step is running on, and so the lock is released regardless of which build releases it.


pip freeze snippet:
```
buildbot==3.1.0
buildbot-worker==3.1.0
buildbot-www==3.1.0
```


`master.cfg`:
```
from buildbot.plugins import *

c = BuildmasterConfig = {}

c['workers'] = [
    worker.LocalWorker('worker1'),
    worker.LocalWorker('worker2'),
    worker.LocalWorker('worker3'),
]

c['protocols'] = {'pb': {'port': 9989}}

c['schedulers'] = [
    schedulers.ForceScheduler(
        name='scheduler',
        builderNames=['builder'],
    )
]

locks = [util.MasterLock(f'lock').access('exclusive')]

factory = util.BuildFactory()
factory.addStep(steps.ShellCommand(
    command=['sleep', 3600],
    locks=locks,
))

c['builders'] = [
    util.BuilderConfig(
        name="builder",
        workernames=['worker1', 'worker2', 'worker3'],
        factory=factory,
    )
]

c['buildbotURL'] = "http://localhost:8010/"
c['www'] = dict(port=8010)
c['db'] = {'db_url': "sqlite:///state.sqlite"}
```


Discussion:
The steps are cloned when the build starts, so it shouldn't be that a step of one build could release a lock owned by a step of another build as these steps are different instances. Though maybe there are surprises in this part of the code.

Thanks for investigation and clear reproduction steps.
I have a comparable problem.
We're running multi-master and only want one build, with triggerable substeps, to run at a given time.
With multi-master I'm able to trigger to builds at once. 

This config works when running single master:
```
build_lock = util.MasterLock("Build")

c["builders"] = []
c["builders"].append(
    util.BuilderConfig(
        name="build",
        workernames=WORKER_NAMES,
        factory=build_factory,
        locks=[build_lock.access("exclusive")],
    )
)
```
Let me know if this is the same problem, or if I should create a new ticket.

Resolving patch:
diff --git a/master/buildbot/locks.py b/master/buildbot/locks.py
--- a/master/buildbot/locks.py
+++ b/master/buildbot/locks.py
@@ -45,9 +45,9 @@ def __init__(self, name, maxCount=1):
 
         # Name of the lock
         self.lockName = name
-        # Current queue, tuples (waiter, LockAccess, deferred)
+        # Current queue, tuples (waiter_id, LockAccess, deferred)
         self.waiting = []
-        # Current owners, tuples (owner, LockAccess)
+        # Current owners, tuples (owner_id, LockAccess)
         self.owners = []
         # maximal number of counting owners
         self.maxCount = maxCount
@@ -77,7 +77,7 @@ def setMaxCount(self, count):
 
     def _find_waiting(self, requester):
         for idx, waiter in enumerate(self.waiting):
-            if waiter[0] is requester:
+            if waiter[0] == id(requester):
                 return idx
         return None
 
@@ -104,7 +104,7 @@ def isAvailable(self, requester, access):
         return not num_excl and not num_counting and not ahead
 
     def _addOwner(self, owner, access):
-        self.owners.append((owner, access))
+        self.owners.append((id(owner), access))
         if access.mode == 'counting':
             self._claimed_counting += access.count
         else:
@@ -116,7 +116,7 @@ def _addOwner(self, owner, access):
     def _removeOwner(self, owner, access):
         # returns True if owner removed, False if the lock has been already
         # released
-        entry = (owner, access)
+        entry = (id(owner), access)
         if entry not in self.owners:
             return False
 
@@ -143,7 +143,7 @@ def claim(self, owner, access):
         if not access.count:
             return
 
-        self.waiting = [w for w in self.waiting if w[0] is not owner]
+        self.waiting = [w for w in self.waiting if w[0] != id(owner)]
         self._addOwner(owner, access)
 
         debuglog(f" {self} is claimed '{access.mode}', {access.count} units")
@@ -175,7 +175,7 @@ def _tryWakeUp(self):
         # Break out of the loop when the first waiting client should not be
         # awakened.
         num_excl, num_counting = self._claimed_excl, self._claimed_counting
-        for i, (w_owner, w_access, d) in enumerate(self.waiting):
+        for i, (w_owner_id, w_access, d) in enumerate(self.waiting):
             if w_access.mode == 'counting':
                 if num_excl > 0 or num_counting >= self.maxCount:
                     break
@@ -189,7 +189,7 @@ def _tryWakeUp(self):
             # If the waiter has a deferred, wake it up and clear the deferred
             # from the wait queue entry to indicate that it has been woken.
             if d:
-                self.waiting[i] = (w_owner, w_access, None)
+                self.waiting[i] = (w_owner_id, w_access, None)
                 eventually(d.callback, self)
 
     def waitUntilMaybeAvailable(self, owner, access):
@@ -217,9 +217,9 @@ def waitUntilMaybeAvailable(self, owner, access):
             _, _, old_d = self.waiting[w_index]
             assert old_d is None, "waitUntilMaybeAvailable() must not be called again before the " \
                                   "previous deferred fired"
-            self.waiting[w_index] = (owner, access, d)
+            self.waiting[w_index] = (id(owner), access, d)
         else:
-            self.waiting.append((owner, access, d))
+            self.waiting.append((id(owner), access, d))
         return d
 
     def stopWaitingUntilAvailable(self, owner, access, d):
@@ -245,7 +245,7 @@ def stopWaitingUntilAvailable(self, owner, access, d):
             self._tryWakeUp()
 
     def isOwner(self, owner, access):
-        return (owner, access) in self.owners
+        return (id(owner), access) in self.owners
 
 
 class RealMasterLock(BaseLock, service.SharedService):


Test patch:
diff --git a/master/buildbot/test/unit/process/test_buildstep.py b/master/buildbot/test/unit/process/test_buildstep.py
--- a/master/buildbot/test/unit/process/test_buildstep.py
+++ b/master/buildbot/test/unit/process/test_buildstep.py
@@ -19,7 +19,6 @@
 
 from twisted.internet import defer
 from twisted.internet import error
-from twisted.internet import reactor
 from twisted.internet.task import deferLater
 from twisted.python import failure
 from twisted.python import log
@@ -71,6 +70,24 @@ def run(self):
         return self.action()
 
 
+def _is_lock_owned_by_step(step, lock):
+    accesses = [
+        step_access for step_lock, step_access in step._locks_to_acquire if step_lock == lock
+    ]
+    if not accesses:
+        return False
+    return lock.isOwner(step, accesses[0])
+
+
+def _is_lock_available_for_step(step, lock):
+    accesses = [
+        step_access for step_lock, step_access in step._locks_to_acquire if step_lock == lock
+    ]
+    if not accesses:
+        return False
+    return lock.isAvailable(step, accesses[0])
+
+
 class TestBuildStep(TestBuildStepMixin, config.ConfigErrorsMixin,
                     TestReactorMixin,
                     unittest.TestCase):
@@ -300,22 +317,7 @@ def test_regular_locks_skip_step(self):
         yield self.run_step()
 
     @defer.inlineCallbacks
-    def test_cancelWhileLocksAvailable(self):
-
-        def _owns_lock(step, lock):
-            access = [
-                step_access for step_lock, step_access in step._locks_to_acquire
-                if step_lock == lock
-            ][0]
-            return lock.isOwner(step, access)
-
-        def _lock_available(step, lock):
-            access = [
-                step_access for step_lock, step_access in step._locks_to_acquire
-                if step_lock == lock
-            ][0]
-            return lock.isAvailable(step, access)
-
+    def test_acquire_multiple_locks_after_not_available(self):
         lock1 = locks.MasterLock("masterlock1")
         lock2 = locks.MasterLock("masterlock2")
 
@@ -330,76 +332,128 @@ def _lock_available(step, lock):
             locks.LockAccess(lock1, 'exclusive'),
             locks.LockAccess(lock2, 'exclusive')
         ]))
-        stepd = self.setup_step(self.FakeBuildStep(locks=[
-            locks.LockAccess(lock1, 'exclusive'),
-            locks.LockAccess(lock2, 'exclusive')
-        ]))
 
         yield stepa._setup_locks()
         yield stepb._setup_locks()
         yield stepc._setup_locks()
-        yield stepd._setup_locks()
 
-        real_lock1 = stepd._locks_to_acquire[0][0]
-        real_lock2 = stepd._locks_to_acquire[1][0]
+        real_lock1 = stepc._locks_to_acquire[0][0]
+        real_lock2 = stepc._locks_to_acquire[1][0]
 
-        # Start all the steps
         yield stepa.acquireLocks()
         yield stepb.acquireLocks()
         c_d = stepc.acquireLocks()
-        d_d = stepd.acquireLocks()
-
-        # Check that step a and step b have the locks
-        self.assertTrue(_owns_lock(stepa, real_lock1))
-        self.assertTrue(_owns_lock(stepb, real_lock2))
 
-        # Check that step c does not have a lock
-        self.assertFalse(_owns_lock(stepc, real_lock1))
-        self.assertFalse(_owns_lock(stepc, real_lock2))
+        self.assertTrue(_is_lock_owned_by_step(stepa, real_lock1))
+        self.assertFalse(_is_lock_owned_by_step(stepa, real_lock2))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock1))
+        self.assertTrue(_is_lock_owned_by_step(stepb, real_lock2))
+        self.assertFalse(_is_lock_owned_by_step(stepc, real_lock1))
+        self.assertFalse(_is_lock_owned_by_step(stepc, real_lock2))
 
-        # Check that step d does not have a lock
-        self.assertFalse(_owns_lock(stepd, real_lock1))
-        self.assertFalse(_owns_lock(stepd, real_lock2))
+        self.assertFalse(_is_lock_available_for_step(stepc, real_lock1))
+        self.assertFalse(_is_lock_available_for_step(stepc, real_lock2))
 
-        # Release lock 1
         stepa.releaseLocks()
-        yield deferLater(reactor, 0, lambda: None)
+        yield deferLater(self.reactor, 0, lambda: None)
+
+        self.assertTrue(_is_lock_available_for_step(stepc, real_lock1))
+        self.assertFalse(_is_lock_available_for_step(stepc, real_lock2))
+
+        stepb.releaseLocks()
+        yield deferLater(self.reactor, 0, lambda: None)
 
-        # lock1 should be available for step c
-        self.assertTrue(_lock_available(stepc, real_lock1))
-        self.assertFalse(_lock_available(stepc, real_lock2))
-        self.assertFalse(_lock_available(stepd, real_lock1))
-        self.assertFalse(_lock_available(stepd, real_lock2))
+        self.assertTrue(_is_lock_owned_by_step(stepc, real_lock1))
+        self.assertTrue(_is_lock_owned_by_step(stepc, real_lock2))
 
-        # Cancel step c
-        stepc.interrupt("cancelling")
         yield c_d
 
-        # Check that step c does not have a lock
-        self.assertFalse(_owns_lock(stepc, real_lock1))
-        self.assertFalse(_owns_lock(stepc, real_lock2))
+    @defer.inlineCallbacks
+    def test_cancel_when_lock_available(self):
+        lock = locks.MasterLock("masterlock1")
+
+        stepa = self.setup_step(self.FakeBuildStep(locks=[locks.LockAccess(lock, 'exclusive')]))
+        stepb = self.setup_step(self.FakeBuildStep(locks=[locks.LockAccess(lock, 'exclusive')]))
+        stepc = self.setup_step(self.FakeBuildStep(locks=[locks.LockAccess(lock, 'exclusive')]))
+
+        yield stepa._setup_locks()
+        yield stepb._setup_locks()
+        yield stepc._setup_locks()
+
+        real_lock = stepc._locks_to_acquire[0][0]
+
+        yield stepa.acquireLocks()
+        b_d = stepb.acquireLocks()
+        c_d = stepc.acquireLocks()
+
+        self.assertTrue(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepc, real_lock))
 
-        # No lock should be available for step c
-        self.assertFalse(_lock_available(stepc, real_lock1))
-        self.assertFalse(_lock_available(stepc, real_lock2))
+        self.assertFalse(_is_lock_available_for_step(stepb, real_lock))
+        self.assertFalse(_is_lock_available_for_step(stepc, real_lock))
 
-        # lock 1 should be available for step d
-        self.assertTrue(_lock_available(stepd, real_lock1))
-        self.assertFalse(_lock_available(stepd, real_lock2))
+        stepa.releaseLocks()
+        yield deferLater(self.reactor, 0, lambda: None)
+
+        self.assertFalse(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertTrue(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertFalse(_is_lock_available_for_step(stepc, real_lock))
 
-        # Release lock 2
+        stepb.interrupt("cancelling")
+        yield b_d
         stepb.releaseLocks()
 
-        # Both locks should be available for step d
-        self.assertTrue(_lock_available(stepd, real_lock1))
-        self.assertTrue(_lock_available(stepd, real_lock2))
+        self.assertFalse(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertTrue(_is_lock_available_for_step(stepc, real_lock))
+
+        yield c_d
+
+        self.assertFalse(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertTrue(_is_lock_owned_by_step(stepc, real_lock))
+
+    @defer.inlineCallbacks
+    def test_cancel_when_lock_not_available(self):
+        lock = locks.MasterLock("masterlock1")
+
+        stepa = self.setup_step(self.FakeBuildStep(locks=[locks.LockAccess(lock, 'exclusive')]))
+        stepb = self.setup_step(self.FakeBuildStep(locks=[locks.LockAccess(lock, 'exclusive')]))
+        stepc = self.setup_step(self.FakeBuildStep(locks=[locks.LockAccess(lock, 'exclusive')]))
+
+        yield stepa._setup_locks()
+        yield stepb._setup_locks()
+        yield stepc._setup_locks()
 
-        # So it should run
-        yield d_d
+        real_lock = stepc._locks_to_acquire[0][0]
+
+        yield stepa.acquireLocks()
+        b_d = stepb.acquireLocks()
+        c_d = stepc.acquireLocks()
+
+        self.assertTrue(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepc, real_lock))
+
+        self.assertFalse(_is_lock_available_for_step(stepb, real_lock))
+        self.assertFalse(_is_lock_available_for_step(stepc, real_lock))
+
+        stepb.interrupt("cancelling")
+        yield b_d
+
+        stepa.releaseLocks()
+        yield deferLater(self.reactor, 0, lambda: None)
+
+        self.assertFalse(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertTrue(_is_lock_owned_by_step(stepc, real_lock))
+
+        yield c_d
 
-        # Check that step d owns the locks
-        self.assertTrue(_owns_lock(stepd, real_lock1))
-        self.assertTrue(_owns_lock(stepd, real_lock2))
+        self.assertFalse(_is_lock_owned_by_step(stepa, real_lock))
+        self.assertFalse(_is_lock_owned_by_step(stepb, real_lock))
+        self.assertTrue(_is_lock_owned_by_step(stepc, real_lock))
 
     @defer.inlineCallbacks
     def test_multiple_cancel(self):