Repository: buildbot/buildbot

Issue:
If two schedulers have same builder and both are triggered, then they cancel each other
Buildbot version: 2.0.1
Twisted version: 18.9.0

I have the following configuration
```.py

def obox2_repo_filter_fn(repo):
  return re.match("git@.*:[^/]*/obox2.git$", repo)

def branch_not_2_x_repo_filter_fn(branch):
  if re.match(".*-2[.][0-9]$", branch):
    return False
  return True

c['schedulers'].append(schedulers.SingleBranchScheduler(
                       name="obox2-master",
                       builderNames=["obox2-plugin-ci","obox2-plugin-scanbuild-ci","obox2-plugin-coverage-ci"],
                       change_filter=util.ChangeFilter(
                         repository='git@git.dovecot.net:dovecot/obox2.git',
                         branch='master'
                       ),
                       treeStableTimer=10,
                       properties = {
                         'docker_image' : 'registry.dovecot.net/dovecot:stretch-worker'
                       }
))
c['schedulers'].append(schedulers.AnyBranchScheduler(
                       name="obox2-ci",
                       builderNames=["obox2-plugin-ci"],
                       change_filter=util.ChangeFilter(
                         repository_fn=obox2_repo_filter_fn,
                         branch_fn=branch_not_2_x_repo_filter_fn
                       ),
                       treeStableTimer=10,
                       properties = {
                         'docker_image' : 'registry.dovecot.net/dovecot:stretch-worker'
                       }
))
```

When I commit changes into dovecot/obox2 master branch, the `obox2-plugin-ci` builder is not triggered, because both schedulers are triggering the same builder. If I change the branch filter to exclude master, then `obox2-plugin-ci` is triggered on master commit.


Discussion:

Resolving patch:
diff --git a/master/buildbot/db/model.py b/master/buildbot/db/model.py
--- a/master/buildbot/db/model.py
+++ b/master/buildbot/db/model.py
@@ -63,6 +63,9 @@ class Model(base.DBConnectorComponent):
     #
     # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres;
     #   use sa.SmallInteger instead
+    #
+    # * BuildRequest.canBeCollapsed() depends on buildrequest.id being auto-incremented which is
+    #   sqlalchemy default.
 
     # Tables related to build requests
     # --------------------------------
diff --git a/master/buildbot/process/buildrequest.py b/master/buildbot/process/buildrequest.py
--- a/master/buildbot/process/buildrequest.py
+++ b/master/buildbot/process/buildrequest.py
@@ -56,7 +56,7 @@ def _getUnclaimedBrs(self, builderid):
 
     @defer.inlineCallbacks
     def collapse(self):
-        brids = set()
+        brids_to_collapse = set()
 
         for brid in self.brids:
             # Get the BuildRequest object
@@ -80,17 +80,16 @@ def collapse(self):
 
                 canCollapse = yield collapseRequestsFn(self.master, bldr, br, unclaim_br)
                 if canCollapse is True:
-                    brids.add(unclaim_br['buildrequestid'])
+                    brids_to_collapse.add(unclaim_br['buildrequestid'])
 
-        brids = list(brids)
-        if brids:
-            # Claim the buildrequests
-            yield self.master.data.updates.claimBuildRequests(brids)
-            # complete the buildrequest with result SKIPPED.
-            yield self.master.data.updates.completeBuildRequests(brids,
-                                                                 SKIPPED)
+        collapsed_brids = []
+        for brid in brids_to_collapse:
+            claimed = yield self.master.data.updates.claimBuildRequests([brid])
+            if claimed:
+                yield self.master.data.updates.completeBuildRequests([brid], SKIPPED)
+                collapsed_brids.append(brid)
 
-        return brids
+        return collapsed_brids
 
 
 class TempSourceStamp:
@@ -254,23 +253,32 @@ def _make_br(cls, brid, brdict, master):
 
         return buildrequest
 
+    @staticmethod
+    def filter_buildset_props_for_collapsing(bs_props):
+        return {name: value for name, (value, source) in bs_props.items()
+                if name != 'scheduler' and source == 'Scheduler'}
+
     @staticmethod
     @defer.inlineCallbacks
-    def canBeCollapsed(master, br1, br2):
+    def canBeCollapsed(master, new_br, old_br):
         """
         Returns true if both buildrequest can be merged, via Deferred.
 
         This implements Buildbot's default collapse strategy.
         """
         # short-circuit: if these are for the same buildset, collapse away
-        if br1['buildsetid'] == br2['buildsetid']:
+        if new_br['buildsetid'] == old_br['buildsetid']:
             return True
 
-        # get the buidlsets for each buildrequest
-        selfBuildsets = yield master.data.get(
-            ('buildsets', str(br1['buildsetid'])))
-        otherBuildsets = yield master.data.get(
-            ('buildsets', str(br2['buildsetid'])))
+        # the new buildrequest must actually be newer than the old build request, otherwise we
+        # may end up with situations where two build requests submitted at the same time will
+        # cancel each other.
+        if new_br['buildrequestid'] < old_br['buildrequestid']:
+            return False
+
+        # get the buildsets for each buildrequest
+        selfBuildsets = yield master.data.get(('buildsets', str(new_br['buildsetid'])))
+        otherBuildsets = yield master.data.get(('buildsets', str(old_br['buildsetid'])))
 
         # extract sourcestamps, as dictionaries by codebase
         selfSources = dict((ss['codebase'], ss)
@@ -313,6 +321,15 @@ def canBeCollapsed(master, br1, br2):
             if selfSS['revision'] != otherSS['revision']:
                 return False
 
+        # don't collapse build requests if the properties injected by the scheduler differ
+        new_bs_props = yield master.data.get(('buildsets', str(new_br['buildsetid']), 'properties'))
+        old_bs_props = yield master.data.get(('buildsets', str(old_br['buildsetid']), 'properties'))
+
+        new_bs_props = BuildRequest.filter_buildset_props_for_collapsing(new_bs_props)
+        old_bs_props = BuildRequest.filter_buildset_props_for_collapsing(old_bs_props)
+        if new_bs_props != old_bs_props:
+            return False
+
         return True
 
     def mergeSourceStampsWith(self, others):


Test patch:
diff --git a/master/buildbot/test/unit/process/test_buildrequest.py b/master/buildbot/test/unit/process/test_buildrequest.py
--- a/master/buildbot/test/unit/process/test_buildrequest.py
+++ b/master/buildbot/test/unit/process/test_buildrequest.py
@@ -14,6 +14,7 @@
 # Copyright Buildbot Team Members
 
 import datetime
+import json
 
 import mock
 
@@ -141,7 +142,8 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
     #   changes but they have matching revisions.
 
     def makeBuildRequestRows(self, brid, bsid, changeid, ssid, codebase, branch=None,
-                             project=None, repository=None, patchid=None, revision=None):
+                             project=None, repository=None, patchid=None, revision=None,
+                             bs_properties=None):
         rows = [
             fakedb.SourceStamp(id=ssid, codebase=codebase, branch=branch,
                                project=project, repository=repository, patchid=patchid,
@@ -163,15 +165,17 @@ def makeBuildRequestRows(self, brid, bsid, changeid, ssid, codebase, branch=None
                 fakedb.Patch(id=patchid, patch_base64='aGVsbG8sIHdvcmxk',
                  patch_author='bar', patch_comment='foo', subdir='/foo',
                  patchlevel=3))
+        if bs_properties:
+            for prop_name, prop_value in bs_properties.items():
+                rows.append(
+                    fakedb.BuildsetProperty(buildsetid=bsid, property_name=prop_name,
+                                            property_value=json.dumps(prop_value)),
+                )
 
         return rows
 
     @defer.inlineCallbacks
     def test_collapseRequests_collapse_default_with_codebases(self):
-
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
-
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]
@@ -184,11 +188,65 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
         yield self.do_request_collapse(rows, [21], [19, 20])
 
     @defer.inlineCallbacks
-    def test_collapseRequests_collapse_default_with_codebases_branches(self):
+    def test_collapseRequests_collapse_default_does_not_collapse_older(self):
+        rows = [
+            fakedb.Builder(id=77, name='A'),
+        ]
+        rows += self.makeBuildRequestRows(21, 121, None, 221, 'C')
+        rows += self.makeBuildRequestRows(19, 119, None, 210, 'C')
+        rows += self.makeBuildRequestRows(20, 120, None, 220, 'C')
+        self.bldr.getCollapseRequestsFn = lambda: Builder._defaultCollapseRequestFn
+        yield self.do_request_collapse(rows, [19], [])
+        yield self.do_request_collapse(rows, [20], [19])
+        yield self.do_request_collapse(rows, [21], [20])
 
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
+    @defer.inlineCallbacks
+    def test_collapseRequests_collapse_default_does_not_collapse_concurrent_claims(self):
+        rows = [
+            fakedb.Builder(id=77, name='A'),
+        ]
+        rows += self.makeBuildRequestRows(21, 121, None, 221, 'C')
+        rows += self.makeBuildRequestRows(19, 119, None, 210, 'C')
+        rows += self.makeBuildRequestRows(20, 120, None, 220, 'C')
+
+        claimed = []
+
+        @defer.inlineCallbacks
+        def collapse_fn(master, builder, brdict1, brdict2):
+            if not claimed:
+                yield self.master.data.updates.claimBuildRequests([20])
+                claimed.append(20)
+            res = yield Builder._defaultCollapseRequestFn(master, builder, brdict1, brdict2)
+            return res
+
+        self.bldr.getCollapseRequestsFn = lambda: collapse_fn
+        yield self.do_request_collapse(rows, [21], [19])
 
+    @defer.inlineCallbacks
+    def test_collapseRequests_collapse_default_does_not_collapse_scheduler_props(self):
+        rows = [
+            fakedb.Builder(id=77, name='A'),
+        ]
+        rows += self.makeBuildRequestRows(21, 121, None, 221, 'C',
+                                          bs_properties={'prop': ('value', 'Scheduler')})
+        rows += self.makeBuildRequestRows(20, 120, None, 220, 'C',
+                                          bs_properties={'prop': ('value', 'Other source')})
+        rows += self.makeBuildRequestRows(19, 119, None, 219, 'C',
+                                          bs_properties={'prop': ('value2', 'Scheduler')})
+        rows += self.makeBuildRequestRows(18, 118, None, 218, 'C',
+                                          bs_properties={'prop': ('value', 'Scheduler')})
+        rows += self.makeBuildRequestRows(17, 117, None, 217, 'C',
+                                          bs_properties={'prop': ('value3', 'Other source')})
+        rows += self.makeBuildRequestRows(16, 116, None, 216, 'C')
+
+        self.bldr.getCollapseRequestsFn = lambda: Builder._defaultCollapseRequestFn
+        # only the same property coming from a scheduler is matched
+        yield self.do_request_collapse(rows, [21], [18])
+        # only takes into account properties coming from scheduler
+        yield self.do_request_collapse(rows, [20], [16, 17])
+
+    @defer.inlineCallbacks
+    def test_collapseRequests_collapse_default_with_codebases_branches(self):
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]
@@ -202,10 +260,6 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
 
     @defer.inlineCallbacks
     def test_collapseRequests_collapse_default_with_codebases_repository(self):
-
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
-
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]
@@ -219,10 +273,6 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
 
     @defer.inlineCallbacks
     def test_collapseRequests_collapse_default_with_codebases_projects(self):
-
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
-
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]
@@ -237,10 +287,6 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
     # * Neither source stamp has a patch (e.g., from a try scheduler)
     @defer.inlineCallbacks
     def test_collapseRequests_collapse_default_with_a_patch(self):
-
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
-
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]
@@ -255,10 +301,6 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
     # * Either both source stamps are associated with changes..
     @defer.inlineCallbacks
     def test_collapseRequests_collapse_default_with_changes(self):
-
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
-
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]
@@ -273,10 +315,6 @@ def collapseRequests_fn(master, builder, brdict1, brdict2):
     # * ... or neither are associated with changes but they have matching revisions.
     @defer.inlineCallbacks
     def test_collapseRequests_collapse_default_with_non_matching_revision(self):
-
-        def collapseRequests_fn(master, builder, brdict1, brdict2):
-            return buildrequest.BuildRequest.canBeCollapsed(builder.master, brdict1, brdict2)
-
         rows = [
             fakedb.Builder(id=77, name='A'),
         ]