internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: buildbot/buildbot

Issue:
gerritchangesource: add gerritprojects option.
It adds the 'gerritprojects' to GerritChangeSource and
GerritEventLogPoller so that it can be set a  list of
projects which events should be handled. By default
events of any project is handled.


Discussion:

Resolving patch:
diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py
--- a/master/buildbot/steps/source/git.py
+++ b/master/buildbot/steps/source/git.py
@@ -577,6 +577,9 @@ def checkWithPathExists(exists):
             d = self.runCommand(cmd)
 
             def checkWithListdir(_):
+                if 'files' not in cmd.updates:
+                    # no files - directory doesn't exist
+                    return "clone"
                 files = cmd.updates['files'][0]
                 if '.git' in files:
                     return "update"


Test patch:
diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py
--- a/master/buildbot/test/unit/test_steps_source_git.py
+++ b/master/buildbot/test/unit/test_steps_source_git.py
@@ -1671,6 +1671,37 @@ def test_mode_full_logEnviron(self):
         self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Git')
         return self.runStep()
 
+    def test_wkdir_doesnt_exist(self):
+        self.setupStep(
+            git.Git(repourl='http://github.com/buildbot/buildbot.git',
+                    mode='full'))
+        self.expectCommands(
+            ExpectShell(workdir='wkdir',
+                        command=['git', '--version'])
+            + ExpectShell.log('stdio',
+                              stdout='git version 1.7.5')
+            + 0,
+            Expect('stat', dict(file='wkdir/.buildbot-patched',
+                                logEnviron=True))
+            + 1,
+            Expect('listdir', {'dir': 'wkdir', 'logEnviron': True,
+                               'timeout': 1200})
+            + 1,
+            ExpectShell(workdir='wkdir',
+                        command=['git', 'clone',
+                                 'http://github.com/buildbot/buildbot.git',
+                                 '.'])
+            + 0,
+            ExpectShell(workdir='wkdir',
+                        command=['git', 'rev-parse', 'HEAD'])
+            + ExpectShell.log('stdio',
+                              stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d')
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, status_text=["update"])
+        self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Git')
+        return self.runStep()
+
     def test_getDescription(self):
         # clone of: test_mode_incremental
         # only difference is to set the getDescription property
Repository: buildbot/buildbot

Issue:
Fix bytes/unicode issue to fix test on Python 3



Discussion:

Resolving patch:
diff --git a/master/buildbot/db/schedulers.py b/master/buildbot/db/schedulers.py
--- a/master/buildbot/db/schedulers.py
+++ b/master/buildbot/db/schedulers.py
@@ -30,13 +30,13 @@ class SchedulersConnectorComponent(base.DBConnectorComponent):
 
     def classifyChanges(self, schedulerid, classifications):
         def thd(conn):
-            transaction = conn.begin()
             tbl = self.db.model.scheduler_changes
             ins_q = tbl.insert()
             upd_q = tbl.update(
                 ((tbl.c.schedulerid == schedulerid)
                  & (tbl.c.changeid == sa.bindparam('wc_changeid'))))
             for changeid, important in classifications.items():
+                transaction = conn.begin()
                 # convert the 'important' value into an integer, since that
                 # is the column type
                 imp_int = important and 1 or 0
@@ -54,7 +54,7 @@ def thd(conn):
                                  wc_changeid=changeid,
                                  important=imp_int)
 
-            transaction.commit()
+                transaction.commit()
         return self.db.pool.do(thd)
 
     def flushChangeClassifications(self, schedulerid, less_than=None):


Test patch:
diff --git a/master/buildbot/test/unit/test_db_schedulers.py b/master/buildbot/test/unit/test_db_schedulers.py
--- a/master/buildbot/test/unit/test_db_schedulers.py
+++ b/master/buildbot/test/unit/test_db_schedulers.py
@@ -60,16 +60,21 @@ def test_classifyChanges(self):
     @defer.inlineCallbacks
     def test_classifyChanges_again(self):
         # test reclassifying changes, which may happen during some timing
-        # conditions
+        # conditions.  It's important that this test uses multiple changes,
+        # only one of which already exists
         yield self.insertTestData([
             self.ss92,
             self.change3,
+            self.change4,
+            self.change5,
+            self.change6,
             self.scheduler24,
-            fakedb.SchedulerChange(schedulerid=24, changeid=3, important=0),
+            fakedb.SchedulerChange(schedulerid=24, changeid=5, important=0),
         ])
-        yield self.db.schedulers.classifyChanges(24, {3: True})
+        yield self.db.schedulers.classifyChanges(
+                24, {3: True, 4: False, 5: True, 6: False})
         res = yield self.db.schedulers.getChangeClassifications(24)
-        self.assertEqual(res, {3: True})
+        self.assertEqual(res, {3: True, 4: False, 5: True, 6: False})
 
     def test_signature_flushChangeClassifications(self):
         @self.assertArgSpecMatches(
Repository: buildbot/buildbot

Issue:
Fix bytes/unicode issue to fix test on Python 3



Discussion:

Resolving patch:
diff --git a/master/buildbot/db/schedulers.py b/master/buildbot/db/schedulers.py
--- a/master/buildbot/db/schedulers.py
+++ b/master/buildbot/db/schedulers.py
@@ -24,13 +24,13 @@ class SchedulersConnectorComponent(base.DBConnectorComponent):
 
     def classifyChanges(self, objectid, classifications):
         def thd(conn):
-            transaction = conn.begin()
             tbl = self.db.model.scheduler_changes
             ins_q = tbl.insert()
             upd_q = tbl.update(
                 ((tbl.c.objectid == objectid)
                  & (tbl.c.changeid == sa.bindparam('wc_changeid'))))
             for changeid, important in classifications.items():
+                transaction = conn.begin()
                 # convert the 'important' value into an integer, since that
                 # is the column type
                 imp_int = important and 1 or 0
@@ -48,7 +48,7 @@ def thd(conn):
                                  wc_changeid=changeid,
                                  important=imp_int)
 
-            transaction.commit()
+                transaction.commit()
         return self.db.pool.do(thd)
 
     def flushChangeClassifications(self, objectid, less_than=None):


Test patch:
diff --git a/master/buildbot/test/unit/test_db_schedulers.py b/master/buildbot/test/unit/test_db_schedulers.py
--- a/master/buildbot/test/unit/test_db_schedulers.py
+++ b/master/buildbot/test/unit/test_db_schedulers.py
@@ -85,14 +85,21 @@ def thd(conn):
 
     def test_classifyChanges_again(self):
         # test reclassifying changes, which may happen during some timing
-        # conditions
+        # conditions.  It's important that this test uses multiple changes,
+        # only one of which already exists
         d = self.insertTestData([
+        # conditions.  It's important that this test uses multiple changes,
+        # only one of which already exists
             self.change3,
+            self.change4,
+            self.change5,
+            self.change6,
             self.scheduler24,
-            fakedb.SchedulerChange(objectid=24, changeid=3, important=0),
+            fakedb.SchedulerChange(objectid=24, changeid=5, important=0),
         ])
         d.addCallback(lambda _:
-                      self.db.schedulers.classifyChanges(24, {3: True}))
+                      self.db.schedulers.classifyChanges(
+                          24, {3: True, 4: False, 5: True, 6: False}))
 
         def check(_):
             def thd(conn):
@@ -101,7 +108,9 @@ def thd(conn):
                 r = conn.execute(q)
                 rows = [(row.objectid, row.changeid, row.important)
                         for row in r.fetchall()]
-                self.assertEqual(rows, [(24, 3, 1)])
+                self.assertEqual(sorted(rows),
+                                 sorted([(24, 3, 1), (24, 4, 0),
+                                         (24, 5, 1), (24, 6, 0)]))
             return self.db.pool.do(thd)
         d.addCallback(check)
         return d
Repository: buildbot/buildbot

Issue:
minor syntax formating
* [x] I have updated the appropriate documentation




Discussion:

Resolving patch:
diff --git a/master/setup.py b/master/setup.py
--- a/master/setup.py
+++ b/master/setup.py
@@ -25,6 +25,8 @@
 
 from buildbot import version
 from distutils.core import setup
+from distutils.version import LooseVersion
+import pkg_resources
 
 from distutils.command.install_data import install_data
 from distutils.command.sdist import sdist
@@ -357,6 +359,24 @@ def define_plugin_entries(groups):
 if not py_26:
     raise RuntimeError("Buildbot master requires at least Python-2.6")
 
+# pip<1.4 doesn't have the --pre flag, and will thus attempt to install alpha
+# and beta versions of Buildbot.  Prevent that from happening.
+VERSION_MSG = """
+This is a pre-release version of Buildbot, which can only be installed with
+pip-1.4 or later Try installing the latest stable version of Buildbot instead:
+    pip install buildbot==0.8.12
+See https://pypi.python.org/pypi/buildbot to verify the current stable version.
+"""
+if 'a' in version or 'b' in version:
+    try:
+        pip_dist = pkg_resources.get_distribution('pip')
+    except pkg_resources.DistributionNotFound:
+        pip_dist = None
+
+    if pip_dist:
+        if LooseVersion(pip_dist.version) < LooseVersion('1.4'):
+            raise RuntimeError(VERSION_MSG)
+
 try:
     # If setuptools is installed, then we'll add setuptools-specific arguments
     # to the setup args.


Test patch:
Repository: buildbot/buildbot

Issue:
Documentation for Pushover reporter says "sends email"
The documentation for the [Pushover reporter](https://docs.buildbot.net/current/manual/cfg-reporters.html#pushover-notifications) talks about sending emails:

> The following simple example will send an email upon the completion of each build. The email contains a description of the Build, its results, and URLs where more information can be obtained.

It should probably talk about sending a Pushover message or something like that.


Discussion:

Resolving patch:
diff --git a/master/docs/conf.py b/master/docs/conf.py
--- a/master/docs/conf.py
+++ b/master/docs/conf.py
@@ -21,7 +21,19 @@
 sys.path.append(os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
-
+try:
+    import sphinxcontrib.blockdiag
+    assert sphinxcontrib.blockdiag
+except ImportError:
+    raise RuntimeError("sphinxcontrib.blockdiag is not installed. "
+        "Please install documentation dependencies with `pip install buildbot[docs]`")
+
+import pkg_resources
+try:
+    pkg_resources.require('docutils>=0.8')
+except pkg_resources.ResolutionError:
+    raise RuntimeError("docutils is not installed or has incompatible version. "
+        "Please install documentation dependencies with `pip install buildbot[docs]`")
 # If your documentation needs a minimal Sphinx version, state it here.
 needs_sphinx = '1.0'
 
diff --git a/master/setup.py b/master/setup.py
--- a/master/setup.py
+++ b/master/setup.py
@@ -429,6 +429,12 @@ def define_plugin_entries(groups):
         'tls': [
             'Twisted[tls] ' + twisted_ver,
         ],
+        'docs': [
+            # TODO: Pin Sphinx version to workaround http://trac.buildbot.net/ticket/3408
+            'sphinx==1.3.3',
+            'sphinxcontrib-blockdiag',
+            'docutils>=0.8',
+        ],
     }
 
     if os.getenv('NO_INSTALL_REQS'):


Test patch:
Repository: buildbot/buildbot

Issue:
Log scroll bug

* [x] I have created a file in the `master/buildbot/newsfragment` directory (and read the `README.txt` in that directory)



Discussion:

Resolving patch:
diff --git a/common/porttostable.py b/common/porttostable.py
new file mode 100644
--- /dev/null
+++ b/common/porttostable.py
@@ -0,0 +1,34 @@
+import os
+from subprocess import CalledProcessError
+from subprocess import check_output
+
+import requests
+import yaml
+
+s = requests.Session()
+with open(os.path.expanduser('~/.config/hub')) as f:
+    config = yaml.load(f)['github.com'][0]
+    s.auth = config['user'], config['oauth_token']
+
+os.system("git fetch --all")
+r = s.get("https://api.github.com/search/issues?q=label:\"port%20to%20stable\"+repo:buildbot/buildbot")
+to_port = r.json()
+summary = ""
+for pr in to_port['items']:
+    r = s.get("https://api.github.com/repos/buildbot/buildbot/pulls/{number}/commits".format(**pr))
+    commits = r.json()
+    for c in commits:
+        title = c['commit']['message'].split("\n")[0]
+        try:
+            check_output("git cherry-pick {sha} 2>&1".format(**c), shell=True)
+        except CalledProcessError as e:
+            os.system("git diff")
+            os.system("git reset --hard HEAD 2>&1 >/dev/null")
+            if '--allow-empty' in e.output:
+                continue
+            if 'fatal: bad object' in e.output:
+                continue
+            print "cannot automatically cherry-pick", pr['number'], c['sha'], title,e.output
+        else:
+            summary += "\n#{number}: {title}".format(number=pr['number'], title=title , **c)
+print summary
diff --git a/master/setup.py b/master/setup.py
--- a/master/setup.py
+++ b/master/setup.py
@@ -411,9 +411,8 @@ def define_plugin_entries(groups):
     txaio_ver = "== 2.2.2"
 
 else:
-    # twisted 16.3.0 breaks buildbot websockets
-    twisted_ver = ">= 14.0.1, < 16.3.0"
-    autobahn_ver = ">= 0.10.2"
+    twisted_ver = ">= 14.0.1"
+    autobahn_ver = ">= 0.16.0"
     txaio_ver = ">= 2.2.2"
 
 bundle_version = version.split("-")[0]


Test patch:
Repository: buildbot/buildbot

Issue:
Tarball on pypi lacks secrets/providers subdirectory
https://pypi.python.org/pypi/buildbot/0.9.5
The tarball provided there lacks buildbot/master /buildbot/secrets/providers (it does contain secrets itself though). Hence unit tests won't run completely.

```
]$ ls -al buildbot/secrets/
total 9
drwxr-xr-x   2 502  staff     5 Mar 20 14:54 .
drwxr-xr-x  24 502  staff    41 Mar 20 14:54 ..
-rw-r--r--   1 502  staff   705 Mar 20 11:28 __init__.py
-rw-r--r--   1 502  staff  1621 Mar 20 11:28 manager.py
-rw-r--r--   1 502  staff  1713 Mar 20 11:28 secret.py

```


Discussion:
@protatremy fyi
When copying the files manually the same happens, this is probably due to setup.py not listing the package, so it's as simple as:

```
--- setup.py~   2017-03-20 14:45:28.000000000 +0100
+++ setup.py    2017-03-28 23:45:32.795331960 +0200
@@ -180,6 +180,7 @@
         "buildbot.schedulers",
         "buildbot.scripts",
         "buildbot.secrets",
+        "buildbot.secrets.providers",
         "buildbot.statistics",
         "buildbot.statistics.storage_backends",
         "buildbot.status",

```

Resolving patch:
diff --git a/master/setup.py b/master/setup.py
--- a/master/setup.py
+++ b/master/setup.py
@@ -180,6 +180,7 @@ def define_plugin_entries(groups):
         "buildbot.schedulers",
         "buildbot.scripts",
         "buildbot.secrets",
+        "buildbot.secrets.providers",
         "buildbot.statistics",
         "buildbot.statistics.storage_backends",
         "buildbot.status",


Test patch:
Repository: buildbot/buildbot

Issue:
canStartBuild example (RemoteCommand) does not work 
`RemoteCommand` implementation has changed and `canStartBuild` [example](http://docs.buildbot.net/0.9.8/manual/customization.html#canstartbuild-functions) stopped working because `self.step is None` in [here](https://github.com/buildbot/buildbot/blob/a5237a9cc02740e474cb0b3b717ed7296889f708/master/buildbot/process/remotecommand.py#L278)


Discussion:
[permanent link](https://github.com/buildbot/buildbot/blob/a5237a9cc02740e474cb0b3b717ed7296889f708/master/buildbot/process/remotecommand.py#L278)
thanks, updated both links to be permanent
@nazarewk looks like indeed a patch is needed in cleanup so that it is just a nop if self.step.build is None. Would you like to do this patch?

Resolving patch:
diff --git a/master/buildbot/process/remotecommand.py b/master/buildbot/process/remotecommand.py
--- a/master/buildbot/process/remotecommand.py
+++ b/master/buildbot/process/remotecommand.py
@@ -275,7 +275,10 @@ def addToLog(self, logname, data):
     @defer.inlineCallbacks
     def remoteUpdate(self, update):
         def cleanup(data):
+            if self.step is None:
+                return data
             return self.step.build.properties.cleanupTextFromSecrets(data)
+
         if self.debug:
             for k, v in iteritems(update):
                 log.msg("Update[%s]: %s" % (k, v))


Test patch:
Repository: buildbot/buildbot

Issue:
0.9.8: MailNotifier info gets "overwritten" if builders argument is the same
I have a function `mk_MailNotifiers` like this:

```
    red_builders = ["foobuilder"]
    green_builders = ["foobuilder"]
    mns = []
    red_modes = ["failing", "warnings", "exception", "cancelled"]
    if red_builders:
        mns.append(reporters.MailNotifier(
            relayhost='smtp.corp.imvu.com',
            fromaddr='buildbot-red@imvu.com',
            sendToInterestedUsers=False,
            extraRecipients=["larver@imvu.com"],
            lookup="imvu.com",
            builders=red_builders,
            mode=red_modes,
            messageFormatter=formatter))
    if green_builders:
        mns.append(reporters.MailNotifier(
            relayhost='smtp.corp.imvu.com',
            fromaddr='buildbot@imvu.com',
            sendToInterestedUsers=False,
            extraRecipients=["larver@imvu.com"],
            lookup="imvu.com",
            builders=green_builders,
            mode='passing',
            messageFormatter=formatter))
    return mns
```

and I assign it in my buildmaster config to the `c['services']` key.

If I run a build and cancel it, I do not get an email. If the build passes, then I do get an email.

If I reverse the order of the `if` branches, so that the green builder MailNotifier gets added first to the list, then I do get an email if I cancel a build.

So it looks like if the `builders` argument is the same for the two MailNotifiers, the one that comes last overwrites the earlier one(s).

I realize that in my case both `red_builders` and `green_builders` is the same, and that I should probably be using something more like `mode="all"` with just 1 MailNotifier, but I can't because I want red builds to have a different `fromaddr` argument (`buildbot-red@imvu.com`) than green builds.

Is this a legitimate bug or am I not using the MailNotifier correctly?


Discussion:
I tested on 0.9.9 also and the same behavior is present.
As a point of comparison, in Buildbot 0.8.5 (from which my code example above originates from), the behavior works as expected (no overwriting of settings).

Resolving patch:
diff --git a/master/buildbot/reporters/notifier.py b/master/buildbot/reporters/notifier.py
--- a/master/buildbot/reporters/notifier.py
+++ b/master/buildbot/reporters/notifier.py
@@ -80,6 +80,7 @@ def checkConfig(self, mode=("failing", "passing", "warnings"),
                 self.name += "_schedulers_" + "+".join(schedulers)
             if branches is not None:
                 self.name += "_branches_" + "+".join(branches)
+            self.name += "_".join(mode)
 
         if '\n' in subject:
             config.error(


Test patch:
Repository: buildbot/buildbot

Issue:
Tracker for `RolesFromDomain`
This is to track the implementation of `RolesFromDomain`, which implements role setting depending on the email domain of the user.


Discussion:

Resolving patch:
diff --git a/master/buildbot/www/authz/roles.py b/master/buildbot/www/authz/roles.py
--- a/master/buildbot/www/authz/roles.py
+++ b/master/buildbot/www/authz/roles.py
@@ -61,6 +61,24 @@ def getRolesFromUser(self, userDetails):
         return []
 
 
+class RolesFromDomain(RolesFromEmails):
+
+    def __init__(self, **kwargs):
+        RolesFromBase.__init__(self)
+
+        self.domain_roles = {}
+        for role, domains in iteritems(kwargs):
+            for domain in domains:
+                self.domain_roles.setdefault(domain, []).append(role)
+
+    def getRolesFromUser(self, userDetails):
+        if 'email' in userDetails:
+            email = userDetails['email']
+            edomain = email.split('@')[-1]
+            return self.domain_roles.get(edomain, [])
+        return []
+
+
 class RolesFromOwner(RolesFromBase):
 
     def __init__(self, role):


Test patch:
diff --git a/master/buildbot/test/unit/test_www_authz.py b/master/buildbot/test/unit/test_www_authz.py
--- a/master/buildbot/test/unit/test_www_authz.py
+++ b/master/buildbot/test/unit/test_www_authz.py
@@ -28,6 +28,7 @@
 from buildbot.www.authz.endpointmatchers import RebuildBuildEndpointMatcher
 from buildbot.www.authz.endpointmatchers import StopBuildEndpointMatcher
 from buildbot.www.authz.endpointmatchers import ViewBuildsEndpointMatcher
+from buildbot.www.authz.roles import RolesFromDomain
 from buildbot.www.authz.roles import RolesFromEmails
 from buildbot.www.authz.roles import RolesFromGroups
 from buildbot.www.authz.roles import RolesFromOwner
@@ -76,11 +77,13 @@ def setUp(self):
                 RolesFromGroups(groupPrefix="buildbot-"),
                 RolesFromEmails(admins=["homer@springfieldplant.com"],
                                 agents=["007@mi6.uk"]),
-                RolesFromOwner(role="owner")
+                RolesFromOwner(role="owner"),
+                RolesFromDomain(admins=["mi7.uk"])
             ]
         )
         self.users = dict(homer=dict(email="homer@springfieldplant.com"),
                           bond=dict(email="007@mi6.uk"),
+                          moneypenny=dict(email="moneypenny@mi7.uk"),
                           nineuser=dict(email="user@nine.com", groups=["buildbot-nine-mergers",
                                                                        "buildbot-nine-developers"]),
                           eightuser=dict(
@@ -124,6 +127,7 @@ def assertUserForbidden(self, ep, action, options, user):
     @defer.inlineCallbacks
     def test_anyEndpoint(self):
         yield self.assertUserAllowed("foo/bar", "get", {}, "homer")
+        yield self.assertUserAllowed("foo/bar", "get", {}, "moneypenny")
         yield self.assertUserForbidden("foo/bar", "get", {}, "bond")
 
     @defer.inlineCallbacks