Repository: buildbot/buildbot

Issue:
No "Changed files" in Console view while using GerritChangeSource
We have 2 instances of Buildbot. First instance gets changes from Github (using GitPoller) and the second gets it from Gerrit (using GerritChangeSource).
In case of 1st instance, which uses Github, we have filled "changed files" section. Example:
![image](https://user-images.githubusercontent.com/31437274/37398503-cf09bf08-278f-11e8-8a9b-ef8938711432.png)
It is ok.

In case of 2nd instance, which uses Gerrit, "changed files" section is empty:
![image](https://user-images.githubusercontent.com/31437274/37398570-02cfd5c0-2790-11e8-946a-769eeb9d01a5.png)

We use GerritChangeSource like this:
```
from buildbot.plugins import changes

c['change_source'] = changes.GerritChangeSource(
    "gerrit.example.com",
    "gerrit_user",
    handled_events=["patchset-created", "change-merged"]) 
```

Is it a bug?

All Buildbot instances uses version: 0.9.15post1


Discussion:

Resolving patch:
diff --git a/master/buildbot/changes/gerritchangesource.py b/master/buildbot/changes/gerritchangesource.py
--- a/master/buildbot/changes/gerritchangesource.py
+++ b/master/buildbot/changes/gerritchangesource.py
@@ -18,6 +18,7 @@
 
 from twisted.internet import defer
 from twisted.internet import reactor
+from twisted.internet import utils
 from twisted.internet.protocol import ProcessProtocol
 from twisted.python import log
 
@@ -74,7 +75,8 @@ class GerritChangeSourceBase(base.ChangeSource):
     def checkConfig(self,
                     gitBaseURL=None,
                     handled_events=("patchset-created", "ref-updated"),
-                    debug=False):
+                    debug=False,
+                    get_files=False):
 
         if gitBaseURL is None:
             config.error("gitBaseURL must be specified")
@@ -82,9 +84,11 @@ def checkConfig(self,
     def reconfigService(self,
                         gitBaseURL=None,
                         handled_events=("patchset-created", "ref-updated"),
-                        debug=False):
+                        debug=False,
+                        get_files=False):
         self.gitBaseURL = gitBaseURL
         self.handled_events = list(handled_events)
+        self._get_files = get_files
         self.debug = debug
 
     def lineReceived(self, line):
@@ -154,11 +158,19 @@ def getGroupingPolicyFromEvent(self, event):
                               event_change['number'])
         return event_change["branch"]
 
+    @defer.inlineCallbacks
     def addChangeFromEvent(self, properties, event):
-
         if "change" in event and "patchSet" in event:
             event_change = event["change"]
-            return self.addChange({
+
+            files = ["unknown"]
+            if self._get_files:
+                files = yield self.getFiles(
+                    change=event_change["number"],
+                    patchset=event["patchSet"]["number"]
+                )
+
+            yield self.addChange({
                 'author': _gerrit_user_to_author(event_change["owner"]),
                 'project': util.bytes2unicode(event_change["project"]),
                 'repository': "{}/{}".format(
@@ -167,7 +179,7 @@ def addChangeFromEvent(self, properties, event):
                 'revision': event["patchSet"]["revision"],
                 'revlink': event_change["url"],
                 'comments': event_change["subject"],
-                'files': ["unknown"],
+                'files': files,
                 'category': event["type"],
                 'properties': properties})
 
@@ -298,17 +310,49 @@ def streamProcessStopped(self):
             self.startStreamProcess()
             self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
 
+    def _buildGerritCommand(self, *gerrit_args):
+        '''Get an ssh command list which invokes gerrit with the given args on the
+        remote host'''
+
+        cmd = [
+            "ssh",
+            "%s@%s" % (self.username, self.gerritserver),
+            "-p", str(self.gerritport)
+        ]
+
+        if self.identity_file is not None:
+            cmd.extend(["-i", self.identity_file])
+
+        cmd.append("gerrit")
+        cmd.extend(gerrit_args)
+        return cmd
+
     def startStreamProcess(self):
         if self.debug:
             log.msg("starting 'gerrit stream-events'")
+
+        cmd = self._buildGerritCommand("stream-events")
         self.lastStreamProcessStart = util.now()
-        uri = "%s@%s" % (self.username, self.gerritserver)
-        args = [uri, "-p", str(self.gerritport)]
-        if self.identity_file is not None:
-            args = args + ['-i', self.identity_file]
-        self.process = reactor.spawnProcess(
-            self.LocalPP(self), "ssh",
-            ["ssh"] + args + ["gerrit", "stream-events"], env=None)
+        self.process = reactor.spawnProcess(self.LocalPP(self), "ssh", cmd, env=None)
+
+    @defer.inlineCallbacks
+    def getFiles(self, change, patchset):
+        cmd = self._buildGerritCommand("query", str(change), "--format", "JSON",
+                                       "--files", "--patch-sets")
+
+        if self.debug:
+            log.msg("querying gerrit for changed files in change %s/%s: %s" %
+                    (change, patchset, cmd))
+
+        out = yield utils.getProcessOutput(cmd[0], cmd[1:], env=None)
+        out = out.splitlines()[0]
+        res = json.loads(bytes2unicode(out))
+
+        if res.get("rowCount") == 0:
+            return ["unknown"]
+
+        patchsets = {i["number"]: i["files"] for i in res["patchSets"]}
+        return [i["file"] for i in patchsets[int(patchset)]]
 
     def activate(self):
         self.wantProcess = True
@@ -387,6 +431,14 @@ def eventReceived(self, event):
             yield self.master.db.state.setState(self._oid, 'last_event_ts', event['eventCreatedOn'])
         return res
 
+    @defer.inlineCallbacks
+    def getFiles(self, change, patchset):
+        res = yield self._http.get("/changes/%s/revisions/%s/files/" % (change, patchset))
+        res = yield res.content()
+
+        res = res.splitlines()[1].decode('utf8')  # the first line of every response is `)]}'`
+        return list(json.loads(res))
+
     # FIXME this copy the code from PollingChangeSource
     # but as PollingChangeSource and its subclasses need to be ported to reconfigurability
     # we can't use it right now


Test patch:
diff --git a/master/buildbot/test/unit/test_changes_gerritchangesource.py b/master/buildbot/test/unit/test_changes_gerritchangesource.py
--- a/master/buildbot/test/unit/test_changes_gerritchangesource.py
+++ b/master/buildbot/test/unit/test_changes_gerritchangesource.py
@@ -20,6 +20,7 @@
 from twisted.internet import defer
 from twisted.internet import error
 from twisted.internet import reactor
+from twisted.internet import utils
 from twisted.python import failure
 from twisted.trial import unittest
 
@@ -190,10 +191,8 @@ def test_handled_events_filter_true(self):
 
     @defer.inlineCallbacks
     def test_handled_events_filter_false(self):
-        s = self.newChangeSource(
-            'somehost', 'some_choosy_user')
+        s = self.newChangeSource('somehost', 'some_choosy_user')
         yield s.lineReceived(json.dumps(self.change_merged_event))
-
         self.assertEqual(len(self.master.data.updates.changesAdded), 0)
 
     @defer.inlineCallbacks
@@ -230,6 +229,75 @@ def spawnProcess(pp, cmd, argv, env):
 
         s.startStreamProcess()
 
+    # -------------------------------------------------------------------------
+    # Test data for getFiles()
+    # -------------------------------------------------------------------------
+    query_files_success_line1 = {
+        "patchSets": [
+            {
+                "number": 1,
+                "files": [
+                    {"file": "/COMMIT_MSG", "type": "ADDED", "insertions": 13, "deletions": 0},
+                ],
+            },
+            {
+                "number": 13,
+                "files": [
+                    {"file": "/COMMIT_MSG", "type": "ADDED", "insertions": 13, "deletions": 0},
+                    {"file": "file1", "type": "MODIFIED", "insertions": 7, "deletions": 0},
+                    {"file": "file2", "type": "MODIFIED", "insertions": 2, "deletions": -2},
+                ],
+            }
+        ]
+    }
+
+    query_files_success_line2 = {
+        "type": "stats", "rowCount": 1
+    }
+
+    query_files_success = '\n'.join([
+        json.dumps(query_files_success_line1),
+        json.dumps(query_files_success_line2)
+    ]).encode('utf8')
+
+    query_files_failure = b'{"type":"stats","rowCount":0}'
+
+    @defer.inlineCallbacks
+    def test_getFiles(self):
+        s = self.newChangeSource('host', 'user', gerritport=2222)
+        exp_argv = [
+            'ssh', 'user@host', '-p', '2222', 'gerrit', 'query', '1000',
+            '--format', 'JSON', '--files', '--patch-sets'
+        ]
+
+        def getoutput_success(cmd, argv, env):
+            self.assertEqual([cmd, argv], [exp_argv[0], exp_argv[1:]])
+            return self.query_files_success
+
+        def getoutput_failure(cmd, argv, env):
+            return self.query_files_failure
+
+        self.patch(utils, 'getProcessOutput', getoutput_success)
+        res = yield s.getFiles(1000, 13)
+        self.assertEqual(set(res), {'/COMMIT_MSG', 'file1', 'file2'})
+
+        self.patch(utils, 'getProcessOutput', getoutput_failure)
+        res = yield s.getFiles(1000, 13)
+        self.assertEqual(res, ['unknown'])
+
+    @defer.inlineCallbacks
+    def test_getFilesFromEvent(self):
+        s = self.newChangeSource('host', 'user', get_files=True,
+                                 handled_events=["change-merged"])
+
+        def getoutput(cmd, argv, env):
+            return self.query_files_success
+        self.patch(utils, 'getProcessOutput', getoutput)
+
+        yield s.lineReceived(json.dumps(self.change_merged_event))
+        c = self.master.data.updates.changesAdded[0]
+        self.assertEqual(set(c['files']), {'/COMMIT_MSG', 'file1', 'file2'})
+
 
 class TestGerritEventLogPoller(changesource.ChangeSourceMixin,
                                TestReactorMixin,
@@ -288,7 +356,7 @@ def test_lineReceived_patchset_created(self):
         self.master.db.insertTestData([
             fakedb.Object(id=self.OBJECTID, name='GerritEventLogPoller:gerrit',
                           class_name='GerritEventLogPoller')])
-        yield self.newChangeSource()
+        yield self.newChangeSource(get_files=True)
         self.changesource.now = lambda: datetime.datetime.utcfromtimestamp(
             self.NOW_TIMESTAMP)
         self._http.expect(method='get', ep='/plugins/events-log/events/',
@@ -307,15 +375,27 @@ def test_lineReceived_patchset_created(self):
                               eventCreatedOn=self.EVENT_TIMESTAMP,
                               patchSet=dict(revision="abcdef", number="12")))
 
+        self._http.expect(
+            method='get',
+            ep='/changes/4321/revisions/12/files/',
+            content=self.change_revision_resp,
+        )
+
         yield self.startChangeSource()
         yield self.changesource.poll()
+
         self.assertEqual(len(self.master.data.updates.changesAdded), 1)
+
         c = self.master.data.updates.changesAdded[0]
         for k, v in c.items():
+            if k == 'files':
+                continue
             self.assertEqual(TestGerritChangeSource.expected_change[k], v)
         self.master.db.state.assertState(
             self.OBJECTID, last_event_ts=self.EVENT_TIMESTAMP)
 
+        self.assertEqual(set(c['files']), {'/COMMIT_MSG', 'file1'})
+
         # do a second poll, it should ask for the next events
         self._http.expect(method='get', ep='/plugins/events-log/events/',
                           params={'t1': self.EVENT_FORMATTED},
@@ -333,10 +413,36 @@ def test_lineReceived_patchset_created(self):
                               eventCreatedOn=self.EVENT_TIMESTAMP + 1,
                               patchSet=dict(revision="abcdef", number="12")))
 
+        self._http.expect(
+            method='get',
+            ep='/changes/4321/revisions/12/files/',
+            content=self.change_revision_resp,
+        )
+
         yield self.changesource.poll()
         self.master.db.state.assertState(
             self.OBJECTID, last_event_ts=self.EVENT_TIMESTAMP + 1)
 
+    change_revision_dict = {
+        '/COMMIT_MSG': {'status': 'A', 'lines_inserted': 9, 'size_delta': 1, 'size': 1},
+        'file1': {'lines_inserted': 9, 'lines_deleted': 2, 'size_delta': 1, 'size': 1},
+    }
+    change_revision_resp = b')]}\n' + json.dumps(change_revision_dict).encode('utf8')
+
+    @defer.inlineCallbacks
+    def test_getFiles(self):
+        yield self.newChangeSource(get_files=True)
+        yield self.startChangeSource()
+
+        self._http.expect(
+            method='get',
+            ep='/changes/100/revisions/1/files/',
+            content=self.change_revision_resp,
+        )
+
+        files = yield self.changesource.getFiles(100, 1)
+        self.assertEqual(set(files), {'/COMMIT_MSG', 'file1'})
+
 
 class TestGerritChangeFilter(unittest.TestCase):