Repository: buildbot/buildbot

Issue:
Cannot get steps via HttpStatusPush
I'm working on adding Buildbot 0.9 support to Homu, and have stumbled across the following issue.

Usage:

```py
def homuStatusUpdate(build):
    build['homu-secret'] = HOMU_SECRET
    return build

c['services'] = [
    HttpStatusPush(
        serverUrl='http://{{ homu_host }}:{{ homu_port }}/buildbot',
        format_fn=homuStatusUpdate,
        wantProperties=True,
        wantSteps=True,
    ),
]
```

Log output (after instrumenting Buildbot 0.9.6):
```
2017-05-08 21:39:34-0700 [-] Build w/o details: {'buildrequestid': 35, 'complete_at': datetime.datetime(2017, 5, 9, 4, 39, 34, 779074, tzinfo=tzutc()), 'complete': True, 'builderid': 18, 'buildid': 37, 'workerid': 1, 'results': 6, 'number': 13, 'masterid': 1, 'state_string': u'finished', 'started_at': datetime.datetime(2017, 5, 9, 4, 37, 18, 350768, tzinfo=tzutc()), 'properties': {}}
2017-05-08 21:39:35-0700 [-] Build w/ details:  {'buildrequestid': 35, 'complete_at': datetime.datetime(2017, 5, 9, 4, 39, 34, 779074, tzinfo=tzutc()), 'complete': True, 'builderid': 18, 'buildid': 37, 'workerid': 1, 'builder': {'masterids': [1], 'tags': [], 'description': None, 'name': u'expected-fail', 'builderid': 18}, 'results': 6, 'number': 13, 'masterid': 1, 'steps': ListResult([{'complete': True, 'buildid': 37, 'results': 0, 'number': 0, 'started_at': datetime.datetime(2017, 5, 9, 4, 38, 45, 775999, tzinfo=tzutc()), 'stepid': 164, 'complete_at': datetime.datetime(2017, 5, 9, 4, 38, 57, 624931, tzinfo=tzutc()), 'name': u'git', 'urls': [], 'state_string': u'update', 'hidden': False}, {'complete': True, 'buildid': 37, 'results': 0, 'number': 1, 'started_at': datetime.datetime(2017, 5, 9, 4, 38, 59, 34691, tzinfo=tzutc()), 'stepid': 165, 'complete_at': datetime.datetime(2017, 5, 9, 4, 39, 0, 124917, tzinfo=tzutc()), 'name': u'git clean', 'urls': [], 'state_string': u"'git clean ...'", 'hidden': False}, {'complete': True, 'buildid': 37, 'results': 4, 'number': 2, 'started_at': datetime.datetime(2017, 5, 9, 4, 39, 0, 341593, tzinfo=tzutc()), 'stepid': 166, 'complete_at': datetime.datetime(2017, 5, 9, 4, 39, 34, 554919, tzinfo=tzutc()), 'name': u'build.sh', 'urls': [], 'state_string': u'build.sh', 'hidden': False}], offset=None, total=3, limit=None), 'url': 'https://ci.z.cash/#builders/18/builds/13', 'buildrequest': {'buildrequestid': 35, 'complete': False, 'waited_for': False, 'claimed_at': datetime.datetime(2017, 5, 9, 4, 37, 18, tzinfo=tzutc()), 'results': -1, 'claimed': True, 'buildsetid': 32, 'complete_at': None, 'submitted_at': datetime.datetime(2017, 5, 9, 4, 37, 18, tzinfo=tzutc()), 'builderid': 18, 'claimed_by_masterid': 1, 'priority': 0}, 'buildset': {'bsid': 32, 'complete_at': None, 'complete': False, 'sourcestamps': [{'project': u'', 'codebase': u'', 'ssid': 1, 'branch': u'', 'repository': u'', 'patch': None, 'created_at': datetime.datetime(2017, 4, 26, 3, 37, 28, 256620, tzinfo=tzutc()), 'revision': u''}], 'parent_buildid': None, 'results': -1, 'parent_relationship': None, 'reason': u"A build was forced by 'zc': force build", 'external_idstring': None, 'submitted_at': 1494304638}, 'state_string': u'finished', 'started_at': datetime.datetime(2017, 5, 9, 4, 37, 18, 350768, tzinfo=tzutc()), 'properties': {u'github_repo_owner': (u'zcash', u'Builder'), u'owners': ([u'zc'], u'Build'), u'github_repo_name': (u'zcash', u'Builder'), u'repository': (u'', u'Build'), u'buildername': (u'expected-fail', u'Builder'), u'got_revision': (u'20d61ac0ea83ac712b8e3b24ffae7347b4f6a8f3', u'Git'), u'builddir': (u'/home/admin/bbs/expected-fail', u'worker'), u'codebase': (u'', u'Build'), u'project': (u'', u'Build'), u'slavename': (u'latent-1', u'Worker (deprecated)'), u'reason': (u'force build', u'Force Build Form'), u'instance': (u'i-0737aca71992d517c', u'Worker'), u'workername': (u'latent-1', u'Worker'), u'scheduler': (u'force', u'Scheduler'), u'branch': (u'', u'Build'), u'owner': (u'zc', u'Force Build Form'), u'buildnumber': (13, u'Build'), u'revision': (u'', u'Build')}}
2017-05-08 21:39:35-0700 [-] Steps: ListResult([{'complete': True, 'buildid': 37, 'results': 0, 'number': 0, 'started_at': datetime.datetime(2017, 5, 9, 4, 38, 45, 775999, tzinfo=tzutc()), 'stepid': 164, 'complete_at': datetime.datetime(2017, 5, 9, 4, 38, 57, 624931, tzinfo=tzutc()), 'name': u'git', 'urls': [], 'state_string': u'update', 'hidden': False}, {'complete': True, 'buildid': 37, 'results': 0, 'number': 1, 'started_at': datetime.datetime(2017, 5, 9, 4, 38, 59, 34691, tzinfo=tzutc()), 'stepid': 165, 'complete_at': datetime.datetime(2017, 5, 9, 4, 39, 0, 124917, tzinfo=tzutc()), 'name': u'git clean', 'urls': [], 'state_string': u"'git clean ...'", 'hidden': False}, {'complete': True, 'buildid': 37, 'results': 4, 'number': 2, 'started_at': datetime.datetime(2017, 5, 9, 4, 39, 0, 341593, tzinfo=tzutc()), 'stepid': 166, 'complete_at': datetime.datetime(2017, 5, 9, 4, 39, 34, 554919, tzinfo=tzutc()), 'name': u'build.sh', 'urls': [], 'state_string': u'build.sh', 'hidden': False}], offset=None, total=3, limit=None)
2017-05-08 21:39:35-0700 [-] Unable to format event {'log_namespace': 'buildbot.util.httpclientservice', 'log_level': <LogLevel=error>, 'format': '%(log_legacy)s', 'time': 1494304775.082636, 'log_source': None, 'log_format': 'Input json: {\'buildrequestid\': 35, \'complete_at\': datetime.datetime(2017, 5, 9, 4, 39, 34, 779074, tzinfo=tzutc()), \'complete\': True, \'builderid\': 18, \'buildid\': 37, \'workerid\': 1, \'builder\': {\'masterids\': [1], \'tags\': [], \'description\': None, \'name\': u\'expected-fail\', \'builderid\': 18}, \'results\': 6, \'number\': 13, \'masterid\': 1, \'steps\': ListResult([{\'complete\': True, \'buildid\': 37, \'results\': 0, \'number\': 0, \'started_at\': datetime.datetime(2017, 5, 9, 4, 38, 45, 775999, tzinfo=tzutc()), \'stepid\': 164, \'complete_at\': datetime.datetime(2017, 5, 9, 4, 38, 57, 624931, tzinfo=tzutc()), \'name\': u\'git\', \'urls\': [], \'state_string\': u\'update\', \'hidden\': False}, {\'complete\': True, \'buildid\': 37, \'results\': 0, \'number\': 1, \'started_at\': datetime.datetime(2017, 5, 9, 4, 38, 59, 34691, tzinfo=tzutc()), \'stepid\': 165, \'complete_at\': datetime.datetime(2017, 5, 9, 4, 39, 0, 124917, tzinfo=tzutc()), \'name\': u\'git clean\', \'urls\': [], \'state_string\': u"\'git clean ...\'", \'hidden\': False}, {\'complete\': True, \'buildid\': 37, \'results\': 4, \'number\': 2, \'started_at\': datetime.datetime(2017, 5, 9, 4, 39, 0, 341593, tzinfo=tzutc()), \'stepid\': 166, \'complete_at\': datetime.datetime(2017, 5, 9, 4, 39, 34, 554919, tzinfo=tzutc()), \'name\': u\'build.sh\', \'urls\': [], \'state_string\': u\'build.sh\', \'hidden\': False}], offset=None, total=3, limit=None), \'url\': \'https://ci.z.cash/#builders/18/builds/13\', \'buildrequest\': {\'buildrequestid\': 35, \'complete\': False, \'waited_for\': False, \'claimed_at\': datetime.datetime(2017, 5, 9, 4, 37, 18, tzinfo=tzutc()), \'results\': -1, \'claimed\': True, \'buildsetid\': 32, \'complete_at\': None, \'submitted_at\': datetime.datetime(2017, 5, 9, 4, 37, 18, tzinfo=tzutc()), \'builderid\': 18, \'claimed_by_masterid\': 1, \'priority\': 0}, \'buildset\': {\'bsid\': 32, \'complete_at\': None, \'complete\': False, \'sourcestamps\': [{\'project\': u\'\', \'codebase\': u\'\', \'ssid\': 1, \'branch\': u\'\', \'repository\': u\'\', \'patch\': None, \'created_at\': datetime.datetime(2017, 4, 26, 3, 37, 28, 256620, tzinfo=tzutc()), \'revision\': u\'\'}], \'parent_buildid\': None, \'results\': -1, \'parent_relationship\': None, \'reason\': u"A build was forced by \'zc\': force build", \'external_idstring\': None, \'submitted_at\': 1494304638}, \'state_string\': u\'finished\', \'homu-secret\': \'REDACTED\', \'started_at\': datetime.datetime(2017, 5, 9, 4, 37, 18, 350768, tzinfo=tzutc()), \'properties\': {u\'github_repo_owner\': (u\'zcash\', u\'Builder\'), u\'owners\': ([u\'zc\'], u\'Build\'), u\'github_repo_name\': (u\'zcash\', u\'Builder\'), u\'repository\': (u\'\', u\'Build\'), u\'buildername\': (u\'expected-fail\', u\'Builder\'), u\'got_revision\': (u\'20d61ac0ea83ac712b8e3b24ffae7347b4f6a8f3\', u\'Git\'), u\'builddir\': (u\'/home/admin/bbs/expected-fail\', u\'worker\'), u\'codebase\': (u\'\', u\'Build\'), u\'project\': (u\'\', u\'Build\'), u\'slavename\': (u\'latent-1\', u\'Worker (deprecated)\'), u\'reason\': (u\'force build\', u\'Force Build Form\'), u\'instance\': (u\'i-0737aca71992d517c\', u\'Worker\'), u\'workername\': (u\'latent-1\', u\'Worker\'), u\'scheduler\': (u\'force\', u\'Scheduler\'), u\'branch\': (u\'\', u\'Build\'), u\'owner\': (u\'zc\', u\'Force Build Form\'), u\'buildnumber\': (13, u\'Build\'), u\'revision\': (u\'\', u\'Build\')}}', 'system': '-', 'log_logger': <Logger 'buildbot.util.httpclientservice'>, 'message': (), 'log_time': 1494304775.082636}: u"'buildrequestid'"
2017-05-08 21:39:35-0700 [-] Unable to format event {'log_namespace': 'buildbot.util.httpclientservice', 'log_level': <LogLevel=error>, 'format': '%(log_legacy)s', 'time': 1494304775.08308, 'log_source': None, 'log_format': 'Dumped json: {"buildrequestid": 35, "complete_at": 1494304774, "complete": true, "builderid": 18, "buildid": 37, "workerid": 1, "builder": {"masterids": [1], "tags": [], "description": null, "name": "expected-fail", "builderid": 18}, "results": 6, "number": 13, "masterid": 1, "steps": null, "url": "https://ci.z.cash/#builders/18/builds/13", "buildrequest": {"buildrequestid": 35, "complete": false, "waited_for": false, "claimed_at": 1494304638, "results": -1, "claimed": true, "buildsetid": 32, "complete_at": null, "submitted_at": 1494304638, "builderid": 18, "claimed_by_masterid": 1, "priority": 0}, "buildset": {"bsid": 32, "complete_at": null, "complete": false, "sourcestamps": [{"project": "", "codebase": "", "ssid": 1, "branch": "", "repository": "", "patch": null, "created_at": 1493177848, "revision": ""}], "parent_buildid": null, "results": -1, "parent_relationship": null, "reason": "A build was forced by \'zc\': force build", "external_idstring": null, "submitted_at": 1494304638}, "state_string": "finished", "homu-secret": "REDACTED", "started_at": 1494304638, "properties": {"github_repo_owner": ["zcash", "Builder"], "owners": [["zc"], "Build"], "github_repo_name": ["zcash", "Builder"], "repository": ["", "Build"], "buildername": ["expected-fail", "Builder"], "got_revision": ["20d61ac0ea83ac712b8e3b24ffae7347b4f6a8f3", "Git"], "builddir": ["/home/admin/bbs/expected-fail", "worker"], "codebase": ["", "Build"], "project": ["", "Build"], "slavename": ["latent-1", "Worker (deprecated)"], "reason": ["force build", "Force Build Form"], "instance": ["i-0737aca71992d517c", "Worker"], "workername": ["latent-1", "Worker"], "scheduler": ["force", "Scheduler"], "branch": ["", "Build"], "owner": ["zc", "Force Build Form"], "buildnumber": [13, "Build"], "revision": ["", "Build"]}}', 'system': '-', 'log_logger': <Logger 'buildbot.util.httpclientservice'>, 'message': (), 'log_time': 1494304775.08308}: u'"buildrequestid"'
```

Looking at the `Unable to format event` lines (which is a bug in my log command, but still prints the right info), you can see the following:
```
Input json: {..., 'steps': ListResult([{...}, {...}, {...}], offset=None, total=3, limit=None), ...}
Dumped json: {..., "steps": null, ...}
```

These log lines were wrapping [this line](https://github.com/buildbot/buildbot/blob/master/master/buildbot/util/httpclientservice.py#L154). Thus there seems to be a problem in rendering `ListResult` as JSON.


Discussion:
I see that `ListResult` is a subclass of `UserList`:

```py
>>> import UserList
>>> import json
>>> l = [1, 2, 3]
>>> json.dumps(l)
'[1, 2, 3]'
>>> json.dumps(UserList.UserList(l))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/lib/python2.7/json/__init__.py", line 244, in dumps
    return _default_encoder.encode(obj)
  File "/usr/lib/python2.7/json/encoder.py", line 207, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/lib/python2.7/json/encoder.py", line 270, in iterencode
    return _iterencode(o, 0)
  File "/usr/lib/python2.7/json/encoder.py", line 184, in default
    raise TypeError(repr(o) + " is not JSON serializable")
TypeError: [1, 2, 3] is not JSON serializable
```
I believe that the simplest fix is to handle this [here](https://github.com/buildbot/buildbot/blob/master/master/buildbot/reporters/utils.py#L132):

```py
        if wantSteps:
            # Ensure we pass back a list, not a ListResult
            build['steps'] = [step for step in steps]
```
You can just say ``build['steps'] = list(steps)`` rather than the list comprehension.
a merge would be great!
also logs have the same problem

Resolving patch:
diff --git a/master/buildbot/reporters/utils.py b/master/buildbot/reporters/utils.py
--- a/master/buildbot/reporters/utils.py
+++ b/master/buildbot/reporters/utils.py
@@ -111,7 +111,8 @@ def getDetailsForBuilds(master, buildset, builds, wantProperties=False, wantStep
              for build in builds])
         if wantLogs:
             for s in flatten(buildsteps, types=(list, UserList)):
-                s['logs'] = yield master.data.get(("steps", s['stepid'], 'logs'))
+                logs = yield master.data.get(("steps", s['stepid'], 'logs'))
+                s['logs'] = list(logs)
                 for l in s['logs']:
                     l['content'] = yield master.data.get(("logs", l['logid'], 'contents'))
 
@@ -129,7 +130,7 @@ def getDetailsForBuilds(master, buildset, builds, wantProperties=False, wantStep
             build['properties'] = properties
 
         if wantSteps:
-            build['steps'] = steps
+            build['steps'] = list(steps)
 
         if wantPreviousBuild:
             build['prev_build'] = prev


Test patch: