Repository: buildbot/buildbot

Issue:
Github change hook incorrectly treats payload with bytes
Using the Github change hook, it seems like the decoded payload contains bytes instead of strings:

```
2017-07-18 13:37:22+0200 [-] {b'payload': [b'{"ref":"refs/heads/master","before":"3c41557e1a77291ca8b109d'
2017-07-18 13:37:22+0200 [-]               b'd267cbd80dfa647ae","after":"2d28d01c433ea620d4808ea0e7fb5ba3'
...
```

So indexing with a regular string `"payload"` throws a KeyError:

```
        Traceback (most recent call last):
          File "/opt/buildbot/lib/python3.5/site-packages/twisted/web/server.py", line 235, in render
            body = resrc.render(self)
          File "/opt/buildbot/lib/python3.5/site-packages/twisted/web/resource.py", line 250, in render
            return m(request)
          File "/opt/buildbot/lib/python3.5/site-packages/buildbot/www/change_hook.py", line 81, in render_POST
            d = self.getAndSubmitChanges(request)
          File "/opt/buildbot/lib/python3.5/site-packages/twisted/internet/defer.py", line 1532, in unwindGenerator
            return _inlineCallbacks(None, gen, Deferred())
        --- <exception caught here> ---
          File "/opt/buildbot/lib/python3.5/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
            result = result.throwExceptionIntoGenerator(g)
          File "/opt/buildbot/lib/python3.5/site-packages/twisted/python/failure.py", line 393, in throwExceptionIntoGenerator
            return g.throw(self.type, self.value, self.tb)
          File "/opt/buildbot/lib/python3.5/site-packages/buildbot/www/change_hook.py", line 107, in getAndSubmitChanges
            changes, src = yield self.getChanges(request)
        builtins.KeyError: 'payload'
```

Changing the source [here](https://github.com/buildbot/buildbot/blob/08cf30663f317ad3c089a2d94b8bca5f98d830ef/master/buildbot/www/hooks/github.py#L101) to index with bytes and decode the results (ie. `json.loads(bytes2NativeString(request.args[b'payload'][0]))`) makes the change hook work again, but I lack the python experience to know how good of a fix that is.

```
buildbot==0.9.9.post2
buildbot-console-view==0.9.9.post2
buildbot-waterfall-view==0.9.9.post2
buildbot-worker==0.9.9.post2
buildbot-www==0.9.9.post2

Python 3.5.3

Debian Jessie
```


Discussion:
Hi @maleadt that change looks good. can you submit a patch?

You may have to fix the unit tests:
http://docs.buildbot.net/latest/developer/tests.html
@maleadt I was able to reproduce your problem.  The problem occurs if
your GitHub webhook is configured with *application/x-www-form-urlencoded*.
If you change your webhook to be configured with *application/json*, then the problem
does not occur and everything works.

I'll see if I can fix this.
One other thing to note.
In Python 3.5, this:

```
json.loads(b'{}')
```

gives an error:

```
TypeError: the JSON object must be str, not 'bytes'
```

On Python 3.6, json.loads(b'{}') does not give an error, i.e. it accepts an argument of bytes.

Resolving patch:
diff --git a/master/buildbot/www/hooks/github.py b/master/buildbot/www/hooks/github.py
--- a/master/buildbot/www/hooks/github.py
+++ b/master/buildbot/www/hooks/github.py
@@ -28,7 +28,7 @@
 from twisted.python import log
 
 from buildbot.changes.github import PullRequestMixin
-from buildbot.util import bytes2NativeString
+from buildbot.util import bytes2unicode
 from buildbot.util import httpclientservice
 from buildbot.util import unicode2bytes
 from buildbot.www.hooks.base import BaseHookHandler
@@ -75,7 +75,7 @@ def process(self, request):
         payload = self._get_payload(request)
 
         event_type = request.getHeader(_HEADER_EVENT)
-        event_type = bytes2NativeString(event_type)
+        event_type = bytes2unicode(event_type)
         log.msg("X-GitHub-Event: {}".format(
             event_type), logLevel=logging.DEBUG)
 
@@ -89,10 +89,10 @@ def process(self, request):
 
     def _get_payload(self, request):
         content = request.content.read()
-        content = bytes2NativeString(content)
+        content = bytes2unicode(content)
 
         signature = request.getHeader(_HEADER_SIGNATURE)
-        signature = bytes2NativeString(signature)
+        signature = bytes2unicode(signature)
 
         if not signature and self._strict:
             raise ValueError('Request has no required signature')
@@ -120,7 +120,7 @@ def _get_payload(self, request):
         if content_type == b'application/json':
             payload = json.loads(content)
         elif content_type == b'application/x-www-form-urlencoded':
-            payload = json.loads(request.args['payload'][0])
+            payload = json.loads(bytes2unicode(request.args[b'payload'][0]))
         else:
             raise ValueError('Unknown content type: {}'.format(content_type))
 
diff --git a/master/buildbot/www/hooks/gitlab.py b/master/buildbot/www/hooks/gitlab.py
--- a/master/buildbot/www/hooks/gitlab.py
+++ b/master/buildbot/www/hooks/gitlab.py
@@ -23,7 +23,7 @@
 
 from twisted.python import log
 
-from buildbot.util import bytes2NativeString
+from buildbot.util import bytes2unicode
 from buildbot.www.hooks.base import BaseHookHandler
 
 _HEADER_EVENT = b'X-Gitlab-Event'
@@ -141,15 +141,16 @@ def getChanges(self, request):
         expected_secret = isinstance(self.options, dict) and self.options.get('secret')
         if expected_secret:
             received_secret = request.getHeader(_HEADER_GITLAB_TOKEN)
-            received_secret = bytes2NativeString(received_secret)
+            received_secret = bytes2unicode(received_secret)
             if received_secret != expected_secret:
                 raise ValueError("Invalid secret")
         try:
-            payload = json.load(request.content)
+            content = request.content.read()
+            payload = json.loads(bytes2unicode(content))
         except Exception as e:
             raise ValueError("Error loading JSON: " + str(e))
         event_type = request.getHeader(_HEADER_EVENT)
-        event_type = bytes2NativeString(event_type)
+        event_type = bytes2unicode(event_type)
         # newer version of gitlab have a object_kind parameter,
         # which allows not to use the http header
         event_type = payload.get('object_kind', event_type)


Test patch:
diff --git a/master/buildbot/test/fake/web.py b/master/buildbot/test/fake/web.py
--- a/master/buildbot/test/fake/web.py
+++ b/master/buildbot/test/fake/web.py
@@ -16,10 +16,11 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+from io import BytesIO
+
 from mock import Mock
 
 from twisted.internet import defer
-from twisted.python.compat import NativeStringIO
 from twisted.web import server
 
 from buildbot.test.fake import fakemaster
@@ -52,14 +53,14 @@ class FakeRequest(Mock):
     redirected_to = None
     failure = None
 
-    def __init__(self, args=None, content=''):
+    def __init__(self, args=None, content=b''):
         Mock.__init__(self)
 
         if args is None:
             args = {}
 
         self.args = args
-        self.content = NativeStringIO(content)
+        self.content = BytesIO(content)
         self.site = Mock()
         self.site.buildbot_service = Mock()
         self.uri = b'/'
diff --git a/master/buildbot/test/unit/test_www_hooks_github.py b/master/buildbot/test/unit/test_www_hooks_github.py
--- a/master/buildbot/test/unit/test_www_hooks_github.py
+++ b/master/buildbot/test/unit/test_www_hooks_github.py
@@ -17,15 +17,12 @@
 from __future__ import absolute_import
 from __future__ import print_function
 from future.utils import PY3
-from future.utils import string_types
-from future.utils import text_type
 
 import hmac
 from calendar import timegm
 from copy import deepcopy
 from hashlib import sha1
 from io import BytesIO
-from io import StringIO
 
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -43,7 +40,7 @@
 # Added "modified" and "removed", and change email
 # Added "head_commit"
 #   https://developer.github.com/v3/activity/events/types/#webhook-payload-example-26
-gitJsonPayload = """
+gitJsonPayload = b"""
 {
   "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d",
   "repository": {
@@ -103,7 +100,7 @@
 }
 """
 
-gitJsonPayloadCiSkipTemplate = """
+gitJsonPayloadCiSkipTemplate = u"""
 {
   "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d",
   "repository": {
@@ -163,7 +160,7 @@
 }
 """
 
-gitJsonPayloadTag = """
+gitJsonPayloadTag = b"""
 {
   "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d",
   "repository": {
@@ -223,70 +220,7 @@
 }
 """
 
-gitJsonPayloadTagUnicode = u"""
-{
-  "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d",
-  "repository": {
-    "url": "http://github.com/defunkt/github",
-    "html_url": "http://github.com/defunkt/github",
-    "name": "github",
-    "full_name": "defunkt/github",
-    "description": "You're lookin' at it.",
-    "watchers": 5,
-    "forks": 2,
-    "private": 1,
-    "owner": {
-      "email": "julian.rueth@fsfe.org",
-      "name": "Julian Rüth"
-    }
-  },
-  "commits": [
-    {
-      "id": "41a212ee83ca127e3c8cf465891ab7216a705f59",
-      "distinct": true,
-      "url": "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59",
-      "author": {
-        "name": "Julian Rüth",
-        "email": "julian.rueth@fsfe.org",
-        "username": "saraedum"
-      },
-      "message": "okay i give in",
-      "timestamp": "2008-02-15T14:57:17-08:00",
-      "added": ["filepath.rb"]
-    },
-    {
-      "id": "de8251ff97ee194a289832576287d6f8ad74e3d0",
-      "url": "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0",
-      "author": {
-        "name": "Julian Rüth",
-        "email": "julian.rueth@fsfe.org",
-        "username": "saraedum"
-      },
-      "message": "update pricing a tad",
-      "timestamp": "2008-02-15T14:36:34-08:00",
-      "modified": ["modfile"],
-      "removed": ["removedFile"]
-    }
-  ],
-  "head_commit": {
-    "id": "de8251ff97ee194a289832576287d6f8ad74e3d0",
-    "url": "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0",
-    "author": {
-      "name": "Julian Rüth",
-      "email": "julian.rueth@fsfe.org",
-      "username": "saraedum"
-    },
-    "message": "update pricing a tad",
-    "timestamp": "2008-02-15T14:36:34-08:00",
-    "modified": ["modfile"],
-    "removed": ["removedFile"]
-  },
-  "after": "de8251ff97ee194a289832576287d6f8ad74e3d0",
-  "ref": "refs/tags/v1.0.0"
-}
-"""
-
-gitJsonPayloadNonBranch = """
+gitJsonPayloadNonBranch = b"""
 {
   "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d",
   "repository": {
@@ -322,7 +256,7 @@
 }
 """
 
-gitJsonPayloadPullRequest = """
+gitJsonPayloadPullRequest = b"""
 {
   "action": "opened",
   "number": 50,
@@ -491,7 +425,7 @@
     'event': 'pull_request'
 }
 
-gitJsonPayloadEmpty = """
+gitJsonPayloadEmpty = b"""
 {
   "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d",
   "repository": {
@@ -540,11 +474,11 @@ def _prepare_request(event, payload, _secret=None, headers=None):
         _HEADER_EVENT: event
     }
 
-    if isinstance(payload, (string_types, bytes)):
-        if isinstance(payload, text_type):
-            request.content = StringIO(payload)
-        elif isinstance(payload, bytes):
-            request.content = BytesIO(payload)
+    assert isinstance(payload, (bytes, list)), \
+        "payload can only be bytes or list, not {}".format(type(payload))
+
+    if isinstance(payload, bytes):
+        request.content = BytesIO(payload)
         request.received_headers[_HEADER_CT] = _CT_JSON
 
         if _secret is not None:
@@ -554,7 +488,7 @@ def _prepare_request(event, payload, _secret=None, headers=None):
             request.received_headers[_HEADER_SIGNATURE] = \
                 'sha1={}'.format(signature.hexdigest())
     else:
-        request.args['payload'] = payload
+        request.args[b'payload'] = payload
         request.received_headers[_HEADER_CT] = _CT_ENCODED
 
     request.received_headers.update(headers)
@@ -614,10 +548,10 @@ def _check_ping(self, payload):
         self.assertEqual(len(self.changeHook.master.addedChanges), 0)
 
     def test_ping_encoded(self):
-        self._check_ping(['{}'])
+        self._check_ping([b'{}'])
 
     def test_ping_json(self):
-        self._check_ping('{}')
+        self._check_ping(b'{}')
 
     @defer.inlineCallbacks
     def test_git_with_push_tag(self):
@@ -630,17 +564,6 @@ def test_git_with_push_tag(self):
                          "Fred Flinstone <fred@flinstone.org>")
         self.assertEqual(change["branch"], "v1.0.0")
 
-    @defer.inlineCallbacks
-    def test_git_with_push_tag_unicode(self):
-        self.request = _prepare_request(b'push', gitJsonPayloadTagUnicode)
-        yield self.request.test_render(self.changeHook)
-
-        self.assertEqual(len(self.changeHook.master.addedChanges), 2)
-        change = self.changeHook.master.addedChanges[0]
-        self.assertEqual(change["author"],
-                         u"Julian Rüth <julian.rueth@fsfe.org>")
-        self.assertEqual(change["branch"], "v1.0.0")
-
     # Test 'base' hook with attributes. We should get a json string
     # representing a Change object as a dictionary. All values show be set.
     @defer.inlineCallbacks
@@ -702,8 +625,8 @@ def test_git_with_change_json(self):
     # the commit before.
     @defer.inlineCallbacks
     def testGitWithDistinctFalse(self):
-        self.request = _prepare_request(b'push', [gitJsonPayload.replace('"distinct": true,',
-                                                                        '"distinct": false,')])
+        self.request = _prepare_request(b'push', [gitJsonPayload.replace(b'"distinct": true,',
+                                                                        b'"distinct": false,')])
 
         yield self.request.test_render(self.changeHook)
         self.assertEqual(len(self.changeHook.master.addedChanges), 2)
@@ -826,9 +749,9 @@ def _check_git_push_with_skip_message(self, payload):
 
     def test_git_push_with_skip_message(self):
         gitJsonPayloadCiSkips = [
-            (gitJsonPayloadCiSkipTemplate % {'skip': '[ci skip]'}),
-            (gitJsonPayloadCiSkipTemplate % {'skip': '[skip ci]'}),
-            (gitJsonPayloadCiSkipTemplate % {'skip': '[  ci skip   ]'}),
+            unicode2bytes(gitJsonPayloadCiSkipTemplate % {'skip': '[ci skip]'}),
+            unicode2bytes(gitJsonPayloadCiSkipTemplate % {'skip': '[skip ci]'}),
+            unicode2bytes(gitJsonPayloadCiSkipTemplate % {'skip': '[  ci skip   ]'}),
         ]
 
         for payload in gitJsonPayloadCiSkips:
@@ -879,8 +802,8 @@ def _check_push_with_skip_message(self, payload):
 
     def test_push_with_skip_message(self):
         gitJsonPayloadCiSkips = [
-            (gitJsonPayloadCiSkipTemplate % {'skip': '[bb skip]'}),
-            (gitJsonPayloadCiSkipTemplate % {'skip': '[  bb skip   ]'}),
+            unicode2bytes(gitJsonPayloadCiSkipTemplate % {'skip': '[bb skip]'}),
+            unicode2bytes(gitJsonPayloadCiSkipTemplate % {'skip': '[  bb skip   ]'}),
         ]
 
         for payload in gitJsonPayloadCiSkips:
@@ -896,6 +819,7 @@ def test_push_no_ci_skip(self):
         # user overrode the skip pattern already,
         # so the default patterns should not work.
         payload = gitJsonPayloadCiSkipTemplate % {'skip': '[ci skip]'}
+        payload = unicode2bytes(payload)
         self._check_push_no_ci_skip(payload)
 
     @defer.inlineCallbacks
diff --git a/master/buildbot/test/unit/test_www_hooks_gitlab.py b/master/buildbot/test/unit/test_www_hooks_gitlab.py
--- a/master/buildbot/test/unit/test_www_hooks_gitlab.py
+++ b/master/buildbot/test/unit/test_www_hooks_gitlab.py
@@ -32,7 +32,7 @@
 
 # Sample GITLAB commit payload from https://docs.gitlab.com/ce/user/project/integrations/webhooks.html
 # Added "modified" and "removed", and change email
-gitJsonPayload = """
+gitJsonPayload = b"""
 {
   "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
   "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
@@ -70,7 +70,7 @@
   "total_commits_count": 2
 }
 """
-gitJsonPayloadTag = """
+gitJsonPayloadTag = b"""
 {
   "object_kind": "tag_push",
   "before": "0000000000000000000000000000000000000000",
@@ -113,13 +113,13 @@
    "total_commits_count": 2
 }
 """
-gitJsonPayloadMR = """
+gitJsonPayloadMR = b"""
 {
   "object_kind": "merge_request",
   "user": {
     "name": "Administrator",
     "username": "root",
-    "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon"
+    "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon"
   },
   "object_attributes": {
     "id": 99,
@@ -187,7 +187,7 @@
     "assignee": {
       "name": "User1",
       "username": "user1",
-      "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon"
+      "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon"
     }
   }
 }
@@ -272,7 +272,7 @@ def testGitWithChange(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_push_event(res)
 
@@ -281,7 +281,7 @@ def testGitWithChange_WithProjectToo(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'project': ['MyProject']}
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_push_event(res, project="MyProject")
@@ -291,7 +291,7 @@ def testGitWithChange_WithCodebaseToo(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_push_event(res, codebase="MyCodebase")
@@ -301,7 +301,7 @@ def testGitWithChange_WithPushTag(self):
         self.request = FakeRequest(content=gitJsonPayloadTag)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_tag_event(res, codebase="MyCodebase")
@@ -310,7 +310,7 @@ def testGitWithNoJson(self):
         self.request = FakeRequest()
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         d = self.request.test_render(self.changeHook)
 
         def check_changes(r):
@@ -324,7 +324,7 @@ def check_changes(r):
     @defer.inlineCallbacks
     def test_event_property(self):
         self.request = FakeRequest(content=gitJsonPayload)
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
         yield self.request.test_render(self.changeHook)
@@ -338,7 +338,7 @@ def testGitWithChange_WithMR(self):
         self.request = FakeRequest(content=gitJsonPayloadMR)
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
-        self.request.received_headers[_HEADER_EVENT] = "Merge Request Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Merge Request Hook"
         self.request.method = b"POST"
         res = yield self.request.test_render(self.changeHook)
         self.check_changes_mr_event(res, codebase="MyCodebase")
@@ -361,7 +361,7 @@ def test_missing_secret(self):
         self.request.uri = b"/change_hook/gitlab"
         self.request.args = {'codebase': ['MyCodebase']}
         self.request.method = b"POST"
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         yield self.request.test_render(self.changeHook)
         expected = b'Invalid secret'
         self.assertEqual(self.request.written, expected)
@@ -371,7 +371,7 @@ def test_missing_secret(self):
     def test_valid_secret(self):
         self.request = FakeRequest(content=gitJsonPayload)
         self.request.received_headers[_HEADER_GITLAB_TOKEN] = self._SECRET
-        self.request.received_headers[_HEADER_EVENT] = "Push Hook"
+        self.request.received_headers[_HEADER_EVENT] = b"Push Hook"
         self.request.uri = b"/change_hook/gitlab"
         self.request.method = b"POST"
         yield self.request.test_render(self.changeHook)