Repository: buildbot/buildbot

Issue:
Base web hook fails with non ascii characters in payload
buildbot-0.9.7

```
...buildbot-contrib/master/contrib$ ./post_build_request.py -w "привет привет не работает" -H bb-master -r kernel -P kernel -c "не asci комментарий" -R "test"
```
and in `master/twisted.log`

```
2017-06-07 14:28:38+0300 [_GenericHTTPChannelProtocol,10,127.0.0.1] Attempting to load module buildbot.www.hooks.base
2017-06-07 14:28:38+0300 [_GenericHTTPChannelProtocol,10,127.0.0.1] Got the following changes [{'category': None, 'codebase': None, 'repository': 'kernel', 'author': '\xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbd\xd0\xb5 \xd1\x80\xd0\xb0\xd0\xb1\xd0\xbe\xd1\x82\xd0\xb0\xd0\xb5\xd1\x82', 'when': None, 'comments': u'\u043d\u0435 asci \u043a\u043e\u043c\u043c\u0435\u043d\u0442\u0430\u0440\u0438\u0439', 'project': 'kernel', 'branch': None, 'revlink': None, 'properties': {}, 'files': [], 'revision': 'test'}]
2017-06-07 14:28:38+0300 [_GenericHTTPChannelProtocol,10,127.0.0.1] Payload: {'project': ['kernel'], 'revision': ['test'], 'repository': ['kernel'], 'comments': ['\xd0\xbd\xd0\xb5 asci \xd0\xba\xd0\xbe\xd0\xbc\xd0\xbc\xd0\xb5\xd0\xbd\xd1\x82\xd0\xb0\xd1\x80\xd0\xb8\xd0\xb9'], 'author': ['\xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbd\xd0\xb5 \xd1\x80\xd0\xb0\xd0\xb1\xd0\xbe\xd1\x82\xd0\xb0\xd0\xb5\xd1\x82']}
2017-06-07 14:28:38+0300 [_GenericHTTPChannelProtocol,10,127.0.0.1] WARNING: change source is using deprecated self.master.addChange method; this method will disappear in Buildbot-1.0.0
2017-06-07 14:28:38+0300 [_GenericHTTPChannelProtocol,10,127.0.0.1] adding changes from web hook
        Traceback (most recent call last):
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/twisted/internet/defer.py", line 1447, in unwindGenerator
            return _inlineCallbacks(None, gen, Deferred())
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/twisted/internet/defer.py", line 1301, in _inlineCallbacks
            result = g.send(result)
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/buildbot/www/change_hook.py", line 158, in submitChanges
            change = yield self.master.addChange(src=src, **chdict)
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/twisted/internet/defer.py", line 1447, in unwindGenerator
            return _inlineCallbacks(None, gen, Deferred())
        --- <exception caught here> ---
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/twisted/internet/defer.py", line 1301, in _inlineCallbacks
            result = g.send(result)
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/buildbot/master.py", line 485, in addChange
            kwargs[k] = ascii2unicode(kwargs[k])
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/buildbot/util/__init__.py", line 239, in ascii2unicode
            return bytes2unicode(x, encoding='ascii', errors=errors)
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/buildbot/util/__init__.py", line 235, in bytes2unicode
            return text_type(x, encoding, errors)
        exceptions.UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in position 0: ordinal not in range(128)
```


Discussion:
we should use bytes2unicode instead of ascii2unicode. Would you like to make a patch?
Where exactly..? I do not see anything in `master/buildbot/changes/base.py`
but see 
```
$ grep -nRi 'ascii2unicode' .|wc -l
85
```
in a lot of other places
well, you can start with the place in the stack trace. maybe you will get more
```
          File "/home/builder/buildbot/sandbox/lib/python2.7/site-packages/buildbot/master.py", line 485, in addChange
            kwargs[k] = ascii2unicode(kwargs[k])
```

Resolving patch:
diff --git a/master/buildbot/master.py b/master/buildbot/master.py
--- a/master/buildbot/master.py
+++ b/master/buildbot/master.py
@@ -52,7 +52,7 @@
 from buildbot.schedulers.manager import SchedulerManager
 from buildbot.secrets.manager import SecretManager
 from buildbot.status.master import Status
-from buildbot.util import ascii2unicode
+from buildbot.util import bytes2unicode
 from buildbot.util import check_functional_environment
 from buildbot.util import datetime2epoch
 from buildbot.util import service
@@ -477,12 +477,12 @@ def handle_deprec(oldname, newname):
         for k in ('comments', 'author', 'revision', 'branch', 'category',
                   'revlink', 'repository', 'codebase', 'project'):
             if k in kwargs:
-                kwargs[k] = ascii2unicode(kwargs[k])
+                kwargs[k] = bytes2unicode(kwargs[k])
         if kwargs.get('files'):
-            kwargs['files'] = [ascii2unicode(f)
+            kwargs['files'] = [bytes2unicode(f)
                                for f in kwargs['files']]
         if kwargs.get('properties'):
-            kwargs['properties'] = dict((ascii2unicode(k), v)
+            kwargs['properties'] = dict((bytes2unicode(k), v)
                                         for k, v in iteritems(kwargs['properties']))
 
         # pass the converted call on to the data API


Test patch:
diff --git a/master/buildbot/test/unit/test_master.py b/master/buildbot/test/unit/test_master.py
--- a/master/buildbot/test/unit/test_master.py
+++ b/master/buildbot/test/unit/test_master.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 # This file is part of Buildbot.  Buildbot is free software: you can
 # redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation, version 2.
@@ -114,6 +115,16 @@ def test_addChange_args_author(self):
             kwargs=dict(who='me'),
             exp_data_kwargs=dict(author='me'))
 
+    def test_addChange_args_cyrillic(self):
+        return self.do_test_addChange_args(
+            kwargs=dict(who=b'\xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82',
+                        files=[b'\xd0\xbd\xd0\xb5'],
+                        properties={b'\xd1\x80\xd0\xb0\xd0\xb1\xd0\xbe\xd1\x82\xd0\xb0\xd0\xb5\xd1\x82':
+                                    'a'}),
+            exp_data_kwargs=dict(author=u'привет',
+                                 files=[u'не'],
+                                 properties={u'работает': 'a'}))
+
     def test_addChange_args_when(self):
         # when should come through as when_timestamp, as a datetime
         return self.do_test_addChange_args(