Repository: buildbot/buildbot

Issue:
UnicodeEncodeError in SVNPoller (0.9.15)
I just upgraded from 0.9.11 to 0.9.15, but this may not be a new issue. The SVN log message in r663 is: "Adding some of François's stuff".

```
2018-01-02 22:50:18+0000 [-] SVNPoller: _process_changes 662 .. 663
2018-01-02 22:50:18+0000 [-] Adding change revision 663
2018-01-02 22:50:18+0000 [-] SVNPoller: Error in  while polling
        Traceback (most recent call last):
          File "/home/chris/continuous-build/buildbot/buildbotvenv/lib/python3.4/site-packages/twisted/internet/_baseprocess.py", line 64, in maybeCallProcessEnded
            proto.processEnded(Failure(reason))
          File "/home/chris/continuous-build/buildbot/buildbotvenv/lib/python3.4/site-packages/twisted/internet/utils.py", line 93, in processEnded
            self.deferred.callback(self.s.getvalue())
          File "/home/chris/continuous-build/buildbot/buildbotvenv/lib/python3.4/site-packages/twisted/internet/defer.py", line 459, in callback
            self._startRunCallbacks(result)
          File "/home/chris/continuous-build/buildbot/buildbotvenv/lib/python3.4/site-packages/twisted/internet/defer.py", line 567, in _startRunCallbacks
            self._runCallbacks()
        --- <exception caught here> ---
          File "/home/chris/continuous-build/buildbot/buildbotvenv/lib/python3.4/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
            current.result = callback(current.result, *args, **kw)
          File "/home/chris/continuous-build/buildbot/buildbotvenv/lib/python3.4/site-packages/buildbot/changes/svnpoller.py", line 360, in create_changes
            path = path.encode("ascii")
        builtins.UnicodeEncodeError: 'ascii' codec can't encode character '\u0327' in position 43: ordinal not in range(128)

2018-01-02 22:51:11+0000 [-] SVNPoller: polling
```


Discussion:
Looks like the cause is the file path not the comment:

```
/UE4/StarryExpanse/Content/Developers/François/
```
Can you provide the full raw log of r663 in XML format?

something like:

```
svn log --xml -r663
``` 
According to http://svnbook.red-bean.com/en/1.7/svn.advanced.l10n.html , Subversion internally  stores paths encoded as UTF-8, so we need to improve the code in buildbot, and maybe
add some tests with some paths which have non-ASCII characters.

Resolving patch:
diff --git a/master/buildbot/changes/svnpoller.py b/master/buildbot/changes/svnpoller.py
--- a/master/buildbot/changes/svnpoller.py
+++ b/master/buildbot/changes/svnpoller.py
@@ -30,7 +30,6 @@
 
 from buildbot import util
 from buildbot.changes import base
-from buildbot.util import bytes2NativeString
 from buildbot.util import bytes2unicode
 
 # these split_file_* functions are available for use as values to the
@@ -355,11 +354,7 @@ def create_changes(self, new_logentries):
             for p in pathlist.getElementsByTagName("path"):
                 kind = p.getAttribute("kind")
                 action = p.getAttribute("action")
-                path = "".join([t.data for t in p.childNodes])
-                # Convert the path from unicode to bytes
-                path = path.encode("ascii")
-                # Convert path from bytes to native string.  Needed for Python 3.
-                path = bytes2NativeString(path, "ascii")
+                path = u"".join([t.data for t in p.childNodes])
                 if path.startswith("/"):
                     path = path[1:]
                 if kind == "dir" and not path.endswith("/"):


Test patch:
diff --git a/master/buildbot/test/unit/test_changes_svnpoller.py b/master/buildbot/test/unit/test_changes_svnpoller.py
--- a/master/buildbot/test/unit/test_changes_svnpoller.py
+++ b/master/buildbot/test/unit/test_changes_svnpoller.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 # This file is part of Buildbot.  Buildbot is free software: you can
 # redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation, version 2.
@@ -156,7 +157,7 @@
 <date>2006-10-01T19:35:10.215692Z</date>
 <paths>
 <path
-   action="M">/sample/branch/main.c</path>
+   action="M">/sample/branch/c\xcc\xa7main.c</path>
 </paths>
 <msg>commit_on_branch</msg>
 </logentry>
@@ -379,7 +380,7 @@ def test_create_changes(self):
         self.assertEqual(changes[0]['project'], '')
         self.assertEqual(changes[0]['repository'], base)
         self.assertEqual(changes[1]['branch'], "branch")
-        self.assertEqual(changes[1]['files'], ["main.c"])
+        self.assertEqual(changes[1]['files'], [u"çmain.c"])
         self.assertEqual(changes[1]['revision'], '3')
         self.assertEqual(changes[1]['project'], '')
         self.assertEqual(changes[1]['repository'], base)
@@ -388,7 +389,7 @@ def test_create_changes(self):
         self.assertEqual(len(changes), 1)
         self.assertEqual(changes[0]['branch'], None)
         self.assertEqual(changes[0]['revision'], '4')
-        self.assertEqual(changes[0]['files'], ["version.c"])
+        self.assertEqual(changes[0]['files'], [u"version.c"])
 
         # r5 should *not* create a change as it's a branch deletion
         changes = s.create_changes([logentries[5]])
@@ -399,7 +400,7 @@ def test_create_changes(self):
         self.assertEqual(len(changes), 1)
         self.assertEqual(changes[0]['branch'], 'branch')
         self.assertEqual(changes[0]['revision'], '6')
-        self.assertEqual(changes[0]['files'], ["version.c"])
+        self.assertEqual(changes[0]['files'], [u"version.c"])
 
     def makeInfoExpect(self, password='bbrocks'):
         args = ['svn', 'info', '--xml', '--non-interactive', sample_base,
@@ -443,7 +444,7 @@ def custom_split_file(path):
         self.assertEqual(changes[0]['codebase'], "overridden-codebase")
 
         self.assertEqual(changes[1]['branch'], "branch")
-        self.assertEqual(changes[1]['files'], ["main.c"])
+        self.assertEqual(changes[1]['files'], [u'çmain.c'])
         self.assertEqual(changes[1]['revision'], '3')
         self.assertEqual(changes[1]['project'], "overridden-project")
         self.assertEqual(changes[1]['repository'], "overridden-repository")
@@ -489,7 +490,7 @@ def check_third(_):
                 'category': None,
                 'codebase': None,
                 'comments': u'make_branch',
-                'files': [''],
+                'files': [u''],
                 'project': '',
                 'properties': {},
                 'repository': 'file:///usr/home/warner/stuff/Projects/Buildbot/trees/misc/_trial_temp/test_vc/repositories/SVN-Repository/sample',
@@ -514,7 +515,7 @@ def check_fourth(_):
                 'category': None,
                 'codebase': None,
                 'comments': u'commit_on_branch',
-                'files': ['main.c'],
+                'files': [u'çmain.c'],
                 'project': '',
                 'properties': {},
                 'repository': 'file:///usr/home/warner/stuff/Projects/Buildbot/trees/misc/_trial_temp/test_vc/repositories/SVN-Repository/sample',
@@ -528,7 +529,7 @@ def check_fourth(_):
                 'category': None,
                 'codebase': None,
                 'comments': u'revised_to_2',
-                'files': ['version.c'],
+                'files': [u'version.c'],
                 'project': '',
                 'properties': {},
                 'repository': 'file:///usr/home/warner/stuff/Projects/Buildbot/trees/misc/_trial_temp/test_vc/repositories/SVN-Repository/sample',