Repository: buildbot/buildbot

Issue:
[0.9.10]: SVN.purge fails when modified files contain non-ascii characters
Version info:

* Python version: 2.7.9
* Buildbot version: 0.9.10
* Twisted version: 17.5.0
* Client: macOS Sierra

I have a master configuration with the following:

```python
steps.SVN(name="SVN Source",                                                                                                                  
          repourl="http://the.host.com/svn/repository",                                                            
          mode='full', method='fresh')
```

with a modified file as so:

```xml
<entry
   path="Path/To/Content/Developers/François">
<wc-status
   item="unversioned"
   props="none">
</wc-status>
</entry>
```

Fails as so:

```
Traceback (most recent call last):
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1442, in gotResult
    _inlineCallbacks(r, g, deferred)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/python/failure.py", line 393, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
--- <exception caught here> ---
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/buildbot/process/buildstep.py", line 575, in startStep
    self.results = yield self.run()
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1386, in _inlineCallbacks
    result = g.send(result)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/buildbot/process/buildstep.py", line 705, in run
    results = yield self._start_deferred
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/python/failure.py", line 393, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/buildbot/steps/source/svn.py", line 131, in mode_full
    yield self.fresh()
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/build/buildbot/sandbox/local/lib/python2.7/site-packages/buildbot/steps/source/svn.py", line 356, in parseAndRemove
    filename = self.workdir + '/' + str(filename)
exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\u0327' in position 42: ordinal not in range(128)

```


Discussion:

Resolving patch:
diff --git a/master/buildbot/steps/source/svn.py b/master/buildbot/steps/source/svn.py
--- a/master/buildbot/steps/source/svn.py
+++ b/master/buildbot/steps/source/svn.py
@@ -32,6 +32,7 @@
 from buildbot.process import buildstep
 from buildbot.process import remotecommand
 from buildbot.steps.source.base import Source
+from buildbot.util import unicode2NativeString
 
 
 class SVN(Source):
@@ -353,7 +354,7 @@ def purge(self, ignore_ignores):
         def parseAndRemove(stdout):
             files = []
             for filename in self.getUnversionedFiles(stdout, self.keep_on_purge):
-                filename = self.workdir + '/' + str(filename)
+                filename = unicode2NativeString(self.build.path_module.join(self.workdir, filename))
                 files.append(filename)
             if not files:
                 d = defer.succeed(0)


Test patch:
diff --git a/master/buildbot/test/unit/test_steps_source_svn.py b/master/buildbot/test/unit/test_steps_source_svn.py
--- a/master/buildbot/test/unit/test_steps_source_svn.py
+++ b/master/buildbot/test/unit/test_steps_source_svn.py
@@ -1,3 +1,4 @@
+# -*- coding: utf8 -*-
 # This file is part of Buildbot.  Buildbot is free software: you can
 # redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation, version 2.
@@ -47,7 +48,7 @@ class TestSVN(sourcesteps.SourceStepMixin, unittest.TestCase):
                     <wc-status props="none" item="unversioned">
                     </wc-status>
                 </entry>
-                <entry path="svn_external_path/unversioned_file2">
+                <entry path="svn_external_path/unversioned_file2_uniçode">
                     <wc-status props="none" item="unversioned">
                     </wc-status>
                 </entry>
@@ -894,7 +895,7 @@ def test_mode_full_fresh_keep_on_purge(self):
                               stdout=self.svn_st_xml)
             + 0,
             Expect('rmdir', {'dir':
-                             ['wkdir/svn_external_path/unversioned_file2'],
+                             ['wkdir/svn_external_path/unversioned_file2_uniçode'],
                              'logEnviron': True,
                              'timeout': 1200})
             + 0,
@@ -1091,7 +1092,7 @@ def test_mode_full_clean_old_rmdir(self):
                              'timeout': 1200})
             + 0,
             Expect('rmdir', {'dir':
-                             'wkdir/svn_external_path/unversioned_file2',
+                             'wkdir/svn_external_path/unversioned_file2_uniçode',
                              'logEnviron': True,
                              'timeout': 1200})
             + 0,
@@ -1140,7 +1141,7 @@ def test_mode_full_clean_new_rmdir(self):
             + 0,
             Expect('rmdir', {'dir':
                              ['wkdir/svn_external_path/unversioned_file1',
-                              'wkdir/svn_external_path/unversioned_file2'],
+                              'wkdir/svn_external_path/unversioned_file2_uniçode'],
                              'logEnviron': True,
                              'timeout': 1200})
             + 0,
@@ -1308,7 +1309,7 @@ def test_mode_full_export_patch(self):
                               stdout=self.svn_st_xml)
             + 0,
             Expect('rmdir', dict(dir=['wkdir/svn_external_path/unversioned_file1',
-                                      'wkdir/svn_external_path/unversioned_file2'],
+                                      'wkdir/svn_external_path/unversioned_file2_uniçode'],
                                  logEnviron=True,
                                  timeout=1200))
             + 0,
@@ -1383,7 +1384,7 @@ def test_mode_full_export_patch_worker_2_16(self):
                               stdout=self.svn_st_xml)
             + 0,
             Expect('rmdir', dict(dir=['wkdir/svn_external_path/unversioned_file1',
-                                      'wkdir/svn_external_path/unversioned_file2'],
+                                      'wkdir/svn_external_path/unversioned_file2_uniçode'],
                                  logEnviron=True,
                                  timeout=1200))
             + 0,
@@ -1841,7 +1842,7 @@ def test_rmdir_fails_purge(self):
                               stdout=self.svn_st_xml)
             + 0,
             Expect('rmdir', {'dir':
-                             ['wkdir/svn_external_path/unversioned_file2'],
+                             ['wkdir/svn_external_path/unversioned_file2_uniçode'],
                              'logEnviron': True,
                              'timeout': 1200})
             + 1,
@@ -2025,6 +2026,24 @@ def test_getUnversionedFiles_no_item(self):
         self.assertEqual(
             ["svn_external_path/unversioned_file"], unversioned_files)
 
+    def test_getUnversionedFiles_unicode(self):
+        svn_st_xml = """<?xml version="1.0"?>
+        <status>
+            <target path=".">
+                <entry
+                   path="Path/To/Content/Developers/François">
+                    <wc-status
+                       item="unversioned"
+                       props="none">
+                    </wc-status>
+                </entry>
+            </target>
+        </status>
+        """
+        unversioned_files = list(svn.SVN.getUnversionedFiles(svn_st_xml, []))
+        self.assertEqual(
+            [u"Path/To/Content/Developers/François"], unversioned_files)
+
 
 class TestSvnUriCanonicalize(unittest.TestCase):
     # svn.SVN.svnUriCanonicalize() test method factory