Repository: buildbot/buildbot

Issue:
utf8 decode problem with 0.9.12 worker
We have some issues with 0.9.12 workers, where it gets a backtrace like:

```
2017-10-13 13:21:37+0100 [-] Unhandled Error
        Traceback (most recent call last):
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/python/log.py", line 103, in callWithLogger
            return callWithContext({"system": lp}, func, *args, **kw)
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/python/log.py", line 86, in callWithContext
            return context.call({ILogContext: newCtx}, func, *args, **kw)
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/python/context.py", line 122, in callWithContext
            return self.currentContext().callWithContext(ctx, func, *args, **kw)
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/python/context.py", line 85, in callWithContext
            return func(*args,**kw)
        --- <exception caught here> ---
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
            why = selectable.doRead()
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/internet/process.py", line 1118, in doRead
            lambda data: self.proto.childDataReceived(1, data))
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/internet/fdesc.py", line 94, in readFromFD
            callback(output)
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/internet/process.py", line 1118, in <lambda>
            lambda data: self.proto.childDataReceived(1, data))
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/twisted/internet/protocol.py", line 636, in childDataReceived
            self.outReceived(data)
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/buildbot_worker/runprocess.py", line 216, in outReceived
            data, self.command.builder.unicode_encoding)
          File "/home/builder/sandbox/local/lib/python2.7/site-packages/buildbot_worker/compat.py", line 87, in bytes2unicode
            return text_type(x, encoding, errors)
          File "/home/builder/sandbox/lib/python2.7/encodings/utf_8.py", line 16, in decode
            return codecs.utf_8_decode(input, errors, True)
        exceptions.UnicodeDecodeError: 'utf8' codec can't decode byte 0xe2 in position 288: unexpected end of data
        
2017-10-13 13:21:37+0100 [-] command finished with signal 1, exit code None, elapsedTime: 1828.524536
```

This happens when building qtwebkit, which is a very long build, and some of the warnings do have utf8 multi-byte characters (in particular, the quotes that it uses). Here is an example of a build that gets this:  https://flathub.org/builds/#/builders/6/builds/633



Discussion:
@rodrigc looks like a regression with 0.9.12. Can you look at this?
I have a similar exception with 0.9.12 worker (different backtrace):
```
2017-10-13T19:34:16+0200 [-] Unhandled Error
        Traceback (most recent call last):
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 1243, in run
            self.mainLoop()
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 1252, in mainLoop
            self.runUntilCurrent()
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 878, in runUntilCurrent
            call.func(*call.args, **call.kw)
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/task.py", line 239, in __call__
            d = defer.maybeDeferred(self.f, *self.a, **self.kw)
        --- <exception caught here> ---
          File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 150, in maybeDeferred
            result = f(*args, **kw)
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/runprocess.py", line 163, in poll
            self.command.addLogfile(self.name, data)
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/runprocess.py", line 760, in addLogfile
            self._addToBuffers(('log', name), data)
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/runprocess.py", line 736, in _addToBuffers
            self._sendBuffers()
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/runprocess.py", line 718, in _sendBuffers
            self._sendMessage(msg)
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/runprocess.py", line 664, in _sendMessage
            msg = self._collapseMsg(msg)
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/runprocess.py", line 650, in _collapseMsg
            m = bytes2unicode(m, self.builder.unicode_encoding)
          File "/usr/local/lib/python2.7/dist-packages/buildbot_worker/compat.py", line 87, in bytes2unicode
            return text_type(x, encoding, errors)
          File "/usr/lib/python2.7/encodings/utf_8.py", line 16, in decode
            return codecs.utf_8_decode(input, errors, True)
        exceptions.UnicodeDecodeError: 'utf8' codec can't decode bytes in position 9998-9999: unexpected end of data
```

Resolving patch:
diff --git a/worker/buildbot_worker/runprocess.py b/worker/buildbot_worker/runprocess.py
--- a/worker/buildbot_worker/runprocess.py
+++ b/worker/buildbot_worker/runprocess.py
@@ -32,6 +32,7 @@
 import subprocess
 import sys
 import traceback
+from codecs import getincrementaldecoder
 from collections import deque
 from tempfile import NamedTemporaryFile
 
@@ -186,6 +187,10 @@ def __init__(self, command):
         self.pending_stdin = b""
         self.stdin_finished = False
         self.killed = False
+        decoderFactory = getincrementaldecoder(
+            self.command.builder.unicode_encoding)
+        self.stdoutDecode = decoderFactory(errors='replace')
+        self.stderrDecode = decoderFactory(errors='replace')
 
     def setStdin(self, data):
         assert not self.connected
@@ -212,16 +217,14 @@ def connectionMade(self):
     def outReceived(self, data):
         if self.debug:
             log.msg("RunProcessPP.outReceived")
-        data = bytes2unicode(
-            data, self.command.builder.unicode_encoding)
-        self.command.addStdout(data)
+        decodedData = self.stdoutDecode.decode(data)
+        self.command.addStdout(decodedData)
 
     def errReceived(self, data):
         if self.debug:
             log.msg("RunProcessPP.errReceived")
-        data = bytes2unicode(
-            data, self.command.builder.unicode_encoding)
-        self.command.addStderr(data)
+        decodedData = self.stderrDecode.decode(data)
+        self.command.addStderr(decodedData)
 
     def processEnded(self, status_object):
         if self.debug:


Test patch:
diff --git a/worker/buildbot_worker/test/unit/test_runprocess.py b/worker/buildbot_worker/test/unit/test_runprocess.py
--- a/worker/buildbot_worker/test/unit/test_runprocess.py
+++ b/worker/buildbot_worker/test/unit/test_runprocess.py
@@ -169,6 +169,26 @@ def check(ign):
         d.addCallback(check)
         return d
 
+    def test_incrementalDecoder(self):
+        b = FakeWorkerForBuilder(self.basedir)
+        b.unicode_encoding = "utf-8"
+        s = runprocess.RunProcess(
+            b, stderrCommand("hello"), self.basedir, sendStderr=True)
+        pp = runprocess.RunProcessPP(s)
+        # u"\N{SNOWMAN} when encoded to utf-8 bytes is b"\xe2\x98\x83"
+        pp.outReceived(b"\xe2")
+        pp.outReceived(b"\x98\x83")
+        pp.errReceived(b"\xe2")
+        pp.errReceived(b"\x98\x83")
+        d = s.start()
+
+        def check(ign):
+            self.assertTrue({'stderr': u"\N{SNOWMAN}"} in b.updates)
+            self.assertTrue({'stdout': u"\N{SNOWMAN}"} in b.updates)
+            self.assertTrue({'rc': 0} in b.updates, b.show())
+        d.addCallback(check)
+        return d
+
     def testKeepStderr(self):
         b = FakeWorkerForBuilder(self.basedir)
         s = runprocess.RunProcess(