Repository: buildbot/buildbot

Issue:
FakeTransport error while starting buildbot.
I'm trying to understand why I'm getting the following error, even though the master seems to start and run normally. Googling for this message was fruitless. I'm using buildbot 2.9.3 and Twisted 20.3.0 (both latest). Do I need to downgrade Twisted?

```
(venv) alex@solitude:~/build_bot$ buildbot start master
Following twistd.log until startup finished..
Unhandled Error
Traceback (most recent call last):
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/python/log.py", line 103, in callWithLogger
    return callWithContext({"system": lp}, func, *args, **kw)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/python/log.py", line 86, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
--- <exception caught here> ---
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
    why = selectable.doRead()
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/internet/process.py", line 252, in doRead
    return fdesc.readFromFD(self.fd, self.dataReceived)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/internet/fdesc.py", line 94, in readFromFD
    callback(output)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/internet/process.py", line 256, in dataReceived
    self.proc.childDataReceived(self.name, data)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/internet/process.py", line 922, in childDataReceived
    self.proto.childDataReceived(name, data)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/internet/protocol.py", line 640, in childDataReceived
    self.outReceived(data)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/buildbot/scripts/logwatcher.py", line 49, in outReceived
    self.lw.dataReceived(data)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/protocols/basic.py", line 452, in dataReceived
    return self.lineLengthExceeded(line)
  File "/home/alex/build_bot/venv/lib/python3.7/site-packages/twisted/protocols/basic.py", line 484, in lineLengthExceeded
    return self.transport.loseConnection()
builtins.AttributeError: 'FakeTransport' object has no attribute 'loseConnection'

The buildmaster took more than 10.0 seconds to start, so we were unable to
confirm that it started correctly. Please 'tail twistd.log' and look for a line
that says 'BuildMaster is running' to verify correct startup.
```

_Originally posted by @alexreinking in https://github.com/buildbot/buildbot/discussions/5702_


Discussion:
Thanks for the bug report. I haven't yet looked deeply into Twisted, but it seems that we were relying on their non-documented behavior, which has changed.

The error is harmless.
> Thanks for the bug report. I haven't yet looked deeply into Twisted, but it seems that we were relying on their non-documented behavior, which has changed.

Do you know which is the most recent version that does not display this error?

> The error is harmless.

Good to know, thanks.
> Do you know which is the most recent version that does not display this error?

Through trial and error, it appears it is 17.9.0

Resolving patch:
diff --git a/master/buildbot/scripts/logwatcher.py b/master/buildbot/scripts/logwatcher.py
--- a/master/buildbot/scripts/logwatcher.py
+++ b/master/buildbot/scripts/logwatcher.py
@@ -21,7 +21,6 @@
 from twisted.internet import error
 from twisted.internet import protocol
 from twisted.internet import reactor
-from twisted.protocols.basic import LineOnlyReceiver
 from twisted.python.failure import Failure
 
 from buildbot.util import unicode2bytes
@@ -49,10 +48,41 @@ def outReceived(self, data):
         self.lw.dataReceived(data)
 
     def errReceived(self, data):
-        print("ERR: '{}'".format(data))
+        self.lw.print_output("ERR: '{}'".format(data))
+
+
+class LineOnlyLongLineReceiver(protocol.Protocol):
+    """
+    This is almost the same as Twisted's LineOnlyReceiver except that long lines are handled
+    appropriately.
+    """
+    _buffer = b''
+    delimiter = b'\r\n'
+    MAX_LENGTH = 16384
+
+    def dataReceived(self, data):
+        lines = (self._buffer + data).split(self.delimiter)
+        self._buffer = lines.pop(-1)
+        for line in lines:
+            if self.transport.disconnecting:
+                # this is necessary because the transport may be told to lose
+                # the connection by a line within a larger packet, and it is
+                # important to disregard all the lines in that packet following
+                # the one that told it to close.
+                return
+            if len(line) > self.MAX_LENGTH:
+                self.lineLengthExceeded(line)
+            else:
+                self.lineReceived(line)
+
+    def lineReceived(self, line):
+        raise NotImplementedError
+
+    def lineLengthExceeded(self, line):
+        raise NotImplementedError
 
 
-class LogWatcher(LineOnlyReceiver):
+class LogWatcher(LineOnlyLongLineReceiver):
     POLL_INTERVAL = 0.1
     TIMEOUT_DELAY = 10.0
     delimiter = unicode2bytes(os.linesep)
@@ -69,8 +99,7 @@ def __init__(self, logfile, timeout=None, _reactor=reactor):
 
     def start(self):
         # If the log file doesn't exist, create it now.
-        if not os.path.exists(self.logfile):
-            open(self.logfile, 'a').close()
+        self.create_logfile(self.logfile)
 
         # return a Deferred that fires when the reconfig process has
         # finished. It errbacks with TimeoutError if the startup has not
@@ -121,6 +150,17 @@ def finished(self, results):
         self.in_reconfig = False
         self.d.callback(results)
 
+    def create_logfile(self, path):  # pragma: no cover
+        if not os.path.exists(path):
+            open(path, 'a').close()
+
+    def print_output(self, output):  # pragma: no cover
+        print(output)
+
+    def lineLengthExceeded(self, line):
+        msg = 'Got an a very long line in the log (length {} bytes), ignoring'.format(len(line))
+        self.print_output(msg)
+
     def lineReceived(self, line):
         if not self.running:
             return None
@@ -130,7 +170,7 @@ def lineReceived(self, line):
             self.in_reconfig = True
 
         if self.in_reconfig:
-            print(line.decode())
+            self.print_output(line.decode())
 
         # certain lines indicate progress, so we "cancel" the timeout
         # and it will get re-added when it fires


Test patch:
diff --git a/master/buildbot/test/unit/scripts/test_logwatcher.py b/master/buildbot/test/unit/scripts/test_logwatcher.py
--- a/master/buildbot/test/unit/scripts/test_logwatcher.py
+++ b/master/buildbot/test/unit/scripts/test_logwatcher.py
@@ -26,10 +26,26 @@
 from buildbot.scripts.logwatcher import ReconfigError
 from buildbot.test.util import dirs
 from buildbot.test.util.misc import TestReactorMixin
+from buildbot.util import unicode2bytes
+
+
+class MockedLogWatcher(LogWatcher):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.printed_output = []
+        self.created_paths = []
+
+    def create_logfile(self, path):
+        self.created_paths.append(path)
+
+    def print_output(self, output):
+        self.printed_output.append(output)
 
 
 class TestLogWatcher(unittest.TestCase, dirs.DirsMixin, TestReactorMixin):
 
+    delimiter = unicode2bytes(os.linesep)
+
     def setUp(self):
         self.setUpDirs('workdir')
         self.addCleanup(self.tearDownDirs)
@@ -39,17 +55,17 @@ def setUp(self):
         self.reactor.spawnProcess = mock.Mock(return_value=self.spawned_process)
 
     def test_start(self):
-        lw = LogWatcher('workdir/test.log', _reactor=self.reactor)
+        lw = MockedLogWatcher('workdir/test.log', _reactor=self.reactor)
         lw._start = mock.Mock()
 
         lw.start()
         self.reactor.spawnProcess.assert_called()
-        self.assertTrue(os.path.exists('workdir/test.log'))
+        self.assertEqual(lw.created_paths, ['workdir/test.log'])
         self.assertTrue(lw.running)
 
     @defer.inlineCallbacks
     def test_success_before_timeout(self):
-        lw = LogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
+        lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
         d = lw.start()
         self.reactor.advance(4.9)
         lw.lineReceived(b'BuildMaster is running')
@@ -58,7 +74,7 @@ def test_success_before_timeout(self):
 
     @defer.inlineCallbacks
     def test_failure_after_timeout(self):
-        lw = LogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
+        lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
         d = lw.start()
         self.reactor.advance(5.1)
         lw.lineReceived(b'BuildMaster is running')
@@ -67,7 +83,7 @@ def test_failure_after_timeout(self):
 
     @defer.inlineCallbacks
     def test_progress_restarts_timeout(self):
-        lw = LogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
+        lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
         d = lw.start()
         self.reactor.advance(4.9)
         lw.lineReceived(b'added builder')
@@ -76,6 +92,42 @@ def test_progress_restarts_timeout(self):
         res = yield d
         self.assertEqual(res, 'buildmaster')
 
+    @defer.inlineCallbacks
+    def test_handles_very_long_lines(self):
+        lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
+        d = lw.start()
+        lw.dataReceived(b't' * lw.MAX_LENGTH * 2 + self.delimiter + b'BuildMaster is running' +
+                        self.delimiter)
+        res = yield d
+        self.assertEqual(lw.printed_output, [
+            'Got an a very long line in the log (length 32768 bytes), ignoring'
+        ])
+        self.assertEqual(res, 'buildmaster')
+
+    @defer.inlineCallbacks
+    def test_handles_very_long_lines_separate_packet(self):
+        lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
+        d = lw.start()
+        lw.dataReceived(b't' * lw.MAX_LENGTH * 2)
+        lw.dataReceived(self.delimiter + b'BuildMaster is running' + self.delimiter)
+        res = yield d
+        self.assertEqual(lw.printed_output, [
+            'Got an a very long line in the log (length 32768 bytes), ignoring'
+        ])
+        self.assertEqual(res, 'buildmaster')
+
+    @defer.inlineCallbacks
+    def test_handles_very_long_lines_separate_packet_with_newline(self):
+        lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
+        d = lw.start()
+        lw.dataReceived(b't' * lw.MAX_LENGTH * 2 + self.delimiter)
+        lw.dataReceived(b'BuildMaster is running' + self.delimiter)
+        res = yield d
+        self.assertEqual(lw.printed_output, [
+            'Got an a very long line in the log (length 32768 bytes), ignoring'
+        ])
+        self.assertEqual(res, 'buildmaster')
+
     @defer.inlineCallbacks
     def test_matches_lines(self):
         lines_and_expected = [
@@ -90,8 +142,7 @@ def test_matches_lines(self):
         ]
 
         for line, expected in lines_and_expected:
-            lw = LogWatcher('workdir/test.log', timeout=5,
-                            _reactor=self.reactor)
+            lw = MockedLogWatcher('workdir/test.log', timeout=5, _reactor=self.reactor)
             d = lw.start()
             lw.lineReceived(line)