Repository: pantsbuild/pants

Issue:
./pants commands fail with TruncatedHeaderError after pantsd log rotation until daemon is manually killed
for users with highly active filesystems, it appears that the built in python log rotation causes the underlying stream in the daemon's `_LoggerStream` to go away which leads to this traceback in the pantsd.log:

```
F0117 15:10:04.500231 12664 process_manager.py:463] Traceback (most recent call last):
  File "/Users/redacted/.pex/install/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl.ac6c80da047e6ed09228262b774906b65ebf6918/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl/pants/pantsd/process_manager.py", line 461, in daemonize
    self.post_fork_child(**post_fork_child_opts or {})
  File "/Users/redacted/.pex/install/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl.ac6c80da047e6ed09228262b774906b65ebf6918/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl/pants/bin/daemon_pants_runner.py", line 172, in post_fork_child
    self._exiter.set_except_hook(sys.stderr)
  File "/Users/redacted/.pex/install/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl.ac6c80da047e6ed09228262b774906b65ebf6918/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl/pants/base/exiter.py", line 123, in set_except_hook
    self._setup_faulthandler(trace_stream or sys.stderr)
  File "/Users/redacted/.pex/install/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl.ac6c80da047e6ed09228262b774906b65ebf6918/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl/pants/base/exiter.py", line 117, in _setup_faulthandler
    faulthandler.enable(trace_stream)
  File "/Users/redacted/.pex/install/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl.ac6c80da047e6ed09228262b774906b65ebf6918/pantsbuild.pants-1.4.0.dev20+3346017507-cp27-none-macosx_10_8_x86_64.whl/pants/pantsd/pants_daemon.py", line 61, in fileno
    return self._stream.fileno()
ValueError: I/O operation on closed file
```

and this symptom on the front-end:

```
Exception caught: (<class 'pants.java.nailgun_client.NailgunError'>)
  File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 73, in <module>
    main()
  File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 69, in main
    PantsLoader.run()
  File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 65, in run
    cls.load_and_execute(entrypoint)
  File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 58, in load_and_execute
    entrypoint_main()
  File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_exe.py", line 32, in main
    PantsRunner(exiter).run()
  File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_runner.py", line 38, in run
    return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run()
  File "/Users/kwilson/dev/pants/src/python/pants/bin/remote_pants_runner.py", line 118, in run
    self._connect_and_execute(port)
  File "/Users/kwilson/dev/pants/src/python/pants/bin/remote_pants_runner.py", line 107, in _connect_and_execute
    result = client.execute(self.PANTS_COMMAND, *self._args, **modified_env)
  File "/Users/kwilson/dev/pants/src/python/pants/java/nailgun_client.py", line 185, in execute
    .format(self._host, self._port, e))

Exception message: Problem in nailgun protocol with nailgun server at 127.0.0.1:61991: TruncatedHeaderError(u"Failed to read nailgun chunk header (TruncatedRead(u'Expected 5 bytes before socket shutdown, instead received 0',)).",)
```

repro:

first, drop the max value [here](https://github.com/pantsbuild/pants/blob/e3a227601de8c4179eae852e77c6673cc334b075/src/python/pants/logging/setup.py#L66) down lower than 10MB, then:

window 1:
```
$ cd $REPO && watch -n1 "find . -exec touch {} \;"
```

window 2 (in a loop):
```
$ cd $REPO && PANTS_ENABLE_PANTSD=True ./pants list :
```




Discussion:

Resolving patch:
diff --git a/src/python/pants/logging/setup.py b/src/python/pants/logging/setup.py
--- a/src/python/pants/logging/setup.py
+++ b/src/python/pants/logging/setup.py
@@ -9,13 +9,12 @@
 import os
 import time
 from collections import namedtuple
-from logging import Formatter, StreamHandler
-from logging.handlers import RotatingFileHandler
+from logging import FileHandler, Formatter, StreamHandler
 
 from pants.util.dirutil import safe_mkdir
 
 
-class LoggingSetupResult(namedtuple('LoggingSetupResult', ['log_filename', 'log_stream'])):
+class LoggingSetupResult(namedtuple('LoggingSetupResult', ['log_filename', 'log_handler'])):
   """A structured result for logging setup."""
 
 
@@ -48,7 +47,7 @@ def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name
   # standard logging knob.
 
   log_filename = None
-  log_stream = None
+  file_handler = None
 
   logger = logging.getLogger(scope)
   for handler in logger.handlers:
@@ -63,28 +62,29 @@ def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name
   if log_dir:
     safe_mkdir(log_dir)
     log_filename = os.path.join(log_dir, log_name or 'pants.log')
-    file_handler = RotatingFileHandler(log_filename, maxBytes=10 * 1024 * 1024, backupCount=4)
-    log_stream = file_handler.stream
+    file_handler = FileHandler(log_filename)
 
     class GlogFormatter(Formatter):
       LEVEL_MAP = {
-          logging.FATAL: 'F',
-          logging.ERROR: 'E',
-          logging.WARN: 'W',
-          logging.INFO: 'I',
-          logging.DEBUG: 'D'}
+        logging.FATAL: 'F',
+        logging.ERROR: 'E',
+        logging.WARN: 'W',
+        logging.INFO: 'I',
+        logging.DEBUG: 'D'
+      }
 
       def format(self, record):
         datetime = time.strftime('%m%d %H:%M:%S', time.localtime(record.created))
         micros = int((record.created - int(record.created)) * 1e6)
         return '{levelchar}{datetime}.{micros:06d} {process} {filename}:{lineno}] {msg}'.format(
-            levelchar=self.LEVEL_MAP[record.levelno],
-            datetime=datetime,
-            micros=micros,
-            process=record.process,
-            filename=record.filename,
-            lineno=record.lineno,
-            msg=record.getMessage())
+          levelchar=self.LEVEL_MAP[record.levelno],
+          datetime=datetime,
+          micros=micros,
+          process=record.process,
+          filename=record.filename,
+          lineno=record.lineno,
+          msg=record.getMessage()
+        )
 
     file_handler.setFormatter(GlogFormatter())
     file_handler.setLevel(level)
@@ -95,4 +95,4 @@ def format(self, record):
   # This routes warnings through our loggers instead of straight to raw stderr.
   logging.captureWarnings(True)
 
-  return LoggingSetupResult(log_filename, log_stream)
+  return LoggingSetupResult(log_filename, file_handler)
diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py
--- a/src/python/pants/pantsd/pants_daemon.py
+++ b/src/python/pants/pantsd/pants_daemon.py
@@ -36,16 +36,16 @@
 class _LoggerStream(object):
   """A sys.{stdout,stderr} replacement that pipes output to a logger."""
 
-  def __init__(self, logger, log_level, logger_stream):
+  def __init__(self, logger, log_level, handler):
     """
     :param logging.Logger logger: The logger instance to emit writes to.
     :param int log_level: The log level to use for the given logger.
-    :param file logger_stream: The underlying file object the logger is writing to, for
-                               determining the fileno to support faulthandler logging.
+    :param Handler handler: The underlying log handler, for determining the fileno
+                            to support faulthandler logging.
     """
     self._logger = logger
     self._log_level = log_level
-    self._stream = logger_stream
+    self._handler = handler
 
   def write(self, msg):
     for line in msg.rstrip().splitlines():
@@ -58,7 +58,7 @@ def isatty(self):
     return False
 
   def fileno(self):
-    return self._stream.fileno()
+    return self._handler.stream.fileno()
 
 
 class PantsDaemon(FingerprintedProcessManager):
@@ -238,11 +238,11 @@ def _pantsd_logging(self):
       # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`.
       # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements
       # for `1,2`, and allow them to be used via `stdio_as`.
-      sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_stream)
-      sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_stream)
+      sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler)
+      sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler)
 
       self._logger.debug('logging initialized')
-      yield result.log_stream
+      yield result.log_handler.stream
 
   def _setup_services(self, services):
     assert self._lifecycle_lock is not None, 'PantsDaemon lock has not been set!'


Test patch: