Repository: pantsbuild/pants

Issue:
pantsd startup timeout likely on a busy machine
On a machine with a lot of CPU usage, hitting this timeout is pretty easy:
https://github.com/pantsbuild/pants/blob/ab40cc17389859ac0ab6879e826c766c5d5f8eb3/src/python/pants/pantsd/pants_daemon.py#L322-L323

We should do one or more of:

1) allowing that timeout to be configured
2) making it a multiple of some other known timeout that approximates repository size
3) logging a "still waiting for X" message that renders after a few iterations of https://github.com/pantsbuild/pants/blob/1148b07af07456d7e0495f4613803cb34993d689/src/python/pants/pantsd/process_manager.py#L109


Discussion:
Have started this one.

Resolving patch:
diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py
--- a/src/python/pants/pantsd/pants_daemon.py
+++ b/src/python/pants/pantsd/pants_daemon.py
@@ -319,8 +319,8 @@ def maybe_launch(self):
         self.terminate(include_watchman=False)
         self._logger.debug('launching pantsd')
         self.daemon_spawn()
-        # Wait up to 10 seconds for pantsd to write its pidfile so we can display the pid to the user.
-        self.await_pid(10)
+        # Wait up to 60 seconds for pantsd to write its pidfile.
+        self.await_pid(60)
       listening_port = self.read_named_socket('pailgun', int)
       pantsd_pid = self.pid
     self._logger.debug('released lock: {}'.format(self.process_lock))
diff --git a/src/python/pants/pantsd/process_manager.py b/src/python/pants/pantsd/process_manager.py
--- a/src/python/pants/pantsd/process_manager.py
+++ b/src/python/pants/pantsd/process_manager.py
@@ -74,7 +74,8 @@ class ProcessMetadataManager(object):
   class MetadataError(Exception): pass
   class Timeout(Exception): pass
 
-  FILE_WAIT_SEC = 10
+  FAIL_WAIT_SEC = 10
+  INFO_INTERVAL_SEC = 5
   WAIT_INTERVAL_SEC = .1
 
   def __init__(self, metadata_base_dir=None):
@@ -105,38 +106,47 @@ def _maybe_cast(item, caster):
       return item
 
   @classmethod
-  def _deadline_until(cls, closure, timeout, wait_interval=WAIT_INTERVAL_SEC):
+  def _deadline_until(cls, closure, action_msg, timeout=FAIL_WAIT_SEC,
+                      wait_interval=WAIT_INTERVAL_SEC, info_interval=INFO_INTERVAL_SEC):
     """Execute a function/closure repeatedly until a True condition or timeout is met.
 
     :param func closure: the function/closure to execute (should not block for long periods of time
                          and must return True on success).
+    :param str action_msg: a description of the action that is being executed, to be rendered as
+                           info while we wait, and as part of any rendered exception.
     :param float timeout: the maximum amount of time to wait for a true result from the closure in
                           seconds. N.B. this is timing based, so won't be exact if the runtime of
                           the closure exceeds the timeout.
     :param float wait_interval: the amount of time to sleep between closure invocations.
+    :param float info_interval: the amount of time to wait before and between reports via info
+                                logging that we're still waiting for the closure to succeed.
     :raises: :class:`ProcessManager.Timeout` on execution timeout.
     """
-    deadline = time.time() + timeout
+    now = time.time()
+    deadline = now + timeout
+    info_deadline = now + info_interval
     while 1:
       if closure():
         return True
-      elif time.time() > deadline:
-        raise cls.Timeout('exceeded timeout of {} seconds for {}'.format(timeout, closure))
+
+      now = time.time()
+      if now > deadline:
+        raise cls.Timeout('exceeded timeout of {} seconds while waiting for {}'.format(timeout, action_msg))
+
+      if now > info_deadline:
+        logger.info('waiting for {}...'.format(action_msg))
+        info_deadline = info_deadline + info_interval
       elif wait_interval:
         time.sleep(wait_interval)
 
   @classmethod
-  def _wait_for_file(cls, filename, timeout=FILE_WAIT_SEC, want_content=True):
+  def _wait_for_file(cls, filename, timeout=FAIL_WAIT_SEC, want_content=True):
     """Wait up to timeout seconds for filename to appear with a non-zero size or raise Timeout()."""
     def file_waiter():
       return os.path.exists(filename) and (not want_content or os.path.getsize(filename))
 
-    try:
-      return cls._deadline_until(file_waiter, timeout)
-    except cls.Timeout:
-      # Re-raise with a more helpful exception message.
-      raise cls.Timeout('exceeded timeout of {} seconds while waiting for file {} to appear'
-                         .format(timeout, filename))
+    action_msg = 'file {} to appear'.format(filename)
+    return cls._deadline_until(file_waiter, action_msg, timeout=timeout)
 
   def _get_metadata_dir_by_name(self, name):
     """Retrieve the metadata dir by name.
@@ -407,7 +417,7 @@ def terminate(self, signal_chain=KILL_CHAIN, kill_wait=KILL_WAIT_SEC, purge=True
 
         # Wait up to kill_wait seconds to terminate or move onto the next signal.
         try:
-          if self._deadline_until(self.is_dead, kill_wait):
+          if self._deadline_until(self.is_dead, 'daemon to exit', timeout=kill_wait):
             alive = False
             logger.debug('successfully terminated pid {}'.format(pid))
             break


Test patch:
diff --git a/tests/python/pants_test/base_test.py b/tests/python/pants_test/base_test.py
--- a/tests/python/pants_test/base_test.py
+++ b/tests/python/pants_test/base_test.py
@@ -6,9 +6,11 @@
                         unicode_literals, with_statement)
 
 import itertools
+import logging
 import os
 import unittest
 from collections import defaultdict
+from contextlib import contextmanager
 from tempfile import mkdtemp
 from textwrap import dedent
 
@@ -455,3 +457,31 @@ def get_bootstrap_options(self, cli_options=()):
     # Can't parse any options without a pants.ini.
     self.create_file('pants.ini')
     return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope()
+
+  class LoggingRecorder(object):
+    """Simple logging handler to record warnings."""
+
+    def __init__(self):
+      self._records = []
+      self.level = logging.DEBUG
+
+    def handle(self, record):
+      self._records.append(record)
+
+    def _messages_for_level(self, levelname):
+      return ['{}: {}'.format(record.name, record.getMessage())
+              for record in self._records if record.levelname == levelname]
+
+    def infos(self):
+      return self._messages_for_level('INFO')
+
+    def warnings(self):
+      return self._messages_for_level('WARNING')
+
+  @contextmanager
+  def captured_logging(self):
+    handler = self.LoggingRecorder()
+    logger = logging.getLogger('')
+    logger.addHandler(handler)
+    yield handler
+    logger.removeHandler(handler)
diff --git a/tests/python/pants_test/pantsd/test_process_manager.py b/tests/python/pants_test/pantsd/test_process_manager.py
--- a/tests/python/pants_test/pantsd/test_process_manager.py
+++ b/tests/python/pants_test/pantsd/test_process_manager.py
@@ -141,7 +141,10 @@ def test_readwrite_metadata_by_name(self):
 
   def test_deadline_until(self):
     with self.assertRaises(self.pmm.Timeout):
-      self.pmm._deadline_until(lambda: False, timeout=.1)
+      with self.captured_logging() as captured:
+        self.pmm._deadline_until(lambda: False, 'the impossible', timeout=.5, info_interval=.1)
+    self.assertTrue(4 <= len(captured.infos()) <= 6,
+                    'Expected between 4 and 6 infos, got: {}'.format(captured.infos()))
 
   def test_wait_for_file(self):
     with temporary_dir() as td:
diff --git a/tests/python/pants_test/util/BUILD b/tests/python/pants_test/util/BUILD
--- a/tests/python/pants_test/util/BUILD
+++ b/tests/python/pants_test/util/BUILD
@@ -109,6 +109,7 @@ python_tests(
   sources = ['test_osutil.py'],
   dependencies = [
     'src/python/pants/util:osutil',
+    'tests/python/pants_test:base_test',
   ]
 )
 
diff --git a/tests/python/pants_test/util/test_osutil.py b/tests/python/pants_test/util/test_osutil.py
--- a/tests/python/pants_test/util/test_osutil.py
+++ b/tests/python/pants_test/util/test_osutil.py
@@ -5,30 +5,11 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-import logging
-import unittest
-from contextlib import contextmanager
-
 from pants.util.osutil import OS_ALIASES, known_os_names, normalize_os_name
+from pants_test.base_test import BaseTest
 
 
-class OsutilTest(unittest.TestCase):
-
-  class WarningRecorder(object):
-    """Simple logging handler to record warnings."""
-
-    def __init__(self):
-      self.warning_list = []
-      self.level = logging.WARNING
-
-    def handle(self, record):
-      self.warning_list.append('{}: {}'.format(record.name, record.getMessage()))
-
-  @contextmanager
-  def warnings(self):
-    handler = self.WarningRecorder()
-    logging.getLogger('').addHandler(handler)
-    yield handler.warning_list
+class OsutilTest(BaseTest):
 
   def test_alias_normalization(self):
     for normal_os, aliases in OS_ALIASES.items():
@@ -41,14 +22,14 @@ def test_keys_in_aliases(self):
 
   def test_no_warnings_on_known_names(self):
     for name in known_os_names():
-      with self.warnings() as warning_list:
+      with self.captured_logging() as captured:
         normalize_os_name(name)
-        self.assertEqual(0, len(warning_list),
-                         'Recieved unexpected warnings: {}'.format(warning_list))
+        self.assertEqual(0, len(captured.warnings()),
+                         'Recieved unexpected warnings: {}'.format(captured.warnings()))
 
   def test_warnings_on_unknown_names(self):
     name = 'I really hope no one ever names an operating system with this string.'
-    with self.warnings() as warning_list:
+    with self.captured_logging() as captured:
       normalize_os_name(name)
-      self.assertEqual(1, len(warning_list),
-                       'Expected exactly one warning, but got: {}'.format(warning_list))
+      self.assertEqual(1, len(captured.warnings()),
+                       'Expected exactly one warning, but got: {}'.format(captured.warnings()))