Repository: pantsbuild/pants

Issue:
TestPantsDaemonIntegration.test_pantsd_sigterm is flaky
To my great shame, this test, added in #6574, has flaked: see https://travis-ci.org/pantsbuild/pants/jobs/510547772.
```
                     ==================== FAILURES ====================
                     _ TestPantsDaemonIntegration.test_pantsd_sigterm _
                     
                     self = <pants_test.pantsd.test_pantsd_integration.TestPantsDaemonIntegration testMethod=test_pantsd_sigterm>
                     
                         def test_pantsd_sigterm(self):
                           self._assert_pantsd_keyboardinterrupt_signal(
                             signal.SIGTERM,
                     >       ['Signal {signum} (SIGTERM) was raised. Exiting with failure.'.format(signum=signal.SIGTERM)])
                     
                     .pants.d/pyprep/sources/62676ab0dd65698bc674cce2191134fbe580f110/pants_test/pantsd/test_pantsd_integration.py:464: 
                     _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
                     .pants.d/pyprep/sources/62676ab0dd65698bc674cce2191134fbe580f110/pants_test/pantsd/test_pantsd_integration.py:450: in _assert_pantsd_keyboardinterrupt_signal
                         self.assertIn(msg, waiter_run.stderr_data)
                     E   AssertionError: 'Signal 15 (SIGTERM) was raised. Exiting with failure.' not found in 'INFO] waiting for file /home/travis/build/pantsbuild/pants/.pants.d/tmp/tmpnxjru4og.pants.d/.pids/pantsd/pid to appear...\nInterrupted by user:\nInterrupted by user over pailgun client!\n'
```

Not sure how to repro this yet, but running it enough times locally might work. I believe this happens when the timeout for reading more content from the nailgun client completes before all the output from the subprocess is written. It's not clear why the timeout being hit isn't causing a `warn` log, as it is expected to.


Discussion:
Happened again. I think the answer might actually be the opposite of the above, in that the subprocess likely completes before the timeout completes, so the `self._exit_reason` is raised, but there's no signal sent to the `pantsd-runner` subprocess. How can we stop this subprocess from completing before our timeout? We could either make this timeout smaller (probably the right idea), or we could figure out how to make our subprocess wait longer.

Resolving patch:
diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py
--- a/src/python/pants/bin/daemon_pants_runner.py
+++ b/src/python/pants/bin/daemon_pants_runner.py
@@ -337,6 +337,10 @@ def post_fork_child(self):
           'Encountered graceful termination exception {}; exiting'.format(e))
         self._exiter.exit(e.exit_code)
       except Exception:
+        # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
+        # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
+        # happening here, so something is probably overriding the excepthook. By catching Exception
+        # and calling this method, we emulate the normal, expected sys.excepthook override.
         ExceptionSink._log_unhandled_exception_and_exit()
       else:
         self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
diff --git a/src/python/pants/bin/remote_pants_runner.py b/src/python/pants/bin/remote_pants_runner.py
--- a/src/python/pants/bin/remote_pants_runner.py
+++ b/src/python/pants/bin/remote_pants_runner.py
@@ -32,20 +32,23 @@ def __init__(self, pailgun_client, timeout=1, *args, **kwargs):
     self._timeout = timeout
     super(PailgunClientSignalHandler, self).__init__(*args, **kwargs)
 
-  def _forward_signal_with_timeout(self, signum):
+  def _forward_signal_with_timeout(self, signum, signame):
+    logger.info(
+      'Sending {} to pantsd-runner with pid {}, waiting up to {} seconds before sending SIGKILL...'
+      .format(signame, self._pailgun_client._maybe_last_pid(), self._timeout))
     self._pailgun_client.set_exit_timeout(
       timeout=self._timeout,
       reason=KeyboardInterrupt('Interrupted by user over pailgun client!'))
     self._pailgun_client.maybe_send_signal(signum)
 
   def handle_sigint(self, signum, _frame):
-    self._forward_signal_with_timeout(signum)
+    self._forward_signal_with_timeout(signum, 'SIGINT')
 
   def handle_sigquit(self, signum, _frame):
-    self._forward_signal_with_timeout(signum)
+    self._forward_signal_with_timeout(signum, 'SIGQUIT')
 
   def handle_sigterm(self, signum, _frame):
-    self._forward_signal_with_timeout(signum)
+    self._forward_signal_with_timeout(signum, 'SIGTERM')
 
 
 class RemotePantsRunner(object):
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -276,7 +276,8 @@ def register_bootstrap_options(cls, register):
              help='The host to bind the pants nailgun server to.')
     register('--pantsd-pailgun-port', advanced=True, type=int, default=0,
              help='The port to bind the pants nailgun server to. Defaults to a random port.')
-    register('--pantsd-pailgun-quit-timeout', advanced=True, type=float, default=1.0,
+    # TODO(#7514): Make this default to 1.0 seconds if stdin is a tty!
+    register('--pantsd-pailgun-quit-timeout', advanced=True, type=float, default=5.0,
              help='The length of time (in seconds) to wait for further output after sending a '
                   'signal to the remote pantsd-runner process before killing it.')
     register('--pantsd-log-dir', advanced=True, default=None,


Test patch:
diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py
--- a/tests/python/pants_test/pantsd/test_pantsd_integration.py
+++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py
@@ -418,8 +418,15 @@ def test_pantsd_multiple_parallel_runs(self):
       self.assert_success(creator_handle.join())
       self.assert_success(waiter_handle.join())
 
-  def _assert_pantsd_keyboardinterrupt_signal(self, signum, messages, regexps=[],
-                                              quit_timeout=None):
+  def _assert_pantsd_keyboardinterrupt_signal(self, signum, regexps=[], quit_timeout=None):
+    """Send a signal to the thin pailgun client and observe the error messaging.
+
+    :param int signum: The signal to send.
+    :param regexps: Assert that all of these regexps match somewhere in stderr.
+    :type regexps: list of str
+    :param float quit_timeout: The duration of time to wait for the pailgun client to flush all of
+                               its output and die after being killed.
+    """
     # TODO: This tests that pantsd-runner processes actually die after the thin client receives the
     # specified signal.
     with self.pantsd_test_context() as (workdir, config, checker):
@@ -446,8 +453,6 @@ def _assert_pantsd_keyboardinterrupt_signal(self, signum, messages, regexps=[],
       waiter_run = waiter_handle.join()
       self.assert_failure(waiter_run)
 
-      for msg in messages:
-        self.assertIn(msg, waiter_run.stderr_data)
       for regexp in regexps:
         assertRegex(self, waiter_run.stderr_data, regexp)
 
@@ -461,20 +466,54 @@ def _assert_pantsd_keyboardinterrupt_signal(self, signum, messages, regexps=[],
   def test_pantsd_sigterm(self):
     self._assert_pantsd_keyboardinterrupt_signal(
       signal.SIGTERM,
-      ['Signal {signum} (SIGTERM) was raised. Exiting with failure.'.format(signum=signal.SIGTERM)])
-
-  def test_keyboardinterrupt_signals_with_pantsd(self):
-    for interrupt_signal in [signal.SIGINT, signal.SIGQUIT]:
-      self._assert_pantsd_keyboardinterrupt_signal(
-        interrupt_signal,
-        ['\nInterrupted by user over pailgun client!\n'])
+      regexps=[
+        '^INFO] Sending SIGTERM to pantsd-runner with pid [0-9]+, waiting up to 5\\.0 seconds before sending SIGKILL\\.\\.\\.',
+        re.escape("\nSignal {signum} (SIGTERM) was raised. Exiting with failure.\n"
+                  .format(signum=signal.SIGTERM)),
+        """
+Interrupted by user:
+Interrupted by user over pailgun client!
+$"""
+      ])
+
+  def test_pantsd_sigquit(self):
+    self._assert_pantsd_keyboardinterrupt_signal(
+      signal.SIGQUIT,
+      regexps=[
+        '^INFO] Sending SIGQUIT to pantsd-runner with pid [0-9]+, waiting up to 5\\.0 seconds before sending SIGKILL\\.\\.\\.',
+        re.escape("\nSignal {signum} (SIGQUIT) was raised. Exiting with failure.\n"
+                  .format(signum=signal.SIGQUIT)),
+        """
+Interrupted by user:
+Interrupted by user over pailgun client!
+$"""])
+
+  def test_pantsd_sigint(self):
+    self._assert_pantsd_keyboardinterrupt_signal(
+      signal.SIGINT,
+      regexps=["""\
+^INFO] Sending SIGINT to pantsd-runner with pid [0-9]+, waiting up to 5\\.0 seconds before sending SIGKILL\\.\\.\\.
+Interrupted by user.
+Interrupted by user:
+Interrupted by user over pailgun client!
+$"""])
 
   def test_signal_pailgun_stream_timeout(self):
+    # NB: The actual timestamp has the date and time at sub-second granularity. The date is just
+    # used here since that is known in advance in order to assert that the timestamp is well-formed.
     today = datetime.date.today().isoformat()
     self._assert_pantsd_keyboardinterrupt_signal(
       signal.SIGINT,
-      messages=['\nInterrupted by user over pailgun client!\n'],
-      regexps=["""WARN\\] timed out when attempting to gracefully shut down the remote client executing "'pantsd-runner.*'"\\. sending SIGKILL to the remote client at pid: [0-9]+\\. message: iterating over bytes from nailgun timed out with timeout interval 0\\.01 starting at {today}T[^\n]+, overtime seconds: [^\n]+"""
+      regexps=["""\
+^INFO] Sending SIGINT to pantsd-runner with pid [0-9]+, waiting up to 0\\.01 seconds before sending SIGKILL\\.\\.\\.
+Interrupted by user\\.
+WARN\\] timed out when attempting to gracefully shut down the remote client executing \
+"'pantsd-runner.*'"\\. sending SIGKILL to the remote client at pid: [0-9]+\\. message: iterating \
+over bytes from nailgun timed out with timeout interval 0\\.01 starting at {today}T[^\n]+, \
+overtime seconds: [^\n]+
+Interrupted by user:
+Interrupted by user over pailgun client!
+"""
                .format(today=re.escape(today))],
       quit_timeout=0.01,
     )