Repository: pantsbuild/pants

Issue:
The client error when pantsd restarts is not self-explanatory
When `pantsd` restarts (generally because its code or plugin code has been edited), the error is not self explanatory.

Under v2 this might look like:
```
pants.bin.remote_pants_runner.RemotePantsRunner.Terminated: abruptly lost active connection to pantsd runner: NailgunError('Problem talking to nailgun server (address: 127.0.0.1:61905): TruncatedHeaderError("Failed to read nailgun chunk header (TruncatedRead(\'Expected 5 bytes before socket shutdown, instead received 0\',)).",)', TruncatedHeaderError("Failed to read nailgun chunk header (TruncatedRead('Expected 5 bytes before socket shutdown, instead received 0',)).",))
Remote exception:
timestamp: 2020-05-29T16:03:45.528163
process title: pantsd [/Users/stuhood/src/pants]
sys.argv: ['/Users/stuhood/src/pants/src/python/pants/bin/pants_loader.py', '--no-v1', '--v2', 'binary', 'src/python/pants/bin:pants_local_binary']
pid: 25924
Exception caught: (pants.pantsd.pants_daemon.RuntimeFailure)
  File "/Users/stuhood/src/pants/src/python/pants/bin/pants_loader.py", line 98, in <module>
    main()
  File "/Users/stuhood/src/pants/src/python/pants/bin/pants_loader.py", line 94, in main
    PantsLoader.run()
  File "/Users/stuhood/src/pants/src/python/pants/bin/pants_loader.py", line 90, in run
    cls.load_and_execute(entrypoint)
  File "/Users/stuhood/src/pants/src/python/pants/bin/pants_loader.py", line 83, in load_and_execute
    entrypoint_main()
  File "/Users/stuhood/src/pants/src/python/pants/pantsd/pants_daemon.py", line 583, in launch
    PantsDaemon.Factory.create(OptionsBootstrapper.create()).run_sync()
  File "/Users/stuhood/src/pants/src/python/pants/pantsd/pants_daemon.py", line 503, in run_sync
    self._run_services(self._services)
  File "/Users/stuhood/src/pants/src/python/pants/pantsd/pants_daemon.py", line 415, in _run_services
    f"service failure for {service}, shutting down!"

Exception message: service failure for <pants.pantsd.service.scheduler_service.SchedulerService object at 0x10e603828>, shutting down!
```
whereas under v1 it might look like:
```sh
 ./pants test src/python/toolchain/satresolver::                                                                            

17:37:42 00:00 [main]
               See a report at: http://localhost:51055/run/pants_run_2019_08_22_17_37_42_471_0f9540ac8f01464f997a4a29df197279
17:37:42 00:00   [setup]
17:37:42 00:00     [parse]21:37:42 [WARN] /Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/option/options.py:460: DeprecationWarning: DEPRECATED: scope compile.zinc will be removed in version 1.20.0.dev2.
  Use scope compile.rsc instead (options: args, use_classpath_jars, incremental_caching)
  self._check_and_apply_deprecations(scope, values)


               Executing tasks in goals: bootstrap -> imports -> unpack-wheels -> unpack-jars -> deferred-sources -> native-compile -> link -> jvm-platform-validate -> gen -> pyprep -> resolve -> compile -> resources -> test
17:37:42 00:00   [bootstrap]
17:37:42 00:00     [substitute-aliased-targets]
17:37:42 00:00     [jar-dependency-management]timestamp: 2019-08-22T17:37:42.879907
Exception caught: (builtins.OSError)
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/bin/daemon_pants_runner.py", line 256, in run
    runner.run()
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/bin/local_pants_runner.py", line 231, in run
    self._run()
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/bin/local_pants_runner.py", line 303, in _run
    run_tracker_result = self._run_tracker.end()
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/goal/run_tracker.py", line 499, in end
    with self.new_workunit("complete"):
  File "/Users/tansypants/.pyenv/versions/3.7.3/lib/python3.7/contextlib.py", line 112, in __enter__
    return next(self.gen)
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/goal/run_tracker.py", line 312, in new_workunit
    log_config=log_config) as workunit:
  File "/Users/tansypants/.pyenv/versions/3.7.3/lib/python3.7/contextlib.py", line 112, in __enter__
    return next(self.gen)
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/goal/run_tracker.py", line 340, in new_workunit_under_parent
    self.report.start_workunit(workunit)
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/reporting/report.py", line 94, in start_workunit
    reporter.start_workunit(workunit)
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/reporting/plaintext_reporter.py", line 142, in start_workunit
    self.flush()
  File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.20.0.dev1_py37/lib/python3.7/site-packages/pants/reporting/plaintext_reporter.py", line 194, in flush
    self.settings.outfile.flush()

Exception message: [Errno 9] Bad file descriptor
```


Discussion:
I have a diff to fix this which I will put up later today.
Found this in the log:
```
21:13:59 [WARN]   File "/Users/tansypants/.cache/pants/setup/bootstrap-Darwin-x86_64/1.17.0rc0_py36/lib/python3.6/site-packages/pants/util/socket.py", line 33, in safe_select
21:13:59 [WARN]     if e[0] != errno.EINTR:
21:13:59 [WARN] TypeError: 'OSError' object is not subscriptable
21:13:59 [INFO] terminating pantsd service: <pants.pantsd.service.store_gc_service.StoreGCService object at 0x10e900f28>
21:13:59 [INFO] terminating pantsd
21:13:59 [WARN] Encountered error when trying to log this message: b'timestamp: 2019-06-20T17:13:59.734720\nException caught: (pants.pantsd.pants_daemon.RuntimeFailure)\n  File "
 exception: write to closed file
 out: <_io.BufferedWriter name=2>
```


I am seeing this too.
```

⠈
15:34:54:815 [ERROR]
15:34:54:815 [ERROR] lost active connection to pantsd!
15:34:54:816 [ERROR] abruptly lost active connection to pantsd runner: NailgunError('Problem talking to nailgun server (address: 127.0.0.1:57502): TruncatedHeaderError("Failed to read nailgun chunk header (TruncatedRead(\'Expected 5 bytes before socket shutdown, instead received 0\')).")', TruncatedHeaderError("Failed to read nailgun chunk header (TruncatedRead('Expected 5 bytes before socket shutdown, instead received 0'))."))
Remote exception:
timestamp: 2020-06-02T08:34:53.717969
process title: pantsd [/Users/asher/projects/toolchain]
sys.argv: ['/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/bin/pants', '--pants-config-files=+["/Users/asher/projects/toolchain/toolchain.toml"]', '--pants-bin-name=./tc', 'test', 'src/python/toolchain/buildsense/::', 'src/python/toolchain/django/site/::']
pid: 37190
Exception caught: (pants.pantsd.pants_daemon.RuntimeFailure)
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/bin/pants", line 8, in <module>
    sys.exit(main())
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/bin/pants_loader.py", line 94, in main
    PantsLoader.run()
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/bin/pants_loader.py", line 90, in run
    cls.load_and_execute(entrypoint)
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/bin/pants_loader.py", line 83, in load_and_execute
    entrypoint_main()
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/pantsd/pants_daemon.py", line 583, in launch
    PantsDaemon.Factory.create(OptionsBootstrapper.create()).run_sync()
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/pantsd/pants_daemon.py", line 503, in run_sync
    self._run_services(self._services)
  File "/Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/pantsd/pants_daemon.py", line 414, in _run_services
    raise PantsDaemon.RuntimeFailure(

Exception message: service failure for <pants.pantsd.service.scheduler_service.SchedulerService object at 0x10eb697c0>, shutting down!


(Use --print-exception-stacktrace to see more error details.)
```
In the middle of a test session run.

@stuhood 
pantsd log:
```
15:34:52:462 [INFO] notify invalidation: cleared 2 and dirtied 208 nodes for: {"src/python/toolchain/buildsense", "src/python/toolchain/buildsense/views_api_test.py"}
15:34:52:466 [INFO] notify invalidation: cleared 0 and dirtied 0 nodes for: {"src/python/toolchain/buildsense/views_api_test.py", "src/python/toolchain/buildsense"}
15:34:53:218 [INFO] terminating pantsd service: <pants.pantsd.service.scheduler_service.SchedulerService object at 0x10eb697c0>
15:34:53:218 [INFO] terminating pantsd service: <pants.pantsd.service.pailgun_service.PailgunService object at 0x10eb69fa0>
15:34:53:711 [INFO] pailgun service on shutting down
15:34:53:711 [INFO] terminating pantsd service: <pants.pantsd.service.store_gc_service.StoreGCService object at 0x10eb69dc0>
15:34:53:711 [INFO] terminating pantsd
15:34:53:720 [WARN] /Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/base/exception_sink.py:359: DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats
  process_title=setproctitle.getproctitle(),

15:34:53:721 [ERROR] service failure for <pants.pantsd.service.scheduler_service.SchedulerService object at 0x10eb697c0>, shutting down!

(Use --print-exception-stacktrace to see more error details.)



15:38:27:475 [INFO] setting up service <pants.pantsd.service.scheduler_service.SchedulerService object at 0x1155c39a0>
15:38:27:484 [INFO] notify invalidation: cleared 0 and dirtied 0 nodes for: {".pids", ".pids/pantsd"}
15:38:27:485 [INFO] notify invalidation: cleared 0 and dirtied 0 nodes for: {".pids/pantsd", ".pids/pantsd/socket_pailgun"}
15:38:27:485 [INFO] notify invalidation: cleared 0 and dirtied 0 nodes for: {".pids/pantsd/socket_pailgun", ".pids/pantsd"}
```
@asherf : That isn't quite enough of the pantsd log, but I'm assuming that somewhere shortly above those lines there was a warning about the invalidation globs having been matched?
```
15:34:41:175 [INFO] notify invalidation: cleared 0 and dirtied 0 nodes for: {".pids/pantsd", ".pids/pantsd/fingerprint"}
15:34:41:233 [INFO] handling request: `--pants-config-files=+["/Users/asher/projects/toolchain/toolchain.toml"] --pants-bin-name=./tc test src/python/toolchain/buildsense/:: src/python/toolchain/django/site/::`
15:34:51:786 [INFO] notify invalidation: cleared 2 and dirtied 208 nodes for: {"src/python/toolchain/buildsense", "src/python/toolchain/buildsense/views_api_test.py"}
15:34:52:317 [ERROR] saw filesystem changes covered by invalidation globs: content changed (Digest(fingerprint='fdfee1ee8e0b7b42f9991d462b9d69fd80daa3b550e75ddb184a444969d7d74a', serialized_bytes_length=407) fs Digest(fingerprint='f35d669e0e15ea641c97a6bb5add4863bd77dae0fe97578ec3ba320ae48af969', serialized_bytes_length=407)). terminating the daemon.
15:34:52:462 [INFO] notify invalidation: cleared 2 and dirtied 208 nodes for: {"src/python/toolchain/buildsense", "src/python/toolchain/buildsense/views_api_test.py"}
15:34:52:466 [INFO] notify invalidation: cleared 0 and dirtied 0 nodes for: {"src/python/toolchain/buildsense/views_api_test.py", "src/python/toolchain/buildsense"}
15:34:53:218 [INFO] terminating pantsd service: <pants.pantsd.service.scheduler_service.SchedulerService object at 0x10eb697c0>
15:34:53:218 [INFO] terminating pantsd service: <pants.pantsd.service.pailgun_service.PailgunService object at 0x10eb69fa0>
15:34:53:711 [INFO] pailgun service on shutting down
15:34:53:711 [INFO] terminating pantsd service: <pants.pantsd.service.store_gc_service.StoreGCService object at 0x10eb69dc0>
15:34:53:711 [INFO] terminating pantsd
15:34:53:720 [WARN] /Users/asher/.cache/pants/setup/bootstrap-Darwin-x86_64/1.29.0.dev1_py38/lib/python3.8/site-packages/pants/base/exception_sink.py:359: DeprecationWarning: PY_SSIZE_T_CLEAN will be required for '#' formats
  process_title=setproctitle.getproctitle(),

15:34:53:721 [ERROR] service failure for <pants.pantsd.service.scheduler_service.SchedulerService object at 0x10eb697c0>, shutting down!

(Use --print-exception-stacktrace to see more error details.)
15:38:27:475 [INFO] setting up service <pants.pantsd.service.scheduler_service.SchedulerService object at 0x1155c39a0>
```

Resolving patch:
diff --git a/src/python/pants/engine/internals/native.py b/src/python/pants/engine/internals/native.py
--- a/src/python/pants/engine/internals/native.py
+++ b/src/python/pants/engine/internals/native.py
@@ -189,13 +189,12 @@ def override_thread_logging_destination_to_just_stderr(self):
     def match_path_globs(self, path_globs: PathGlobs, paths: Iterable[str]) -> bool:
         return cast(bool, self.lib.match_path_globs(path_globs, tuple(paths)))
 
-    def nailgun_server_await_bound(self, nailgun_server) -> int:
-        """Blocks until the server has bound a port, and then returns the port.
+    def nailgun_server_await_shutdown(self, nailgun_server) -> None:
+        """Blocks until the server has shut down.
 
-        Returns the actual port the server has successfully bound to, or raises an exception if the
-        server has exited.
+        Raises an exception if the server exited abnormally
         """
-        return cast(int, self.lib.nailgun_server_await_bound(self._executor, nailgun_server))
+        self.lib.nailgun_server_await_shutdown(self._executor, nailgun_server)
 
     def new_nailgun_server(self, port: int, runner: RawFdRunner):
         """Creates a nailgun server with a requested port.
diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py
--- a/src/python/pants/pantsd/pants_daemon.py
+++ b/src/python/pants/pantsd/pants_daemon.py
@@ -255,7 +255,7 @@ def _pantsd_logging(self) -> Iterator[IO[str]]:
 
     def _write_nailgun_port(self):
         """Write the nailgun port to a well known file."""
-        self.write_socket(self._native.nailgun_server_await_bound(self._server))
+        self.write_socket(self._server.port())
 
     def _initialize_pid(self):
         """Writes out our pid and metadata.
@@ -325,7 +325,9 @@ def run_sync(self):
                 time.sleep(self.JOIN_TIMEOUT_SECONDS)
 
             # We're exiting: join the server to avoid interrupting ongoing runs.
-            # TODO: This will happen via #8200.
+            self._logger.info("waiting for ongoing runs to complete before exiting...")
+            self._native.nailgun_server_await_shutdown(self._server)
+            self._logger.info("exiting.")
 
 
 def launch():


Test patch:
diff --git a/src/rust/engine/nailgun/src/tests.rs b/src/rust/engine/nailgun/src/tests.rs
--- a/src/rust/engine/nailgun/src/tests.rs
+++ b/src/rust/engine/nailgun/src/tests.rs
@@ -1,12 +1,17 @@
 use crate::Server;
 
 use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
 
+use futures::future;
 use nails::client_handle_connection;
 use nails::execution::{child_channel, ChildInput, ChildOutput, Command, ExitCode};
 use task_executor::Executor;
 use tokio::net::TcpStream;
 use tokio::runtime::Handle;
+use tokio::sync::Notify;
+use tokio::time::delay_for;
 
 #[tokio::test]
 async fn spawn_and_bind() {
@@ -14,7 +19,8 @@ async fn spawn_and_bind() {
     .await
     .unwrap();
   // Should have bound a random port.
-  assert!(0 != server.await_bound().await.unwrap());
+  assert!(0 != server.port());
+  server.shutdown().await.unwrap();
 }
 
 #[tokio::test]
@@ -23,10 +29,56 @@ async fn accept() {
   let server = Server::new(Executor::new(Handle::current()), 0, move |_| exit_code)
     .await
     .unwrap();
-  let server_port = server.await_bound().await.unwrap();
 
   // And connect with a client. This Nail will ignore the content of the command, so we're
   // only validating the exit code.
+  let actual_exit_code = run_client(server.port()).await.unwrap();
+  assert_eq!(exit_code, actual_exit_code);
+  server.shutdown().await.unwrap();
+}
+
+#[tokio::test]
+async fn shutdown_awaits_ongoing() {
+  // A server that waits for a signal to complete a connection.
+  let connection_accepted = Arc::new(Notify::new());
+  let should_complete_connection = Arc::new(Notify::new());
+  let exit_code = ExitCode(42);
+  let server = Server::new(Executor::new(Handle::current()), 0, {
+    let connection_accepted = connection_accepted.clone();
+    let should_complete_connection = should_complete_connection.clone();
+    move |_| {
+      connection_accepted.notify();
+      Handle::current().block_on(should_complete_connection.notified());
+      exit_code
+    }
+  })
+  .await
+  .unwrap();
+
+  // Spawn a connection in the background, and once it has been established, kick off shutdown of
+  // the server.
+  let mut client_completed = tokio::spawn(run_client(server.port()));
+  connection_accepted.notified().await;
+  let mut server_shutdown = tokio::spawn(server.shutdown());
+
+  // Confirm that the client doesn't return, and that the server doesn't shutdown.
+  match future::select(client_completed, delay_for(Duration::from_millis(500))).await {
+    future::Either::Right((_, c_c)) => client_completed = c_c,
+    x => panic!("Client should not have completed: {:?}", x),
+  }
+  match future::select(server_shutdown, delay_for(Duration::from_millis(500))).await {
+    future::Either::Right((_, s_s)) => server_shutdown = s_s,
+    x => panic!("Server should not have shut down: {:?}", x),
+  }
+
+  // Then signal completion of the connection, and confirm that both the client and server exit
+  // cleanly.
+  should_complete_connection.notify();
+  assert_eq!(exit_code, client_completed.await.unwrap().unwrap());
+  server_shutdown.await.unwrap().unwrap();
+}
+
+async fn run_client(port: u16) -> Result<ExitCode, String> {
   let cmd = Command {
     command: "nothing".to_owned(),
     args: vec![],
@@ -35,11 +87,8 @@ async fn accept() {
   };
   let (stdio_write, _stdio_read) = child_channel::<ChildOutput>();
   let (_stdin_write, stdin_read) = child_channel::<ChildInput>();
-  let stream = TcpStream::connect(("127.0.0.1", server_port))
+  let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap();
+  client_handle_connection(stream, cmd, stdio_write, stdin_read)
     .await
-    .unwrap();
-  let actual_exit_code = client_handle_connection(stream, cmd, stdio_write, stdin_read)
-    .await
-    .unwrap();
-  assert_eq!(exit_code, actual_exit_code);
+    .map_err(|e| e.to_string())
 }