Repository: pantsbuild/pants

Issue:
[pantsd] Pantsd test harness wrongly identifies some processes as pantsd lingering processes.
When running under the `no_lingering_processes()` context manager, it will wrongly identify some processes as lingering processes:

```
# In one terminal
$ watch -n1 'ps aux | grep "pantsd.* \[$( pwd )\] \| pantsd-runner \[" | grep -v watch | grep -v tail | grep -v grep'

# In another terminal:
$ ./pants test tests/python/pants_test/pantsd:: -- -s -k test_pantsd_aligned_output
```
This will fail, identifying some of the grep processes as `pantsd-runner` processes.

A potential fix is to improve the matching regex of the process tracker to be more strict, but this is likely a different regex pre and post #7596, so holding off on the PR for now. 


Discussion:

Resolving patch:
diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py
--- a/build-support/bin/generate_travis_yml.py
+++ b/build-support/bin/generate_travis_yml.py
@@ -510,7 +510,7 @@ def lint(python_version: PythonVersion) -> Dict:
             # local execution has finished.
             (
                 "travis-wait-enhanced --timeout 40m --interval 9m -- ./build-support/bin/ci.py "
-                f"--remote-execution-enabled --lint --python-version {python_version.decimal}"
+                f"--lint --python-version {python_version.decimal}"
             ),
         ],
     }
@@ -557,7 +557,7 @@ def unit_tests(python_version: PythonVersion) -> Dict:
         "name": f"Unit tests (Python {python_version.decimal})",
         "script": [
             "travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py "
-            f"--unit-tests --remote-execution-enabled --python-version {python_version.decimal}"
+            f"--unit-tests --python-version {python_version.decimal}"
         ],
     }
     safe_append(shard, "env", f"CACHE_NAME=unit_tests.py{python_version.number}")
@@ -618,16 +618,15 @@ def build_wheels_osx() -> Dict:
 def integration_tests(python_version: PythonVersion) -> Dict:
     shard = {
         **linux_shard(python_version=python_version, install_travis_wait=True),
-        "name": f"Integration tests - V2 (Python {python_version.decimal})",
+        "name": f"Integration tests (Python {python_version.decimal})",
         "script": [
             (
                 "travis-wait-enhanced --timeout 65m --interval 9m -- ./build-support/bin/ci.py "
-                "--integration-tests --remote-execution-enabled --python-version "
-                f"{python_version.decimal}"
+                f"--integration-tests --python-version {python_version.decimal}"
             ),
         ],
     }
-    safe_append(shard, "env", f"CACHE_NAME=integration.v2.py{python_version.number}")
+    safe_append(shard, "env", f"CACHE_NAME=integration.py{python_version.number}")
     return shard
 
 


Test patch:
diff --git a/tests/python/pants_test/init/BUILD b/tests/python/pants_test/init/BUILD
--- a/tests/python/pants_test/init/BUILD
+++ b/tests/python/pants_test/init/BUILD
@@ -26,7 +26,7 @@ python_tests(
     'src/python/pants/util:dirutil',
     'src/python/pants/util:logging',
   ],
-  timeout = 180,
+  timeout = 240,
 )
 
 python_tests(
diff --git a/tests/python/pants_test/pantsd/pantsd_integration_test_base.py b/tests/python/pants_test/pantsd/pantsd_integration_test_base.py
--- a/tests/python/pants_test/pantsd/pantsd_integration_test_base.py
+++ b/tests/python/pants_test/pantsd/pantsd_integration_test_base.py
@@ -17,10 +17,6 @@
     kill_daemon,
     read_pantsd_log,
 )
-from pants.testutil.process_test_util import (
-    TrackedProcessesContext,
-    no_lingering_process_by_command,
-)
 from pants.testutil.retry import attempts
 from pants.util.collections import recursively_update
 from pants.util.contextutil import temporary_dir
@@ -33,13 +29,8 @@ def banner(s):
 
 
 class PantsDaemonMonitor(ProcessManager):
-    def __init__(self, runner_process_context: TrackedProcessesContext, metadata_base_dir=None):
-        """
-        :param runner_process_context: A TrackedProcessContext that can be used to inspect live
-          pantsd instances created in this context.
-        """
+    def __init__(self, metadata_base_dir=None):
         super().__init__(name="pantsd", metadata_base_dir=metadata_base_dir)
-        self.runner_process_context = runner_process_context
 
     def _log(self):
         print(magenta(f"PantsDaemonMonitor: pid is {self._pid} is_alive={self.is_alive()}"))
@@ -117,30 +108,29 @@ def use_pantsd_env_var(cls):
     def pantsd_test_context(
         self, *, log_level: str = "info", extra_config: Optional[Dict[str, Any]] = None
     ) -> Iterator[Tuple[str, Dict[str, Any], PantsDaemonMonitor]]:
-        with no_lingering_process_by_command("pantsd") as runner_process_context:
-            with temporary_dir(root_dir=os.getcwd()) as workdir_base:
-                pid_dir = os.path.join(workdir_base, ".pids")
-                workdir = os.path.join(workdir_base, ".workdir.pants.d")
-                print(f"\npantsd log is {workdir}/pantsd/pantsd.log")
-                pantsd_config = {
-                    "GLOBAL": {"pantsd": True, "level": log_level, "pants_subprocessdir": pid_dir,}
-                }
-
-                if extra_config:
-                    recursively_update(pantsd_config, extra_config)
-                print(f">>> config: \n{pantsd_config}\n")
-
-                checker = PantsDaemonMonitor(runner_process_context, pid_dir)
+        with temporary_dir(root_dir=os.getcwd()) as workdir_base:
+            pid_dir = os.path.join(workdir_base, ".pids")
+            workdir = os.path.join(workdir_base, ".workdir.pants.d")
+            print(f"\npantsd log is {workdir}/pantsd/pantsd.log")
+            pantsd_config = {
+                "GLOBAL": {"pantsd": True, "level": log_level, "pants_subprocessdir": pid_dir,}
+            }
+
+            if extra_config:
+                recursively_update(pantsd_config, extra_config)
+            print(f">>> config: \n{pantsd_config}\n")
+
+            checker = PantsDaemonMonitor(pid_dir)
+            kill_daemon(pid_dir)
+            try:
+                yield (workdir, pantsd_config, checker)
                 kill_daemon(pid_dir)
-                try:
-                    yield (workdir, pantsd_config, checker)
-                    kill_daemon(pid_dir)
-                    checker.assert_stopped()
-                finally:
-                    banner("BEGIN pantsd.log")
-                    for line in read_pantsd_log(workdir):
-                        print(line)
-                    banner("END pantsd.log")
+                checker.assert_stopped()
+            finally:
+                banner("BEGIN pantsd.log")
+                for line in read_pantsd_log(workdir):
+                    print(line)
+                banner("END pantsd.log")
 
     @contextmanager
     def pantsd_successful_run_context(self, *args, **kwargs) -> Iterator[PantsdRunContext]:
diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py
--- a/tests/python/pants_test/pantsd/test_pantsd_integration.py
+++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py
@@ -13,7 +13,6 @@
 import pytest
 
 from pants.testutil.pants_run_integration_test import read_pantsd_log
-from pants.testutil.process_test_util import no_lingering_process_by_command
 from pants.util.contextutil import environment_as, temporary_dir, temporary_file
 from pants.util.dirutil import rm_rf, safe_file_dump, safe_mkdir, safe_open, touch
 from pants_test.pantsd.pantsd_integration_test_base import PantsDaemonIntegrationTestBase
@@ -58,22 +57,20 @@ def test_pantsd_broken_pipe(self):
             checker.assert_started()
 
     def test_pantsd_pantsd_runner_doesnt_die_after_failed_run(self):
-        # Check for no stray pantsd processes.
-        with no_lingering_process_by_command("pantsd"):
-            with self.pantsd_test_context() as (workdir, pantsd_config, checker):
-                # Run target that throws an exception in pants.
-                self.assert_failure(
-                    self.run_pants_with_workdir(
-                        ["lint", "testprojects/src/python/unicode/compilation_failure"],
-                        workdir,
-                        pantsd_config,
-                    )
+        with self.pantsd_test_context() as (workdir, pantsd_config, checker):
+            # Run target that throws an exception in pants.
+            self.assert_failure(
+                self.run_pants_with_workdir(
+                    ["lint", "testprojects/src/python/unicode/compilation_failure"],
+                    workdir,
+                    pantsd_config,
                 )
-                checker.assert_started()
+            )
+            checker.assert_started()
 
-                # Assert pantsd is in a good functional state.
-                self.assert_success(self.run_pants_with_workdir(["help"], workdir, pantsd_config))
-                checker.assert_running()
+            # Assert pantsd is in a good functional state.
+            self.assert_success(self.run_pants_with_workdir(["help"], workdir, pantsd_config))
+            checker.assert_running()
 
     def test_pantsd_lifecycle_invalidation(self):
         """Run with different values of daemon=True options, which should trigger restarts."""
@@ -465,8 +462,6 @@ def _assert_pantsd_keyboardinterrupt_signal(self, signum, regexps=[], quit_timeo
             checker.assert_started()
             checker.assert_pantsd_runner_started(client_pid)
 
-            # Get all the pantsd processes while they're still around.
-            pantsd_runner_processes = checker.runner_process_context.current_processes()
             # This should kill the pantsd processes through the RemotePantsRunner signal handler.
             os.kill(client_pid, signum)
             waiter_run = waiter_handle.join()
@@ -475,12 +470,8 @@ def _assert_pantsd_keyboardinterrupt_signal(self, signum, regexps=[], quit_timeo
             for regexp in regexps:
                 self.assertRegex(waiter_run.stderr_data, regexp)
 
-            time.sleep(1)
-            for proc in pantsd_runner_processes:
-                # TODO: we could be checking the return codes of the subprocesses, but psutil is currently
-                # limited on non-Windows hosts -- see https://psutil.readthedocs.io/en/latest/#processes.
-                # The pantsd processes should be dead, and they should have exited with 1.
-                self.assertFalse(proc.is_running())
+            time.sleep(5)
+            checker.assert_stopped()
 
     @unittest.skip("flaky: https://github.com/pantsbuild/pants/issues/7554")
     def test_pantsd_sigterm(self):