Repository: Parsl/parsl

Issue:
test_htex does not shut down all of its test htexes
while looking at post-test resource cleanup (#3077) enforced through more aggressive pytest fixtures, I noticed that this test is leaving un-shutdown htex executors around, including non-shutdown interchanges.

I mentioned this to @rjmello (who wrote that bit of the test) on Slack but also noting in GitHub in this issue.

parsl/tests/test_htex/test_htex.py::test_htex_start_encrypted[False-False] ERROR                                       [ 51%]

```
=========================================================== ERRORS ===========================================================
________________________________ ERROR at teardown of test_htex_start_encrypted[False-False] _________________________________

    @pytest.fixture(autouse=True, scope='module')
    def assert_no_multiprocessing():
        import multiprocessing
    
        yield
>       assert multiprocessing.active_children() == []
E       AssertionError: assert [<ForkProcess...arted daemon>] == []
E         Left contains 3 more items, first extra item: <ForkProcess name='HTEX-Interchange' pid=34823 parent=33945 started daemon>
E         Full diff:
E           [
E         -  ,
E         +  <ForkProcess name='HTEX-Interchange' pid=34823 parent=33945 started daemon>,
E         +  <ForkProcess name='HTEX-Interchange' pid=34795 parent=33945 started daemon>,
E         +  <ForkProcess name='HTEX-Interchange' pid=34845 parent=33945 started daemon>,
E           ]

parsl/tests/conftest.py:321: AssertionError

```


Discussion:

Resolving patch:
diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py
--- a/parsl/executors/high_throughput/executor.py
+++ b/parsl/executors/high_throughput/executor.py
@@ -6,7 +6,7 @@
 import queue
 import datetime
 import pickle
-from multiprocessing import Queue
+from multiprocessing import Process, Queue
 from typing import Dict, Sequence
 from typing import List, Optional, Tuple, Union, Callable
 import math
@@ -290,6 +290,7 @@ def __init__(self,
         self.hub_port = None  # set to the correct hub port in dfk
         self.worker_ports = worker_ports
         self.worker_port_range = worker_port_range
+        self.interchange_proc: Optional[Process] = None
         self.interchange_port_range = interchange_port_range
         self.heartbeat_threshold = heartbeat_threshold
         self.heartbeat_period = heartbeat_period
@@ -766,12 +767,28 @@ def status(self) -> Dict[str, JobStatus]:
                     )
         return job_status
 
-    def shutdown(self):
+    def shutdown(self, timeout: float = 10.0):
         """Shutdown the executor, including the interchange. This does not
         shut down any workers directly - workers should be terminated by the
         scaling mechanism or by heartbeat timeout.
+
+        Parameters
+        ----------
+
+        timeout : float
+            Amount of time to wait for the Interchange process to terminate before
+            we forcefully kill it.
         """
+        if self.interchange_proc is None:
+            logger.info("HighThroughputExecutor has not started; skipping shutdown")
+            return
 
         logger.info("Attempting HighThroughputExecutor shutdown")
+
         self.interchange_proc.terminate()
+        self.interchange_proc.join(timeout=timeout)
+        if self.interchange_proc.is_alive():
+            logger.info("Unable to terminate Interchange process; sending SIGKILL")
+            self.interchange_proc.kill()
+
         logger.info("Finished HighThroughputExecutor shutdown attempt")
diff --git a/parsl/multiprocessing.py b/parsl/multiprocessing.py
--- a/parsl/multiprocessing.py
+++ b/parsl/multiprocessing.py
@@ -5,17 +5,16 @@
 import multiprocessing
 import multiprocessing.queues
 import platform
+from multiprocessing.context import ForkProcess as ForkProcessType
 
-from typing import Callable, Type
+from typing import Callable
 
 logger = logging.getLogger(__name__)
 
 ForkContext = multiprocessing.get_context("fork")
 SpawnContext = multiprocessing.get_context("spawn")
 
-# maybe ForkProcess should be: Callable[..., Process] so as to make
-# it clear that it returns a Process always to the type checker?
-ForkProcess: Type = ForkContext.Process
+ForkProcess: Callable[..., ForkProcessType] = ForkContext.Process
 
 
 class MacSafeQueue(multiprocessing.queues.Queue):


Test patch:
diff --git a/parsl/tests/test_htex/test_htex.py b/parsl/tests/test_htex/test_htex.py
--- a/parsl/tests/test_htex/test_htex.py
+++ b/parsl/tests/test_htex/test_htex.py
@@ -1,18 +1,40 @@
 import pathlib
+from unittest import mock
 
 import pytest
 
 from parsl import curvezmq
 from parsl import HighThroughputExecutor
+from parsl.multiprocessing import ForkProcess
+
+_MOCK_BASE = "parsl.executors.high_throughput.executor"
+
+
+@pytest.fixture
+def encrypted(request: pytest.FixtureRequest):
+    if hasattr(request, "param"):
+        return request.param
+    return True
+
+
+@pytest.fixture
+def htex(encrypted: bool):
+    htex = HighThroughputExecutor(encrypted=encrypted)
+
+    yield htex
+
+    htex.shutdown()
 
 
 @pytest.mark.local
-@pytest.mark.parametrize("encrypted", (True, False))
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 @pytest.mark.parametrize("cert_dir_provided", (True, False))
 def test_htex_start_encrypted(
-    encrypted: bool, cert_dir_provided: bool, tmpd_cwd: pathlib.Path
+    encrypted: bool,
+    cert_dir_provided: bool,
+    htex: HighThroughputExecutor,
+    tmpd_cwd: pathlib.Path,
 ):
-    htex = HighThroughputExecutor(encrypted=encrypted)
     htex.run_dir = str(tmpd_cwd)
     if cert_dir_provided:
         provided_base_dir = tmpd_cwd / "provided"
@@ -44,3 +66,44 @@ def test_htex_start_encrypted(
         assert htex.outgoing_q.zmq_context.cert_dir is None
         assert htex.incoming_q.zmq_context.cert_dir is None
         assert htex.command_client.zmq_context.cert_dir is None
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("started", (True, False))
+@pytest.mark.parametrize("timeout_expires", (True, False))
+@mock.patch(f"{_MOCK_BASE}.logger")
+def test_htex_shutdown(
+    mock_logger: mock.MagicMock,
+    started: bool,
+    timeout_expires: bool,
+    htex: HighThroughputExecutor,
+):
+    mock_ix_proc = mock.Mock(spec=ForkProcess)
+
+    if started:
+        htex.interchange_proc = mock_ix_proc
+        mock_ix_proc.is_alive.return_value = True
+
+    if not timeout_expires:
+        # Simulate termination of the Interchange process
+        def kill_interchange(*args, **kwargs):
+            mock_ix_proc.is_alive.return_value = False
+
+        mock_ix_proc.terminate.side_effect = kill_interchange
+
+    htex.shutdown()
+
+    mock_logs = mock_logger.info.call_args_list
+    if started:
+        assert mock_ix_proc.terminate.called
+        assert mock_ix_proc.join.called
+        assert {"timeout": 10} == mock_ix_proc.join.call_args[1]
+        if timeout_expires:
+            assert "Unable to terminate Interchange" in mock_logs[1][0][0]
+            assert mock_ix_proc.kill.called
+        assert "Attempting" in mock_logs[0][0][0]
+        assert "Finished" in mock_logs[-1][0][0]
+    else:
+        assert not mock_ix_proc.terminate.called
+        assert not mock_ix_proc.join.called
+        assert "has not started" in mock_logs[0][0][0]