Repository: Parsl/parsl

Issue:
zmq channels should not close their contexts
**Describe the bug**

zmq pipes implemented in zmq_pipes.py close the zmq context when the pipe is closed - eg here: https://github.com/Parsl/parsl/blob/23afcb1315ee4188646df71a041929401c7a274b/parsl/executors/high_throughput/zmq_pipes.py#L82

This conflicts with using ZMQ contexts for multiple sockets (for example, I've seen elsewhere people say to imagine one ZMQ context for an entire program). That doesn't happen right now with these sockets because the initialisation code makes one context per socket: https://github.com/Parsl/parsl/blob/23afcb1315ee4188646df71a041929401c7a274b/parsl/executors/high_throughput/executor.py#L417

... but I think it would be reasonable to expect this to all behave nicely if one context was shared between all three zmq pipes created there and I am suspicious because of this pipe-closes-context behaviour that this is not true.

See this review for discovering context:
https://github.com/Parsl/parsl/pull/3441/files#r1605802597

@rjmello noted that he was interested in fixing this so I'm assigning this issue to him.


Discussion:
On second thought, I think there is some benefit to using one ZMQ context per pipe and terminating it in the `.close()` method. Namely, this simplifies handling the pipe objects and helps us avoid doing things such as creating a socket in a different thread than the context, which can be problematic.

Unless someone has an objection, I will make our intentions more explicit by simply moving the context instantiation within each pipe's `__init__` method.
moving the context instantiation into init sounds like the right thing to do if a context is to be per-pipe.

this doesn't entirely avoid sockets being used across threads though - see #3376 

Resolving patch:
diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py
--- a/parsl/executors/high_throughput/executor.py
+++ b/parsl/executors/high_throughput/executor.py
@@ -415,13 +415,13 @@ def start(self):
             )
 
         self.outgoing_q = zmq_pipes.TasksOutgoing(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
         self.incoming_q = zmq_pipes.ResultsIncoming(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
         self.command_client = zmq_pipes.CommandClient(
-            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+            "127.0.0.1", self.interchange_port_range, self.cert_dir
         )
 
         self._queue_management_thread = None
diff --git a/parsl/executors/high_throughput/zmq_pipes.py b/parsl/executors/high_throughput/zmq_pipes.py
--- a/parsl/executors/high_throughput/zmq_pipes.py
+++ b/parsl/executors/high_throughput/zmq_pipes.py
@@ -4,6 +4,7 @@
 import logging
 import threading
 import time
+from typing import Optional
 
 from parsl import curvezmq
 from parsl.errors import InternalConsistencyError
@@ -15,20 +16,23 @@
 class CommandClient:
     """ CommandClient
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.ip_address = ip_address
         self.port_range = port_range
         self.port = None
@@ -119,20 +123,23 @@ def close(self):
 class TasksOutgoing:
     """ Outgoing task queue from the executor to the Interchange
     """
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.zmq_socket = self.zmq_context.socket(zmq.DEALER)
         self.zmq_socket.set_hwm(0)
         self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
@@ -172,20 +179,23 @@ class ResultsIncoming:
     """ Incoming results queue from the Interchange to the executor
     """
 
-    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
+    def __init__(self, ip_address, port_range, cert_dir: Optional[str] = None):
         """
         Parameters
         ----------
 
-        zmq_context: curvezmq.ClientContext
-            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
+
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
+        cert_dir: str | None
+            Path to the certificate directory. Setting this to None will disable encryption.
+            default: None
+
         """
-        self.zmq_context = zmq_context
+        self.zmq_context = curvezmq.ClientContext(cert_dir)
         self.results_receiver = self.zmq_context.socket(zmq.DEALER)
         self.results_receiver.set_hwm(0)
         self.port = self.results_receiver.bind_to_random_port("tcp://{}".format(ip_address),


Test patch:
diff --git a/parsl/tests/test_htex/test_command_client_timeout.py b/parsl/tests/test_htex/test_command_client_timeout.py
--- a/parsl/tests/test_htex/test_command_client_timeout.py
+++ b/parsl/tests/test_htex/test_command_client_timeout.py
@@ -18,10 +18,8 @@
 def test_command_not_sent() -> None:
     """Tests timeout on command send.
     """
-    ctx = curvezmq.ClientContext(None)
-
     # RFC6335 ephemeral port range
-    cc = CommandClient(ctx, "127.0.0.1", (49152, 65535))
+    cc = CommandClient("127.0.0.1", (49152, 65535))
 
     # cc will now wait for a connection, but we won't do anything to make the
     # other side of the connection exist, so any command given to cc should
@@ -43,10 +41,8 @@ def test_command_ignored() -> None:
     htex makes multithreaded use of the command client: see issue #3376 about
     that lack of thread safety.
     """
-    ctx = curvezmq.ClientContext(None)
-
     # RFC6335 ephemeral port range
-    cc = CommandClient(ctx, "127.0.0.1", (49152, 65535))
+    cc = CommandClient("127.0.0.1", (49152, 65535))
 
     ic_ctx = curvezmq.ServerContext(None)
     ic_channel = ic_ctx.socket(zmq.REP)
@@ -63,7 +59,4 @@ def test_command_ignored() -> None:
         cc.run("ANOTHER_COMMAND")
 
     cc.close()
-    ctx.term()
-
     ic_channel.close()
-    ic_ctx.term()