Repository: Parsl/parsl

Issue:
Usage tracking with certain missing network causes 20s startup delay.
Usage tracking appears to happen synchronously when something like this is called:

    dfk = parsl.DataFlowKernel(executors=[workers])

When something is wrong with the network that causes long delays (for example, slow/timing-out name resolution - for example, in a container with a weird network configuration), startup of the DataFlowKernel is delayed, before the program then successfully continues onwards.

In my case, I'm running in a docker container where the `/etc/resolv.conf` resolver points to a not-responding IP address, and the startup delay is 20 seconds.

No log message appears to be output describing the cause of the delay.

This occurs with several versions, including 48d46ba04afcd86c984953c1d75b7818101cf87f



Discussion:
@benclifford What should be the correct behavior ?
1. We can try to send a message asynchronously and abort if it doesn't work
2. We can inform the user of the delay if we wait for sending to complete.
I think usage tracking should be async and never significantly delay workflow start or cause any other workflow problems in the case of problems.

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -561,6 +561,7 @@ def cleanup(self):
 
         # Send final stats
         self.usage_tracker.send_message()
+        self.usage_tracker.close()
         # We do not need to cleanup if the executors are managed outside
         # the DFK
         if not self._executors_managed:
diff --git a/parsl/dataflow/usage_tracking/usage.py b/parsl/dataflow/usage_tracking/usage.py
--- a/parsl/dataflow/usage_tracking/usage.py
+++ b/parsl/dataflow/usage_tracking/usage.py
@@ -7,6 +7,7 @@
 import logging
 import socket
 import sys
+import multiprocessing as mp
 
 from parsl.dataflow.states import States
 from parsl.version import VERSION as PARSL_VERSION
@@ -14,6 +15,55 @@
 logger = logging.getLogger(__name__)
 
 
+def async_process(fn):
+    """ Decorator function to launch a function as a separate process """
+
+    def run(*args, **kwargs):
+        proc = mp.Process(target=fn, args=args, kwargs=kwargs)
+        proc.start()
+        return proc
+
+    return run
+
+
+@async_process
+def udp_messenger(domain_name, UDP_IP, UDP_PORT, sock_timeout, message):
+    """Send UDP messages to usage tracker asynchronously
+
+    This multiprocessing based messenger was written to overcome the limitations
+    of signalling/terminating a thread that is blocked on a system call. This
+    messenger is created as a separate process, and initialized with 2 queues,
+    to_send to receive messages to be sent to the internet, and done to let the
+    main thread know return status.
+
+    Args:
+          - domain_name (str) : Domain name string
+          - UDP_IP (str) : IP address YYY.YYY.YYY.YYY
+          - UDP_PORT (int) : UDP port to send out on
+          - sock_timeout (int) : Socket timeout
+          - to_send (multiprocessing.Queue) : Queue of outgoing messages to internet
+          - done (multiprocessing.Queue) : Queue of status to be received by main process
+    """
+
+    if domain_name:
+        try:
+            UDP_IP = socket.gethostbyname(domain_name)
+        except Exception:
+            # (False, "Domain lookup failed, defaulting to {0}".format(UDP_IP))
+            pass
+
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
+        sock.settimeout(sock_timeout)
+        sock.sendto(bytes(message, "utf-8"), (UDP_IP, UDP_PORT))
+        sock.close()
+
+    except socket.timeout:
+        done.put_nowait((False, "Sending usage-tracking info timed-out"))
+    except OSError as e:
+        done.put_nowait((False, "Unable to reach the network to send usage data {}".format(e)))
+
+
 class UsageTracker (object):
     """Anonymized Usage Tracking for Parsl.
 
@@ -27,6 +77,11 @@ def __init__(self, dfk, ip='52.3.111.203', port=50077,
                  domain_name='tracking.parsl-project.org'):
         """Initialize usage tracking unless the user has opted-out.
 
+        We will try to resolve the hostname specified in kwarg:domain_name
+        and if that fails attempt to use the kwarg:ip. Determining the
+        IP and sending message is threaded to avoid slowing down DFK
+        initialization.
+
         Tracks usage stats by inspecting the internal state of the dfk.
 
         Args:
@@ -38,15 +93,14 @@ def __init__(self, dfk, ip='52.3.111.203', port=50077,
              - domain_name (string) : Domain name, will override IP
                   Default: tracking.parsl-project.org
         """
-        if domain_name:
-            try:
-                self.UDP_IP = socket.gethostbyname(domain_name)
-            except Exception:
-                logging.debug("Could not lookup domain_name, defaulting to 52.3.111.203")
-                self.UDP_IP = ip
-        else:
-            self.UDP_IP = ip
+
+        self.domain_name = domain_name
+        self.ip = ip
+        # The sock timeout will only apply to UDP send and not domain resolution
+        self.sock_timeout = 5
         self.UDP_PORT = port
+        self.UDP_IP = None
+        self.procs = []
         self.dfk = dfk
         self.config = self.dfk.config
         self.uuid = str(uuid.uuid4())
@@ -134,14 +188,13 @@ def construct_end_message(self):
 
     def send_UDP_message(self, message):
         """Send UDP message."""
+        x = 0
         if self.tracking_enabled:
             try:
-                sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
-                x = sock.sendto(bytes(message, "utf-8"), (self.UDP_IP, self.UDP_PORT))
-                sock.close()
-            except OSError:
-                logger.debug("Unable to reach the network to send usage data")
-                x = 0
+                proc = udp_messenger(self.domain_name, self.UDP_IP, self.UDP_PORT, self.sock_timeout, message)
+                self.procs.append(proc)
+            except Exception as e:
+                logger.debug("Usage tracking failed: {}".format(e))
         else:
             x = -1
 
@@ -163,22 +216,26 @@ def send_message(self):
         else:
             message = self.construct_end_message()
 
-        x = self.send_UDP_message(message)
+        self.send_UDP_message(message)
         end = time.time()
 
-        return x, end - start
+        return end - start
+
+    def __del__(self):
+        return self.close()
+
+    def close(self):
+        """We terminate (SIGTERM) the processes added to the self.procs list """
+        for proc in self.procs:
+            proc.terminate()
 
 
 if __name__ == '__main__':
 
     from parsl import *
 
+    set_stream_logger()
     workers = ThreadPoolExecutor(max_workers=4)
     dfk = DataFlowKernel(executors=[workers])
 
-    # ut = UsageTracker(dfk, ip='52.3.111.203')
-    ut = UsageTracker(dfk, domain_name='tracking.parsl-project.org')
-
-    for i in range(0, 2):
-        x = ut.send_message()
-        print(x)
+    dfk.cleanup()


Test patch:
diff --git a/parsl/tests/test_checkpointing/checkpointed.py b/parsl/tests/test_checkpointing/checkpointed.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_checkpointing/checkpointed.py
@@ -0,0 +1,42 @@
+from parsl import App, DataFlowKernel
+import time
+import argparse
+
+
+def run_checkpointed(n=2, mode="task_exit", sleep_dur=0):
+    """ This test runs n apps that will fail with Division by zero error,
+    followed by 1 app that will succeed. The checkpoint should only have 1 task
+    """
+
+    from parsl.configs.local import localThreads as config
+    config["globals"]["checkpointMode"] = mode
+    dfk = DataFlowKernel(config=config)
+
+    @App('python', dfk, cache=True)
+    def cached_rand(x, sleep_dur=0):
+        import random
+        import time
+        time.sleep(sleep_dur)
+        return random.randint(0, 10000)
+
+    items = []
+    for i in range(0, n):
+        x = cached_rand(i, sleep_dur=sleep_dur)
+        items.append(x)
+
+    time.sleep(10)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--mode", help="DFK checkpointing mode", required=True)
+    parser.add_argument("-n", "--niter", default="2",
+                        help="Number of iterations")
+    parser.add_argument("-s", "--sleep", default="0", help="seconds to sleep")
+    parser.add_argument("-d", "--dir", help="Checkpoint dir")
+    args = parser.parse_args()
+
+    run_checkpointed(n=int(args.niter),
+                     sleep_dur=float(args.sleep),
+                     mode=args.mode)
diff --git a/parsl/tests/test_checkpointing/test_regression_232.py b/parsl/tests/test_checkpointing/test_regression_232.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_checkpointing/test_regression_232.py
@@ -0,0 +1,77 @@
+import pickle
+import time
+import subprocess
+import signal
+import os
+
+
+def kill():
+    my_pid = os.getpid()
+    os.kill(my_pid, signal.SIGINT)
+    print("Killing self")
+
+
+def test_regress_232_task_exit(count=2):
+    """Recovering from a run that was SIGINT'ed with task_exit checkpointing
+    """
+    cwd = os.path.dirname(os.path.realpath(__file__))
+    checkpoint_file = os.path.join(cwd, 'checkpointed.py')
+
+    proc = subprocess.Popen("python3 {} -n {} -m task_exit".format(checkpoint_file, count),
+                            shell=True)
+    time.sleep(0.3)
+    proc.send_signal(signal.SIGINT)
+    proc.wait()
+    # We need to wait after the signal to make sure files were closed and such
+    last = os.path.abspath(
+        'runinfo/{0}/checkpoint'.format(sorted(os.listdir('runinfo/'))[-1]))
+    checkpoint_file = "{}/tasks.pkl".format(last)
+
+    with open(checkpoint_file, 'rb') as f:
+        tasks = []
+        try:
+            while f:
+                tasks.append(pickle.load(f))
+                print
+        except EOFError:
+            print("Caught error")
+            pass
+        print("Tasks from cache : ", tasks)
+        assert len(tasks) == count, "Expected {} checkpoint events, got {}".format(
+            1, len(tasks))
+
+
+def test_regress_232_dfk_exit(count=2):
+    """Recovering from a run that was SIGINT'ed with dfk_exit checkpointing
+    """
+    cwd = os.path.dirname(os.path.realpath(__file__))
+    checkpoint_file = os.path.join(cwd, 'checkpointed.py')
+
+    proc = subprocess.Popen("python3 {} -n {} -m dkf_exit".format(checkpoint_file, count),
+                            shell=True)
+    proc.send_signal(signal.SIGINT)
+    # We need to wait after the signal to make sure files were closed and such
+    proc.wait()
+
+    last = os.path.abspath(
+        'runinfo/{0}/checkpoint'.format(sorted(os.listdir('runinfo/'))[-1]))
+    checkpoint_file = "{}/tasks.pkl".format(last)
+
+    print(checkpoint_file, "now exists")
+    with open(checkpoint_file, 'rb') as f:
+        tasks = []
+        try:
+            while f:
+                tasks.append(pickle.load(f))
+                print
+        except EOFError:
+            pass
+        print("Tasks from cache : ", tasks)
+        assert len(tasks) == count, "Expected {} checkpoint events, got {}".format(
+            1, len(tasks))
+
+
+if __name__ == "__main__":
+
+    test_regress_232_task_exit()
+    test_regress_232_dfk_exit()
diff --git a/parsl/tests/test_manual/test_regression_220.py b/parsl/tests/test_manual/test_regression_220.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_manual/test_regression_220.py
@@ -0,0 +1,28 @@
+from parsl import DataFlowKernel, set_stream_logger
+import time
+from parsl.configs.local import localThreads as config
+
+
+def test_220():
+    """Test async usage_tracking behavior for issue #220 """
+
+    print("This test assumes /etc/resolv being misconfigured")
+    with open("/etc/resolv.conf", 'r') as f:
+        for line in f.readlines():
+            line = line.strip()
+            print("Line: [{}]".format(line))
+            if line.startswith("nameserver") and line != "nameserver 2.2.2.2":
+                assert False, "/etc/resolv.conf should be misconfigured"
+
+    start = time.time()
+    set_stream_logger()
+    dfk = DataFlowKernel(config=config)
+    delta = time.time() - start
+    print("Time taken : ", delta)
+    assert delta < 1, "DFK took too much time to start, delta:{}".format(delta)
+    dfk.cleanup()
+
+
+if __name__ == "__main__":
+
+    test_220()