Repository: Parsl/parsl

Issue:
Clarify warning message when a manager goes missing
Currently in the parsl log, if a manager goes missing, the user sees:
```
Remote exception description: b'da6348e1c3f1'
```

While in the interchange log you see the more helpful
```
2018-12-11 14:50:09 interchange:376 [WARNING]  [MAIN] Too many heartbeats missed for manager b'da6348e1c3f1'
```

We should improve the former.


Discussion:

Resolving patch:
diff --git a/parsl/executors/high_throughput/interchange.py b/parsl/executors/high_throughput/interchange.py
--- a/parsl/executors/high_throughput/interchange.py
+++ b/parsl/executors/high_throughput/interchange.py
@@ -32,17 +32,24 @@ def __init__(self):
     def __repr__(self):
         return "Shutdown request received at {}".format(self.tstamp)
 
+    def __str__(self):
+        return self.__repr__()
+
 
 class ManagerLost(Exception):
     ''' Task lost due to worker loss. Worker is considered lost when multiple heartbeats
     have been missed.
     '''
-    def __init__(self, worker_id):
+    def __init__(self, worker_id, hostname):
         self.worker_id = worker_id
         self.tstamp = time.time()
+        self.hostname = hostname
 
     def __repr__(self):
-        return "Task failure due to loss of worker {}".format(self.worker_id)
+        return "Task failure due to loss of Manager {} on host {}".format(self.worker_id, self.hostname)
+
+    def __str__(self):
+        return self.__repr__()
 
 
 class BadRegistration(Exception):
@@ -54,8 +61,12 @@ def __init__(self, worker_id, critical=False):
         self.handled = "critical" if critical else "suppressed"
 
     def __repr__(self):
-        return "Manager:{} caused a {} failure".format(self.worker_id,
-                                                       self.handled)
+        return "Manager {} attempted to register with a bad registration message. Caused a {} failure".format(
+            self.worker_id,
+            self.handled)
+
+    def __str__(self):
+        return self.__repr__()
 
 
 class Interchange(object):
@@ -358,6 +369,7 @@ def start(self, poll_period=None):
                                                           'free_capacity': 0,
                                                           'block_id': None,
                                                           'max_capacity': 0,
+                                                          'worker_count': 0,
                                                           'active': True,
                                                           'tasks': []}
                     if reg_flag is True:
@@ -373,7 +385,7 @@ def start(self, poll_period=None):
                             if self.suppress_failure is False:
                                 logger.debug("Setting kill event")
                                 self._kill_event.set()
-                                e = ManagerLost(manager)
+                                e = ManagerLost(manager, self._ready_manager_queue[manager]['hname'])
                                 result_package = {'task_id': -1, 'exception': serialize_object(e)}
                                 pkl_package = pickle.dumps(result_package)
                                 self.results_outgoing.send(pkl_package)
@@ -461,7 +473,6 @@ def start(self, poll_period=None):
                     logger.debug("[MAIN] Current tasks: {}".format(self._ready_manager_queue[manager]['tasks']))
                 logger.debug("[MAIN] leaving results_incoming section")
 
-            logger.debug("[MAIN] entering bad_managers section")
             bad_managers = [manager for manager in self._ready_manager_queue if
                             time.time() - self._ready_manager_queue[manager]['last'] > self.heartbeat_threshold]
             for manager in bad_managers:
@@ -470,15 +481,13 @@ def start(self, poll_period=None):
 
                 for tid in self._ready_manager_queue[manager]['tasks']:
                     try:
-                        raise ManagerLost(manager)
+                        raise ManagerLost(manager, self._ready_manager_queue[manager]['hname'])
                     except Exception:
                         result_package = {'task_id': tid, 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}
                         pkl_package = pickle.dumps(result_package)
                         self.results_outgoing.send(pkl_package)
                         logger.warning("[MAIN] Sent failure reports, unregistering manager")
                 self._ready_manager_queue.pop(manager, 'None')
-            logger.debug("[MAIN] leaving bad_managers section")
-            logger.debug("[MAIN] ending one main loop iteration")
 
         delta = time.time() - start
         logger.info("Processed {} tasks in {} seconds".format(count, delta))


Test patch:
diff --git a/parsl/tests/manual_tests/test_htex_worker_loss.py b/parsl/tests/manual_tests/test_htex_worker_loss.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/manual_tests/test_htex_worker_loss.py
@@ -0,0 +1,139 @@
+import argparse
+import time
+import parsl
+
+
+parsl.set_stream_logger()
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SingleNodeLauncher
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+try:
+    from parsl.addresses import address_by_interface
+    address = address_by_interface('enx00e112002c62')
+except OSError:
+    print("***************ERROR****************************")
+    print("Interface doesn't exist, falling back to default, 127.0.0.1")
+    print("***************ERROR****************************")
+    address = '127.0.01'
+
+
+config = Config(
+    executors=[
+        HighThroughputExecutor(
+            poll_period=1,
+            address=address,
+            label="htex_local",
+            worker_debug=True,
+            cores_per_worker=1,
+            heartbeat_period=1,
+            heartbeat_threshold=2,
+            worker_ports=(54759, 54330),
+            suppress_failure=True,
+            provider=LocalProvider(
+                channel=LocalChannel(),
+                init_blocks=0,
+                max_blocks=0,
+                # tasks_per_node=1,  # For HighThroughputExecutor, this option should in most cases be 1
+                launcher=SingleNodeLauncher(),
+            ),
+        )
+    ],
+    strategy=None,
+)
+
+
+# config.executors[0].provider.tasks_per_node = 4
+parsl.load(config)
+from parsl.app.app import python_app  # , bash_app
+
+
+@python_app
+def double(x):
+    return x * 2
+
+
+@python_app
+def platform(sleep=10, stdout=None):
+    import platform
+    import time
+    time.sleep(sleep)
+    return platform.uname()
+
+
+def test_simple(n=2, dur=10):
+    """ Tests whether the right exception is returned when a manager is lost.
+    Check whether the error string reports manager loss on node X.
+
+    """
+    start = time.time()
+
+    # Prime a worker
+    double(5).result()
+
+    # Launch the slow task to a failing node.
+    x = platform(sleep=dur)
+
+    time.sleep(0.1)
+
+    dfk = parsl.dfk()
+    job_ids = dfk.executors['htex_local'].provider.resources.keys()
+    dfk.executors['htex_local'].provider.cancel(job_ids)
+    print("Result : ", x.result())
+    print("Duration : {0}s".format(time.time() - start))
+    print("[TEST STATUS] test_parallel_for [SUCCESS]")
+    return True
+
+
+def test_manager_fail(n=2, dur=10):
+    """ Test manager failure due to intermittent n/w loss.
+    Run parsl on laptop, start the worker on a remote node.
+    Once connected and task is executed, kill the network
+    confirm that the interchange does not error out.
+    """
+    start = time.time()
+
+    # Prime a worker
+    double(5).result()
+
+    # Launch the slow task to a failing node.
+    x = platform(sleep=dur)
+    x.result()
+
+    # At this point we know that the worker is connected.
+    # Now we need to kill the network to mimic a n/w failure.
+    time.sleep(0.1)
+    print('*' * 80)
+    print("Manually kill the network to create a n/w failure")
+    print("We should see suppression of the failure")
+    print('*' * 80)
+
+    print("Duration : {0}s".format(time.time() - start))
+    print("[TEST STATUS] test_parallel_for [SUCCESS]")
+    dfk = parsl.dfk()
+    for i in range(200):
+        print(dfk.executors['htex_local'].connected_workers)
+        time.sleep(1)
+
+    return True
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-s", "--sleep", default="4")
+    parser.add_argument("-c", "--count", default="10",
+                        help="Count of apps to launch")
+    parser.add_argument("-d", "--debug", action='store_true',
+                        help="Count of apps to launch")
+    args = parser.parse_args()
+
+    if args.debug:
+        parsl.set_stream_logger()
+
+    # x = test_simple(int(args.count))
+    # x = test_imports()
+    # x = test_simple(int(args.count), int(args.sleep))
+    x = test_manager_fail(int(args.count), int(args.sleep))