Repository: Parsl/parsl

Issue:
Make all swift tests pass
Currently `test_except` fails even if everything is working properly.


Discussion:

Resolving patch:
diff --git a/parsl/executors/swift_t.py b/parsl/executors/swift_t.py
--- a/parsl/executors/swift_t.py
+++ b/parsl/executors/swift_t.py
@@ -58,12 +58,12 @@ def runner(incoming_q, outgoing_q):
     logger.debug("[RUNNER] Starting")
 
     def execute_task(bufs):
-        ''' Deserialize the buf, and execute the task.
-        Returns the serialized result/exception
+        ''' Deserialize the buffer and execute the task.
+
+        Returns the serialized result or exception.
         '''
-        all_names = dir(__builtins__)
         user_ns = locals()
-        user_ns.update({'__builtins__': {k: getattr(__builtins__, k) for k in all_names}})
+        user_ns.update({'__builtins__': __builtins__})
 
         f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)
 
@@ -83,31 +83,28 @@ def execute_task(bufs):
                                                argname, kwargname)
 
         try:
-
-            print("[RUNNER] Executing : {0}".format(code))
+            logger.debug("[RUNNER] Executing: {0}".format(code))
             exec(code, user_ns, user_ns)
 
         except Exception as e:
-            logger.warning("Caught errors but will not handled %s", e)
+            logger.warning("Caught exception; will raise it: {}".format(e))
             raise e
 
         else:
-            # print("Done : {0}".format(locals()))
-            print("[RUNNER] Result    : {0}".format(user_ns.get(resultname)))
+            logger.debug("[RUNNER] Result: {0}".format(user_ns.get(resultname)))
             return user_ns.get(resultname)
 
     while True:
         try:
             # Blocking wait on the queue
             msg = incoming_q.get(block=True, timeout=10)
-            # logger.debug("[RUNNER] Got message : %s", msg)
 
         except queue.Empty:
-            # Handle case where no items were on queue
-            logger.debug("[RUNNER] got nothing")
+            # Handle case where no items were in the queue
+            logger.debug("[RUNNER] Queue is empty")
 
-        except IOError as ioerror:
-            logger.debug("[RUNNER] broken pipe, error: %s", ioerror)
+        except IOError as e:
+            logger.debug("[RUNNER] Broken pipe: {}".format(e))
             try:
                 # Attempt to send a stop notification to the management thread
                 outgoing_q.put(None)
@@ -118,7 +115,7 @@ def execute_task(bufs):
             break
 
         except Exception as e:
-            logger.debug("[RUNNER] caught unknown exception : %s", e)
+            logger.debug("[RUNNER] Caught unknown exception: {}".format(e))
 
         else:
             # Handle received message
@@ -129,17 +126,17 @@ def execute_task(bufs):
                 break
             else:
                 # Received a valid message, handle it
-                logger.debug("[RUNNER] Got a valid task : %s", msg["task_id"])
+                logger.debug("[RUNNER] Got a valid task with ID {}".format(msg["task_id"]))
                 try:
                     response_obj = execute_task(msg['buffer'])
                     response = {"task_id": msg["task_id"],
                                 "result": serialize_object(response_obj)}
 
-                    logger.warning("[RUNNER] Returing result : %s",
-                                   deserialize_object(response["result"]))
+                    logger.debug("[RUNNER] Returing result: {}".format(
+                                   deserialize_object(response["result"])))
 
                 except Exception as e:
-                    logger.debug("[RUNNER] Caught task exception")
+                    logger.debug("[RUNNER] Caught task exception: {}".format(e))
                     response = {"task_id": msg["task_id"],
                                 "exception": serialize_object(e)}
 
@@ -152,7 +149,7 @@ class TurbineExecutor(ParslExecutor):
     ''' The Turbine executor. Bypass the Swift/T language and run on top off the Turbine engines
     in an MPI environment.
 
-    Here's a simple diagram
+    Here is a diagram
 
     .. code:: python
 
@@ -172,10 +169,10 @@ class TurbineExecutor(ParslExecutor):
     '''
 
     def _queue_management_worker(self):
-        ''' The queue management worker is responsible for listening to the incoming_q
-        for task status messages and updating tasks with results/exceptions/updates
+        '''Listen to the queue for task status messages and handle them.
 
-        It expects the following messages:
+        Depending on the message, tasks will be updated with results, exceptions,
+        or updates. It expects the following messages:
 
         .. code:: python
 
@@ -190,7 +187,7 @@ def _queue_management_worker(self):
                "exception" : serialized exception object, on failure
             }
 
-        We don't support these yet, but they could be added easily as heartbeat.
+        We do not support these yet, but they could be added easily.
 
         .. code:: python
 
@@ -202,9 +199,7 @@ def _queue_management_worker(self):
                "started"  : tstamp
             }
 
-        The None message is a die request.
-        None
-
+        The `None` message is a die request.
         '''
 
         while True:
@@ -213,15 +208,15 @@ def _queue_management_worker(self):
                 msg = self.incoming_q.get(block=True, timeout=1)
 
             except queue.Empty as e:
-                # timed out.
+                # Timed out.
                 pass
 
             except IOError as e:
-                logger.debug("[MTHREAD] caught broken queue : %s : errno:%s", e, e.errno)
+                logger.debug("[MTHREAD] Caught broken queue with exception code {}: {}".format(e.errno, e))
                 return
 
             except Exception as e:
-                logger.debug("[MTHREAD] caught unknown exception : %s", e)
+                logger.debug("[MTHREAD] Caught unknown exception: {}".format(e))
 
             else:
 
@@ -230,7 +225,7 @@ def _queue_management_worker(self):
                     return
 
                 else:
-                    logger.debug("[MTHREAD] Got message : %s", msg)
+                    logger.debug("[MTHREAD] Received message: {}".format(msg))
                     task_fut = self.tasks[msg['task_id']]
                     if 'result' in msg:
                         result, _ = deserialize_object(msg['result'])
@@ -240,7 +235,7 @@ def _queue_management_worker(self):
                         exception, _ = deserialize_object(msg['exception'])
                         task_fut.set_exception(exception)
 
-            if not self.isAlive:
+            if not self.is_alive:
                 break
 
     # When the executor gets lost, the weakref callback will wake up
@@ -271,7 +266,7 @@ def shutdown(self):
         ''' Shutdown method, to kill the threads and workers.
         '''
 
-        self.isAlive = False
+        self.is_alive = False
         logging.debug("Waking management thread")
         self.incoming_q.put(None)  # Wake up the thread
         self._queue_management_thread.join()  # Force join
@@ -288,11 +283,11 @@ def __init__(self, swift_attribs=None, config=None, **kwargs):
 
         '''
         self.config = config
-        logger.debug("In __init__")
+        logger.debug("Initializing TurbineExecutor")
         self.mp_manager = mp.Manager()
         self.outgoing_q = self.mp_manager.Queue()
         self.incoming_q = self.mp_manager.Queue()
-        self.isAlive = True
+        self.is_alive = True
 
         self._queue_management_thread = None
         self._start_queue_management_thread()
@@ -321,7 +316,7 @@ def submit(self, func, *args, **kwargs):
         '''
         task_id = uuid.uuid4()
 
-        logger.debug("Before pushing to queue : func:%s func_args:%s", func, args)
+        logger.debug("Pushing function {} to queue with args {}".format(func, args))
 
         self.tasks[task_id] = Future()
 


Test patch:
diff --git a/parsl/tests/test_swift.py b/parsl/tests/test_swift.py
--- a/parsl/tests/test_swift.py
+++ b/parsl/tests/test_swift.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3.5
+from nose.tools import assert_raises
 
 import parsl
 from parsl import *
@@ -27,36 +28,36 @@ def test_simple():
     print("Start")
     tex = TurbineExecutor()
     x = tex.submit(foo, 5, 10)
-    print("Got : ", x)
-    print("X result : ", x.result())
+    print("Got: ", x)
+    print("X result: ", x.result())
     assert x.result() == 50, "X != 50"
     print("done")
 
 
-def test_except():
-    print("Start")
+def test_slow():
+    futs = {}
     tex = TurbineExecutor()
-    x = tex.submit(bad_foo, 5, 10)
-    print("Got : ", x)
+    for i in range(0, 3):
+        futs[i] = tex.submit(slow_foo, 1, 2)
 
-    print("X exception : ", x.exception())
-    print("X result : ", x.result())
+    total = sum([futs[i].result(timeout=10) for i in futs])
+    assert total == 6, "expected 6, got {}".format(total)
 
-    print("done")
+
+def test_except():
+    def get_bad_result():
+        tex = TurbineExecutor()
+        x = tex.submit(bad_foo, 5, 10)
+
+        return x.result()
+
+    assert_raises(NameError, get_bad_result)
 
 
 if __name__ == "__main__":
 
     # test_simple()
+    # test_slow()
     test_except()
-    exit(0)
-    futs = {}
-    for i in range(0, 1):
-        futs[i] = tex.submit(slow_foo, 3, 10)
-
-    x.result(timeout=10)
-    for x in range(0, 10):
-        print(futs)
-        time.sleep(4)
 
     print("Done")