Repository: Parsl/parsl

Issue:
Race condition in task checkpointing
With `config["globals"]["checkpointMode"] = "task_exit"`, the DFK attempts to checkpoint every task at task completion. However in the case of fast apps without dependencies, the task returns before the AppFuture is created. This results in a race condition between the checkpoint method and the task update.

If the checkpoint method is called first we get :
```
Traceback (most recent call last):
  File "test_regression_233.py", line 38, in <module>
    test_checkpoint_availability()
  File "test_regression_233.py", line 27, in test_checkpoint_availability
    original = run_checkpointed([])
  File "test_regression_233.py", line 18, in run_checkpointed
    x = cached_rand(i)
  File "/home/yadu/src/parsl/parsl/app/app_factory.py", line 76, in __call__
    return app_obj(*args, **kwargs)
  File "/home/yadu/src/parsl/parsl/app/python_app.py", line 46, in __call__
    **kwargs)
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 499, in submit
    task_id, func, *new_args, **kwargs)
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 322, in launch_task
    exec_fu.add_done_callback(partial(self.handle_update, task_id))
  File "/usr/lib/python3.5/concurrent/futures/_base.py", line 376, in add_done_callback
    fn(self)
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 218, in handle_update
    self.checkpoint(tasks=[task_id])
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 617, in checkpoint
    if self.tasks[task_id]['app_fu'].done() and \
AttributeError: 'NoneType' object has no attribute 'done'
```


Race condition in task checkpointing
With `config["globals"]["checkpointMode"] = "task_exit"`, the DFK attempts to checkpoint every task at task completion. However in the case of fast apps without dependencies, the task returns before the AppFuture is created. This results in a race condition between the checkpoint method and the task update.

If the checkpoint method is called first we get :
```
Traceback (most recent call last):
  File "test_regression_233.py", line 38, in <module>
    test_checkpoint_availability()
  File "test_regression_233.py", line 27, in test_checkpoint_availability
    original = run_checkpointed([])
  File "test_regression_233.py", line 18, in run_checkpointed
    x = cached_rand(i)
  File "/home/yadu/src/parsl/parsl/app/app_factory.py", line 76, in __call__
    return app_obj(*args, **kwargs)
  File "/home/yadu/src/parsl/parsl/app/python_app.py", line 46, in __call__
    **kwargs)
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 499, in submit
    task_id, func, *new_args, **kwargs)
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 322, in launch_task
    exec_fu.add_done_callback(partial(self.handle_update, task_id))
  File "/usr/lib/python3.5/concurrent/futures/_base.py", line 376, in add_done_callback
    fn(self)
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 218, in handle_update
    self.checkpoint(tasks=[task_id])
  File "/home/yadu/src/parsl/parsl/dataflow/dflow.py", line 617, in checkpoint
    if self.tasks[task_id]['app_fu'].done() and \
AttributeError: 'NoneType' object has no attribute 'done'
```




Discussion:
I also see this error - but in my case, as far as I can tell, it is from a bash app that is slow-ish (a 5 second sleep) - with no dependencies.

This might be a different bug. I'm able to run 1s python apps with no issue.
Yes, I have a different backtrace further up the stack more closely, now I look. It looks like it is something happening at the *start* of a call, not the end. I'll try to make a smaller test case.
I also see this error - but in my case, as far as I can tell, it is from a bash app that is slow-ish (a 5 second sleep) - with no dependencies.

This might be a different bug. I'm able to run 1s python apps with no issue.
Yes, I have a different backtrace further up the stack more closely, now I look. It looks like it is something happening at the *start* of a call, not the end. I'll try to make a smaller test case.

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -21,6 +21,7 @@
 from parsl.data_provider.data_manager import DataManager
 from parsl.execution_provider.provider_factory import ExecProviderFactory as EPF
 from parsl.utils import get_version
+from parsl.app.errors import RemoteException
 
 # from parsl.dataflow.start_controller import Controller
 # Exceptions
@@ -89,8 +90,15 @@ def __init__(self, config=None, executors=None, lazyErrors=True, appCache=True,
         self.usage_tracker = UsageTracker(self)
         self.usage_tracker.send_message()
 
-        # Load checkpoints if any
-        cpts = self.load_checkpoints(checkpointFiles)
+        # Load Memoizer with checkpoints before we start the run.
+        if checkpointFiles:
+            checkpoint_src = checkpointFiles
+        elif self._config and self._config["globals"]["checkpointFiles"]:
+            checkpoint_src = self._config["globals"]["checkpointFiles"]
+        else:
+            checkpoint_src = None
+
+        cpts = self.load_checkpoints(checkpoint_src)
         # Initialize the memoizer
         self.memoizer = Memoizer(self, memoize=appCache, checkpoint=cpts)
         self.checkpointed_tasks = 0
@@ -175,49 +183,47 @@ def handle_update(self, task_id, future, memo_cbk=False):
              memo_cbk(Bool) : Indicates that the call is coming from a memo update,
              that does not require additional memo updates.
         """
-        # TODO : Remove, this check is redundant
         final_state_flag = False
 
-        if future.done():
-
-            try:
-                future.result()
-
-            except Exception as e:
-                logger.exception("Task {} failed".format(task_id))
+        try:
+            res = future.result()
+            if isinstance(res, RemoteException):
+                res.reraise()
 
-                # We keep the history separately, since the future itself could be
-                # tossed.
-                self.tasks[task_id]['fail_history'].append(future._exception)
-                self.tasks[task_id]['fail_count'] += 1
+        except Exception as e:
+            logger.exception("Task {} failed".format(task_id))
 
-                if not self.lazy_fail:
-                    logger.debug("Eager fail, skipping retry logic")
-                    raise e
+            # We keep the history separately, since the future itself could be
+            # tossed.
+            self.tasks[task_id]['fail_history'].append(future._exception)
+            self.tasks[task_id]['fail_count'] += 1
 
-                if self.tasks[task_id]['fail_count'] <= self.fail_retries:
-                    logger.debug("Task {} marked for retry".format(task_id))
-                    self.tasks[task_id]['status'] = States.pending
+            if not self.lazy_fail:
+                logger.debug("Eager fail, skipping retry logic")
+                raise e
 
-                else:
-                    logger.info("Task {} failed after {} retry attempts".format(task_id,
-                                                                                self.fail_retries))
-                    self.tasks[task_id]['status'] = States.failed
-                    final_state_flag = True
+            if self.tasks[task_id]['fail_count'] <= self.fail_retries:
+                logger.debug("Task {} marked for retry".format(task_id))
+                self.tasks[task_id]['status'] = States.pending
 
             else:
-                logger.info("Task {} completed".format(task_id))
-                self.tasks[task_id]['status'] = States.done
+                logger.info("Task {} failed after {} retry attempts".format(task_id,
+                                                                            self.fail_retries))
+                self.tasks[task_id]['status'] = States.failed
                 final_state_flag = True
 
+        else:
+            logger.info("Task {} completed".format(task_id))
+            self.tasks[task_id]['status'] = States.done
+            final_state_flag = True
+
         if not memo_cbk and final_state_flag is True:
             # Update the memoizer with the new result if this is not a
             # result from a memo lookup and the task has reached a terminal state.
             self.memoizer.update_memo(task_id, self.tasks[task_id], future)
 
             if self.checkpoint_mode is 'task_exit':
-                self.checkpoint()
-                logger.debug("Task {} checkpoint created at task exit".format(task_id))
+                self.checkpoint(tasks=[task_id])
 
         # Identify tasks that have resolved dependencies and launch
         for tid in list(self.tasks):
@@ -318,6 +324,7 @@ def launch_task(self, task_id, executable, *args, **kwargs):
                 self.tasks[task_id]['func'].__name__))
 
         exec_fu = executor.submit(executable, *args, **kwargs)
+        self.tasks[task_id]['status'] = States.running
         exec_fu.retries_left = self.fail_retries - \
             self.tasks[task_id]['fail_count']
         exec_fu.add_done_callback(partial(self.handle_update, task_id))
@@ -502,7 +509,6 @@ def submit(self, func, *args, parsl_sites='all', fn_hash=None, cache=False, **kw
                                                           tid=task_id,
                                                           stdout=task_stdout,
                                                           stderr=task_stderr)
-                self.tasks[task_id]['status'] = States.running
                 logger.debug("Task {} launched with AppFut:{}".format(task_id,
                                                                       task_def['app_fu']))
 
@@ -573,12 +579,16 @@ def cleanup(self):
 
         logger.info("DFK cleanup complete")
 
-    def checkpoint(self):
+    def checkpoint(self, tasks=None):
         """Checkpoint the dfk incrementally to a checkpoint file.
 
         When called, every task that has been completed yet not
         checkpointed is checkpointed to a file.
 
+        Kwargs:
+            - tasks (List of task ids) : List of task ids to checkpoint. Default=None
+                                         if set to None, we iterate over all tasks held by the DFK.
+
         .. note::
             Checkpointing only works if memoization is enabled
 
@@ -587,7 +597,12 @@ def checkpoint(self):
             By default the checkpoints are written to the RUNDIR of the current
             run under RUNDIR/checkpoints/{tasks.pkl, dfk.pkl}
         """
-        logger.info("Checkpointing.. ")
+
+        checkpoint_queue = None
+        if tasks:
+            checkpoint_queue = tasks
+        else:
+            checkpoint_queue = self.tasks
 
         checkpoint_dir = '{0}/checkpoint'.format(self.rundir)
         checkpoint_dfk = checkpoint_dir + '/dfk.pkl'
@@ -607,10 +622,12 @@ def checkpoint(self):
             pickle.dump(state, f)
 
         count = 0
+
         with open(checkpoint_tasks, 'ab') as f:
-            for task_id in self.tasks:
-                if self.tasks[task_id]['app_fu'].done() and \
-                   not self.tasks[task_id]['checkpoint']:
+            for task_id in checkpoint_queue:
+                if not self.tasks[task_id]['checkpoint'] and \
+                   self.tasks[task_id]['status'] == States.done:
+
                     hashsum = self.tasks[task_id]['hashsum']
                     if not hashsum:
                         continue
@@ -665,6 +682,7 @@ def _load_checkpoints(self, checkpointDirs):
         memo_lookup_table = {}
 
         for checkpoint_dir in checkpointDirs:
+            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
             checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
             try:
                 with open(checkpoint_file, 'rb') as f:


Test patch:
diff --git a/parsl/tests/test_checkpointing/test_dfk_exit.py b/parsl/tests/test_checkpointing/test_dfk_exit.py
deleted file mode 100644
--- a/parsl/tests/test_checkpointing/test_dfk_exit.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import parsl
-from parsl import *
-import argparse
-
-config = {
-    "sites": [
-        {"site": "Local_Threads",
-         "auth": {"channel": None},
-         "execution": {
-             "executor": "threads",
-             "provider": None,
-             "maxThreads": 2,
-         }
-        }],
-    "globals": {"lazyErrors": True,
-                "memoize": True,
-                "checkpointMode": "dfk_exit",
-    }
-}
-dfk = DataFlowKernel(config=config)
-
-
-@App('python', dfk, cache=True)
-def slow_double(x, sleep_dur=1):
-    import time
-    time.sleep(sleep_dur)
-    return x * 2
-
-
-def test_at_task_exit(n=4):
-    """Test checkpointing at task_exit behavior
-    """
-
-    d = {}
-
-    print("Launching : ", n)
-    for i in range(0, n):
-        d[i] = slow_double(i)
-    print("Done launching")
-
-    for i in range(0, n):
-        d[i].result()
-    print("Done sleeping")
-
-    dfk.cleanup()
-    print("Rundir : ", dfk.rundir)
-    with open("{}/parsl.log".format(dfk.rundir), 'r') as f:
-        expected_msg = "Done checkpointing {} tasks".format(n)
-        lines = [line for line in f.readlines() if expected_msg in line]
-        assert len(lines) == 1, "Expected {} checkpoint events, got {}".format(1, len(lines))
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-c", "--count", default="10",
-                        help="Count of apps to launch")
-    parser.add_argument("-d", "--debug", action='store_true',
-                        help="Count of apps to launch")
-    args = parser.parse_args()
-
-    if args.debug:
-        parsl.set_stream_logger()
-
-    x = test_at_task_exit(n=4)
diff --git a/parsl/tests/test_checkpointing/test_regression_233.py b/parsl/tests/test_checkpointing/test_regression_233.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_checkpointing/test_regression_233.py
@@ -0,0 +1,76 @@
+from parsl import *
+# set_stream_logger()
+
+
+def run_checkpointed(cpts):
+    # set_stream_logger()
+    from parsl.configs.local import localThreads as config
+    config["globals"]["checkpointMode"] = "task_exit"
+    config["globals"]["checkpointFiles"] = cpts
+    dfk = DataFlowKernel(config=config)
+
+    @App('python', dfk, cache=True)
+    def cached_rand(x):
+        import random
+        return random.randint(0, 10000)
+
+    items = []
+    for i in range(0, 5):
+        x = cached_rand(i)
+        items.append(x)
+
+    dfk.cleanup()
+    return [i.result() for i in items]
+
+
+def run_race(sleep_dur):
+
+    from parsl.configs.local import localThreads as config
+    config["globals"]["checkpointMode"] = "task_exit"
+    dfk = DataFlowKernel(config=config)
+
+    @App('python', dfk, cache=True)
+    def cached_rand(x, sleep_dur=0):
+        import random
+        import time
+        time.sleep(sleep_dur)
+        return random.randint(0, 10000)
+
+    items = []
+    for i in range(0, 5):
+        x = cached_rand(i, sleep_dur=sleep_dur)
+        items.append(x)
+
+    dfk.cleanup()
+    return [i.result for i in items]
+
+
+def test_regress_234():
+    """Test task_exit checkpointing with fast tasks"""
+    run_race(0)
+
+
+def test_slower_apps():
+    """Test task_exit tests with slow apps"""
+    run_race(0.5)
+
+
+def test_checkpoint_availability():
+    import os
+
+    original = run_checkpointed([])
+    last_checkpoint = os.path.abspath('runinfo/{0}/checkpoint'.format(sorted(os.listdir('runinfo/'))[-1]))
+    print(last_checkpoint)
+    cached = run_checkpointed([last_checkpoint])
+
+    print(cached)
+    print(original)
+
+    assert cached == original, "All tasks were not cached"
+
+
+if __name__ == "__main__":
+
+    test_checkpoint_availability()
+    # test_regress_234()
+    # test_slower_apps()
diff --git a/parsl/tests/test_checkpointing/test_regression_239.py b/parsl/tests/test_checkpointing/test_regression_239.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_checkpointing/test_regression_239.py
@@ -0,0 +1,85 @@
+from parsl import App, DataFlowKernel
+import pickle
+import time
+
+
+def run_checkpointed(n=2, mode="task_exit"):
+    """ This test runs n apps that will fail with Division by zero error,
+    followed by 1 app that will succeed. THe checkpoint should only have 1 task
+    """
+
+    from parsl.configs.local import localThreads as config
+    config["globals"]["checkpointMode"] = mode
+    dfk = DataFlowKernel(config=config)
+
+    @App('python', dfk, cache=True)
+    def cached_rand(x):
+        import random
+        return random.randint(0, 10000)
+
+    @App('python', dfk, cache=True)
+    def cached_failing(x):
+        5 / 0
+        return 1
+
+    items = []
+    for i in range(0, n):
+        x = cached_failing(0)
+        items.append(x)
+        try:
+            x.result()
+        except Exception as e:
+            print("Ignoring failure of task")
+            pass
+
+    x = cached_rand(1)
+    print(x.result())
+    rundir = dfk.rundir
+    # Call cleanup *only* for dfk_exit to ensure that a checkpoint is written
+    # at all
+    if mode == "dfk_exit":
+        dfk.cleanup()
+    return rundir
+
+
+def test_regress_239():
+    """Ensure failed tasks are not cached with task_exit mode. Tests #239
+    Also tests task_exit behavior.
+    """
+
+    rundir = run_checkpointed()
+    time.sleep(0.5)
+    with open("{}/checkpoint/tasks.pkl".format(rundir), 'rb') as f:
+        tasks = []
+        try:
+            while f:
+                tasks.append(pickle.load(f))
+                print
+        except EOFError:
+            pass
+        print("Tasks from cache : ", tasks)
+        assert len(tasks) == 1, "Expected {} checkpoint events, got {}".format(1, len(tasks))
+
+
+def test_checkpointing_at_dfk_exit():
+    """Ensure failed tasks are not cached with dfk_exit mode. Tests #239
+    """
+
+    rundir = run_checkpointed(mode="dfk_exit")
+    print("Back in test fn")
+    time.sleep(0.5)
+    with open("{}/checkpoint/tasks.pkl".format(rundir), 'rb') as f:
+        tasks = []
+        try:
+            while f:
+                tasks.append(pickle.load(f))
+                print
+        except EOFError:
+            pass
+        print("Tasks from cache : ", tasks)
+        assert len(tasks) == 1, "Expected {} checkpoint events, got {}".format(1, len(tasks))
+
+
+if __name__ == "__main__":
+    test_regress_239()
+    test_checkpointing_at_dfk_exit()
diff --git a/parsl/tests/test_checkpointing/test_task_exit.py b/parsl/tests/test_checkpointing/test_task_exit.py
--- a/parsl/tests/test_checkpointing/test_task_exit.py
+++ b/parsl/tests/test_checkpointing/test_task_exit.py
@@ -1,4 +1,5 @@
 import parsl
+import pickle
 import time
 from parsl import *
 import argparse
@@ -42,12 +43,17 @@ def test_at_task_exit(n=4):
     for i in range(0, n):
         d[i].result()
 
-    time.sleep(1)
+    time.sleep(0.3)
+    print("Loading results")
+    with open("{}/checkpoint/tasks.pkl".format(dfk.rundir), 'rb') as f:
+        tasks = []
+        try:
+            while f:
+                tasks.append(pickle.load(f))
+        except EOFError:
+            pass
 
-    print("Rundir : ", dfk.rundir)
-    with open("{}/parsl.log".format(dfk.rundir), 'r') as f:
-        lines = [line for line in f.readlines() if "Checkpointing.." in line]
-        assert len(lines) == n, "Expected {} checkpoint events, got {}".format(n, len(lines))
+        assert len(tasks) == n, "Expected {} checkpoint events, got {}".format(n, len(tasks))
 
 
 if __name__ == '__main__':
diff --git a/parsl/tests/test_cleanup/test_cleanup.py b/parsl/tests/test_cleanup/test_cleanup.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_cleanup/test_cleanup.py
@@ -0,0 +1,34 @@
+
+from parsl import *
+from parsl.configs.local import localThreads as config
+dfk = DataFlowKernel(config=config)
+
+
+@App('python', dfk)
+def slow_double(x, dur=0.1):
+    import time
+    time.sleep(dur)
+    return x * 5
+
+
+def test_cleanup_behavior_221():
+    """ A1 A2 A3   -> cleanup
+        B1 B2 B3
+
+    """
+    round_1 = []
+    for i in range(0, 10):
+        f = slow_double(i)
+        round_1.append(f)
+
+    round_2 = []
+    for i in round_1:
+        f = slow_double(i)
+        round_2.append(f)
+
+    dfk.cleanup()
+
+
+if __name__ == "__main__":
+
+    test_cleanup_behavior_221()
diff --git a/parsl/tests/test_sites/test_ec2/ec2.py b/parsl/tests/test_sites/test_ec2/ec2.py
deleted file mode 100644
--- a/parsl/tests/test_sites/test_ec2/ec2.py
+++ /dev/null
@@ -1,91 +0,0 @@
-ec2OnDemand = {
-    "sites": [
-        {"site": "ec2",
-         "auth": {
-             "channel": None,
-             "profile": "default"
-         },
-         "execution":
-         {"executor": "ipp",
-          "provider": "aws",
-             "channel": None,
-             "block": {
-                 "initBlocks": 1,
-                 "maxBlocks": 1,
-                 "minBlocks": 0,
-                 "taskBlocks": 1,
-                 "nodes": 1,
-                 "walltime": "00:25:00",
-                 "options": {
-                     "region": "us-east-2",
-                     "imageId": 'ami-82f4dae7',
-                     "stateFile": "awsproviderstate.json",
-                     "keyName": "parsl.test"
-                 }
-             }
-          }
-         }
-    ]
-}
-
-spotNode = {
-    "sites": [
-        {"site": "ec2",
-         "auth": {
-             "channel": None,
-             "profile": "default"
-         },
-         "execution":
-         {"executor": "ipp",
-          "provider": "aws",
-             "channel": None,
-             "block": {
-                 "initBlocks": 1,
-                 "maxBlocks": 1,
-                 "minBlocks": 0,
-                 "taskBlocks": 1,
-                 "nodes": 1,
-                 "walltime": "00:25:00",
-                 "options": {
-                     "region": "us-east-2",
-                     "imageId": 'ami-82f4dae7',
-                     "stateFile": "awsproviderstate.json",
-                     "keyName": "parsl.test",
-                     "spotMaxBid": 1.0
-                 }
-             }
-          }
-         }
-    ]
-}
-
-badSpotConfig = {
-    "sites": [
-        {"site": "ec2",
-         "auth": {
-             "channel": None,
-             "profile": "default"
-         },
-         "execution":
-         {"executor": "ipp",
-          "provider": "aws",
-             "channel": None,
-             "block": {
-                 "initBlocks": 1,
-                 "maxBlocks": 1,
-                 "minBlocks": 0,
-                 "taskBlocks": 1,
-                 "nodes": 1,
-                 "walltime": "00:25:00",
-                 "options": {
-                     "region": "us-east-2",
-                     "imageId": 'ami-82f4dae7',
-                     "stateFile": "awsproviderstate.json",
-                     "keyName": "parsl.test",
-                     "spotMaxBid": 0.001  # <--- Price too low
-                 }
-             }
-          }
-         }
-    ]
-}
diff --git a/parsl/tests/test_sites/test_ec2/ec2.py b/parsl/tests/test_sites/test_ec2/ec2.py
new file mode 120000
--- /dev/null
+++ b/parsl/tests/test_sites/test_ec2/ec2.py
@@ -0,0 +1 @@
+../configs/ec2.py
\ No newline at end of file