Repository: Parsl/parsl

Issue:
Internal consistency error
Running a large number of tests with the thread local executor, the tasks complete with `done` state, but the log shows (i) an internal consistency error, and (ii) the task logged as launched *after* it has completed.

That consistency check is probably not actually required to be true, but it is worrying as it *should* be true.

```
2018-10-30 16:29:48 parsl.dataflow.dflow:640 [INFO]  Task 0 submitted for App foo, waiting on tasks [
]
2018-10-30 16:29:48 parsl.dataflow.dflow:646 [DEBUG]  Task 0 set to pending state with AppFuture: <Ap
pFuture at 0x7f23a75ac518 state=pending>
2018-10-30 16:29:48 parsl.dataflow.dflow:304 [INFO]  Task 0 completed
2018-10-30 16:29:48 parsl.dataflow.dflow:310 [ERROR]  Internal consistency error: app_fu is not done for task 0
2018-10-30 16:29:48 parsl.dataflow.dflow:447 [INFO]  Task 0 launched on executor threads
```


Discussion:
Addressing (i):

`handle_update` is attached to `exec_fu`, not to `app_fu`.

This has to happen after `exec_fu`, not after `app_fu`, because retries are firing this way (when exec_fu is done, but app_fu defininitely isn't).

The consistency check is maybe not a thing that we can expect to be consistent in this way now

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -233,11 +233,12 @@ def config(self):
         """
         return self._config
 
-    def handle_update(self, task_id, future, memo_cbk=False):
-        """This function is called only as a callback from a task being done.
+    def handle_exec_update(self, task_id, future):
+        """This function is called only as a callback from an execution
+        attempt reaching a final state (either successfully or failing).
 
-        Move done task from runnable -> done
-        Move newly doable tasks from pending -> runnable , and launch
+        It will launch retries if necessary, and update the task
+        structure.
 
         Args:
              task_id (string) : Task id which is a uuid string
@@ -248,7 +249,6 @@ def handle_update(self, task_id, future, memo_cbk=False):
              memo_cbk(Bool) : Indicates that the call is coming from a memo update,
              that does not require additional memo updates.
         """
-        final_state_flag = False
 
         try:
             res = future.result()
@@ -282,7 +282,6 @@ def handle_update(self, task_id, future, memo_cbk=False):
                 logger.info("Task {} failed after {} retry attempts".format(task_id,
                                                                             self._config.retries))
                 self.tasks[task_id]['status'] = States.failed
-                final_state_flag = True
                 self.tasks_failed_count += 1
 
                 self.tasks[task_id]['time_returned'] = time.time()
@@ -292,7 +291,6 @@ def handle_update(self, task_id, future, memo_cbk=False):
 
         else:
             self.tasks[task_id]['status'] = States.done
-            final_state_flag = True
             self.tasks_completed_count += 1
 
             logger.info("Task {} completed".format(task_id))
@@ -300,10 +298,37 @@ def handle_update(self, task_id, future, memo_cbk=False):
             if self.monitoring_config is not None:
                 task_log_info = self._create_task_log_info(task_id)
                 self.db_logger.info("Task Done", extra=task_log_info)
-            if not self.tasks[task_id]['app_fu'].done():
-                logger.error("Internal consistency error: app_fu is not done for task {}".format(task_id))
 
-        if not memo_cbk and final_state_flag is True:
+        # it might be that in the course of the update, we've gone back to being
+        # pending - in which case, we should consider ourself for relaunch
+        if self.tasks[task_id]['status'] == States.pending:
+            self.launch_if_ready(task_id)
+
+        return
+
+    def handle_app_update(self, task_id, future, memo_cbk=False):
+        """This function is called as a callback when an AppFuture
+        is in its final state.
+
+        It will trigger post-app processing such as checkpointing
+        and stageout.
+
+        Args:
+             task_id (string) : Task id
+             future (Future) : The relevant app future (which should be
+                 consistent with the task structure 'app_fu' entry
+
+        KWargs:
+             memo_cbk(Bool) : Indicates that the call is coming from a memo update,
+             that does not require additional memo updates.
+        """
+
+        if not self.tasks[task_id]['app_fu'].done():
+            logger.error("Internal consistency error: app_fu is not done for task {}".format(task_id))
+        if not self.tasks[task_id]['app_fu'] == future:
+            logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(task_id))
+
+        if not memo_cbk:
             # Update the memoizer with the new result if this is not a
             # result from a memo lookup and the task has reached a terminal state.
             self.memoizer.update_memo(task_id, self.tasks[task_id], future)
@@ -313,7 +338,8 @@ def handle_update(self, task_id, future, memo_cbk=False):
 
         # Submit _*_stage_out tasks for output data futures that correspond with remote files
         if (self.tasks[task_id]['app_fu'] and
-            self.tasks[task_id]['status'] == States.done and
+            self.tasks[task_id]['app_fu'].done() and
+            self.tasks[task_id]['app_fu'].exception() is None and
             self.tasks[task_id]['executor'] != 'data_manager' and
             self.tasks[task_id]['func_name'] != '_file_stage_in' and
             self.tasks[task_id]['func_name'] != '_ftp_stage_in' and
@@ -323,11 +349,6 @@ def handle_update(self, task_id, future, memo_cbk=False):
                 if isinstance(f, File) and f.is_remote():
                     f.stage_out(self.tasks[task_id]['executor'])
 
-        # it might be that in the course of the update, we've gone back to being
-        # pending - in which case, we should consider ourself for relaunch
-        if self.tasks[task_id]['status'] == States.pending:
-            self.launch_if_ready(task_id)
-
         return
 
     def launch_if_ready(self, task_id):
@@ -420,7 +441,8 @@ def launch_task(self, task_id, executable, *args, **kwargs):
         hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
         if hit:
             logger.info("Reusing cached result for task {}".format(task_id))
-            self.handle_update(task_id, memo_fu, memo_cbk=True)
+            self.handle_exec_update(task_id, memo_fu)
+            self.handle_app_update(task_id, memo_fu, memo_cbk=True)
             return memo_fu
 
         executor_label = self.tasks[task_id]["executor"]
@@ -437,7 +459,7 @@ def launch_task(self, task_id, executable, *args, **kwargs):
             self.db_logger.info("Task Launch", extra=task_log_info)
         exec_fu.retries_left = self._config.retries - \
             self.tasks[task_id]['fail_count']
-        exec_fu.add_done_callback(partial(self.handle_update, task_id))
+        exec_fu.add_done_callback(partial(self.handle_exec_update, task_id))
         logger.info("Task {} launched on executor {}".format(task_id, executor.label))
         return exec_fu
 
@@ -633,9 +655,12 @@ def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwar
                                                                                task_def['func_name'],
                                                                                [fu.tid for fu in depends]))
 
-        self.tasks[task_id]['app_fu'] = AppFuture(None, tid=task_id,
-                                                  stdout=task_stdout,
-                                                  stderr=task_stderr)
+        app_fu = AppFuture(None, tid=task_id,
+                           stdout=task_stdout,
+                           stderr=task_stderr)
+
+        self.tasks[task_id]['app_fu'] = app_fu
+        app_fu.add_done_callback(partial(self.handle_app_update, task_id))
         self.tasks[task_id]['status'] = States.pending
         logger.debug("Task {} set to pending state with AppFuture: {}".format(task_id, task_def['app_fu']))
 
@@ -831,7 +856,8 @@ def checkpoint(self, tasks=None):
             with open(checkpoint_tasks, 'ab') as f:
                 for task_id in checkpoint_queue:
                     if not self.tasks[task_id]['checkpoint'] and \
-                       self.tasks[task_id]['status'] == States.done:
+                       self.tasks[task_id]['app_fu'].done() and \
+                       self.tasks[task_id]['app_fu'].exception() is None:
                         hashsum = self.tasks[task_id]['hashsum']
                         if not hashsum:
                             continue


Test patch: