Repository: Parsl/parsl

Issue:
Make sure tasks in `dep_fail` state are retried
Reported by @djf604.

As shown in the log excerpt below, task 11 is submitted, fails due to an ipengine failure and later completes successfully, but the tasks which are dependent on it are not retried.
```
 2018-08-20 13:07:38 parsl.dataflow.dflow:192 [ERROR]  Task 11 failed
Traceback (most recent call last):
  File "/cephfs/users/woodard/.conda/lib/python3.6/site-packages/parsl/dataflow/dflow.py", line 187, in handle_update
    res = future.result()
  File "/cephfs/users/woodard/.conda/lib/python3.6/concurrent/futures/_base.py", line 425, in result
    return self.__get_result()
  File "/cephfs/users/woodard/.conda/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/cephfs/users/woodard/.conda/lib/python3.6/site-packages/ipyparallel/client/asyncresult.py", line 226, in _resolve_result
    raise r
ipyparallel.error.RemoteError: EngineError(Engine b'442172ce-705b79a69261d1ce48f04e9b' died while running task '2455efd9-fccebdd26277783cbcfb9738')
2018-08-20 13:07:38 parsl.dataflow.dflow:211 [DEBUG]  Task 11 marked for retry
2018-08-20 13:07:38 parsl.dataflow.dflow:363 [INFO]  Task 11 launched on executor eight

[snip]

2018-08-20 13:09:19 parsl.dataflow.dflow:234 [INFO]  Task 11 completed

[snip]

2018-08-20 13:10:25 parsl.dataflow.dflow:295 [INFO]  Task 21 deferred due to dependency failure
2018-08-20 13:10:25 parsl.dataflow.dflow:295 [INFO]  Task 22 deferred due to dependency failure
2018-08-20 13:10:25 parsl.dataflow.dflow:295 [INFO]  Task 23 deferred due to dependency failure
2018-08-20 13:10:25 parsl.dataflow.dflow:295 [INFO]  Task 24 deferred due to dependency failure
2018-08-20 13:10:25 parsl.dataflow.dflow:676 [INFO]  DFK cleanup initiated
2018-08-20 13:10:25 parsl.dataflow.dflow:623 [INFO]  Summary of tasks in DFK:
2018-08-20 13:10:25 parsl.dataflow.dflow:654 [INFO]  Tasks in state States.done: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20
2018-08-20 13:10:25 parsl.dataflow.dflow:654 [INFO]  Tasks in state States.dep_fail: 12, 21, 22, 23, 24
```

I am able to reproduce this issue by running Dominic's workflow on his cluster, but it is non-trivial to set up. If needed I will try to prepare a MWE.


Discussion:

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -141,16 +141,15 @@ def __init__(self, config=Config()):
 
         atexit.register(self.atexit_cleanup)
 
-    @staticmethod
-    def _count_deps(depends, task_id):
+    def _count_deps(self, depends, task_id):
         """Internal.
 
         Count the number of unresolved futures in the list depends.
         """
         count = 0
         for dep in depends:
-            if isinstance(dep, Future) or issubclass(type(dep), Future):
-                if not dep.done():
+            if isinstance(dep, Future):
+                if self.tasks[dep.tid]['status'] not in [States.done, States.failed, States.dep_fail]:
                     count += 1
 
         return count
@@ -383,11 +382,8 @@ def _add_input_deps(self, executor, args, kwargs):
             if isinstance(f, File) and f.is_remote():
                 inputs[idx] = f.stage_in(executor)
 
-    @staticmethod
-    def _count_all_deps(task_id, args, kwargs):
-        """Internal.
-
-        Count the number of unresolved futures in the list depends.
+    def _count_all_deps(self, task_id, args, kwargs):
+        """Count the number of unresolved futures on which a task depends.
 
         Args:
             - task_id (uuid string) : Task_id
@@ -402,31 +398,30 @@ def _count_all_deps(task_id, args, kwargs):
         depends = []
         count = 0
         for dep in args:
-            if isinstance(dep, Future) or issubclass(dep.__class__, Future):
-                if not dep.done():
+            if isinstance(dep, Future):
+                if self.tasks[dep.tid]['status'] not in [States.done, States.failed, States.dep_fail]:
                     count += 1
                 depends.extend([dep])
 
         # Check for explicit kwargs ex, fu_1=<fut>
         for key in kwargs:
             dep = kwargs[key]
-            if isinstance(dep, Future) or issubclass(dep.__class__, Future):
-                if not dep.done():
+            if isinstance(dep, Future):
+                if self.tasks[dep.tid]['status'] not in [States.done, States.failed, States.dep_fail]:
                     count += 1
                 depends.extend([dep])
 
         # Check for futures in inputs=[<fut>...]
         for dep in kwargs.get('inputs', []):
-            if issubclass(dep.__class__, Future) or isinstance(dep, Future):
-                if not dep.done():
+            if isinstance(dep, Future):
+                if self.tasks[dep.tid]['status'] not in [States.done, States.failed, States.dep_fail]:
                     count += 1
                 depends.extend([dep])
 
         # logger.debug("Task:{0}   dep_cnt:{1}  deps:{2}".format(task_id, count, depends))
         return count, depends
 
-    @staticmethod
-    def sanitize_and_wrap(task_id, args, kwargs):
+    def sanitize_and_wrap(self, task_id, args, kwargs):
         """This function should be called **ONLY** when all the futures we track have been resolved.
 
         If the user hid futures a level below, we will not catch
@@ -447,32 +442,35 @@ def sanitize_and_wrap(task_id, args, kwargs):
         # Replace item in args
         new_args = []
         for dep in args:
-            if isinstance(dep, Future) or issubclass(type(dep), Future):
+            if isinstance(dep, Future):
                 try:
                     new_args.extend([dep.result()])
                 except Exception as e:
-                    dep_failures.extend([e])
+                    if self.tasks[dep.tid]['status'] in [States.failed, States.dep_fail]:
+                        dep_failures.extend([e])
             else:
                 new_args.extend([dep])
 
         # Check for explicit kwargs ex, fu_1=<fut>
         for key in kwargs:
             dep = kwargs[key]
-            if isinstance(dep, Future) or issubclass(type(dep), Future):
+            if isinstance(dep, Future):
                 try:
                     kwargs[key] = dep.result()
                 except Exception as e:
-                    dep_failures.extend([e])
+                    if self.tasks[dep.tid]['status'] in [States.failed, States.dep_fail]:
+                        dep_failures.extend([e])
 
         # Check for futures in inputs=[<fut>...]
         if 'inputs' in kwargs:
             new_inputs = []
             for dep in kwargs['inputs']:
-                if isinstance(dep, Future) or issubclass(type(dep), Future):
+                if isinstance(dep, Future):
                     try:
                         new_inputs.extend([dep.result()])
                     except Exception as e:
-                        dep_failures.extend([e])
+                        if self.tasks[dep.tid]['status'] in [States.failed, States.dep_fail]:
+                            dep_failures.extend([e])
 
                 else:
                     new_inputs.extend([dep])
@@ -485,11 +483,8 @@ def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwar
 
         If the app task has the executors attributes not set (default=='all')
         the task will be launched on a randomly selected executor from the
-        list of executors. This behavior could later be updated to support
-        binding to executors based on user specified criteria.
-
-        If the app task specifies a particular set of executors, it will be
-        targetted at those specific executors.
+        list of executors. If the app task specifies a particular set of
+        executors, it will be targeted at the specified executors.
 
         >>> IF all deps are met:
         >>>   send to the runnable queue and launch the task
diff --git a/parsl/dataflow/futures.py b/parsl/dataflow/futures.py
--- a/parsl/dataflow/futures.py
+++ b/parsl/dataflow/futures.py
@@ -40,24 +40,6 @@ class AppFuture(Future):
 
     """
 
-    def parent_callback(self, executor_fu):
-        """Callback from executor future to update the parent.
-
-        Args:
-            - executor_fu (Future): Future returned by the executor along with callback
-
-        Returns:
-            - None
-
-        Updates the super() with the result() or exception()
-        """
-        # print("[RETRY:TODO] parent_Callback for {0}".format(executor_fu))
-        if executor_fu.done() is True:
-            try:
-                super().set_result(executor_fu.result())
-            except Exception as e:
-                super().set_exception(e)
-
     def __init__(self, parent, tid=None, stdout=None, stderr=None):
         """Initialize the AppFuture.
 
@@ -82,6 +64,24 @@ def __init__(self, parent, tid=None, stdout=None, stderr=None):
         self._stdout = stdout
         self._stderr = stderr
 
+    def parent_callback(self, executor_fu):
+        """Callback from executor future to update the parent.
+
+        Args:
+            - executor_fu (Future): Future returned by the executor along with callback
+
+        Returns:
+            - None
+
+        Updates the super() with the result() or exception()
+        """
+        # print("[RETRY:TODO] parent_Callback for {0}".format(executor_fu))
+        if executor_fu.done() is True:
+            try:
+                super().set_result(executor_fu.result())
+            except Exception as e:
+                super().set_exception(e)
+
     @property
     def stdout(self):
         return self._stdout
diff --git a/parsl/dataflow/strategy.py b/parsl/dataflow/strategy.py
--- a/parsl/dataflow/strategy.py
+++ b/parsl/dataflow/strategy.py
@@ -209,14 +209,18 @@ def _strategy_simple(self, tasks, *args, kind=None, **kwargs):
                     # We want to make sure that max_idletime is reached
                     # before killing off resources
                     if not self.executors[executor.label]['idle_since']:
-                        logger.debug("Strategy: Scale_in, tasks=0 starting kill timer")
+                        logger.debug("Executor {} has 0 active tasks; starting kill timer (if idle time exceeds {}s, resources will be removed)".format(
+                            label, self.max_idletime)
+                        )
                         self.executors[executor.label]['idle_since'] = time.time()
 
                     idle_since = self.executors[executor.label]['idle_since']
                     if (time.time() - idle_since) > self.max_idletime:
                         # We have resources idle for the max duration,
                         # we have to scale_in now.
-                        logger.debug("Strategy: Scale_in, tasks=0")
+                        logger.debug("Idle time has reached {}s for executor {}; removing resources".format(
+                            self.max_idletime, label)
+                        )
                         executor.scale_in(active_blocks - min_blocks)
 
                     else:


Test patch: