Repository: Parsl/parsl

Issue:
End of execution task summary is broken by task garbage collection
**Describe the bug**
At DFK shutdown a summary record of final task statuses is logged.
Task garbage collection PR #1543 deletes that information before it is needed by the summary generator, resulting in missing summary information, such as:
```
INFO     parsl.dataflow.dflow:dflow.py:803 Summary of tasks in DFK:
INFO     parsl.dataflow.dflow:dflow.py:841 End of summary
```

**To Reproduce**
```
pytest parsl/tests/test_python_apps/test_memoize_1.py  --config parsl/tests/configs/local_threads.py  --log-cli-level=DEBUG
```
and observe summary info.

**Expected behavior**
The task summary should report all tasks.

This behaviour is broken in commit a33d465b8024d261b1fbe79cafc506053386fcef but works in the immediate parent commit 92a4bbc06c6cd5e46cdc307212caebd546f2db9b

```
pytest parsl/tests/test_python_apps/test_memoize_1.py  --config parsl/tests/configs/local_threads.py  --log-cli-level=DEBUG
...
INFO     parsl.dataflow.dflow:dflow.py:779 Summary of tasks in DFK:
INFO     parsl.dataflow.dflow:dflow.py:810 Tasks in state States.done: 0, 1, 2
INFO     parsl.dataflow.dflow:dflow.py:817 End of summary
```


**Environment**
my laptop, python 3.7, master @ ee2e39b2c



Discussion:
See PR https://github.com/Parsl/parsl/pull/1560 for a regression test for this
Do we want the summary to list all the task ids in each of the states, or would a count do ?

A count is probably sufficient

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -1,5 +1,4 @@
 import atexit
-import itertools
 import logging
 import os
 import pathlib
@@ -96,6 +95,7 @@ def __init__(self, config=Config()):
         self.run_id = str(uuid4())
         self.tasks_completed_count = 0
         self.tasks_failed_count = 0
+        self.tasks_dep_fail_count = 0
 
         self.monitoring = config.monitoring
         # hub address and port for interchange to connect
@@ -431,6 +431,8 @@ def launch_if_ready(self, task_id):
                     "Task {} failed due to dependency failure".format(task_id))
                 # Raise a dependency exception
                 task_record['status'] = States.dep_fail
+                self.tasks_dep_fail_count += 1
+
                 if self.monitoring is not None:
                     task_log_info = self._create_task_log_info(task_id, 'lazy')
                     self.monitoring.send(MessageType.TASK_INFO, task_log_info)
@@ -802,42 +804,23 @@ def callback_adapter(dep_fut):
     def log_task_states(self):
         logger.info("Summary of tasks in DFK:")
 
-        total_summarised = 0
+        keytasks = {state: 0 for state in States}
 
-        keytasks = []
         for tid in self.tasks:
-            keytasks.append((self.tasks[tid]['status'], tid))
-
-        def first(t):
-            return t[0]
-
-        sorted_keytasks = sorted(keytasks, key=first)
-
-        grouped_sorted_keytasks = itertools.groupby(sorted_keytasks, key=first)
-
-        # caution: g is an iterator that also advances the
-        # grouped_sorted_tasks iterator, so looping over
-        # both grouped_sorted_keytasks and g can only be done
-        # in certain patterns
-
-        for k, g in grouped_sorted_keytasks:
-
-            ts = []
-
-            for t in g:
-                tid = t[1]
-                ts.append(str(tid))
-                total_summarised = total_summarised + 1
-
-            tids_string = ", ".join(ts)
-
-            logger.info("Tasks in state {}: {}".format(str(k), tids_string))
-
-        total_in_tasks = len(self.tasks)
-        if total_summarised != total_in_tasks:
-            logger.error("Task count summarisation was inconsistent: summarised {} tasks, but tasks list contains {} tasks".format(
-                total_summarised, total_in_tasks))
-
+            keytasks[self.tasks[tid]['status']] += 1
+        # Fetch from counters since tasks get wiped
+        keytasks[States.done] = self.tasks_completed_count
+        keytasks[States.failed] = self.tasks_failed_count
+        keytasks[States.dep_fail] = self.tasks_dep_fail_count
+
+        for state in States:
+            if keytasks[state]:
+                logger.info("Tasks in state {}: {}".format(str(state), keytasks[state]))
+
+        total_summarized = sum(keytasks.values())
+        if total_summarized != self.task_count:
+            logger.error("Task count summarisation was inconsistent: summarised {} tasks, but task counters registered {} tasks".format(
+                total_summarized, self.task_count))
         logger.info("End of summary")
 
     def _create_remote_dirs_over_channel(self, provider, channel):


Test patch: