Repository: Parsl/parsl

Issue:
Add a new column in task table of monitoring to show whether result is from memo
**Is your feature request related to a problem? Please describe.**
Currently the `task_memoize` field in the `task` table of monitoring indicates whether the caching for an `app` is enabled or not. But this does not reflect whether the result of a specific task is from memo.

**Describe the solution you'd like**
Add a new column in the `task` table to indicate whether a task's result is from memo or not.




Discussion:

Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -91,6 +91,7 @@ def __init__(self, config=Config()):
         # Monitoring
         self.run_id = str(uuid4())
         self.tasks_completed_count = 0
+        self.tasks_memo_completed_count = 0
         self.tasks_failed_count = 0
         self.tasks_dep_fail_count = 0
 
@@ -199,6 +200,8 @@ def _create_task_log_info(self, task_record):
         task_log_info['task_status_name'] = task_record['status'].name
         task_log_info['tasks_failed_count'] = self.tasks_failed_count
         task_log_info['tasks_completed_count'] = self.tasks_completed_count
+        task_log_info['tasks_memo_completed_count'] = self.tasks_memo_completed_count
+        task_log_info['from_memo'] = task_record['from_memo']
         task_log_info['task_inputs'] = str(task_record['kwargs'].get('inputs', None))
         task_log_info['task_outputs'] = str(task_record['kwargs'].get('outputs', None))
         task_log_info['task_stdin'] = task_record['kwargs'].get('stdin', None)
@@ -308,11 +311,15 @@ def handle_exec_update(self, task_id, future):
                     task_record['app_fu'].set_exception(e)
 
         else:
+            if task_record['from_memo']:
+                task_record['status'] = States.memo_done
+                self.tasks_memo_completed_count += 1
+                logger.info("Task {} completed by memoization".format(task_id))
+            else:
+                task_record['status'] = States.exec_done
+                self.tasks_completed_count += 1
+                logger.info("Task {} completed by execution".format(task_id))
 
-            task_record['status'] = States.done
-            self.tasks_completed_count += 1
-
-            logger.info("Task {} completed".format(task_id))
             task_record['time_returned'] = datetime.datetime.now()
 
             with task_record['app_fu']._update_lock:
@@ -473,8 +480,10 @@ def launch_task(self, task_id, executable, *args, **kwargs):
         memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
         if memo_fu:
             logger.info("Reusing cached result for task {}".format(task_id))
+            self.tasks[task_id]['from_memo'] = True
             return memo_fu
 
+        self.tasks[task_id]['from_memo'] = False
         executor_label = self.tasks[task_id]["executor"]
         try:
             executor = self.executors[executor_label]
@@ -734,6 +743,7 @@ def submit(self, func, app_args, executors='all', fn_hash=None, cache=False, ign
                     'exec_fu': None,
                     'fail_count': 0,
                     'fail_history': [],
+                    'from_memo': None,
                     'ignore_for_cache': ignore_for_cache,
                     'status': States.unsched,
                     'try_id': 0,
@@ -832,7 +842,8 @@ def log_task_states(self):
         for tid in self.tasks:
             keytasks[self.tasks[tid]['status']] += 1
         # Fetch from counters since tasks get wiped
-        keytasks[States.done] = self.tasks_completed_count
+        keytasks[States.exec_done] = self.tasks_completed_count
+        keytasks[States.memo_done] = self.tasks_memo_completed_count
         keytasks[States.failed] = self.tasks_failed_count
         keytasks[States.dep_fail] = self.tasks_dep_fail_count
 
diff --git a/parsl/dataflow/states.py b/parsl/dataflow/states.py
--- a/parsl/dataflow/states.py
+++ b/parsl/dataflow/states.py
@@ -11,15 +11,16 @@ class States(IntEnum):
     # state; but the monitoring database may represent a task in this state
     # based on non-DFK information received from monitor_wrapper.
 
-    done = 3
+    exec_done = 3
     failed = 4
     dep_fail = 5
     launched = 7
     fail_retryable = 8
+    memo_done = 9
 
 
 # states from which we will never move to another state
-FINAL_STATES = [States.done, States.failed, States.dep_fail]
+FINAL_STATES = [States.exec_done, States.memo_done, States.failed, States.dep_fail]
 
 # states which are final and which indicate a failure. This must
 # be a subset of FINAL_STATES
diff --git a/parsl/monitoring/visualization/plots/default/workflow_plots.py b/parsl/monitoring/visualization/plots/default/workflow_plots.py
--- a/parsl/monitoring/visualization/plots/default/workflow_plots.py
+++ b/parsl/monitoring/visualization/plots/default/workflow_plots.py
@@ -56,7 +56,8 @@ def task_gantt_plot(df_task, df_status, time_completed=None):
               'running': 'rgb(0, 0, 255)',
               'dep_fail': 'rgb(255, 128, 255)',
               'failed': 'rgb(200, 0, 0)',
-              'done': 'rgb(0, 200, 0)',
+              'exec_done': 'rgb(0, 200, 0)',
+              'memo_done': 'rgb(64, 200, 64)',
               'fail_retryable': 'rgb(200, 128,128)'
              }
 
@@ -210,8 +211,9 @@ def workflow_dag_plot(df_tasks, group_by_apps=True):
                        "running": (3, 'rgb(0, 0, 255)'),
                        "dep_fail": (4, 'rgb(255, 128, 255)'),
                        "failed": (5, 'rgb(200, 0, 0)'),
-                       "done": (6, 'rgb(0, 200, 0)'),
-                       "fail_retryable": (7, 'rgb(200, 128,128)')
+                       "exec_done": (6, 'rgb(0, 200, 0)'),
+                       "memo_done": (7, 'rgb(64, 200, 64)'),
+                       "fail_retryable": (8, 'rgb(200, 128,128)')
                       }
 
     node_traces = [...] * len(groups_list)


Test patch:
diff --git a/parsl/tests/test_summary.py b/parsl/tests/test_summary.py
--- a/parsl/tests/test_summary.py
+++ b/parsl/tests/test_summary.py
@@ -25,5 +25,5 @@ def test_summary(caplog):
     parsl.clear()
 
     assert "Summary of tasks in DFK:" in caplog.text
-    assert "Tasks in state States.done: 1" in caplog.text
+    assert "Tasks in state States.exec_done: 1" in caplog.text
     assert "Tasks in state States.failed: 1" in caplog.text