Repository: Parsl/parsl

Issue:
Get the proper exception logs of a task when the task is retried
Looking at the code at https://github.com/Parsl/parsl/blob/185283452bafdd477a638c5882f259a962abf5d3/parsl/dataflow/dflow.py#L254, 

It does not seem to work. It is always `None`.
It would be great to get the exception so that we can log that to monitoring.db.

Probably related to #111.





Discussion:
I suspect what is happening here is that the exceptions you are getting are returned inside RemoteExceptionWrapper as a regular result of the future, rather than as the future going into an exception state.

The code around line 245,

```
            res = future.result()
            if isinstance(res, RemoteExceptionWrapper):
                res.reraise()
```

ensures that if an exception arrives by the regular Future exception way, or by the RemoteExceptionWrapper mechanism, it will be raised as an exception; but only in the first way will `Future` be storing an exception. In the second way, it will store a result (that is the `RemoteExceptionWrapper`)
Possibly catching the exception using `except Exception as e:` instead of just `except Exception` and using the newly bound variable `e` instead of `future._exception` will give you the value that you want in either exception handling path.


Resolving patch:
diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py
--- a/parsl/dataflow/dflow.py
+++ b/parsl/dataflow/dflow.py
@@ -178,7 +178,7 @@ def _create_task_log_info(self, task_id, fail_mode=None):
         """
 
         info_to_monitor = ['func_name', 'fn_hash', 'memoize', 'checkpoint', 'fail_count',
-                           'fail_history', 'status', 'id', 'time_submitted', 'time_returned', 'executor']
+                           'status', 'id', 'time_submitted', 'time_returned', 'executor']
 
         task_log_info = {"task_" + k: self.tasks[task_id][k] for k in info_to_monitor}
         task_log_info['run_id'] = self.run_id
@@ -191,6 +191,9 @@ def _create_task_log_info(self, task_id, fail_mode=None):
         task_log_info['task_stdin'] = self.tasks[task_id]['kwargs'].get('stdin', None)
         task_log_info['task_stdout'] = self.tasks[task_id]['kwargs'].get('stdout', None)
         task_log_info['task_stderr'] = self.tasks[task_id]['kwargs'].get('stderr', None)
+        task_log_info['task_fail_history'] = None
+        if self.tasks[task_id]['fail_history'] is not None:
+            task_log_info['task_fail_history'] = ",".join(self.tasks[task_id]['fail_history'])
         task_log_info['task_depends'] = None
         if self.tasks[task_id]['depends'] is not None:
             task_log_info['task_depends'] = ",".join([str(t._tid) for t in self.tasks[task_id]['depends']])
@@ -246,12 +249,12 @@ def handle_exec_update(self, task_id, future):
             if isinstance(res, RemoteExceptionWrapper):
                 res.reraise()
 
-        except Exception:
+        except Exception as e:
             logger.exception("Task {} failed".format(task_id))
 
             # We keep the history separately, since the future itself could be
             # tossed.
-            self.tasks[task_id]['fail_history'].append(future._exception)
+            self.tasks[task_id]['fail_history'].append(str(e))
             self.tasks[task_id]['fail_count'] += 1
 
             if not self._config.lazy_errors:
diff --git a/parsl/monitoring/db_manager.py b/parsl/monitoring/db_manager.py
--- a/parsl/monitoring/db_manager.py
+++ b/parsl/monitoring/db_manager.py
@@ -136,6 +136,7 @@ class Task(Base):
         task_stdout = Column('task_stdout', Text, nullable=True)
         task_stderr = Column('task_stderr', Text, nullable=True)
         task_fail_count = Column('task_fail_count', Integer, nullable=False)
+        task_fail_history = Column('task_fail_history', Text, nullable=True)
         __table_args__ = (
             PrimaryKeyConstraint('task_id', 'run_id'),
         )
@@ -296,7 +297,8 @@ def start(self, priority_queue, resource_queue):
                     self._update(table=TASK,
                                  columns=['task_time_returned',
                                           'task_elapsed_time', 'run_id', 'task_id',
-                                          'task_fail_count'],
+                                          'task_fail_count',
+                                          'task_fail_history'],
                                  messages=update_messages)
                 self._insert(table=STATUS, messages=all_messages)
 


Test patch: