Repository: AgnostiqHQ/covalent (language: python)

Issue:
`post_process` fails if there is a conflict in node names
An example of this is:

```python
@ct.electron
def a(x):
    return x,x**2

@ct.lattice
def p(x):
    result,b=a(x=x)
    for _ in range(1):
        result,b=a(x=result)
    return b,result
```

Here, since `post_process` takes in a list of node outputs which also contain non executable electron nodes such as of `:parameter:` type, it fails because it will look for the function name of `a` and find its occurrence in `:parameter:` and then use the output of that for post processing.

This can be avoided if we use an `OrderedDictionary` to store the nodes and their outputs and then pass it to `post_process`.


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dfe1c0862..71e97be68 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.24.16] - 2022-02-12
+
+### Added
+
+- Errors now contain the traceback as well as the error message in the result object.
+- Added test for `_post_process` in `tests/covalent_dispatcher_tests/_core/execution_test.py`.
+
+### Changed
+
+- Post processing logic in `electron` and dispatcher now relies on the order of execution in the transport graph rather than node's function names to allow for a more reliable pairing of nodes and their outputs.
+
+- Renamed `init_test.py` in `tests/covalent_dispatcher_tests/_core/` to `execution_test.py`.
+
+### Removed
+
+- `exclude_from_postprocess` list which contained some non executable node types removed since only executable nodes are post processed now.
+
 ## [0.24.15] - 2022-02-11
 
 ### Fixed
@@ -23,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.24.14] - 2022-02-11
 
 ### Added
+
 - UI: add dashboard cards
 - UI: add scaling dots background
 
@@ -33,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - UI: format executor parameters as YAML code
 - UI: update syntax highlighting scheme
 - UI: update index.html description meta tag
+
 ## [0.24.13] - 2022-02-11
 
 ### Added
diff --git a/VERSION b/VERSION
index c323a9e72..cc31a06f9 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-0.24.15
+0.24.16
diff --git a/covalent/_shared_files/defaults.py b/covalent/_shared_files/defaults.py
index 1a17fbc49..22062cf6f 100644
--- a/covalent/_shared_files/defaults.py
+++ b/covalent/_shared_files/defaults.py
@@ -32,8 +32,6 @@
 sublattice_prefix = f"{prefix_separator}sublattice{prefix_separator}"
 attr_prefix = f"{prefix_separator}attribute{prefix_separator}"
 
-exclude_from_postprocess = [attr_prefix, subscript_prefix, generator_prefix]
-
 # Default configuration settings
 _DEFAULT_CONFIG = {
     "sdk": {
diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py
index efaf70a9e..210002b6c 100644
--- a/covalent/_workflow/electron.py
+++ b/covalent/_workflow/electron.py
@@ -332,19 +332,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]:
             kwargs.update(dict(zip(list(inspect.signature(self.function).parameters), args)))
 
         if active_lattice.post_processing:
-            key = [k for k in active_lattice.electron_outputs if f"{self.function.__name__}" in k][
-                0
-            ]
-
-            if not key:
-                raise ValueError(
-                    "dispatch_id and lattice mismatch. Please try "
-                    "again with the correct dispatch_id."
-                )
-
-            output = active_lattice.electron_outputs[key]
-            del active_lattice.electron_outputs[key]
-            return output
+            return active_lattice.electron_outputs.pop(0)
 
         # Setting metadata for default values according to lattice's metadata
         # If metadata is default, then set it to lattice's default
diff --git a/covalent_dispatcher/_core/execution.py b/covalent_dispatcher/_core/execution.py
index ebf6c2438..6f77f0e6b 100644
--- a/covalent_dispatcher/_core/execution.py
+++ b/covalent_dispatcher/_core/execution.py
@@ -22,6 +22,7 @@
 Defines the core functionality of the dispatcher
 """
 
+import traceback
 from datetime import datetime, timezone
 from typing import Any, Coroutine, Dict, List
 
@@ -38,9 +39,9 @@
     attr_prefix,
     electron_dict_prefix,
     electron_list_prefix,
-    exclude_from_postprocess,
     generator_prefix,
     parameter_prefix,
+    prefix_separator,
     sublattice_prefix,
     subscript_prefix,
 )
@@ -97,7 +98,7 @@ def _get_task_inputs(
     return task_input
 
 
-def _post_process(lattice: Lattice, node_outputs: dict) -> Any:
+def _post_process(lattice: Lattice, node_outputs: Dict, execution_order: List[List]) -> Any:
     """
     Post processing function to be called after the lattice execution.
     This takes care of executing statements that were not an electron
@@ -113,14 +114,27 @@ def _post_process(lattice: Lattice, node_outputs: dict) -> Any:
     Args:
         lattice: Lattice object that was dispatched.
         node_outputs: Dictionary containing the output of all the nodes.
+        execution_order: List of lists containing the order of execution of the nodes.
 
     Reurns:
         result: The result of the lattice function.
     """
 
+    keys_of_outputs = list(node_outputs.keys())
+    values_of_outputs = list(node_outputs.values())
+
+    ordered_node_outputs = []
+    for node_id_list in execution_order:
+        for node_id in node_id_list:
+            # Here we only need outputs of nodes which are executable
+            if not keys_of_outputs[node_id].startswith(prefix_separator) or keys_of_outputs[
+                node_id
+            ].startswith(sublattice_prefix):
+                ordered_node_outputs.append(values_of_outputs[node_id])
+
     with active_lattice_manager.claim(lattice):
         lattice.post_processing = True
-        lattice.electron_outputs = node_outputs
+        lattice.electron_outputs = ordered_node_outputs
         result = lattice.workflow_function(**lattice.kwargs)
         lattice.post_processing = False
         return result
@@ -238,7 +252,7 @@ def _run_task(
             end_time,
             Result.FAILED,
             None,
-            ex,
+            "".join(traceback.TracebackException.from_exception(ex).format()),
         )
 
     result_object.save()
@@ -315,7 +329,7 @@ def _run_planned_workflow(result_object: Result) -> Result:
             if result_object._get_node_status(node_id) == Result.FAILED:
                 result_object._status = Result.FAILED
                 result_object._end_time = datetime.now(timezone.utc)
-                result_object._error = f"Node {result_object._get_node_name(node_id)} failed: {result_object._get_node_error(node_id)}"
+                result_object._error = f"Node {result_object._get_node_name(node_id)} failed: \n{result_object._get_node_error(node_id)}"
                 result_object.save()
                 result_webhook.send_update(result_object)
                 return
@@ -328,12 +342,9 @@ def _run_planned_workflow(result_object: Result) -> Result:
                 return
 
     # post process the lattice
-    node_outputs_for_post_processing = {
-        k: v
-        for k, v in result_object.get_all_node_outputs().items()
-        if all(prefix not in k for prefix in exclude_from_postprocess)
-    }
-    result_object._result = _post_process(result_object.lattice, node_outputs_for_post_processing)
+    result_object._result = _post_process(
+        result_object.lattice, result_object.get_all_node_outputs(), order
+    )
 
     result_object._status = Result.COMPLETED
     result_object._end_time = datetime.now(timezone.utc)
@@ -411,7 +422,7 @@ def run_workflow(dispatch_id: str, results_dir: str) -> None:
     except Exception as ex:
         result_object._status = Result.FAILED
         result_object._end_time = datetime.now(timezone.utc)
-        result_object._error = ex
+        result_object._error = "".join(traceback.TracebackException.from_exception(ex).format())
         result_object.save()
         raise
 


Test patch:
diff --git a/tests/covalent_dispatcher_tests/_core/init_test.py b/tests/covalent_dispatcher_tests/_core/execution_test.py
similarity index 71%
rename from tests/covalent_dispatcher_tests/_core/init_test.py
rename to tests/covalent_dispatcher_tests/_core/execution_test.py
index 55ccf2525..f0daf1081 100644
--- a/tests/covalent_dispatcher_tests/_core/init_test.py
+++ b/tests/covalent_dispatcher_tests/_core/execution_test.py
@@ -26,11 +26,24 @@
 
 import covalent as ct
 from covalent._results_manager import Result
-from covalent_dispatcher._core.execution import _plan_workflow
+from covalent_dispatcher._core.execution import _plan_workflow, _post_process
 
 TEST_RESULTS_DIR = "/tmp/results"
 
 
+@ct.electron
+def a(x):
+    return x, x ** 2
+
+
+@ct.lattice
+def p(x):
+    result, b = a(x=x)
+    for _ in range(1):
+        result, b = a(x=result)
+    return b, result
+
+
 def get_mock_result() -> Result:
     """Construct a mock result object corresponding to a lattice."""
 
@@ -65,3 +78,22 @@ def test_plan_workflow():
     updated_tg = pickle.loads(mock_result.lattice.transport_graph.serialize(metadata_only=True))
 
     assert "results_dir" in updated_tg["nodes"][0]["exec_plan"].execution_args
+
+
+def test_post_process():
+    """Test post-processing of results."""
+
+    p.build_graph(x=2)
+    order = p.transport_graph.get_topologically_sorted_graph()
+    node_outputs = {
+        "a(0)": (2, 4),
+        ":parameter:2(1)": 2,
+        ":generated:a()[0](2)": 2,
+        ":generated:a()[1](3)": 4,
+        "a(4)": (2, 4),
+        ":generated:a()[0](5)": 2,
+        ":generated:a()[1](6)": 4,
+    }
+
+    execution_result = _post_process(p, node_outputs, order)
+    assert execution_result == (4, 2)