Repository: AgnostiqHQ/covalent (python)

Issue:
Removal of `results_dir` from UX
References to the `results_dir` need to be removed from covalent. Since it might be a big change requiring modifications in a lot of files, in this issue the objective should be to remove the references to `results_dir` and its assignment etc. from the UX's side.

So, removal from the `results.py`, `electron.py`, `lattice.py`, RTDs (especially "How To"s ) and such files should be considered the acceptance criteria for resolution of this issue.

When the server constructs a `Result` object from the client's submission, its `results_dir` should be initialized to (in descending order of priority):
1) the value of the `COVALENT_DATA_DIR` env variable, if set, or
2) the value of the config key `dispatcher.results_dir`, which shall default to `~/.local/share/covalent/data`

### Acceptance Criteria
- [x] The SDK no longer makes any reference to `results_dir`.
- [x] When a workflow is submitted to Covalent and run, the `results_dir` property of the `Result` object respects the `COVALENT_DATA_DIR` environment variable if that is set. If that is unset, the config value of `dispatcher.results_dir` shall be used. **The value of `Results.results_dir` must not depend on either 1) the cwd of the Covalent dispatcher or 2) the directory from which the client submits the workflow**
- [x] The config default for `dispatcher.results_dir` is updated to `$HOME/.local/share/covalent/data`.

Pull request:
update references to results_dir
- [ ] I have added the tests to cover my changes.
- [ ] I have updated the documentation and CHANGELOG accordingly.
- [ ] I have read the CONTRIBUTING document.

closes #1382 

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3ba93fd..b6ed6c44 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Changed
+
+- Updated all references to `results_dir` to use the environment variable `COVALENT_DATA_DIR` if set or fall back to `~/.local/share/covalent/data`
+
 ### Docs
 
 - Adding `Deployment` section for self-hosting guide
diff --git a/covalent/_results_manager/result.py b/covalent/_results_manager/result.py
index 356adea0..0aa32a12 100644
--- a/covalent/_results_manager/result.py
+++ b/covalent/_results_manager/result.py
@@ -19,11 +19,12 @@
 # Relief from the License may be granted by purchasing a commercial license.
 
 """Result object."""
-
+import os
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict, List, Set, Union
 
 from .._shared_files import logger
+from .._shared_files.config import get_config
 from .._shared_files.context_managers import active_lattice_manager
 from .._shared_files.defaults import prefix_separator, sublattice_prefix
 from .._shared_files.util_classes import RESULT_STATUS, Status
@@ -71,12 +72,14 @@ class Result:
     FAILED = RESULT_STATUS.FAILED
     CANCELLED = RESULT_STATUS.CANCELLED
 
-    def __init__(self, lattice: Lattice, results_dir: str, dispatch_id: str = None) -> None:
+    def __init__(self, lattice: Lattice, dispatch_id: str = "") -> None:
 
         self._start_time = None
         self._end_time = None
 
-        self._results_dir = results_dir
+        self._results_dir = os.environ.get("COVALENT_DATA_DIR") or get_config(
+            "dispatcher.results_dir"
+        )
 
         self._lattice = lattice
         self._dispatch_id = dispatch_id
diff --git a/covalent/_results_manager/results_manager.py b/covalent/_results_manager/results_manager.py
index 9ffa0753..454e215a 100644
--- a/covalent/_results_manager/results_manager.py
+++ b/covalent/_results_manager/results_manager.py
@@ -110,7 +110,7 @@ def _get_result_from_dispatcher(
 
 def _delete_result(
     dispatch_id: str,
-    results_dir: str = get_config("dispatcher.results_dir"),
+    results_dir: str = os.environ.get("COVALENT_DATA_DIR") or get_config("dispatcher.results_dir"),
     remove_parent_directory: bool = False,
 ) -> None:
     """
diff --git a/covalent/_shared_files/defaults.py b/covalent/_shared_files/defaults.py
index 5ce0fd2f..c8c9a83d 100644
--- a/covalent/_shared_files/defaults.py
+++ b/covalent/_shared_files/defaults.py
@@ -71,7 +71,8 @@ def get_default_dispatcher_config():
         "port": int(os.environ.get("COVALENT_SVC_PORT", 48008)),
         "cache_dir": os.environ.get("COVALENT_CACHE_DIR")
         or ((os.environ.get("XDG_CACHE_HOME") or (os.environ["HOME"] + "/.cache")) + "/covalent"),
-        "results_dir": os.environ.get("COVALENT_RESULTS_DIR", "results"),
+        "results_dir": os.environ.get("COVALENT_DATA_DIR")
+        or os.path.join(os.environ["HOME"], ".local/share/covalent/data"),
         "log_dir": os.environ.get("COVALENT_LOGDIR")
         or ((os.environ.get("XDG_CACHE_HOME") or (os.environ["HOME"] + "/.cache")) + "/covalent"),
         "db_path": os.environ.get("COVALENT_DATABASE")
diff --git a/covalent/_workflow/lattice.py b/covalent/_workflow/lattice.py
index 9d3883df..be4dd304 100644
--- a/covalent/_workflow/lattice.py
+++ b/covalent/_workflow/lattice.py
@@ -28,11 +28,9 @@ from contextlib import redirect_stdout
 from copy import deepcopy
 from dataclasses import asdict
 from functools import wraps
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union
 
 from .._shared_files import logger
-from .._shared_files.config import get_config
 from .._shared_files.context_managers import active_lattice_manager
 from .._shared_files.defaults import DefaultMetadataValues
 from .._shared_files.utils import get_named_params, get_serialized_function_str
@@ -314,7 +312,6 @@ def lattice(
     *,
     backend: Optional[str] = None,
     executor: Optional[Union[List[Union[str, "BaseExecutor"]], Union[str, "BaseExecutor"]]] = None,
-    results_dir: Optional[str] = get_config("dispatcher.results_dir"),
     workflow_executor: Optional[
         Union[List[Union[str, "BaseExecutor"]], Union[str, "BaseExecutor"]]
     ] = None,
@@ -337,7 +334,6 @@ def lattice(
             executor is used by default.
         workflow_executor: Executor for postprocessing the workflow. Defaults to the built-in dask executor or
             the local executor depending on whether Covalent is started with the `--no-cluster` option.
-        results_dir: Directory to store the results
         deps_bash: An optional DepsBash object specifying a list of shell commands to run before `_func`
         deps_pip: An optional DepsPip object specifying a list of PyPI packages to install before running `_func`
         call_before: An optional list of DepsCall objects specifying python functions to invoke before the electron
@@ -354,8 +350,6 @@ def lattice(
         )
         executor = backend
 
-    results_dir = str(Path(results_dir).expanduser().resolve())
-
     deps = {}
 
     if isinstance(deps_bash, DepsBash):
@@ -376,7 +370,6 @@ def lattice(
 
     constraints = {
         "executor": executor,
-        "results_dir": results_dir,
         "workflow_executor": workflow_executor,
         "deps": deps,
         "call_before": call_before,
diff --git a/covalent_dispatcher/_core/execution.py b/covalent_dispatcher/_core/execution.py
index 906216ef..0d399b16 100644
--- a/covalent_dispatcher/_core/execution.py
+++ b/covalent_dispatcher/_core/execution.py
@@ -864,7 +864,7 @@ def initialize_result_object(
 
     dispatch_id = get_unique_id()
     lattice = Lattice.deserialize_from_json(json_lattice)
-    result_object = Result(lattice, lattice.metadata["results_dir"], dispatch_id)
+    result_object = Result(lattice, dispatch_id)
     if parent_result_object:
         result_object._root_dispatch_id = parent_result_object._root_dispatch_id
 
diff --git a/covalent_dispatcher/_service/app.py b/covalent_dispatcher/_service/app.py
index 5871fefb..bc84a263 100644
--- a/covalent_dispatcher/_service/app.py
+++ b/covalent_dispatcher/_service/app.py
@@ -101,7 +101,6 @@ def _result_from(lattice_record: Lattice) -> Result:
     name = lattice_record.name
     executor = lattice_record.executor
     workflow_executor = lattice_record.workflow_executor
-    results_dir = lattice_record.results_dir
     num_nodes = lattice_record.electron_num
 
     attributes = {
@@ -117,7 +116,6 @@ def _result_from(lattice_record: Lattice) -> Result:
             "deps": deps,
             "call_before": call_before,
             "call_after": call_after,
-            "results_dir": results_dir,
         },
         "args": inputs["args"],
         "kwargs": inputs["kwargs"],
@@ -138,7 +136,6 @@ def _result_from(lattice_record: Lattice) -> Result:
 
     result = Result(
         lat,
-        results_dir,
         dispatch_id=lattice_record.dispatch_id,
     )
     result._root_dispatch_id = lattice_record.root_dispatch_id


Test patch:
diff --git a/tests/covalent_dispatcher_tests/_core/execution_test.py b/tests/covalent_dispatcher_tests/_core/execution_test.py
index b9a6ce0c..056d93bf 100644
--- a/tests/covalent_dispatcher_tests/_core/execution_test.py
+++ b/tests/covalent_dispatcher_tests/_core/execution_test.py
@@ -78,7 +78,7 @@ def get_mock_result() -> Result:
         print("Error!", file=sys.stderr)
         return x
 
-    @ct.lattice(results_dir=TEST_RESULTS_DIR, deps_bash=ct.DepsBash(["ls"]))
+    @ct.lattice(deps_bash=ct.DepsBash(["ls"]))
     def pipeline(x):
         res1 = task(x)
         res2 = task(res1)
@@ -86,9 +86,7 @@ def get_mock_result() -> Result:
 
     pipeline.build_graph(x="absolute")
     received_workflow = Lattice.deserialize_from_json(pipeline.serialize_to_json())
-    result_object = Result(
-        received_workflow, pipeline.metadata["results_dir"], "pipeline_workflow"
-    )
+    result_object = Result(received_workflow, "pipeline_workflow")
 
     return result_object
 
@@ -133,7 +131,7 @@ def test_plan_workflow():
 
     workflow.metadata["schedule"] = True
     received_workflow = Lattice.deserialize_from_json(workflow.serialize_to_json())
-    result_object = Result(received_workflow, "/tmp", "asdf")
+    result_object = Result(received_workflow, "asdf")
     _plan_workflow(result_object=result_object)
 
     # Updated transport graph post planning
@@ -252,7 +250,7 @@ def test_get_task_inputs():
     tg.set_node_value(3, "output", ct.TransportableObject(2))
     tg.set_node_value(4, "output", ct.TransportableObject(3))
 
-    result_object = Result(lattice=list_workflow, results_dir="/tmp", dispatch_id="asdf")
+    result_object = Result(lattice=list_workflow, dispatch_id="asdf")
     task_inputs = _get_task_inputs(1, tg.get_node_value(1, "name"), result_object)
 
     expected_inputs = {"args": [], "kwargs": {"x": ct.TransportableObject(serialized_args)}}
@@ -271,7 +269,7 @@ def test_get_task_inputs():
     tg.set_node_value(2, "output", ct.TransportableObject(1))
     tg.set_node_value(3, "output", ct.TransportableObject(2))
 
-    result_object = Result(lattice=dict_workflow, results_dir="/tmp", dispatch_id="asdf")
+    result_object = Result(lattice=dict_workflow, dispatch_id="asdf")
     task_inputs = _get_task_inputs(1, tg.get_node_value(1, "name"), result_object)
     expected_inputs = {"args": [], "kwargs": {"x": ct.TransportableObject(serialized_args)}}
 
@@ -283,7 +281,7 @@ def test_get_task_inputs():
     # Check arg order
     multivar_workflow.build_graph(1, 2)
     received_lattice = Lattice.deserialize_from_json(multivar_workflow.serialize_to_json())
-    result_object = Result(lattice=received_lattice, results_dir="/tmp", dispatch_id="asdf")
+    result_object = Result(lattice=received_lattice, dispatch_id="asdf")
     tg = received_lattice.transport_graph
 
     assert list(tg._graph.nodes) == [0, 1, 2, 3, 4, 5, 6, 7]
@@ -326,7 +324,7 @@ def test_gather_deps():
     workflow.build_graph(5)
 
     received_workflow = Lattice.deserialize_from_json(workflow.serialize_to_json())
-    result_object = Result(received_workflow, "/tmp", "asdf")
+    result_object = Result(received_workflow, "asdf")
 
     before, after = _gather_deps(result_object, 0)
     assert len(before) == 1
@@ -506,7 +504,7 @@ async def test_run_workflow_with_failing_nonleaf(mocker):
     json_lattice = workflow.serialize_to_json()
     dispatch_id = "asdf"
     lattice = Lattice.deserialize_from_json(json_lattice)
-    result_object = Result(lattice, lattice.metadata["results_dir"])
+    result_object = Result(lattice)
     result_object._dispatch_id = dispatch_id
     result_object._root_dispatch_id = dispatch_id
     result_object._initialize_nodes()
@@ -558,7 +556,7 @@ async def test_run_workflow_with_failing_leaf(mocker):
     json_lattice = workflow.serialize_to_json()
     dispatch_id = "asdf"
     lattice = Lattice.deserialize_from_json(json_lattice)
-    result_object = Result(lattice, lattice.metadata["results_dir"])
+    result_object = Result(lattice)
     result_object._dispatch_id = dispatch_id
     result_object._root_dispatch_id = dispatch_id
     result_object._initialize_nodes()
diff --git a/tests/covalent_dispatcher_tests/_db/update_test.py b/tests/covalent_dispatcher_tests/_db/update_test.py
index 77ada122..70f00018 100644
--- a/tests/covalent_dispatcher_tests/_db/update_test.py
+++ b/tests/covalent_dispatcher_tests/_db/update_test.py
@@ -35,7 +35,8 @@ from covalent_dispatcher._db.models import Electron, ElectronDependency, Lattice
 from covalent_dispatcher._db.write_result_to_db import load_file
 from covalent_dispatcher._service.app import _result_from
 
-TEMP_RESULTS_DIR = "/tmp/results"
+# TEMP_RESULTS_DIR = "/tmp/results"
+TEMP_RESULTS_DIR = os.environ.get("COVALENT_DATA_DIR") or ct.get_config("dispatcher.results_dir")
 le = LocalExecutor(log_stdout="/tmp/stdout.log")
 
 
@@ -57,7 +58,7 @@ def result_1():
     def task_2(x, y):
         return x + y
 
-    @ct.lattice(executor=le, workflow_executor=le, results_dir=TEMP_RESULTS_DIR)
+    @ct.lattice(executor=le, workflow_executor=le)
     def workflow_1(a, b):
         """Docstring"""
         res_1 = task_1(a, b)
@@ -66,10 +67,8 @@ def result_1():
     Path(f"{TEMP_RESULTS_DIR}/dispatch_1").mkdir(parents=True, exist_ok=True)
     workflow_1.build_graph(a=1, b=2)
     received_lattice = LatticeClass.deserialize_from_json(workflow_1.serialize_to_json())
-    result = Result(
-        lattice=received_lattice, results_dir=TEMP_RESULTS_DIR, dispatch_id="dispatch_1"
-    )
-    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
+    result = Result(lattice=received_lattice, dispatch_id="dispatch_1")
+    #    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
     result._initialize_nodes()
     return result
 
diff --git a/tests/covalent_dispatcher_tests/_db/upsert_test.py b/tests/covalent_dispatcher_tests/_db/upsert_test.py
index a76eb5dd..e82e10f7 100644
--- a/tests/covalent_dispatcher_tests/_db/upsert_test.py
+++ b/tests/covalent_dispatcher_tests/_db/upsert_test.py
@@ -17,6 +17,7 @@
 # FITNESS FOR A PARTICULAR PURPOSE. See the License for more details.
 #
 # Relief from the License may be granted by purchasing a commercial license.
+import os
 from pathlib import Path
 
 import pytest
@@ -34,7 +35,7 @@ from covalent_dispatcher._db.upsert import (
     _electron_data,
 )
 
-TEMP_RESULTS_DIR = "/tmp/results"
+TEMP_RESULTS_DIR = os.environ.get("COVALENT_DATA_DIR") or ct.get_config("dispatcher.results_dir")
 le = LocalExecutor(log_stdout="/tmp/stdout.log")
 
 
@@ -48,7 +49,7 @@ def result_1():
     def task_2(x, y):
         return x + y
 
-    @ct.lattice(executor=le, workflow_executor=le, results_dir=TEMP_RESULTS_DIR)
+    @ct.lattice(executor=le, workflow_executor=le)
     def workflow_1(a, b):
         """Docstring"""
         res_1 = task_1(a, b)
@@ -57,10 +58,8 @@ def result_1():
     Path(f"{TEMP_RESULTS_DIR}/dispatch_1").mkdir(parents=True, exist_ok=True)
     workflow_1.build_graph(a=1, b=2)
     received_lattice = LatticeClass.deserialize_from_json(workflow_1.serialize_to_json())
-    result = Result(
-        lattice=received_lattice, results_dir=TEMP_RESULTS_DIR, dispatch_id="dispatch_1"
-    )
-    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
+    result = Result(lattice=received_lattice, dispatch_id="dispatch_1")
+    #    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
     result._initialize_nodes()
     return result
 
diff --git a/tests/covalent_tests/results_manager_tests/results_test.py b/tests/covalent_tests/results_manager_tests/results_test.py
index 7d8d5cbb..8741cb6c 100644
--- a/tests/covalent_tests/results_manager_tests/results_test.py
+++ b/tests/covalent_tests/results_manager_tests/results_test.py
@@ -20,6 +20,7 @@
 
 """Unit tests for the Result object."""
 
+import os
 from pathlib import Path
 
 import pytest
@@ -29,7 +30,10 @@ from covalent._results_manager.result import Result
 from covalent._workflow.lattice import Lattice as LatticeClass
 from covalent.executor import LocalExecutor
 
-TEMP_RESULTS_DIR = "/tmp/results"
+# TEMP_RESULTS_DIR = "/tmp/results"
+TEMP_RESULTS_DIR = os.environ.get("COVALENT_DATA_DIR") or os.path.join(
+    os.environ["HOME"], ".local/share/covalent/data"
+)
 
 
 def get_mock_result() -> Result:
@@ -43,7 +47,7 @@ def get_mock_result() -> Result:
         print("Error!", file=sys.stderr)
         return x
 
-    @ct.lattice(results_dir=TEMP_RESULTS_DIR)
+    @ct.lattice
     def pipeline(x):
         res1 = task(x)
         res2 = task(res1)
@@ -52,7 +56,8 @@ def get_mock_result() -> Result:
     pipeline.build_graph(x="absolute")
     received_workflow = LatticeClass.deserialize_from_json(pipeline.serialize_to_json())
     result_object = Result(
-        received_workflow, pipeline.metadata["results_dir"], "pipeline_workflow"
+        received_workflow,
+        "pipeline_workflow",  # pipeline.metadata["results_dir"], "pipeline_workflow"
     )
 
     return result_object
@@ -71,7 +76,7 @@ def result_1():
     def task_2(x, y):
         return x + y
 
-    @ct.lattice(executor=le, workflow_executor=le, results_dir=TEMP_RESULTS_DIR)
+    @ct.lattice(executor=le, workflow_executor=le)
     def workflow_1(a, b):
         """Docstring"""
         res_1 = task_1(a, b)
@@ -80,10 +85,8 @@ def result_1():
     Path(f"{TEMP_RESULTS_DIR}/dispatch_1").mkdir(parents=True, exist_ok=True)
     workflow_1.build_graph(a=1, b=2)
     received_lattice = LatticeClass.deserialize_from_json(workflow_1.serialize_to_json())
-    result = Result(
-        lattice=received_lattice, results_dir=TEMP_RESULTS_DIR, dispatch_id="dispatch_1"
-    )
-    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
+    result = Result(lattice=received_lattice, dispatch_id="dispatch_1")
+    #    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
     result._initialize_nodes()
     return result
 
@@ -94,7 +97,7 @@ def result_2():
     def task_1(x, y):
         raise RuntimeError("error")
 
-    @ct.lattice(executor=le, workflow_executor=le, results_dir=TEMP_RESULTS_DIR)
+    @ct.lattice(executor=le, workflow_executor=le)
     def workflow_1(a, b):
         """Docstring"""
         res_1 = task_1(a, b)
@@ -103,10 +106,8 @@ def result_2():
     Path(f"{TEMP_RESULTS_DIR}/dispatch_1").mkdir(parents=True, exist_ok=True)
     workflow_1.build_graph(a=1, b=2)
     received_lattice = LatticeClass.deserialize_from_json(workflow_1.serialize_to_json())
-    result = Result(
-        lattice=received_lattice, results_dir=TEMP_RESULTS_DIR, dispatch_id="dispatch_1"
-    )
-    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
+    result = Result(lattice=received_lattice, dispatch_id="dispatch_1")
+    #    result.lattice.metadata["results_dir"] = TEMP_RESULTS_DIR
     result._initialize_nodes()
     return result
 
@@ -202,7 +203,7 @@ def test_result_post_process(
         k: ct.TransportableObject.make_transportable(v) for k, v in node_outputs.items()
     }
 
-    res = Result(compute_energy, compute_energy.metadata["results_dir"])
+    res = Result(compute_energy)
     res._initialize_nodes()
 
     for i, v in enumerate(encoded_node_outputs.values()):