Repository: nipype/pydra

Issue:
add pydra-specific exceptions
Once the API is more stable, I can add specific error objects like `NodeError` and so on.

Comments:

Patch:
diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py
index 1a441f1..3cd9c65 100644
--- a/pydra/engine/helpers.py
+++ b/pydra/engine/helpers.py
@@ -10,7 +10,7 @@ from hashlib import sha256
 import subprocess as sp
 
 from .specs import Runtime, File, attr_fields
-from .helpers_file import is_existing_file, hash_file, copyfile, is_existing_file
+from .helpers_file import hash_file, copyfile, is_existing_file
 
 
 def ensure_list(obj, tuple2list=False):
@@ -444,12 +444,14 @@ def hash_function(obj):
 
 def hash_value(value, tp=None, metadata=None):
     """calculating hash or returning values recursively"""
+    if metadata is None:
+        metadata = {}
     if isinstance(value, (tuple, list)):
         return [hash_value(el, tp, metadata) for el in value]
     elif isinstance(value, dict):
         dict_hash = {k: hash_value(v, tp, metadata) for (k, v) in value.items()}
         # returning a sorted object
-        return sorted(dict_hash.items(), key=lambda x: x[0])
+        return [list(el) for el in sorted(dict_hash.items(), key=lambda x: x[0])]
     else:  # not a container
         if (
             (tp is File or "pydra.engine.specs.File" in str(tp))
@@ -457,8 +459,6 @@ def hash_value(value, tp=None, metadata=None):
             and "container_path" not in metadata
         ):
             return hash_file(value)
-        elif isinstance(value, tuple):
-            return list(value)
         else:
             return value
 
diff --git a/pydra/engine/helpers_state.py b/pydra/engine/helpers_state.py
index 1be966a..6aad048 100644
--- a/pydra/engine/helpers_state.py
+++ b/pydra/engine/helpers_state.py
@@ -10,6 +10,16 @@ from .helpers import ensure_list
 logger = logging.getLogger("pydra")
 
 
+class PydraStateError(Exception):
+    """Custom error for Pydra State"""
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return "{}".format(self.value)
+
+
 def splitter2rpn(splitter, other_states=None, state_fields=True):
     """
     Translate user-provided splitter into *reverse polish notation*.
@@ -49,7 +59,7 @@ def _ordering(
         if type(el[0]) is str and el[0].startswith("_"):
             node_nm = el[0][1:]
             if node_nm not in other_states and state_fields:
-                raise Exception(
+                raise PydraStateError(
                     "can't ask for splitter from {}, other nodes that are connected: {}".format(
                         node_nm, other_states.keys()
                     )
@@ -64,7 +74,7 @@ def _ordering(
         if type(el[1]) is str and el[1].startswith("_"):
             node_nm = el[1][1:]
             if node_nm not in other_states and state_fields:
-                raise Exception(
+                raise PydraStateError(
                     "can't ask for splitter from {}, other nodes that are connected: {}".format(
                         node_nm, other_states.keys()
                     )
@@ -87,7 +97,7 @@ def _ordering(
         if type(el[0]) is str and el[0].startswith("_"):
             node_nm = el[0][1:]
             if node_nm not in other_states and state_fields:
-                raise Exception(
+                raise PydraStateError(
                     "can't ask for splitter from {}, other nodes that are connected: {}".format(
                         node_nm, other_states.keys()
                     )
@@ -102,7 +112,7 @@ def _ordering(
         if type(el[1]) is str and el[1].startswith("_"):
             node_nm = el[1][1:]
             if node_nm not in other_states and state_fields:
-                raise Exception(
+                raise PydraStateError(
                     "can't ask for splitter from {}, other nodes that are connected: {}".format(
                         node_nm, other_states.keys()
                     )
@@ -125,7 +135,7 @@ def _ordering(
         if el.startswith("_"):
             node_nm = el[1:]
             if node_nm not in other_states and state_fields:
-                raise Exception(
+                raise PydraStateError(
                     "can't ask for splitter from {}, other nodes that are connected: {}".format(
                         node_nm, other_states.keys()
                     )
@@ -156,7 +166,7 @@ def _ordering(
                 state_fields=state_fields,
             )
     else:
-        raise Exception("splitter has to be a string, a tuple or a list")
+        raise PydraStateError("splitter has to be a string, a tuple or a list")
     if i > 0:
         output_splitter.append(current_sign)
 
@@ -702,7 +712,7 @@ def _single_op_splits_groups(op_single, combiner, inner_inputs, groups):
             return [], {}, [], combiner
         else:
             # TODO: probably not needed, should be already check by st.combiner_validation
-            raise Exception(
+            raise PydraStateError(
                 "all fields from the combiner have to be in splitter_rpn: {}, but combiner: {} is set".format(
                     [op_single], combiner
                 )
@@ -733,7 +743,7 @@ def combine_final_groups(combiner, groups, groups_stack, keys):
             elif gr in grs_removed:
                 pass
             else:
-                raise Exception(
+                raise PydraStateError(
                     "input {} not ready to combine, you have to combine {} "
                     "first".format(comb, groups_stack[-1])
                 )
diff --git a/pydra/engine/state.py b/pydra/engine/state.py
index 92df1be..40c8111 100644
--- a/pydra/engine/state.py
+++ b/pydra/engine/state.py
@@ -111,7 +111,9 @@ class State:
     @splitter.setter
     def splitter(self, splitter):
         if splitter and not isinstance(splitter, (str, tuple, list)):
-            raise Exception("splitter has to be a string, a tuple or a list")
+            raise hlpst.PydraStateError(
+                "splitter has to be a string, a tuple or a list"
+            )
         if splitter:
             self._splitter = hlpst.add_name_splitter(splitter, self.name)
         else:
@@ -209,7 +211,7 @@ class State:
     def combiner(self, combiner):
         if combiner:
             if not isinstance(combiner, (str, list)):
-                raise Exception("combiner has to be a string or a list")
+                raise hlpst.PydraStateError("combiner has to be a string or a list")
             self._combiner = hlpst.add_name_combiner(ensure_list(combiner), self.name)
         else:
             self._combiner = []
@@ -251,11 +253,13 @@ class State:
     def other_states(self, other_states):
         if other_states:
             if not isinstance(other_states, dict):
-                raise Exception("other states has to be a dictionary")
+                raise hlpst.PydraStateError("other states has to be a dictionary")
             else:
                 for key, val in other_states.items():
                     if not val:
-                        raise Exception(f"connection from node {key} is empty")
+                        raise hlpst.PydraStateError(
+                            f"connection from node {key} is empty"
+                        )
             self._other_states = other_states
         else:
             self._other_states = {}
@@ -360,7 +364,9 @@ class State:
         ):
             return "[Left, Right]"  # Left and Right parts separated in outer scalar
         else:
-            raise Exception("Left and Right splitters are mixed - splitter invalid")
+            raise hlpst.PydraStateError(
+                "Left and Right splitters are mixed - splitter invalid"
+            )
 
     def set_input_groups(self, state_fields=True):
         """Evaluate groups, especially the final groups that address the combiner."""
@@ -409,7 +415,7 @@ class State:
             ):
                 last_gr = last_gr - 1
             if left_nm[1:] not in self.other_states:
-                raise Exception(
+                raise hlpst.PydraStateError(
                     f"can't ask for splitter from {left_nm[1:]}, other nodes that are connected: {self.other_states}"
                 )
             st = self.other_states[left_nm[1:]][0]
@@ -434,7 +440,7 @@ class State:
                 )
                 self.keys_final += keys_f_st  # st.keys_final
                 if not hasattr(st, "group_for_inputs_final"):
-                    raise Exception("previous state has to run first")
+                    raise hlpst.PydraStateError("previous state has to run first")
                 group_for_inputs = group_for_inputs_f_st
                 groups_stack = groups_stack_f_st
                 self.left_combiner_all += combiner_all_st
@@ -487,7 +493,7 @@ class State:
                 or spl.startswith("_")
                 or spl.split(".")[0] == self.name
             ):
-                raise Exception(
+                raise hlpst.PydraStateError(
                     "can't include {} in the splitter, consider using _{}".format(
                         spl, spl.split(".")[0]
                     )
@@ -497,9 +503,11 @@ class State:
         """ validating if the combiner is correct (after all states are connected)"""
         if self.combiner:
             if not self.splitter:
-                raise Exception("splitter has to be set before setting combiner")
+                raise hlpst.PydraStateError(
+                    "splitter has to be set before setting combiner"
+                )
             if set(self._combiner) - set(self.splitter_rpn):
-                raise Exception("all combiners have to be in the splitter")
+                raise hlpst.PydraStateError("all combiners have to be in the splitter")
 
     def prepare_states(self, inputs, cont_dim=None):
         """


Test patch:
diff --git a/pydra/engine/tests/test_helpers.py b/pydra/engine/tests/test_helpers.py
index 862f6b1..5043735 100644
--- a/pydra/engine/tests/test_helpers.py
+++ b/pydra/engine/tests/test_helpers.py
@@ -4,17 +4,18 @@ import pytest
 import cloudpickle as cp
 
 from .utils import multiply
-from .. import helpers
+from ..helpers import hash_value, hash_function, save, create_pyscript
 from .. import helpers_file
+from ..specs import File
 
 
 def test_save(tmpdir):
     outdir = Path(tmpdir)
     with pytest.raises(ValueError):
-        helpers.save(tmpdir)
+        save(tmpdir)
     foo = multiply(name="mult", x=1, y=2)
     # save task
-    helpers.save(outdir, task=foo)
+    save(outdir, task=foo)
     del foo
     # load saved task
     task_pkl = outdir / "_task.pklz"
@@ -24,7 +25,7 @@ def test_save(tmpdir):
     # execute task and save result
     res = foo()
     assert res.output.out == 2
-    helpers.save(outdir, result=res)
+    save(outdir, result=res)
     del res
     # load saved result
     res_pkl = outdir / "_result.pklz"
@@ -35,10 +36,10 @@ def test_save(tmpdir):
 def test_create_pyscript(tmpdir):
     outdir = Path(tmpdir)
     with pytest.raises(Exception):
-        helpers.create_pyscript(outdir, "checksum")
+        create_pyscript(outdir, "checksum")
     foo = multiply(name="mult", x=1, y=2)
-    helpers.save(outdir, task=foo)
-    pyscript = helpers.create_pyscript(outdir, foo.checksum)
+    save(outdir, task=foo)
+    pyscript = create_pyscript(outdir, foo.checksum)
     assert pyscript.exists()
 
 
@@ -50,3 +51,76 @@ def test_hash_file(tmpdir):
         helpers_file.hash_file(outdir / "test.file")
         == "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
     )
+
+
+def test_hashfun_float():
+    import math
+
+    pi_50 = 3.14159265358979323846264338327950288419716939937510
+    pi_15 = 3.141592653589793
+    pi_10 = 3.1415926536
+    # comparing for x that have the same x.as_integer_ratio()
+    assert (
+        math.pi.as_integer_ratio()
+        == pi_50.as_integer_ratio()
+        == pi_15.as_integer_ratio()
+    )
+    assert hash_function(math.pi) == hash_function(pi_15) == hash_function(pi_50)
+    # comparing for x that have different x.as_integer_ratio()
+    assert math.pi.as_integer_ratio() != pi_10.as_integer_ratio()
+    assert hash_function(math.pi) != hash_function(pi_10)
+
+
+def test_hash_value_dict():
+    dict1 = {"a": 10, "b": 5}
+    dict2 = {"b": 5, "a": 10}
+    assert (
+        hash_value(dict1)
+        == hash_value(dict2)
+        == [["a", hash_value(10)], ["b", hash_value(5)]]
+        == [["a", 10], ["b", 5]]
+    )
+
+
+def test_hash_value_list_tpl():
+    lst = [2, 5.6, "ala"]
+    tpl = (2, 5.6, "ala")
+    assert hash_value(lst) == [hash_value(2), hash_value(5.6), hash_value("ala")] == lst
+    assert hash_value(lst) == hash_value(tpl)
+
+
+def test_hash_value_list_dict():
+    lst = [2, {"a": "ala", "b": 1}]
+    hash_value(lst)
+    assert (
+        hash_value(lst)
+        == [hash_value(2), hash_value([["a", "ala"], ["b", 1]])]
+        == [2, [["a", "ala"], ["b", 1]]]
+    )
+
+
+def test_hash_value_files(tmpdir):
+    file_1 = tmpdir.join("file_1.txt")
+    file_2 = tmpdir.join("file_2.txt")
+    with open(file_1, "w") as f:
+        f.write("hello")
+    with open(file_2, "w") as f:
+        f.write("hello")
+
+    assert hash_value(file_1, tp=File) == hash_value(file_2, tp=File)
+    assert hash_value(file_1, tp=str) != hash_value(file_2, tp=str)
+    assert hash_value(file_1) != hash_value(file_2)
+
+
+def test_hash_value_files_list(tmpdir):
+    file_1 = tmpdir.join("file_1.txt")
+    file_2 = tmpdir.join("file_2.txt")
+    with open(file_1, "w") as f:
+        f.write("hello")
+    with open(file_2, "w") as f:
+        f.write("hi")
+
+    assert hash_value([file_1, file_2], tp=File) == [
+        hash_value(file_1, tp=File),
+        hash_value(file_2, tp=File),
+    ]
diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py
index 67ea41e..da98ff8 100644
--- a/pydra/engine/tests/test_specs.py
+++ b/pydra/engine/tests/test_specs.py
@@ -289,7 +289,7 @@ def test_input_file_hash_4(tmpdir):
 
     # checking specific hash value
     hash1 = inputs(in_file=[{"file": file, "int": 3}]).hash
-    assert hash1 == "0148961a66106e15e55de3ab7e8c49e6227b6db7cdc614dcbae20406feb2548a"
+    assert hash1 == "e0555e78a40a02611674b0f48da97cdd28eee7e9885ecc17392b560c14826f06"
 
     # the same file, but int field changes
     hash1a = inputs(in_file=[{"file": file, "int": 5}]).hash
diff --git a/pydra/engine/tests/test_state.py b/pydra/engine/tests/test_state.py
index 5d451cd..b3c46d2 100644
--- a/pydra/engine/tests/test_state.py
+++ b/pydra/engine/tests/test_state.py
@@ -1,6 +1,7 @@
 import pytest
 
 from ..state import State
+from ..helpers_state import PydraStateError
 
 
 @pytest.mark.parametrize(
@@ -95,27 +96,27 @@ def test_state_1(
 
 
 def test_state_2_err():
-    with pytest.raises(Exception) as exinfo:
+    with pytest.raises(PydraStateError) as exinfo:
         st = State("NA", splitter={"a"})
     assert "splitter has to be a string, a tuple or a list" == str(exinfo.value)
 
 
 def test_state_3_err():
-    with pytest.raises(Exception) as exinfo:
+    with pytest.raises(PydraStateError) as exinfo:
         st = State("NA", splitter=["a", "b"], combiner=("a", "b"))
     assert "combiner has to be a string or a list" == str(exinfo.value)
 
 
 def test_state_4_err():
     st = State("NA", splitter="a", combiner=["a", "b"])
-    with pytest.raises(Exception) as exinfo:
+    with pytest.raises(PydraStateError) as exinfo:
         st.combiner_validation()
     assert "all combiners have to be in the splitter" in str(exinfo.value)
 
 
 def test_state_5_err():
     st = State("NA", combiner="a")
-    with pytest.raises(Exception) as exinfo:
+    with pytest.raises(PydraStateError) as exinfo:
         st.combiner_validation()
     assert "splitter has to be set before" in str(exinfo.value)
 
@@ -166,7 +167,7 @@ def test_state_connect_1b_exception():
     """can't provide explicitly NA.a (should be _NA)"""
     st1 = State(name="NA", splitter="a", other_states={})
     st2 = State(name="NB", splitter="NA.a")
-    with pytest.raises(Exception) as excinfo:
+    with pytest.raises(PydraStateError) as excinfo:
         st2.splitter_validation()
     assert "consider using _NA" in str(excinfo.value)
 
@@ -175,7 +176,7 @@ def test_state_connect_1b_exception():
 def test_state_connect_1c_exception(splitter2, other_states2):
     """can't ask for splitter from node that is not connected"""
     st1 = State(name="NA", splitter="a")
-    with pytest.raises(Exception) as excinfo:
+    with pytest.raises(PydraStateError) as excinfo:
         st2 = State(name="NB", splitter=splitter2, other_states=other_states2)
         st2.splitter_validation()
 
@@ -691,7 +692,7 @@ def test_state_connect_innerspl_1a():
 
 def test_state_connect_innerspl_1b():
     """incorrect splitter - Right & Left parts in scalar splitter"""
-    with pytest.raises(Exception):
+    with pytest.raises(PydraStateError):
         st1 = State(name="NA", splitter="a")
         st2 = State(name="NB", splitter=("_NA", "b"), other_states={"NA": (st1, "b")})
 
@@ -1718,7 +1719,7 @@ def test_connect_splitters(
     ],
 )
 def test_connect_splitters_exception_1(splitter, other_states):
-    with pytest.raises(Exception) as excinfo:
+    with pytest.raises(PydraStateError) as excinfo:
         st = State(name="CN", splitter=splitter, other_states=other_states)
     assert "Left and Right splitters are mixed" in str(excinfo.value)
 
@@ -1729,13 +1730,13 @@ def test_connect_splitters_exception_2():
         splitter="_NB",
         other_states={"NA": (State(name="NA", splitter="a"), "b")},
     )
-    with pytest.raises(Exception) as excinfo:
+    with pytest.raises(PydraStateError) as excinfo:
         st.set_input_groups()
     assert "can't ask for splitter from NB" in str(excinfo.value)
 
 
 def test_connect_splitters_exception_3():
-    with pytest.raises(Exception) as excinfo:
+    with pytest.raises(PydraStateError) as excinfo:
         st = State(
             name="CN",
             splitter="_NB",