Repository: datalad/datalad-next

Issue:
EnsureGeneratorFromFileLike should not stop on exception
Right now any invalid input will cause it to stop and raise.

However for any batch mode application it would be essential to be able to ignore errors and/or simply report on them

The implementation should be changed to yield (not raise) a special value. Likely the internal exception itself. Either to be reraised outside, or ignored, or reported on.

Comments:

Patch:
diff --git a/datalad_next/constraints/compound.py b/datalad_next/constraints/compound.py
index f808c27..31b529a 100644
--- a/datalad_next/constraints/compound.py
+++ b/datalad_next/constraints/compound.py
@@ -222,15 +222,28 @@ class EnsureGeneratorFromFileLike(Constraint):
     existing file to be read from.
     """
 
-    def __init__(self, item_constraint: Callable):
+    def __init__(
+        self,
+        item_constraint: Callable,
+        exc_mode: str = 'raise',
+    ):
         """
         Parameters
         ----------
         item_constraint:
           Each incoming item will be mapped through this callable
           before being yielded by the generator.
+        exc_mode: {'raise', 'yield'}, optional
+          How to deal with exceptions occurring when processing
+          individual lines/items. With 'yield' the respective
+          exception instance is yielded, and processing continues.
+          A caller can then decide whether to ignore, report, or raise
+          the exception. With 'raise', an exception is raised immediately
+          and processing stops.
         """
+        assert exc_mode in ('raise', 'yield')
         self._item_constraint = item_constraint
+        self._exc_mode = exc_mode
         super().__init__()
 
     def __repr__(self):
@@ -267,11 +280,17 @@ class EnsureGeneratorFromFileLike(Constraint):
     def _item_yielder(self, fp, close_file):
         try:
             for line in fp:
-                yield self._item_constraint(
-                    # splitlines() removes the newline at the end of the string
-                    # that is left in by __iter__()
-                    line.splitlines()[0]
-                )
+                try:
+                    yield self._item_constraint(
+                        # splitlines() removes the newline at the end of
+                        # the string that is left in by __iter__()
+                        line.splitlines()[0]
+                    )
+                except Exception as e:
+                    if self._exc_mode == 'raise':
+                        raise
+                    else:
+                        yield e
         finally:
             if close_file:
                 fp.close()


Test patch:
diff --git a/datalad_next/constraints/tests/test_compound.py b/datalad_next/constraints/tests/test_compound.py
index 12fcb38..8244d4a 100644
--- a/datalad_next/constraints/tests/test_compound.py
+++ b/datalad_next/constraints/tests/test_compound.py
@@ -137,10 +137,24 @@ def test_EnsureGeneratorFromFileLike():
     assert list(c) == [{5: True}, {1234: False}]
 
     # item constraint violation
-    c = constraint(StringIO("5::yes\n1234::BANG"))
+    invalid_input = StringIO("1234::BANG\n5::yes")
+    # immediate raise is default
     with pytest.raises(ValueError) as e:
-        list(c)
+        list(constraint(invalid_input))
     assert 'be convertible to boolean' in str(e)
+    # but optionally it yields the exception to be able to
+    # continue and enable a caller to raise/report/ignore
+    # (must redefine `invalid_input` to read from start)
+    invalid_input = StringIO("1234::BANG\n5::yes")
+    res = list(
+        EnsureGeneratorFromFileLike(
+            item_constraint,
+            exc_mode='yield',
+        )(invalid_input)
+    )
+    # we get the result after the exception occurred
+    assert isinstance(res[0], ValueError)
+    assert res[1] == {5: True}
 
     # read from STDIN
     with patch("sys.stdin", StringIO("5::yes\n1234::no")):