Repository: datalad/datalad-next

Issue:
EnsureGeneratorFromFileLike should not stop on exception
Right now any invalid input will cause it to stop and raise.

However for any batch mode application it would be essential to be able to ignore errors and/or simply report on them

The implementation should be changed to yield (not raise) a special value. Likely the internal exception itself. Either to be reraised outside, or ignored, or reported on.

Discussion:

Patch:
diff --git a/datalad_next/constraints/base.py b/datalad_next/constraints/base.py
index cd2bdad..d82c994 100644
--- a/datalad_next/constraints/base.py
+++ b/datalad_next/constraints/base.py
@@ -87,10 +87,6 @@ class _MultiConstraint(Constraint):
             EnsureNone() if c is None else c for c in constraints
         ]
 
-    def __repr__(self):
-        creprs = ', '.join(f'{c!r}' for c in self.constraints)
-        return f"{self.__class__.__name__}({creprs})"
-
     @property
     def constraints(self):
         return self._constraints
diff --git a/datalad_next/constraints/compound.py b/datalad_next/constraints/compound.py
index f808c27..5cf8602 100644
--- a/datalad_next/constraints/compound.py
+++ b/datalad_next/constraints/compound.py
@@ -55,16 +55,6 @@ class EnsureIterableOf(Constraint):
         self._max_len = max_len
         super().__init__()
 
-    def __repr__(self):
-        # not showing iter_type here, will come via class.name
-        # in general
-        return (
-            f'{self.__class__.__name__}('
-            f'item_constraint={self._item_constraint!r}'
-            f', min_len={self._min_len!r}'
-            f', max_len={self._max_len!r})'
-        )
-
     @property
     def item_constraint(self):
         return self._item_constraint
@@ -165,14 +155,6 @@ class EnsureMapping(Constraint):
         self._delimiter = delimiter
         self._allow_length2_sequence = allow_length2_sequence
 
-    def __repr__(self):
-        return (
-            f'{self.__class__.__name__}('
-            f'key={self._key_constraint!r}'
-            f', value={self._value_constraint!r}'
-            f', delimiter={self._delimiter!r})'
-        )
-
     def short_description(self):
         return 'mapping of {} -> {}'.format(
             self._key_constraint.short_description(),
@@ -222,25 +204,30 @@ class EnsureGeneratorFromFileLike(Constraint):
     existing file to be read from.
     """
 
-    def __init__(self, item_constraint: Callable):
+    def __init__(
+        self,
+        item_constraint: Callable,
+        exc_mode: str = 'raise',
+    ):
         """
         Parameters
         ----------
         item_constraint:
           Each incoming item will be mapped through this callable
           before being yielded by the generator.
+        exc_mode: {'raise', 'yield'}, optional
+          How to deal with exceptions occurring when processing
+          individual lines/items. With 'yield' the respective
+          exception instance is yielded, and processing continues.
+          A caller can then decide whether to ignore, report, or raise
+          the exception. With 'raise', an exception is raised immediately
+          and processing stops.
         """
+        assert exc_mode in ('raise', 'yield')
         self._item_constraint = item_constraint
+        self._exc_mode = exc_mode
         super().__init__()
 
-    def __repr__(self):
-        # not showing iter_type here, will come via class.name
-        # in general
-        return (
-            f'{self.__class__.__name__}('
-            f'item_constraint={self._item_constraint!r})'
-        )
-
     def short_description(self):
         return \
             f'items of type "{self._item_constraint.short_description()}" ' \
@@ -267,11 +254,17 @@ class EnsureGeneratorFromFileLike(Constraint):
     def _item_yielder(self, fp, close_file):
         try:
             for line in fp:
-                yield self._item_constraint(
-                    # splitlines() removes the newline at the end of the string
-                    # that is left in by __iter__()
-                    line.splitlines()[0]
-                )
+                try:
+                    yield self._item_constraint(
+                        # splitlines() removes the newline at the end of
+                        # the string that is left in by __iter__()
+                        line.splitlines()[0]
+                    )
+                except Exception as e:
+                    if self._exc_mode == 'raise':
+                        raise
+                    else:
+                        yield e
         finally:
             if close_file:
                 fp.close()


Test patch:
diff --git a/datalad_next/constraints/tests/test_compound.py b/datalad_next/constraints/tests/test_compound.py
index 12fcb38..8244d4a 100644
--- a/datalad_next/constraints/tests/test_compound.py
+++ b/datalad_next/constraints/tests/test_compound.py
@@ -137,10 +137,24 @@ def test_EnsureGeneratorFromFileLike():
     assert list(c) == [{5: True}, {1234: False}]
 
     # item constraint violation
-    c = constraint(StringIO("5::yes\n1234::BANG"))
+    invalid_input = StringIO("1234::BANG\n5::yes")
+    # immediate raise is default
     with pytest.raises(ValueError) as e:
-        list(c)
+        list(constraint(invalid_input))
     assert 'be convertible to boolean' in str(e)
+    # but optionally it yields the exception to be able to
+    # continue and enable a caller to raise/report/ignore
+    # (must redefine `invalid_input` to read from start)
+    invalid_input = StringIO("1234::BANG\n5::yes")
+    res = list(
+        EnsureGeneratorFromFileLike(
+            item_constraint,
+            exc_mode='yield',
+        )(invalid_input)
+    )
+    # we get the result after the exception occurred
+    assert isinstance(res[0], ValueError)
+    assert res[1] == {5: True}
 
     # read from STDIN
     with patch("sys.stdin", StringIO("5::yes\n1234::no")):