Repository: linw1995/data_extractor

Issue:
Make easy to trace exception thrown by complex extractor extracting data.

Comments:

Patch:
diff --git a/data_extractor/abc.py b/data_extractor/abc.py
index 84f6dd4..b41f71e 100644
--- a/data_extractor/abc.py
+++ b/data_extractor/abc.py
@@ -4,19 +4,29 @@ Abstract Base Classes.
 # Standard Library
 import warnings
 
-from abc import ABC, abstractmethod
-from typing import Any
+from abc import abstractmethod
+from typing import Any, Dict, List, Tuple
 
+# Local Folder
+from .utils import sentinel
 
-class __Sentinel:
-    def __repr__(self) -> str:
-        return "sentinel"
 
+class ComplexExtractorMeta(type):
+    """
+    Complex Extractor Meta Class.
+    """
+
+    def __init__(cls, name: str, bases: Tuple[type], attr_dict: Dict[str, Any]):
+        super().__init__(name, bases, attr_dict)
+        field_names: List[str] = []
+        for key, attr in attr_dict.items():
+            if isinstance(type(attr), ComplexExtractorMeta):
+                field_names.append(key)
 
-sentinel = __Sentinel()
+        cls._field_names = field_names
 
 
-class AbstractExtractor(ABC):
+class AbstractExtractor(metaclass=ComplexExtractorMeta):
     def __init__(self, expr: str):
         self.expr = expr
 
@@ -30,6 +40,8 @@ class AbstractExtractor(ABC):
         """
         raise NotImplementedError
 
+
+class ExtractFirstMixin(AbstractExtractor):
     def extract_first(self, element: Any, default: Any = sentinel) -> Any:
         """
         Extract the first data or subelement from `extract` method call result.
@@ -43,11 +55,13 @@ class AbstractExtractor(ABC):
 
         if not rv:
             if default is sentinel:
-                raise ValueError(f"Invalid {self!r}")
+                from .exceptions import ExtractError
+
+                raise ExtractError(self, element)
 
             return default
 
         return rv[0]
 
 
-__all__ = ("AbstractExtractor", "sentinel")
+__all__ = ("AbstractExtractor", "ComplexExtractorMeta", "ExtractFirstMixin")
diff --git a/data_extractor/exceptions.py b/data_extractor/exceptions.py
index ede9879..af6934e 100644
--- a/data_extractor/exceptions.py
+++ b/data_extractor/exceptions.py
@@ -1,8 +1,14 @@
 """
 Exceptions.
 """
+# Standard Library
+import reprlib
+
+from typing import Any
+
 # Local Folder
 from .abc import AbstractExtractor
+from .utils import LazyStr
 
 
 class ExprError(Exception):
@@ -15,7 +21,31 @@ class ExprError(Exception):
         self.exc = exc
 
     def __repr__(self) -> str:
-        return f"{self.__class__.__name__}({self.extractor!r}, {self.exc!r})"
+        return f"{self.__class__.__name__}({self.extractor!r}, exc={self.exc!r})"
+
+
+class ExtractError(Exception):
+    """
+    ExtractError thrown by extractor extracting data.
+    """
+
+    def __init__(self, extractor: AbstractExtractor, element: Any):
+        super().__init__(LazyStr(func=lambda: self._trace_repr))
+        self.element = element
+        self.extractors = [extractor]
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.extractors[0]!r}, element={reprlib.repr(self.element)})"
+
+    def _append(self, extractor: AbstractExtractor) -> None:
+        self.extractors.append(extractor)
+
+    @property
+    def _trace_repr(self) -> str:
+        return f"{self.__repr__()}\n" + "\n".join(
+            "  " * idx + "|-" + repr(extractor)
+            for idx, extractor in enumerate([*self.extractors[::-1], self.element])
+        )
 
 
-__all__ = ("ExprError",)
+__all__ = ("ExprError", "ExtractError")
diff --git a/data_extractor/item.py b/data_extractor/item.py
index 9a4c0f6..f6de68e 100644
--- a/data_extractor/item.py
+++ b/data_extractor/item.py
@@ -4,28 +4,15 @@ Complex Extractor for data extracting.
 # Standard Library
 import warnings
 
-from typing import Any, Dict, Iterator, List, Tuple
+from typing import Any, Iterator
 
 # Local Folder
-from .abc import AbstractExtractor, sentinel
+from .abc import AbstractExtractor
+from .exceptions import ExtractError
+from .utils import sentinel
 
 
-class FieldMeta(type):
-    """
-    Complex Extractor Meta Class.
-    """
-
-    def __init__(cls, name: str, bases: Tuple[type], attr_dict: Dict[str, Any]):
-        super().__init__(name, bases, attr_dict)
-        field_names: List[str] = []
-        for key, attr in attr_dict.items():
-            if isinstance(type(attr), FieldMeta):
-                field_names.append(key)
-
-        cls._field_names = field_names
-
-
-class Field(metaclass=FieldMeta):
+class Field(AbstractExtractor):
     """
     Extract data by cooperating with extractor.
     """
@@ -44,7 +31,7 @@ class Field(metaclass=FieldMeta):
         self.is_many = is_many
 
     def __repr__(self) -> str:
-        return f"{self.__class__.__name__}(extractor={self.extractor!r}, default={self.default!r}, is_many={self.is_many})"
+        return f"{self.__class__.__name__}({self.extractor!r}, default={self.default!r}, is_many={self.is_many})"
 
     def extract(self, element: Any) -> Any:
         """
@@ -65,7 +52,7 @@ class Field(metaclass=FieldMeta):
 
         if not rv:
             if self.default is sentinel:
-                raise ValueError(f"Invalid {self!r}")
+                raise ExtractError(self, element)
 
             return self.default
 
@@ -83,7 +70,11 @@ class Item(Field):
     def _extract(self, element: Any) -> Any:
         rv = {}
         for field in self.field_names():
-            rv[field] = getattr(self, field).extract(element)
+            try:
+                rv[field] = getattr(self, field).extract(element)
+            except ExtractError as exc:
+                exc._append(extractor=self)
+                raise exc
 
         return rv
 
@@ -96,4 +87,4 @@ class Item(Field):
             yield name
 
 
-__all__ = ("Field", "FieldMeta", "Item")
+__all__ = ("Field", "Item")
diff --git a/data_extractor/json.py b/data_extractor/json.py
index b5c2fa5..38aaaae 100644
--- a/data_extractor/json.py
+++ b/data_extractor/json.py
@@ -8,10 +8,10 @@ from typing import Any
 import jsonpath_rw
 
 # Local Folder
-from .abc import AbstractExtractor
+from .abc import ExtractFirstMixin
 
 
-class JSONExtractor(AbstractExtractor):
+class JSONExtractor(ExtractFirstMixin):
     """
     Use JSONPath expression for JSON data extracting.
 
diff --git a/data_extractor/lxml.py b/data_extractor/lxml.py
index 4942444..dd7c74e 100644
--- a/data_extractor/lxml.py
+++ b/data_extractor/lxml.py
@@ -9,11 +9,11 @@ from lxml.etree import XPathEvalError
 from lxml.etree import _Element as Element
 
 # Local Folder
-from .abc import AbstractExtractor
+from .abc import ExtractFirstMixin
 from .exceptions import ExprError
 
 
-class CSSExtractor(AbstractExtractor):
+class CSSExtractor(ExtractFirstMixin):
     """
     Use CSS Selector for XML or HTML data subelements extracting.
 
@@ -27,7 +27,7 @@ class CSSExtractor(AbstractExtractor):
         return element.cssselect(self.expr)
 
 
-class TextCSSExtractor(AbstractExtractor):
+class TextCSSExtractor(ExtractFirstMixin):
     """
     Use CSS Selector for XML or HTML data subelements' text extracting.
 
@@ -41,7 +41,7 @@ class TextCSSExtractor(AbstractExtractor):
         return [ele.text for ele in CSSExtractor(self.expr).extract(element)]
 
 
-class AttrCSSExtractor(AbstractExtractor):
+class AttrCSSExtractor(ExtractFirstMixin):
     """
     Use CSS Selector for XML or HTML data subelements' attribute value extracting.
 
@@ -66,7 +66,7 @@ class AttrCSSExtractor(AbstractExtractor):
         ]
 
 
-class XPathExtractor(AbstractExtractor):
+class XPathExtractor(ExtractFirstMixin):
     """
     Use XPath for XML or HTML data extracting.
 
diff --git a/data_extractor/utils.py b/data_extractor/utils.py
new file mode 100644
index 0000000..eabc2a0
--- /dev/null
+++ b/data_extractor/utils.py
@@ -0,0 +1,27 @@
+# Standard Library
+from typing import Callable
+
+
+class __Sentinel:
+    """ Singleton """
+
+    def __repr__(self) -> str:
+        return "sentinel"
+
+
+sentinel = __Sentinel()
+
+
+class LazyStr:
+    """
+    Lazy String
+    """
+
+    def __init__(self, func: Callable[[], str]):
+        self.func = func
+
+    def __str__(self) -> str:
+        return self.func()
+
+
+__all__ = ("LazyStr", "sentinel")


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..6e6c8e9
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,22 @@
+# Third Party Library
+import pytest
+
+
+@pytest.fixture
+def json0():
+    return {
+        "data": {
+            "users": [
+                {"id": 0, "name": "Vang Stout", "gender": "female"},
+                {"id": 1, "name": "Jeannie Gaines", "gender": "male"},
+                {"id": 2, "name": "Guzman Hunter", "gender": "female"},
+                {"id": 3, "name": "Janine Gross"},
+                {"id": 4, "name": "Clarke Patrick", "gender": "male"},
+                {"id": 5, "name": "Whitney Mcfadden"},
+            ],
+            "start": 0,
+            "size": 5,
+            "total": 100,
+        },
+        "status": 0,
+    }
diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py
new file mode 100644
index 0000000..decde7e
--- /dev/null
+++ b/tests/test_exceptions.py
@@ -0,0 +1,49 @@
+# Standard Library
+import textwrap
+
+# Third Party Library
+import pytest
+
+# First Party Library
+from data_extractor.exceptions import ExtractError
+from data_extractor.item import Field, Item
+from data_extractor.json import JSONExtractor
+
+
+def test_exception_trace(json0):
+    data = json0
+
+    class User(Item):
+        uid = Field(JSONExtractor("id"))
+        name = Field(JSONExtractor("name"))
+        gender = Field(JSONExtractor("gender"))
+
+    class UserResponse(Item):
+        start = Field(JSONExtractor("start"), default=0)
+        size = Field(JSONExtractor("size"))
+        total = Field(JSONExtractor("total"))
+        data = User(JSONExtractor("users[*]"), is_many=True)
+
+    extractor = UserResponse(JSONExtractor("data"))
+    with pytest.raises(ExtractError) as catch:
+        extractor.extract(data)
+
+    exc = catch.value
+    assert len(exc.extractors) == 3
+    assert exc.extractors[0] is User.gender
+    assert exc.extractors[1] is UserResponse.data
+    assert exc.extractors[2] is extractor
+    assert exc.element == {"id": 3, "name": "Janine Gross"}
+
+    assert (
+        str(exc.args[0])
+        == textwrap.dedent(
+            """
+            ExtractError(Field(JSONExtractor('gender'), default=sentinel, is_many=False), element={'id': 3, 'name': 'Janine Gross'})
+            |-UserResponse(JSONExtractor('data'), default=sentinel, is_many=False)
+              |-User(JSONExtractor('users[*]'), default=sentinel, is_many=True)
+                |-Field(JSONExtractor('gender'), default=sentinel, is_many=False)
+                  |-{'id': 3, 'name': 'Janine Gross'}
+            """
+        ).strip()
+    )
diff --git a/tests/test_item.py b/tests/test_item.py
index c8af9e5..e6b518b 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -5,6 +5,7 @@ from pathlib import Path
 import pytest
 
 # First Party Library
+from data_extractor.exceptions import ExtractError
 from data_extractor.item import Field, Item
 from data_extractor.json import JSONExtractor
 from data_extractor.lxml import CSSExtractor, TextCSSExtractor, XPathExtractor
@@ -81,8 +82,14 @@ def test_field_extract_with_default(element0, Extractor, expr, expect):
     ids=repr,
 )
 def test_field_extract_without_default(element0, Extractor, expr):
-    with pytest.raises(ValueError):
-        Field(Extractor(expr)).extract(element0)
+    extractor = Field(Extractor(expr))
+    with pytest.raises(ExtractError) as catch:
+        extractor.extract(element0)
+
+    exc = catch.value
+    assert len(exc.extractors) == 1
+    assert exc.extractors[0] is extractor
+    assert exc.element is element0
 
 
 def test_field_parameters_conflict():
@@ -167,8 +174,15 @@ def element2():
 
 
 def test_item_extract_failure_when_last_field_missing(element2, Article0):
-    with pytest.raises(ValueError):
-        Article0(CSSExtractor("li.article"), is_many=True).extract(element2)
+    extractor = Article0(CSSExtractor("li.article"), is_many=True)
+    with pytest.raises(ExtractError) as catch:
+        extractor.extract(element2)
+
+    exc = catch.value
+    assert len(exc.extractors) == 2
+    assert exc.extractors[0] is Article0.content
+    assert exc.extractors[1] is extractor
+    assert exc.element is element2.xpath("//li[@class='article'][2]")[0]
 
 
 def test_item_extract_success_without_is_many_when_last_field_missing(
@@ -258,23 +272,8 @@ def test_complex_item_extract_xml_data():
     }
 
 
-def test_complex_item_extract_json_data():
-    data = {
-        "data": {
-            "users": [
-                {"id": 0, "name": "Vang Stout", "gender": "female"},
-                {"id": 1, "name": "Jeannie Gaines", "gender": "male"},
-                {"id": 2, "name": "Guzman Hunter", "gender": "female"},
-                {"id": 3, "name": "Janine Gross"},
-                {"id": 4, "name": "Clarke Patrick", "gender": "male"},
-                {"id": 5, "name": "Whitney Mcfadden"},
-            ],
-            "start": 0,
-            "size": 5,
-            "total": 100,
-        },
-        "status": 0,
-    }
+def test_complex_item_extract_json_data(json0):
+    data = json0
 
     class User(Item):
         uid = Field(JSONExtractor("id"))
diff --git a/tests/test_lxml.py b/tests/test_lxml.py
index 28ff66a..058d36b 100644
--- a/tests/test_lxml.py
+++ b/tests/test_lxml.py
@@ -4,7 +4,7 @@ import pytest
 from lxml.etree import XPathEvalError
 
 # First Party Library
-from data_extractor.exceptions import ExprError
+from data_extractor.exceptions import ExprError, ExtractError
 from data_extractor.lxml import AttrCSSExtractor, TextCSSExtractor, XPathExtractor
 
 
@@ -87,9 +87,14 @@ def test_extract_first(element, Extractor, expr, expect):
 )
 def test_extract_first_without_default(element, Extractor, expr):
     extractor = Extractor(expr)
-    with pytest.raises(ValueError):
+    with pytest.raises(ExtractError) as catch:
         extractor.extract_first(element)
 
+    exc = catch.value
+    assert len(exc.extractors) == 1
+    assert exc.extractors[0] is extractor
+    assert exc.element is element
+
 
 @pytest.mark.parametrize(
     "expr,attr,expect",
@@ -128,18 +133,24 @@ def test_attr_css_extract_first(element, expr, attr, expect):
 )
 def test_attr_css_extract_first_without_default(element, expr, attr):
     extractor = AttrCSSExtractor(expr=expr, attr=attr)
-    with pytest.raises(ValueError):
+    with pytest.raises(ExtractError) as catch:
         extractor.extract_first(element)
 
+    exc = catch.value
+    assert len(exc.extractors) == 1
+    assert exc.extractors[0] is extractor
+    assert exc.element is element
+
 
 @pytest.mark.parametrize("expr", ["///", "/text(", ""])
 def test_invalid_xpath_expr(element, expr):
     extractor = XPathExtractor(expr)
-    with pytest.raises(ExprError) as exc_info:
+    with pytest.raises(ExprError) as catch:
         extractor.extract(element)
 
-    assert exc_info.value.extractor is extractor
-    assert isinstance(exc_info.value.exc, XPathEvalError)
+    exc = catch.value
+    assert exc.extractor is extractor
+    assert isinstance(exc.exc, XPathEvalError)
 
 
 def test_xpath_result_not_list(element):
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 0000000..3310399
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,16 @@
+# First Party Library
+from data_extractor.utils import LazyStr
+
+
+def test_lazy_str():
+    string = ""
+
+    def func():
+        nonlocal string
+        return string
+
+    ls = LazyStr(func=func)
+    assert str(ls) == ""
+
+    string = "abc"
+    assert str(ls) == "abc"