Repository: tefra/xsdata
Pull request #546 (2021-06-28 17:32:39)

Implement iterwalk for xml.etree.ElementTree
Bring up to speed native xml event handler 

- [x] parse from element or tree
- [x] xinclude for XmlEventHandler
- [x] py36-py38 xinclude has no base_url


Patch:
diff --git a/xsdata/codegen/container.py b/xsdata/codegen/container.py
index 4fd9d70a..58e92aed 100644
--- a/xsdata/codegen/container.py
+++ b/xsdata/codegen/container.py
@@ -28,7 +28,6 @@ from xsdata.codegen.models import Status
 from xsdata.codegen.utils import ClassUtils
 from xsdata.models.config import GeneratorConfig
 from xsdata.utils import collections
-from xsdata.utils.collections import group_by
 from xsdata.utils.constants import return_true
 
 
@@ -154,7 +153,7 @@ class ClassContainer(ContainerInterface):
 
         candidates = list(filter(should_generate, self))
         if candidates:
-            self.data = group_by(candidates, key=get_qname)
+            self.data = collections.group_by(candidates, key=get_qname)
 
     def add(self, item: Class):
         """Add class item to the container."""
diff --git a/xsdata/codegen/handlers/class_name_conflict.py b/xsdata/codegen/handlers/class_name_conflict.py
index 8d70a80d..ee9bdce8 100644
--- a/xsdata/codegen/handlers/class_name_conflict.py
+++ b/xsdata/codegen/handlers/class_name_conflict.py
@@ -6,10 +6,9 @@ from xsdata.codegen.models import Class
 from xsdata.codegen.models import get_name
 from xsdata.codegen.models import get_qname
 from xsdata.models.config import StructureStyle
+from xsdata.utils import collections
+from xsdata.utils import namespaces
 from xsdata.utils import text
-from xsdata.utils.collections import group_by
-from xsdata.utils.namespaces import build_qname
-from xsdata.utils.namespaces import split_qname
 
 REQUIRE_UNIQUE_NAMES = (StructureStyle.SINGLE_PACKAGE, StructureStyle.CLUSTERS)
 
@@ -27,7 +26,7 @@ class ClassNameConflictHandler(ContainerHandlerInterface):
 
         use_name = self.container.config.output.structure in REQUIRE_UNIQUE_NAMES
         getter = get_name if use_name else get_qname
-        groups = group_by(self.container, lambda x: text.alnum(getter(x)))
+        groups = collections.group_by(self.container, lambda x: text.alnum(getter(x)))
 
         for classes in groups.values():
             if len(classes) > 1:
@@ -51,7 +50,7 @@ class ClassNameConflictHandler(ContainerHandlerInterface):
         that depend on the target class."""
 
         qname = target.qname
-        namespace, name = split_qname(target.qname)
+        namespace, name = namespaces.split_qname(target.qname)
         target.qname = self.next_qname(namespace, name, use_name)
         target.meta_name = name
         self.container.reset(target, qname)
@@ -72,7 +71,7 @@ class ClassNameConflictHandler(ContainerHandlerInterface):
         while True:
             index += 1
             new_name = f"{name}_{index}"
-            qname = build_qname(namespace, new_name)
+            qname = namespaces.build_qname(namespace, new_name)
             cmp = text.alnum(new_name if use_name else qname)
 
             if cmp not in reserved:
diff --git a/xsdata/codegen/mappers/definitions.py b/xsdata/codegen/mappers/definitions.py
index 1ff60212..ece5309f 100644
--- a/xsdata/codegen/mappers/definitions.py
+++ b/xsdata/codegen/mappers/definitions.py
@@ -25,10 +25,8 @@ from xsdata.models.wsdl import PortTypeMessage
 from xsdata.models.wsdl import PortTypeOperation
 from xsdata.models.wsdl import ServicePort
 from xsdata.utils import collections
+from xsdata.utils import namespaces
 from xsdata.utils import text
-from xsdata.utils.collections import first
-from xsdata.utils.namespaces import build_qname
-from xsdata.utils.namespaces import local_name
 
 
 class DefinitionsMapper:
@@ -115,7 +113,7 @@ class DefinitionsMapper:
         assert definitions.location is not None
 
         yield Class(
-            qname=build_qname(definitions.target_namespace, name),
+            qname=namespaces.build_qname(definitions.target_namespace, name),
             status=Status.FLATTENED,
             tag=type(binding_operation).__name__,
             location=definitions.location,
@@ -210,7 +208,7 @@ class DefinitionsMapper:
         assert definitions.location is not None
 
         target = Class(
-            qname=build_qname(definitions.target_namespace, name),
+            qname=namespaces.build_qname(definitions.target_namespace, name),
             meta_name="Envelope",
             tag=Tag.BINDING_MESSAGE,
             location=definitions.location,
@@ -221,7 +219,7 @@ class DefinitionsMapper:
 
         for ext in binding_message.extended_elements:
             assert ext.qname is not None
-            class_name = local_name(ext.qname).title()
+            class_name = namespaces.local_name(ext.qname).title()
             inner = cls.build_inner_class(target, class_name)
 
             if style == "rpc" and class_name == "Body":
@@ -249,7 +247,7 @@ class DefinitionsMapper:
         assert definitions.location is not None
 
         return Class(
-            qname=build_qname(definitions.target_namespace, message_name),
+            qname=namespaces.build_qname(definitions.target_namespace, message_name),
             status=Status.FLATTENED,
             tag=Tag.ELEMENT,
             location=definitions.location,
@@ -268,10 +266,10 @@ class DefinitionsMapper:
         This helper will also create a forward reference attribute for
         the parent class.
         """
-        inner = first(inner for inner in target.inner if inner.name == name)
+        inner = collections.first(inner for inner in target.inner if inner.name == name)
         if not inner:
             inner = Class(
-                qname=build_qname(name),
+                qname=namespaces.build_qname(name),
                 tag=Tag.BINDING_MESSAGE,
                 location=target.location,
                 ns_map=target.ns_map.copy(),
@@ -291,7 +289,9 @@ class DefinitionsMapper:
         prefix, name = text.split(message.message)
         source_namespace = message.ns_map.get(prefix)
         yield cls.build_attr(
-            name, qname=build_qname(source_namespace, name), namespace=namespace
+            name,
+            qname=namespaces.build_qname(source_namespace, name),
+            namespace=namespace,
         )
 
     @classmethod
@@ -307,7 +307,7 @@ class DefinitionsMapper:
             parts.extend(extended.attributes["parts"].split())
 
         if "message" in extended.attributes:
-            message_name = local_name(extended.attributes["message"])
+            message_name = namespaces.local_name(extended.attributes["message"])
         else:
             message_name = text.suffix(message)
 
@@ -340,7 +340,7 @@ class DefinitionsMapper:
 
             ns_map.update(part.ns_map)
             namespace = part.ns_map.get(prefix)
-            type_qname = build_qname(namespace, type_name)
+            type_qname = namespaces.build_qname(namespace, type_name)
             native = namespace == Namespace.XS.uri
             namespace = "" if part.type else namespace
 
@@ -359,7 +359,7 @@ class DefinitionsMapper:
     def attributes(cls, elements: Iterator[AnyElement]) -> Dict:
         """Return all attributes from all extended elements as a dictionary."""
         return {
-            local_name(qname): value
+            namespaces.local_name(qname): value
             for element in elements
             if isinstance(element, AnyElement)
             for qname, value in element.attributes.items()
diff --git a/xsdata/formats/dataclass/parsers/handlers/native.py b/xsdata/formats/dataclass/parsers/handlers/native.py
index 893d0b63..a8e496a4 100644
--- a/xsdata/formats/dataclass/parsers/handlers/native.py
+++ b/xsdata/formats/dataclass/parsers/handlers/native.py
@@ -1,17 +1,22 @@
+import functools
 from typing import Any
 from typing import Dict
 from typing import Iterable
+from typing import Iterator
 from typing import Optional
 from typing import Tuple
 from typing import Type
+from urllib.parse import urljoin
 from xml import sax
-from xml.etree.ElementTree import iterparse
+from xml.etree import ElementInclude as xinclude
+from xml.etree import ElementTree as etree
 
 from xsdata.exceptions import XmlHandlerError
 from xsdata.formats.dataclass.parsers.mixins import PushParser
 from xsdata.formats.dataclass.parsers.mixins import SaxHandler
 from xsdata.formats.dataclass.parsers.mixins import XmlHandler
 from xsdata.models.enums import EventType
+from xsdata.utils import namespaces
 from xsdata.utils.namespaces import build_qname
 
 EVENTS = (EventType.START, EventType.END, EventType.START_NS)
@@ -29,16 +34,35 @@ class XmlEventHandler(XmlHandler):
 
     def parse(self, source: Any) -> Any:
         """
-        Parse an XML document from a system identifier or an InputSource.
+        Parse an XML document from a system identifier or an InputSource or
+        directly from an xml Element or ElementTree.
 
-        :raises XmlHandlerError: If process xinclude config is enabled.
+        When source is an Element or ElementTree the handler will walk
+        over the objects structure.
+
+        When source is a system identifier or an InputSource the parser
+        will ignore comments and recover from errors.
+
+        When config process_xinclude is enabled the handler will parse
+        the whole document and then walk down the element tree.
         """
-        if self.parser.config.process_xinclude:
-            raise XmlHandlerError(
-                f"{type(self).__name__} doesn't support xinclude elements."
-            )
 
-        return self.process_context(iterparse(source, EVENTS))  # nosec
+        if isinstance(source, etree.ElementTree):
+            source = source.getroot()
+
+        if isinstance(source, etree.Element):
+            ctx = iterwalk(source, {})
+        elif self.parser.config.process_xinclude:
+            root = etree.parse(source).getroot()  # nosec
+            base_url = get_base_url(self.parser.config.base_url, source)
+            loader = functools.partial(xinclude_loader, base_url=base_url)
+
+            xinclude.include(root, loader=loader)
+            ctx = iterwalk(root, {})
+        else:
+            ctx = etree.iterparse(source, EVENTS)  # nosec
+
+        return self.process_context(ctx)
 
     def process_context(self, context: Iterable) -> Any:
         """Iterate context and push the events to main parser."""
@@ -154,3 +178,42 @@ class XmlSaxHandler(SaxHandler, sax.handler.ContentHandler):
         :param uri: Namespace uri
         """
         self.ns_map[prefix] = uri or ""
+
+
+def iterwalk(element: etree.Element, ns_map: Dict) -> Iterator[Tuple[str, Any]]:
+    """
+    Walk over the element tree structure and emit start-ns/start/end events.
+
+    The ElementTree doesn't preserve the original namespace prefixes, we
+    have to generate new ones.
+    """
+    uri = namespaces.target_uri(element.tag)
+    if uri is not None:
+        prefix = namespaces.load_prefix(uri, ns_map)
+        yield EventType.START_NS, (prefix, uri)
+
+    yield EventType.START, element
+
+    for child in element:
+        yield from iterwalk(child, ns_map)
+
+    yield EventType.END, element
+
+
+def get_base_url(base_url: Optional[str], source: Any) -> Optional[str]:
+
+    if base_url:
+        return base_url
+
+    return source if isinstance(source, str) else None
+
+
+def xinclude_loader(
+    href: str,
+    parse: str,
+    encoding: Optional[str] = None,
+    base_url: Optional[str] = None,
+) -> Any:
+    """Custom loader for xinclude to support base_url argument that doesn't
+    exist for python < 3.9."""
+    return xinclude.default_loader(urljoin(base_url or "", href), parse, encoding)
diff --git a/xsdata/formats/dataclass/serializers/xml.py b/xsdata/formats/dataclass/serializers/xml.py
index a44e550d..90befcdb 100644
--- a/xsdata/formats/dataclass/serializers/xml.py
+++ b/xsdata/formats/dataclass/serializers/xml.py
@@ -27,7 +27,6 @@ from xsdata.models.enums import QNames
 from xsdata.utils import collections
 from xsdata.utils import namespaces
 from xsdata.utils.constants import EMPTY_MAP
-from xsdata.utils.namespaces import split_qname
 
 NoneStr = Optional[str]
 
@@ -102,7 +101,7 @@ class XmlSerializer(AbstractSerializer):
         meta = self.context.build(obj.__class__, namespace)
         qname = qname or meta.qname
         nillable = nillable or meta.nillable
-        namespace, tag = split_qname(qname)
+        namespace, tag = namespaces.split_qname(qname)
 
         yield XmlWriterEvent.START, qname
 
@@ -211,7 +210,7 @@ class XmlSerializer(AbstractSerializer):
     def write_wildcard(self, value: Any, var: XmlVar, namespace: NoneStr) -> Generator:
         """Produce an element events stream for the given generic object."""
         if value.qname:
-            namespace, tag = split_qname(value.qname)
+            namespace, tag = namespaces.split_qname(value.qname)
             yield XmlWriterEvent.START, value.qname
 
         for key, val in value.attributes.items():
@@ -384,10 +383,7 @@ class XmlSerializer(AbstractSerializer):
         if isinstance(value, (str, QName)) or var is None:
             return value
 
-        if isinstance(value, tuple) and hasattr(value, "_fields"):
-            return converter.serialize(value, format=var.format)
-
-        if isinstance(value, (tuple, list)):
+        if collections.is_array(value):
             return [cls.encode(v, var) for v in value]
 
         if isinstance(value, Enum):


Test patch:
diff --git a/tests/formats/dataclass/parsers/handlers/test_native.py b/tests/formats/dataclass/parsers/handlers/test_native.py
index 71e1b55a..afd67aec 100644
--- a/tests/formats/dataclass/parsers/handlers/test_native.py
+++ b/tests/formats/dataclass/parsers/handlers/test_native.py
@@ -1,7 +1,12 @@
+import sys
 from unittest import mock
 from unittest.case import TestCase
+from xml import etree
+
+import pytest
 
 from tests import fixtures_dir
+from tests.fixtures.books import BookForm
 from tests.fixtures.books import Books
 from tests.fixtures.books.fixtures import books
 from tests.fixtures.books.fixtures import events
@@ -10,6 +15,7 @@ from xsdata.exceptions import XmlHandlerError
 from xsdata.formats.dataclass.parsers.bases import RecordParser
 from xsdata.formats.dataclass.parsers.handlers import XmlEventHandler
 from xsdata.formats.dataclass.parsers.handlers import XmlSaxHandler
+from xsdata.formats.dataclass.parsers.handlers.native import get_base_url
 
 
 class XmlEventHandlerTests(TestCase):
@@ -28,17 +34,6 @@ class XmlEventHandlerTests(TestCase):
         self.assertEqual({None: "urn:books"}, self.parser.ns_map)
         self.assertEqual(events_default_ns, self.parser.events)
 
-    def test_parse_with_xinclude_raises_exception(self):
-        self.parser.config.process_xinclude = True
-        path = fixtures_dir.joinpath("books/books.xml")
-
-        with self.assertRaises(XmlHandlerError) as cm:
-            self.parser.from_path(path, Books)
-
-        self.assertEqual(
-            "XmlEventHandler doesn't support xinclude elements.", str(cm.exception)
-        )
-
     def test_parse_context_with_unhandled_event(self):
         context = [("reverse", None)]
         handler = XmlEventHandler(parser=self.parser, clazz=Books)
@@ -48,6 +43,41 @@ class XmlEventHandlerTests(TestCase):
 
         self.assertEqual("Unhandled event: `reverse`.", str(cm.exception))
 
+    def test_parse_with_element_or_tree(self):
+        path = fixtures_dir.joinpath("books/books.xml")
+        tree = etree.ElementTree.parse(str(path))
+
+        result = self.parser.parse(tree, Books)
+        self.assertEqual(books, result)
+
+        tree = etree.ElementTree.parse(str(path))
+        result = self.parser.parse(tree.find(".//book"), BookForm)
+        self.assertEqual(books.book[0], result)
+
+    @pytest.mark.skipif(sys.platform == "win32", reason="urljoin + path sep")
+    def test_parse_with_xinclude(self):
+        path = fixtures_dir.joinpath("books/books-xinclude.xml")
+        ns_map = {"ns0": "urn:books"}
+
+        self.parser.config.process_xinclude = True
+        self.assertEqual(books, self.parser.parse(str(path), Books))
+        self.assertEqual(ns_map, self.parser.ns_map)
+
+    @pytest.mark.skipif(sys.platform == "win32", reason="urljoin + path sep")
+    def test_parse_with_xinclude_from_memory(self):
+        path = fixtures_dir.joinpath("books/books-xinclude.xml")
+        ns_map = {"ns0": "urn:books"}
+
+        self.parser.config.process_xinclude = True
+        self.parser.config.base_url = str(path)
+        self.assertEqual(books, self.parser.from_string(path.read_text(), Books))
+        self.assertEqual(ns_map, self.parser.ns_map)
+
+    def test_get_base_url(self):
+        self.assertIsNone(get_base_url(None, None))
+        self.assertIsNone(get_base_url(None, None))
+        self.assertEqual("config/", get_base_url("config/", "/tmp/foo.xml"))
+
 
 class SaxHandlerTests(TestCase):
     def setUp(self):