Repository: tefra/xsdata
Pull request #573 (2021-08-01 20:15:00)

Structure style: namespace-clusters


Patch:
diff --git a/docs/codegen.rst b/docs/codegen.rst
index 70c1b97b..2ca6739c 100644
--- a/docs/codegen.rst
+++ b/docs/codegen.rst
@@ -85,6 +85,33 @@ type substitutions, redefines.
     $ xsdata schema.xsd --package models --structure-style namespaces
 
 
+Since v21.8, the generator converts namespaces to packages similar to jaxb in order
+to facilitate runs against multiple schemas from the same vendor.
+
+.. list-table::
+    :widths: 20 20
+    :header-rows: 1
+
+    * - Examples (before naming conventions)
+      -
+    * - http://www.w3.org/XML/1998/namespace
+      - org.w3.xml.1998.namespace
+    * - myNS.tempuri.org
+      - org.tempuri.myNS
+    * - urn:xmlns:25hoursaday-com:address
+      - com.25hoursaday.address
+
+
+**namespace-clusters**
+
+This style combines the clusters and the namespace styles. It will fail if there
+are strongly connected classes in the same graph from different namespaces.
+
+.. code-block:: console
+
+    $ xsdata schema.xsd --package models --structure-style namespace-clusters
+
+
 **single-package**
 
 This style will group all classes together into a single package eliminating imports
diff --git a/xsdata/cli.py b/xsdata/cli.py
index 6e3418e7..7f281351 100644
--- a/xsdata/cli.py
+++ b/xsdata/cli.py
@@ -125,6 +125,8 @@ def download(source: str, output: str):
         "\n\n"
         "clusters: group by strong connected dependencies"
         "\n\n"
+        "namespace-clusters: group by strong connected dependencies and namespaces"
+        "\n\n"
         "single-package: group all classes together"
     ),
     default="filenames",
diff --git a/xsdata/codegen/container.py b/xsdata/codegen/container.py
index aee6748f..0fece64c 100644
--- a/xsdata/codegen/container.py
+++ b/xsdata/codegen/container.py
@@ -98,6 +98,13 @@ class ClassContainer(ContainerInterface):
 
         return inner
 
+    def first(self, qname: str) -> Class:
+        classes = self.data.get(qname)
+        if not classes:
+            raise KeyError(f"Class {qname} not found")
+
+        return classes[0]
+
     def process(self):
         """
         Run all the process handlers.
diff --git a/xsdata/codegen/handlers/class_designate.py b/xsdata/codegen/handlers/class_designate.py
index 12695d2d..286e6527 100644
--- a/xsdata/codegen/handlers/class_designate.py
+++ b/xsdata/codegen/handlers/class_designate.py
@@ -2,7 +2,10 @@ import os
 from collections import defaultdict
 from pathlib import Path
 from typing import Iterable
+from typing import Iterator
 from typing import List
+from typing import Optional
+from typing import Set
 from urllib.parse import urlparse
 
 from toposort import toposort_flatten
@@ -11,10 +14,12 @@ from xsdata.codegen.mixins import ContainerHandlerInterface
 from xsdata.codegen.models import Class
 from xsdata.codegen.models import get_location
 from xsdata.codegen.models import get_target_namespace
+from xsdata.exceptions import CodeGenerationError
 from xsdata.models.config import StructureStyle
 from xsdata.models.enums import COMMON_SCHEMA_DIR
 from xsdata.utils import collections
 from xsdata.utils.graphs import strongly_connected_components
+from xsdata.utils.namespaces import to_package_name
 from xsdata.utils.package import module_name
 
 
@@ -32,6 +37,8 @@ class ClassDesignateHandler(ContainerHandlerInterface):
             self.group_all_together()
         elif structure_style == StructureStyle.CLUSTERS:
             self.group_by_strong_components()
+        elif structure_style == StructureStyle.NAMESPACE_CLUSTERS:
+            self.group_by_namespace_clusters()
         else:
             self.group_by_filenames()
 
@@ -57,10 +64,12 @@ class ClassDesignateHandler(ContainerHandlerInterface):
 
     def group_by_namespace(self):
         """Group classes by their target namespace."""
-        package = self.container.config.output.package
         groups = collections.group_by(self.container, key=get_target_namespace)
         for namespace, classes in groups.items():
-            self.assign(classes, package, namespace or "")
+            parts = self.combine_ns_package(namespace)
+            module = parts.pop()
+            package = ".".join(parts)
+            self.assign(classes, package, module)
 
     def group_all_together(self):
         """Group all classes together in the same module."""
@@ -72,24 +81,35 @@ class ClassDesignateHandler(ContainerHandlerInterface):
 
     def group_by_strong_components(self):
         """Find circular imports and cluster their classes together."""
-        edges = {}
-        class_map = {}
-        for obj in self.container:
-            edges[obj.qname] = set(obj.dependencies(True))
-            class_map[obj.qname] = obj
-
-        groups = strongly_connected_components(edges)
         package = self.container.config.output.package
+        for group in self.strongly_connected_classes():
+            classes = self.sorted_classes(group)
+            module = classes[0].name
+            self.assign(classes, package, module)
+
+    def group_by_namespace_clusters(self):
+        for group in self.strongly_connected_classes():
+            classes = self.sorted_classes(group)
+            if len(set(map(get_target_namespace, classes))) > 1:
+                raise CodeGenerationError(
+                    "Found strongly connected classes from different "
+                    "namespaces, grouping them is impossible!"
+                )
 
-        for group in groups:
-            group_edges = {
-                qname: set(class_map[qname].dependencies()).intersection(group)
-                for qname in group
-            }
-            classes = [class_map[qname] for qname in toposort_flatten(group_edges)]
+            parts = self.combine_ns_package(classes[0].target_namespace)
             module = classes[0].name
+            self.assign(classes, ".".join(parts), module)
 
-            self.assign(classes, package, module)
+    def sorted_classes(self, qnames: Set[str]) -> List[Class]:
+        edges = {
+            qname: set(self.container.first(qname).dependencies()).intersection(qnames)
+            for qname in qnames
+        }
+        return [self.container.first(qname) for qname in toposort_flatten(edges)]
+
+    def strongly_connected_classes(self) -> Iterator[Set[str]]:
+        edges = {obj.qname: list(set(obj.dependencies(True))) for obj in self.container}
+        return strongly_connected_components(edges)
 
     @classmethod
     def assign(cls, classes: Iterable[Class], package: str, module: str):
@@ -118,3 +138,15 @@ class ClassDesignateHandler(ContainerHandlerInterface):
                 groups[index].append(path)
 
         return list(groups.values())
+
+    def combine_ns_package(self, namespace: Optional[str]) -> List[str]:
+        result = self.container.config.output.package.split(".")
+        aliases = self.container.config.aliases.package_name
+        alias = collections.first(x.target for x in aliases if x.source == namespace)
+
+        if alias:
+            result.extend(alias.split("."))
+        else:
+            result.extend(to_package_name(namespace).split("."))
+
+        return list(filter(None, result))
diff --git a/xsdata/codegen/mixins.py b/xsdata/codegen/mixins.py
index 47addf6e..340a1c7a 100644
--- a/xsdata/codegen/mixins.py
+++ b/xsdata/codegen/mixins.py
@@ -33,6 +33,11 @@ class ContainerInterface(abc.ABC):
     def find_inner(self, source: Class, qname: str) -> Class:
         """Search by qualified name for a specific inner class or fail."""
 
+    @abc.abstractmethod
+    def first(self, qname: str) -> Class:
+        """Search by qualified name for a specific class and return the first
+        available."""
+
     @abc.abstractmethod
     def add(self, item: Class):
         """Add class item to the container."""
diff --git a/xsdata/models/config.py b/xsdata/models/config.py
index c0b43937..9b76e6e4 100644
--- a/xsdata/models/config.py
+++ b/xsdata/models/config.py
@@ -33,12 +33,14 @@ class StructureStyle(Enum):
     :cvar NAMESPACES: namespaces
     :cvar CLUSTERS: clusters
     :cvar SINGLE_PACKAGE: single-package
+    :cvar NAMESPACE_CLUSTERS: namespace-clusters
     """
 
     FILENAMES = "filenames"
     NAMESPACES = "namespaces"
     CLUSTERS = "clusters"
     SINGLE_PACKAGE = "single-package"
+    NAMESPACE_CLUSTERS = "namespace-clusters"
 
 
 class NameCase(Enum):
diff --git a/xsdata/utils/namespaces.py b/xsdata/utils/namespaces.py
index 2c4003d1..311e5e07 100644
--- a/xsdata/utils/namespaces.py
+++ b/xsdata/utils/namespaces.py
@@ -150,3 +150,56 @@ def is_uri(uri: Optional[str]) -> bool:
     """Verify given string is a valid uri."""
 
     return bool(URI_REGEX.search(uri)) if uri else False
+
+
+@functools.lru_cache(maxsize=50)
+def to_package_name(uri: Optional[str]) -> str:
+    """Util method to convert a namespace to a dot style package name."""
+
+    if not uri:
+        return ""
+
+    # Remove scheme
+    domain_sep = "."
+    if uri.startswith("http://"):
+        uri = uri[7:]
+    elif uri.startswith("urn:"):
+        uri = uri[4:]
+        domain_sep = "-"
+
+        if uri.startswith("xmlns:"):
+            uri = uri[6:]
+
+        uri = uri.replace(":", "/")
+
+    # Remote target
+    pos = uri.find("#")
+    if pos > 0:
+        uri = uri[:pos]
+
+    tokens = [token for token in uri.split("/") if token.strip()]
+
+    if not tokens:
+        return ""
+
+    # Remove extension
+    if len(tokens) > 1:
+        last = tokens[-1]
+        pos = tokens[-1].rfind(".")
+        if pos > 0:
+            tokens[-1] = last[:pos]
+
+    # Remove port from domain
+    domain = tokens.pop(0)
+    pos = domain.find(":")
+    if pos > 0:
+        domain = domain[:pos]
+
+    # Remove www from domain
+    if domain.startswith("www"):
+        domain = domain[3:]
+
+    for part in domain.split(domain_sep):
+        tokens.insert(0, part)
+
+    return ".".join(token for token in tokens if token)


Test patch:
diff --git a/tests/codegen/handlers/test_class_designate.py b/tests/codegen/handlers/test_class_designate.py
index f98ce4d2..346b98b6 100644
--- a/tests/codegen/handlers/test_class_designate.py
+++ b/tests/codegen/handlers/test_class_designate.py
@@ -1,5 +1,7 @@
 from xsdata.codegen.container import ClassContainer
 from xsdata.codegen.handlers import ClassDesignateHandler
+from xsdata.exceptions import CodeGenerationError
+from xsdata.models.config import GeneratorAlias
 from xsdata.models.config import GeneratorConfig
 from xsdata.models.config import StructureStyle
 from xsdata.models.enums import Namespace
@@ -56,8 +58,8 @@ class ClassDesignateHandlerTests(FactoryTestCase):
 
     def test_group_by_namespace(self):
         classes = [
-            ClassFactory.create(qname="{a}a", location="foo"),
-            ClassFactory.create(qname="{a}b", location="foo"),
+            ClassFactory.create(qname="{myNS.tempuri.org}a", location="foo"),
+            ClassFactory.create(qname="{myNS.tempuri.org}b", location="foo"),
             ClassFactory.create(qname="b", location="bar"),
         ]
 
@@ -66,13 +68,13 @@ class ClassDesignateHandlerTests(FactoryTestCase):
         self.config.output.package = "bar"
 
         self.handler.run()
-        self.assertEqual("bar", classes[0].package)
-        self.assertEqual("bar", classes[1].package)
-        self.assertEqual("bar", classes[2].package)
+        self.assertEqual("bar.org.tempuri", classes[0].package)
+        self.assertEqual("bar.org.tempuri", classes[1].package)
+        self.assertEqual("", classes[2].package)
 
-        self.assertEqual("a", classes[0].module)
-        self.assertEqual("a", classes[1].module)
-        self.assertEqual("", classes[2].module)
+        self.assertEqual("myNS", classes[0].module)
+        self.assertEqual("myNS", classes[1].module)
+        self.assertEqual("bar", classes[2].module)
 
     def test_group_all_together(self):
         classes = [
@@ -128,3 +130,68 @@ class ClassDesignateHandlerTests(FactoryTestCase):
         self.assertEqual("class_E", classes[1].module)
         self.assertEqual("class_E", classes[2].module)
         self.assertEqual("class_E", classes[3].module)
+
+    def test_group_by_namespace_clusters(self):
+        classes = [
+            ClassFactory.create("{urn:foo-bar:com}a"),
+            ClassFactory.create("{urn:foo-bar:add}b"),
+            ClassFactory.create("{urn:foo-bar:add}c"),
+            ClassFactory.create("{urn:foo-bar:add}d"),
+        ]
+
+        classes[0].attrs.append(AttrFactory.reference(classes[1].qname))
+        classes[1].attrs.append(AttrFactory.reference(classes[2].qname))
+        classes[2].attrs.append(AttrFactory.reference(classes[3].qname))
+        classes[3].attrs.append(AttrFactory.reference(classes[1].qname, circular=True))
+
+        self.config.output.structure = StructureStyle.NAMESPACE_CLUSTERS
+        self.config.output.package = "models"
+        self.container.extend(classes)
+
+        self.handler.run()
+        self.assertEqual("a", classes[0].module)
+        self.assertEqual("d", classes[1].module)
+        self.assertEqual("d", classes[2].module)
+        self.assertEqual("d", classes[3].module)
+
+        self.assertEqual("models.bar.foo.com", classes[0].package)
+        self.assertEqual("models.bar.foo.add", classes[1].package)
+        self.assertEqual("models.bar.foo.add", classes[2].package)
+        self.assertEqual("models.bar.foo.add", classes[3].package)
+
+    def test_group_by_namespace_clusters_raises_exception(self):
+        classes = [
+            ClassFactory.create("{urn:foo-bar:com}a"),
+            ClassFactory.create("{urn:foo-bar:add}b"),
+            ClassFactory.create("{urn:foo-bar:exc}c"),
+            ClassFactory.create("{urn:foo-bar:add}d"),
+        ]
+
+        classes[0].attrs.append(AttrFactory.reference(classes[1].qname))
+        classes[1].attrs.append(AttrFactory.reference(classes[2].qname))
+        classes[2].attrs.append(AttrFactory.reference(classes[3].qname))
+        classes[3].attrs.append(AttrFactory.reference(classes[1].qname, circular=True))
+
+        self.config.output.structure = StructureStyle.NAMESPACE_CLUSTERS
+        self.config.output.package = "models"
+        self.container.extend(classes)
+
+        with self.assertRaises(CodeGenerationError) as cm:
+            self.handler.run()
+
+        self.assertEqual(
+            "Found strongly connected classes from different namespaces, "
+            "grouping them is impossible!",
+            str(cm.exception),
+        )
+
+    def test_combine_ns_package(self):
+        namespace = "urn:foo-bar:add"
+        result = self.handler.combine_ns_package(namespace)
+        self.assertEqual(["generated", "bar", "foo", "add"], result)
+
+        alias = GeneratorAlias(source=namespace, target="add.again")
+        self.config.aliases.package_name.append(alias)
+
+        result = self.handler.combine_ns_package(namespace)
+        self.assertEqual(["generated", "add", "again"], result)
diff --git a/tests/codegen/test_container.py b/tests/codegen/test_container.py
index 4a59dcb0..30b667a2 100644
--- a/tests/codegen/test_container.py
+++ b/tests/codegen/test_container.py
@@ -100,6 +100,14 @@ class ClassContainerTests(FactoryTestCase):
         self.assertEqual(second, self.container.find_inner(obj, "{a}b"))
         mock_process_class.assert_called_once_with(first, Steps.FLATTEN)
 
+    def test_first(self):
+        obj = ClassFactory.create()
+        self.container.add(obj)
+        self.assertEqual(obj, self.container.first(obj.qname))
+
+        with self.assertRaises(KeyError) as cm:
+            self.container.first("aa")
+
     def test_process_class(self):
         target = ClassFactory.create(
             inner=[ClassFactory.elements(2), ClassFactory.elements(1)]
diff --git a/tests/utils/test_namespaces.py b/tests/utils/test_namespaces.py
index d5d14d17..56e77099 100644
--- a/tests/utils/test_namespaces.py
+++ b/tests/utils/test_namespaces.py
@@ -12,6 +12,7 @@ from xsdata.utils.namespaces import is_uri
 from xsdata.utils.namespaces import load_prefix
 from xsdata.utils.namespaces import prefix_exists
 from xsdata.utils.namespaces import split_qname
+from xsdata.utils.namespaces import to_package_name
 
 
 class NamespacesTests(TestCase):
@@ -112,3 +113,25 @@ class NamespacesTests(TestCase):
         self.assertTrue(is_default("foo", {"": "foo"}))
         self.assertTrue(is_default("foo", {None: "foo"}))
         self.assertTrue(is_default("foo", {"a": "foo", None: "foo"}))
+
+    def test_to_package_name(self):
+        cases = {
+            "http://www.w3.org/XML/1998/namespace": "org.w3.XML.1998.namespace",
+            "http://www.w3.org/XML/2008/06/xlink.xsd": "org.w3.XML.2008.06.xlink",
+            "http://xsdtesting": "xsdtesting",
+            "http://xsdtesting:8080": "xsdtesting",
+            "http://xsdtesting:8080#target": "xsdtesting",
+            "myNS.tempuri.org": "org.tempuri.myNS",
+            "ElemDecl/disallowedSubst": "ElemDecl.disallowedSubst",
+            "http://xstest-tns/schema11": "xstest-tns.schema11",
+            "http://uri.etsi.org/#": "org.etsi.uri",
+            "urn:xmlns:25hoursaday-com:address": "com.25hoursaday.address",
+            "urn:my.test:SchemaB": "my.test.SchemaB",
+            "http://": "",
+            "": "",
+            " ": "",
+            None: "",
+        }
+
+        for uri, package in cases.items():
+            self.assertEqual(package, to_package_name(uri))
diff --git a/xsdata/utils/testing.py b/xsdata/utils/testing.py
index 2435f1a4..73e3bf1e 100644
--- a/xsdata/utils/testing.py
+++ b/xsdata/utils/testing.py
@@ -99,6 +99,7 @@ class ClassFactory(Factory):
         qname: Optional[str] = None,
         meta_name: Optional[str] = None,
         namespace: Optional[str] = None,
+        target_namespace: Optional[str] = None,
         tag: Optional[str] = None,
         abstract: bool = False,
         mixed: bool = False,