Repository: tefra/xsdata

Issue:
I would like to write huge trees but don't retain the entire tree in memory
Ideally I would like to write out a tree where the data is added _just in time_. The proposal in #1030 has an increasing memory usage, which suggests that the tree is still being build completely in memory. I wanted to add some evidence. Please ignore the timing.

Using the generator method:
![mem-graph-generator](https://github.com/tefra/xsdata/assets/502394/e66c7728-45b8-4355-88c1-481a3a7d2f31)

Materializing into a list first:
![mem-graph](https://github.com/tefra/xsdata/assets/502394/714e73af-af36-4962-853d-f32ac7081902)

Ideally, I wish that the memory consumption wouldn't increase at all, and the data would just been written out as it would be provided. But I guess the graphs do give a clear view where we can make some improvements writing out huge documents.


Discussion:

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7a8d0a35..1117e8f6 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,24 +2,24 @@ exclude: tests/fixtures
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
   - repo: https://github.com/crate-ci/typos
-    rev: v1.24.6
+    rev: v1.26.0
     hooks:
       - id: typos
         exclude: ^tests/|.xsd|xsdata/models/datatype.py$
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.6
+    rev: v0.7.0
     hooks:
       - id: ruff
         args: [ --fix, --show-fixes]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.12.1
     hooks:
       - id: mypy
         files: ^(xsdata/)
diff --git a/docs/codegen/config.md b/docs/codegen/config.md
index b8469bba..446523f2 100644
--- a/docs/codegen/config.md
+++ b/docs/codegen/config.md
@@ -60,7 +60,14 @@ type # typing.Type
 
 **CLI Option:** `--subscriptable-types / --no-subscriptable-types`
 
-**Requirements:** `python>=3.9`
+### `genericCollections`
+
+Use `collections.abc.Iterable` and `collections.abc.Mapping` instead of `List|Tuple` and
+`Dict`
+
+**Default Value:** `False`
+
+**CLI Option:** `--generic-collections / --no-generic-collections`
 
 ### `unionType`
 
diff --git a/docs/installation.md b/docs/installation.md
index f22ae63f..287028c9 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -34,7 +34,7 @@ $ xsdata --help
 
 ## Requirements
 
-!!! Note "xsData relies on these awesome libraries and supports `python >= 3.8`"
+!!! Note "xsData relies on these awesome libraries and supports `python >= 3.9`"
 
     - [lxml](https://lxml.de/) - XML advanced features
     - [requests](https://requests.readthedocs.io/) - Webservice Default Transport
diff --git a/pyproject.toml b/pyproject.toml
index f687878e..126f2dfb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -29,7 +28,7 @@ classifiers = [
     "Topic :: Text Processing :: Markup :: XML",
 ]
 keywords = ["xsd", "wsdl", "schema", "dtd", "binding", "xml", "json", "dataclasses", "generator", "cli"]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
     "typing-extensions>=4.7.0",
 ]
diff --git a/xsdata/formats/dataclass/client.py b/xsdata/formats/dataclass/client.py
index 64a14f6f..98a90ff6 100644
--- a/xsdata/formats/dataclass/client.py
+++ b/xsdata/formats/dataclass/client.py
@@ -51,7 +51,7 @@ class Config:
             for f in fields(cls)
         }
 
-        return cls(**params)
+        return cls(**params)  # type: ignore
 
 
 class TransportTypes:
diff --git a/xsdata/formats/dataclass/filters.py b/xsdata/formats/dataclass/filters.py
index 32715c88..574a47b7 100644
--- a/xsdata/formats/dataclass/filters.py
+++ b/xsdata/formats/dataclass/filters.py
@@ -58,6 +58,7 @@ class Filters:
         "max_line_length",
         "union_type",
         "subscriptable_types",
+        "generic_collections",
         "relative_imports",
         "postponed_annotations",
         "format",
@@ -106,6 +107,7 @@ class Filters:
         self.max_line_length: int = config.output.max_line_length
         self.union_type: bool = config.output.union_type
         self.subscriptable_types: bool = config.output.subscriptable_types
+        self.generic_collections: bool = config.output.generic_collections
         self.relative_imports: bool = config.output.relative_imports
         self.postponed_annotations: bool = config.output.postponed_annotations
         self.format = config.output.format
@@ -758,6 +760,9 @@ class Filters:
             return result
 
         if attr.is_dict:
+            if self.generic_collections:
+                return "Mapping[str, str]"
+
             return "dict[str, str]" if self.subscriptable_types else "Dict[str, str]"
 
         if attr.is_nillable or (
@@ -889,7 +894,10 @@ class Filters:
 
         return "\n".join(collections.unique_sequence(imports))
 
-    def _get_iterable_format(self):
+    def _get_iterable_format(self) -> str:
+        if self.generic_collections:
+            return "Iterable[{}]"
+
         fmt = "Tuple[{}, ...]" if self.format.frozen else "List[{}]"
         return fmt.lower() if self.subscriptable_types else fmt
 
@@ -902,7 +910,7 @@ class Filters:
             "decimal": {"Decimal": type_patterns("Decimal")},
             "enum": {"Enum": ["(Enum)"]},
             "typing": {
-                "Dict": [": Dict"],
+                "Dict": [": Dict["],
                 "List": [": List["],
                 "Optional": ["Optional["],
                 "Tuple": ["Tuple["],
@@ -910,6 +918,10 @@ class Filters:
                 "ForwardRef": [": ForwardRef("],
                 "Any": type_patterns("Any"),
             },
+            "collections.abc": {
+                "Iterable": [": Iterable["],
+                "Mapping": [": Mapping["],
+            },
             "xml.etree.ElementTree": {"QName": type_patterns("QName")},
             "xsdata.models.datatype": {
                 "XmlDate": type_patterns("XmlDate"),
diff --git a/xsdata/formats/dataclass/typing.py b/xsdata/formats/dataclass/typing.py
index 8cb39c6e..b190c0bc 100644
--- a/xsdata/formats/dataclass/typing.py
+++ b/xsdata/formats/dataclass/typing.py
@@ -1,4 +1,5 @@
 import sys
+from collections.abc import Iterable, Mapping
 from typing import (
     Any,
     Callable,
@@ -45,7 +46,7 @@ else:
 
 NONE_TYPE = type(None)
 UNION_TYPES = (Union, UnionType)
-ITERABLE_TYPES = (list, tuple)
+ITERABLE_TYPES = (list, tuple, Iterable)
 
 
 def evaluate(tp: Any, globalns: Any, localns: Any = None) -> Any:
@@ -143,6 +144,9 @@ def evaluate_attribute(annotation: Any, tokens: bool = False) -> Result:
 
         args = analyze_token_args(origin, args)
         tokens_factory = origin
+        if tokens_factory is Iterable:
+            tokens_factory = list
+
         origin = get_origin(args[0])
 
         if origin in UNION_TYPES:
@@ -171,7 +175,7 @@ def evaluate_attributes(annotation: Any, **_: Any) -> Result:
         origin = get_origin(annotation)
         args = get_args(annotation)
 
-        if origin is not dict and annotation is not dict:
+        if origin is not dict and origin is not Mapping:
             raise TypeError
 
     if args and not all(arg is str for arg in args):
@@ -222,6 +226,12 @@ def evaluate_element(annotation: Any, tokens: bool = False) -> Result:
     elif origin:
         raise TypeError
 
+    if factory is Iterable:
+        factory = list
+
+    if tokens_factory is Iterable:
+        tokens_factory = list
+
     return Result(types=types, factory=factory, tokens_factory=tokens_factory)
 
 
@@ -247,7 +257,7 @@ def evaluate_wildcard(annotation: Any, **_: Any) -> Result:
     if origin in UNION_TYPES:
         types = filter_none_type(get_args(annotation))
     elif origin in ITERABLE_TYPES:
-        factory = origin
+        factory = list if origin is Iterable else origin
         types = filter_ellipsis(get_args(annotation))
     elif origin is None:
         types = (annotation,)
diff --git a/xsdata/models/config.py b/xsdata/models/config.py
index 95d51b9e..7b11b9a0 100644
--- a/xsdata/models/config.py
+++ b/xsdata/models/config.py
@@ -225,6 +225,7 @@ class GeneratorOutput:
         max_line_length: Adjust the maximum line length
         subscriptable_types: Use PEP-585 generics for standard
             collections, python>=3.9 Only
+        generic_collections: Use generic collections (Iterable, Mapping)
         union_type: Use PEP-604 union type, python>=3.10 Only
         postponed_annotations: Use 563 postponed evaluation of  annotations
         unnest_classes: Move inner classes to upper level
@@ -243,6 +244,7 @@ class GeneratorOutput:
     wrapper_fields: bool = element(default=False)
     max_line_length: int = attribute(default=79)
     subscriptable_types: bool = attribute(default=False)
+    generic_collections: bool = attribute(default=False)
     union_type: bool = attribute(default=False)
     postponed_annotations: bool = element(default=False)
     unnest_classes: bool = element(default=False)
@@ -255,13 +257,6 @@ class GeneratorOutput:
 
     def validate(self):
         """Reset configuration conflicts."""
-        if self.subscriptable_types and sys.version_info < (3, 9):
-            self.subscriptable_types = False
-            warnings.warn(
-                "Generics PEP 585 requires python >= 3.9, reverting...",
-                CodegenWarning,
-            )
-
         if self.union_type and sys.version_info < (3, 10):
             self.union_type = False
             warnings.warn(
@@ -276,6 +271,13 @@ class GeneratorOutput:
                 CodegenWarning,
             )
 
+        if self.generic_collections and self.format.frozen:
+            self.generic_collections = False
+            warnings.warn(
+                "Generic Collections, requires frozen=False, reverting...",
+                CodegenWarning,
+            )
+
     def update(self, **kwargs: Any):
         """Update instance attributes recursively."""
         objects.update(self, **kwargs)


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 3774d6d5..f91bf540 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -14,7 +14,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {name: Python 3.8, python: '3.8'}
           - {name: Python 3.9, python: '3.9'}
           - {name: Python 3.10, python: '3.10'}
           - {name: Python 3.11, python: '3.11'}
diff --git a/tests/formats/dataclass/cases/__init__.py b/tests/formats/dataclass/cases/__init__.py
index dacb7683..fc8dcd60 100644
--- a/tests/formats/dataclass/cases/__init__.py
+++ b/tests/formats/dataclass/cases/__init__.py
@@ -1,4 +1,3 @@
 import sys
 
-PY39 = sys.version_info[:2] >= (3, 9)
 PY310 = sys.version_info[:2] >= (3, 10)
diff --git a/tests/formats/dataclass/cases/attribute.py b/tests/formats/dataclass/cases/attribute.py
index e4d75a58..9889b97f 100644
--- a/tests/formats/dataclass/cases/attribute.py
+++ b/tests/formats/dataclass/cases/attribute.py
@@ -1,6 +1,7 @@
+from collections.abc import Iterable
 from typing import Dict, List, Literal, Optional, Set, Tuple, Union
 
-from tests.formats.dataclass.cases import PY39, PY310
+from tests.formats.dataclass.cases import PY310
 from xsdata.models.enums import Mode
 
 tokens = [
@@ -12,9 +13,17 @@ tokens = [
     (List[Union[List[int], int]], False),
     (List[List[int]], False),
     (Tuple[int, ...], ((int,), None, tuple)),
+    (Iterable[int], ((int,), None, list)),
     (List[int], ((int,), None, list)),
     (List[Union[str, int]], ((str, int), None, list)),
     (Optional[List[Union[str, int]]], ((str, int), None, list)),
+    (list[int, int], False),
+    (dict[str, str], False),
+    (dict, False),
+    (set[str], False),
+    (tuple[int, ...], ((int,), None, tuple)),
+    (list[int], ((int,), None, list)),
+    (list[Union[str, int]], ((str, int), None, list)),
 ]
 
 not_tokens = [
@@ -25,19 +34,6 @@ not_tokens = [
     (Union[str, Mode], ((str, Mode), None, None)),
 ]
 
-if PY39:
-    tokens.extend(
-        [
-            (list[int, int], False),
-            (dict[str, str], False),
-            (dict, False),
-            (set[str], False),
-            (tuple[int, ...], ((int,), None, tuple)),
-            (list[int], ((int,), None, list)),
-            (list[Union[str, int]], ((str, int), None, list)),
-        ]
-    )
-
 if PY310:
     tokens.extend(
         [
diff --git a/tests/formats/dataclass/cases/attributes.py b/tests/formats/dataclass/cases/attributes.py
index f6d1a155..18050821 100644
--- a/tests/formats/dataclass/cases/attributes.py
+++ b/tests/formats/dataclass/cases/attributes.py
@@ -1,7 +1,6 @@
+from collections.abc import Mapping
 from typing import Dict, List, Set, Tuple
 
-from tests.formats.dataclass.cases import PY39
-
 cases = [
     (int, False),
     (Set, False),
@@ -10,11 +9,6 @@ cases = [
     (Dict[str, int], False),
     (Dict, ((str,), dict, None)),
     (Dict[str, str], ((str,), dict, None)),
+    (Mapping[str, str], ((str,), dict, None)),
+    (dict[str, str], ((str,), dict, None)),
 ]
-
-if PY39:
-    cases.extend(
-        [
-            (dict[str, str], ((str,), dict, None)),
-        ]
-    )
diff --git a/tests/formats/dataclass/cases/element.py b/tests/formats/dataclass/cases/element.py
index 24310ab5..77bfe0a4 100644
--- a/tests/formats/dataclass/cases/element.py
+++ b/tests/formats/dataclass/cases/element.py
@@ -1,7 +1,6 @@
+from collections.abc import Iterable
 from typing import Dict, List, Optional, Set, Tuple, Union
 
-from tests.formats.dataclass.cases import PY39
-
 tokens = [
     (Set, False),
     (Dict[str, int], False),
@@ -12,8 +11,14 @@ tokens = [
     (List[List[str]], ((str,), list, list)),
     (Optional[List[List[Union[str, int]]]], ((str, int), list, list)),
     (List[Tuple[str, ...]], ((str,), list, tuple)),
+    (Iterable[Iterable[str, ...]], ((str,), list, list)),
     (Tuple[List[str], ...], ((str,), tuple, list)),
     (Optional[Tuple[List[str], ...]], ((str,), tuple, list)),
+    (list[str], ((str,), None, list)),
+    (tuple[str, ...], ((str,), None, tuple)),
+    (list[list[str]], ((str,), list, list)),
+    (list[tuple[str, ...]], ((str,), list, tuple)),
+    (tuple[list[str], ...], ((str,), tuple, list)),
 ]
 
 not_tokens = [
@@ -28,22 +33,6 @@ not_tokens = [
     (List[Union[str, int]], ((str, int), list, None)),
     (Optional[List[Union[str, int]]], ((str, int), list, None)),
     (Tuple[str, ...], ((str,), tuple, None)),
+    (list[str], ((str,), list, None)),
+    (tuple[str, ...], ((str,), tuple, None)),
 ]
-
-if PY39:
-    tokens.extend(
-        [
-            (list[str], ((str,), None, list)),
-            (tuple[str, ...], ((str,), None, tuple)),
-            (list[list[str]], ((str,), list, list)),
-            (list[tuple[str, ...]], ((str,), list, tuple)),
-            (tuple[list[str], ...], ((str,), tuple, list)),
-        ]
-    )
-
-    not_tokens.extend(
-        [
-            (list[str], ((str,), list, None)),
-            (tuple[str, ...], ((str,), tuple, None)),
-        ]
-    )
diff --git a/tests/formats/dataclass/cases/elements.py b/tests/formats/dataclass/cases/elements.py
index 49863ad8..61a00881 100644
--- a/tests/formats/dataclass/cases/elements.py
+++ b/tests/formats/dataclass/cases/elements.py
@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional, Tuple, Union
 
-from tests.formats.dataclass.cases import PY39, PY310
+from tests.formats.dataclass.cases import PY310
 
 cases = [
     (Dict, False),
@@ -10,19 +10,12 @@ cases = [
     (Optional[Union[str, int]], ((object,), None, None)),
     (Union[str, int, None], ((object,), None, None)),
     (List[Union[List[str], Tuple[str, ...]]], ((object,), list, None)),
+    (list[str], ((object,), list, None)),
+    (tuple[str, ...], ((object,), tuple, None)),
+    (list[Union[list[str], tuple[str, ...]]], ((object,), list, None)),
 ]
 
 
-if PY39:
-    cases.extend(
-        [
-            (list[str], ((object,), list, None)),
-            (tuple[str, ...], ((object,), tuple, None)),
-            (list[Union[list[str], tuple[str, ...]]], ((object,), list, None)),
-        ]
-    )
-
-
 if PY310:
     cases.extend(
         [
diff --git a/tests/formats/dataclass/cases/wildcard.py b/tests/formats/dataclass/cases/wildcard.py
index d84e8c4e..47940d05 100644
--- a/tests/formats/dataclass/cases/wildcard.py
+++ b/tests/formats/dataclass/cases/wildcard.py
@@ -1,3 +1,4 @@
+from collections.abc import Iterable
 from typing import Dict, List, Literal, Optional, Set, Tuple
 
 from tests.formats.dataclass.cases import PY310
@@ -11,6 +12,7 @@ cases = [
     (object, ((object,), None, None)),
     (List[object], ((object,), list, None)),
     (Tuple[object, ...], ((object,), tuple, None)),
+    (Iterable[object, ...], ((object,), list, None)),
     (Optional[object], ((object,), None, None)),
 ]
 
diff --git a/tests/formats/dataclass/test_filters.py b/tests/formats/dataclass/test_filters.py
index ebd10c60..eeb4ea31 100644
--- a/tests/formats/dataclass/test_filters.py
+++ b/tests/formats/dataclass/test_filters.py
@@ -643,6 +643,9 @@ class FiltersTests(FactoryTestCase):
         self.filters.format.frozen = False
         self.assertEqual('list["A.B.C"]', self.filters.field_type(self.obj, attr))
 
+        self.filters.generic_collections = True
+        self.assertEqual('Iterable["A.B.C"]', self.filters.field_type(self.obj, attr))
+
     def test_field_type_with_token_attr(self):
         attr = AttrFactory.create(
             types=AttrTypeFactory.list(1, qname="foo_bar"),
@@ -710,6 +713,9 @@ class FiltersTests(FactoryTestCase):
         self.filters.subscriptable_types = True
         self.assertEqual("dict[str, str]", self.filters.field_type(self.obj, attr))
 
+        self.filters.generic_collections = True
+        self.assertEqual("Mapping[str, str]", self.filters.field_type(self.obj, attr))
+
     def test_field_type_with_native_type(self):
         attr = AttrFactory.create(
             types=[
@@ -903,6 +909,14 @@ class FiltersTests(FactoryTestCase):
         expected = "from typing import Any"
         self.assertIn(expected, self.filters.default_imports(output))
 
+        output = ": Iterable[str] = "
+        expected = "from collections.abc import Iterable"
+        self.assertIn(expected, self.filters.default_imports(output))
+
+        output = ": Mapping[str, str] = "
+        expected = "from collections.abc import Mapping"
+        self.assertIn(expected, self.filters.default_imports(output))
+
     def test_default_imports_combo(self):
         output = (
             "@dataclass\n"
diff --git a/tests/models/test_config.py b/tests/models/test_config.py
index 656ae7b7..3bfd2c37 100644
--- a/tests/models/test_config.py
+++ b/tests/models/test_config.py
@@ -29,7 +29,7 @@ class GeneratorConfigTests(TestCase):
         expected = (
             '<?xml version="1.0" encoding="UTF-8"?>\n'
             f'<Config xmlns="http://pypi.org/project/xsdata" version="{__version__}">\n'
-            '  <Output maxLineLength="79" subscriptableTypes="false" unionType="false">\n'
+            '  <Output maxLineLength="79" subscriptableTypes="false" genericCollections="false" unionType="false">\n'
             "    <Package>generated</Package>\n"
             '    <Format repr="true" eq="true" order="false" unsafeHash="false" frozen="false" slots="false" kwOnly="false">dataclasses</Format>\n'
             "    <Structure>filenames</Structure>\n"
@@ -89,7 +89,7 @@ class GeneratorConfigTests(TestCase):
         expected = (
             '<?xml version="1.0" encoding="UTF-8"?>\n'
             f'<Config xmlns="http://pypi.org/project/xsdata" version="{__version__}">\n'
-            '  <Output maxLineLength="79" subscriptableTypes="false" unionType="false">\n'
+            '  <Output maxLineLength="79" subscriptableTypes="false" genericCollections="false" unionType="false">\n'
             "    <Package>foo.bar</Package>\n"
             '    <Format repr="true" eq="true" order="false" unsafeHash="false"'
             ' frozen="false" slots="false" kwOnly="false">dataclasses</Format>\n'
@@ -136,23 +136,6 @@ class GeneratorConfigTests(TestCase):
 
         self.assertEqual("Enabling eq because order is true", str(w[-1].message))
 
-    def test_subscriptable_types_requires_390(self):
-        if sys.version_info < (3, 9):
-            with warnings.catch_warnings(record=True) as w:
-                self.assertFalse(
-                    GeneratorOutput(subscriptable_types=True).subscriptable_types
-                )
-
-            self.assertEqual(
-                "Generics PEP 585 requires python >= 3.9, reverting...",
-                str(w[-1].message),
-            )
-
-        else:
-            self.assertTrue(
-                GeneratorOutput(subscriptable_types=True).subscriptable_types
-            )
-
     def test_use_union_type_requires_310_and_postponed_annotations(self):
         if sys.version_info < (3, 10):
             with warnings.catch_warnings(record=True) as w:
@@ -172,6 +155,18 @@ class GeneratorConfigTests(TestCase):
                     str(w[-1].message),
                 )
 
+    def test_generic_collections_requires_frozen_false(self):
+        with warnings.catch_warnings(record=True) as w:
+            output = GeneratorOutput(
+                generic_collections=True, format=OutputFormat(frozen=True)
+            )
+            self.assertFalse(output.generic_collections)
+
+            self.assertEqual(
+                "Generic Collections, requires frozen=False, reverting...",
+                str(w[-1].message),
+            )
+
     def test_format_slots_requires_310(self):
         if sys.version_info < (3, 10):
             self.assertTrue(OutputFormat(slots=True, value="attrs").slots)