Repository: SneaksAndData/adapta (language: python)

Issue:
[FEATURE] Add support for converting xml to dict
**Is your feature request related to a problem? Please describe.**
I'm always frustrated when processing xml data using PySpark, especially for complex xml data with multiple levels of nesting. Because complex and large scale xml data is very likely to put a huge burden on a single worker or even collapse.

**Describe the solution you'd like**
I would like to have a function which converts xml data to dict.


Discussion / hints:

Fix patch:
diff --git a/adapta/utils/__init__.py b/adapta/utils/__init__.py
index 412d40b3..6dd39996 100644
--- a/adapta/utils/__init__.py
+++ b/adapta/utils/__init__.py
@@ -19,3 +19,4 @@
 
 from adapta.utils._common import *
 from adapta.utils.decorators import run_time_metrics
+from adapta.utils.data_structures import *
diff --git a/adapta/utils/data_structures/__init__.py b/adapta/utils/data_structures/__init__.py
new file mode 100644
index 00000000..a3b1980c
--- /dev/null
+++ b/adapta/utils/data_structures/__init__.py
@@ -0,0 +1,5 @@
+"""
+ Utilities module index
+"""
+
+from adapta.utils.data_structures._functions import *
diff --git a/adapta/utils/data_structures/_functions.py b/adapta/utils/data_structures/_functions.py
new file mode 100644
index 00000000..30e83881
--- /dev/null
+++ b/adapta/utils/data_structures/_functions.py
@@ -0,0 +1,129 @@
+"""
+ Module for data structures methods.
+"""
+import os.path
+from pathlib import Path
+from typing import List, Union, TypeVar, Dict
+import xml.etree.ElementTree as ET
+
+XmlNodeT = TypeVar("XmlNodeT")
+
+
+def xmltree_to_dict_collection(xml_source: Union[str, Path], node_type: type[XmlNodeT]) -> List[XmlNodeT]:
+    """
+     Convert a xml source to a list of dict, which can be a path or a xml string
+
+    for example
+        <?xml version="1.0"?>
+        <catalog>
+           <book id="bk101" name="bookname1">
+              <author>author1</author>
+              <price currency="USD">10</price>
+           </book>
+           <book id="bk102" name="bookname2">
+              <author>author2</author>
+              <price currency="USD">6</price>
+           </book>
+        </catalog>
+
+    When node_type is dict, the returned value is
+        [
+         {"book_id": "bk101", "book_name": "bookname1", "author":"author1", "price_currency": "USD", "price": "10"},
+         {"book_id": "bk102", "book_name": "bookname2", "author":"author2", "price_currency": "USD", "price": "6"}
+        ]
+
+    :param xml_source: Valid XML string or a path to a valid xml file
+    :param node_type: The type of each element in returned List, like dict or a created class inheriting from DataClassJsonMixin
+    :return:
+    """
+
+    def node_attributes_to_dict(node: ET.Element) -> Dict:
+        """
+         Get the node's attributes
+
+        for example <date id="15-11-2023" time="12:34">:
+        the return would be: {'date_id': '15-11-2023', 'date_time': '12:34'}
+
+        :param node: Current node
+        :return:
+        """
+        return {f"{node.tag.lower()}_{key.lower()}": value for key, value in node.attrib.items()}
+
+    def merge(node: ET.Element, leaf: ET.Element) -> Dict:
+        """
+         Merge current node's attributes, all the leafs' attributes and text
+
+        :param node: Node
+        :param leaf: Leaf
+        :return:
+        """
+        assert len(leaf) == 0, "Sub-element detected, the expectation is each leaf node should not have sub-tag."
+
+        return node_attributes_to_dict(node) | node_attributes_to_dict(leaf) | {leaf.tag.lower(): leaf.text}
+
+    def node_type_convert(base_node: Dict) -> XmlNodeT:
+        """
+         Convert type of node to XmlNodeT
+
+        :param base_node: Node to be converted to XmlNodeT
+        :return:
+        """
+        return base_node if node_type is dict else node_type.from_dict(base_node)
+
+    def backtrack(node: ET.Element, converted_node: Dict):
+        """
+         Generate all the combinations from root to the node closest to leaves based on the backtracking algorithm
+
+        Base case (reached leaf nodes), there are two possible outcomes:
+               1. all the leaves have the same tag name like "book" leaves in the following example
+                   <catalog>
+                      <book>book_name1</book>
+                      <book>book_name2</book>
+                   </catalog>
+                Then directly append to combinations
+
+               2. each leaf has different tag name
+                   <catalog>
+                      <book>book_name1</book>
+                      <price>10</price>
+                   </catalog>
+                Then merge all the leaves and append to combinations
+
+        Recursive case:
+            Get the attributes of the current node,
+            traverse each child and start a new recursion to generate all the combinations
+
+        :param node: Current node
+        :param converted_node: The combination from root to current node
+        :return:
+        """
+
+        # when the node's children are leaves
+        if len(node) > 0 and len(node[0]) == 0:
+            # all the leaves have the same tag, directly append to combinations
+            if len(node.findall(node[0].tag)) > 1:
+                for leaf in node:
+                    converted_nodes.append(node_type_convert(converted_node | merge(node, leaf)))
+            # each leaf has different tag name, merge all the leaves and append to combinations
+            else:
+                for leaf in node:
+                    converted_node |= merge(node, leaf)
+
+                converted_nodes.append(node_type_convert(converted_node))
+
+        # when the node is far away from leaves
+        else:
+            for child in node:
+                backtrack(child, converted_node | node_attributes_to_dict(child))
+
+    if isinstance(xml_source, Path) and not os.path.isfile(xml_source):
+        raise RuntimeError("Provided path is not a file or does not exist")
+
+    converted_nodes: list[XmlNodeT] = []
+    # read xml and get root node
+    root = ET.parse(str(xml_source)).getroot() if isinstance(xml_source, Path) else ET.fromstring(xml_source)
+
+    if len(root) > 0:
+        backtrack(root, node_attributes_to_dict(root))
+
+    return converted_nodes


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 8e1aa811..d933c21e 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -13,15 +13,18 @@
 #  limitations under the License.
 #
 import os
+import pathlib
 import sys
 
 import time
+from dataclasses import dataclass
 from logging import StreamHandler
 from typing import List, Any, Dict, Optional
 
 import numpy
 import pandas
 import pytest
+from dataclasses_json import DataClassJsonMixin
 
 from adapta.logs import SemanticLogger
 from adapta.logs.models import LogLevel
@@ -33,6 +36,7 @@
     map_column_names,
     run_time_metrics,
     downcast_dataframe,
+    xmltree_to_dict_collection,
 )
 from adapta.utils.concurrent_task_runner import Executable, ConcurrentTaskRunner
 
@@ -360,3 +364,180 @@ def test_downcast_dataframe(dataframe, expected_types, column_filter):
     result = downcast_dataframe(dataframe, columns=column_filter)
     for column in expected_types:
         assert result[column].dtype == expected_types[column]
+
+
+# create classes for node type converting test in xmltree_to_dict_collection
+@dataclass
+class BasicMultipleRows(DataClassJsonMixin):
+    """
+    Class used in basic_multiple_rows.xml test
+    """
+
+    book: str
+
+
+@dataclass
+class Complicated(DataClassJsonMixin):
+    """
+    Class used in complicated.xml test
+    """
+
+    date_id: str
+    time_id: int
+    books_id: int
+    books_listname: str
+    books_database: str
+    book_color: str
+    book_size: float
+    description: str
+    price: float
+    price_currency: Optional[str] = None
+
+
+@pytest.mark.parametrize(
+    "xml_source, expected_result, node_type",
+    [
+        (
+            "<?xml version='1.0'?><catalog><book>book_name1</book><book>book_name2</book></catalog>",
+            [{"book": "book_name1"}, {"book": "book_name2"}],
+            dict,
+        ),
+        ("empty.xml", [], dict),
+        ("root_with_attributes.xml", [{"root_id": "eqweqwre", "child": "data"}], dict),
+        ("basic.xml", [{"child": "data"}], dict),
+        (
+            "basic_with_attributes.xml",
+            [
+                {"book_id": "1", "book_location": "北京", "book": "book_name1"},
+                {"book_id": "2", "book_location": "Copenhagen", "book": None},
+            ],
+            dict,
+        ),
+        (
+            "basic_multiple_rows.xml",
+            [BasicMultipleRows.from_dict(element) for element in [{"book": "book_name1"}, {"book": "book_name2"}]],
+            BasicMultipleRows,
+        ),
+        (
+            "nested_easy.xml",
+            [{"author": "author_name1", "price": "10"}, {"author": "author_name2", "price": "20"}],
+            dict,
+        ),
+        (
+            "nested_single.xml",
+            [
+                {
+                    "book_id": "book_121232",
+                    "book_parentid": "book_1212",
+                    "book_city": "Copenhagen",
+                    "price": "2",
+                    "first_store": None,
+                }
+            ],
+            dict,
+        ),
+        (
+            "nested.xml",
+            [
+                {
+                    "books_year": "2022",
+                    "book_id": "bk101",
+                    "book_name": "bookname1",
+                    "author": "author1",
+                    "price_currency": "USD",
+                    "price": "10",
+                },
+                {
+                    "books_year": "2023",
+                    "book_id": "bk201",
+                    "book_name": "bookname11",
+                    "author": "author11",
+                    "price_currency": "USD",
+                    "price": "20",
+                },
+                {
+                    "books_year": "2023",
+                    "book_id": "bk202",
+                    "book_name": "bookname22",
+                    "author": "author22",
+                    "price": "30",
+                },
+            ],
+            dict,
+        ),
+        (
+            "complicated.xml",
+            (
+                [
+                    Complicated.from_dict(element)
+                    for element in [
+                        {
+                            "date_id": "15.11.2023",
+                            "time_id": "123123",
+                            "books_id": "12345",
+                            "books_listname": "List of book",
+                            "books_database": "database1",
+                            "book_color": "123/234",
+                            "book_size": "10",
+                            "description": "After an inadvertant trip through a Heisenberg, Uncertainty Device, James Salway discovers the problems,of being quantum. The Microsoft MSXML3 parser is covered in\n                      detail, with attention to XML DOM interfaces, XSLT processing, SAX and more.",
+                            "price_currency": "CNY",
+                            "price": "10",
+                        },
+                        {
+                            "date_id": "15.11.2023",
+                            "time_id": "123123",
+                            "books_id": "56789",
+                            "books_listname": "List of book",
+                            "books_database": "database2",
+                            "book_color": "789/101",
+                            "book_size": "100",
+                            "description": "haha",
+                            "price": "80",
+                        },
+                        {
+                            "date_id": "15.11.2023",
+                            "time_id": "123123",
+                            "books_id": "56789",
+                            "books_listname": "List of book",
+                            "books_database": "database2",
+                            "book_color": "121/314",
+                            "book_size": "58",
+                            "description": "enen",
+                            "price": "29",
+                        },
+                        {
+                            "date_id": "15.11.2023",
+                            "time_id": "456456",
+                            "books_id": "101112",
+                            "books_listname": "List of book",
+                            "books_database": "database3",
+                            "book_color": "abc/def",
+                            "book_size": "101",
+                            "description": "hehehe",
+                            "price": "789",
+                        },
+                        {
+                            "date_id": "14.11.2023",
+                            "time_id": "789789",
+                            "books_id": "131415",
+                            "books_listname": "List of book",
+                            "books_database": "database4",
+                            "book_color": "ghi/jkl",
+                            "book_size": "102",
+                            "description": "descriptiondescription",
+                            "price": "300",
+                        },
+                    ]
+                ]
+            ),
+            Complicated,
+        ),
+    ],
+)
+def test_xmltree_to_dict_collection(xml_source, expected_result, node_type):
+    xml_source = (
+        pathlib.Path(f"{pathlib.Path(__file__).parent.resolve()}/xml_files/{xml_source}")
+        if xml_source.endswith(".xml")
+        else xml_source
+    )
+    assert expected_result == xmltree_to_dict_collection(xml_source, node_type)
diff --git a/tests/xml_files/basic.xml b/tests/xml_files/basic.xml
new file mode 100644
index 00000000..a69117ab
--- /dev/null
+++ b/tests/xml_files/basic.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<root>
+   <!-- This is a comment -->
+   <child>data</child>
+</root>
\ No newline at end of file
diff --git a/tests/xml_files/basic_multiple_rows.xml b/tests/xml_files/basic_multiple_rows.xml
new file mode 100644
index 00000000..af1819cd
--- /dev/null
+++ b/tests/xml_files/basic_multiple_rows.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0"?>
+<catalog>
+   <book>book_name1</book>
+   <book>book_name2</book>
+</catalog>
\ No newline at end of file
diff --git a/tests/xml_files/basic_with_attributes.xml b/tests/xml_files/basic_with_attributes.xml
new file mode 100644
index 00000000..78252cdb
--- /dev/null
+++ b/tests/xml_files/basic_with_attributes.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<root>
+   <book id="1" location="北京">book_name1</book>
+   <book id="2" location="Copenhagen"/>
+</root>
\ No newline at end of file
diff --git a/tests/xml_files/complicated.xml b/tests/xml_files/complicated.xml
new file mode 100644
index 00000000..b1ef38b5
--- /dev/null
+++ b/tests/xml_files/complicated.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<complicated>
+	<date id="15.11.2023">
+		<time id="123123">
+			<books id="12345" ListName="List of book" DataBase="database1">
+				<Book Color="123/234" Size="10">
+					<description>After an inadvertant trip through a Heisenberg, Uncertainty Device, James Salway discovers the problems,of being quantum. The Microsoft MSXML3 parser is covered in
+                      detail, with attention to XML DOM interfaces, XSLT processing, SAX and more.</description>
+					<price currency="CNY">10</price>
+				</Book>
+            </books>
+            <books id="56789" ListName="List of book" DataBase="database2">
+				<Book Color="789/101" Size="100">
+					<description>haha</description>
+					<price>80</price>
+				</Book>
+				<Book Color="121/314" Size="58">
+					<description>enen</description>
+					<price>29</price>
+				</Book>
+            </books>
+        </time>
+        <time id="456456">
+			<books id="101112" ListName="List of book" DataBase="database3">
+				<Book Color="abc/def" Size="101">
+					<description>hehehe</description>
+					<price>789</price>
+				</Book>
+            </books>
+        </time>
+    </date>
+
+	<date id="14.11.2023">
+		<time id="789789">
+			<books id="131415" ListName="List of book" DataBase="database4">
+				<Book Color="ghi/jkl" Size="102">
+					<description>descriptiondescription</description>
+					<price>300</price>
+				</Book>
+            </books>
+        </time>
+    </date>
+</complicated>
\ No newline at end of file
diff --git a/tests/xml_files/empty.xml b/tests/xml_files/empty.xml
new file mode 100644
index 00000000..01d70a2d
--- /dev/null
+++ b/tests/xml_files/empty.xml
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<root></root>
\ No newline at end of file
diff --git a/tests/xml_files/nested.xml b/tests/xml_files/nested.xml
new file mode 100644
index 00000000..6ee9ae19
--- /dev/null
+++ b/tests/xml_files/nested.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<catalog>
+    <Books Year="2022">
+        <book id="bk101" Name="bookname1">
+          <author>author1</author>
+          <price currency="USD">10</price>
+       </book>
+    </Books>
+    <Books year="2023">
+        <book id="bk201" Name="bookname11">
+          <author>author11</author>
+          <price currency="USD">20</price>
+       </book>
+       <book id="bk202" Name="bookname22">
+          <author>author22</author>
+          <price>30</price>
+       </book>
+    </Books>
+</catalog>
\ No newline at end of file
diff --git a/tests/xml_files/nested_easy.xml b/tests/xml_files/nested_easy.xml
new file mode 100644
index 00000000..5f602389
--- /dev/null
+++ b/tests/xml_files/nested_easy.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0"?>
+<catalog>
+   <book>
+      <author>author_name1</author>
+      <price>10</price>
+   </book>
+   <book>
+      <author>author_name2</author>
+      <price>20</price>
+   </book>
+</catalog>
\ No newline at end of file
diff --git a/tests/xml_files/nested_single.xml b/tests/xml_files/nested_single.xml
new file mode 100644
index 00000000..7822bad6
--- /dev/null
+++ b/tests/xml_files/nested_single.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<catalog>
+	<book id="book_121232" ParentId="book_1212">
+		<book_city>Copenhagen</book_city>
+		<price>2</price>
+		<first_store/>
+	</book>
+</catalog>
\ No newline at end of file
diff --git a/tests/xml_files/root_with_attributes.xml b/tests/xml_files/root_with_attributes.xml
new file mode 100644
index 00000000..f5439746
--- /dev/null
+++ b/tests/xml_files/root_with_attributes.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<root id="eqweqwre">
+   <child>data</child>
+</root>
\ No newline at end of file