Repository: ssato/python-anyconfig (language: python)

Issue:
Update toml parser
**Is your feature request related to a problem? Please describe.**
The “toml“ project on PyPI is old and unmaintained.

**Describe the solution you'd like**
Use the stdlib’s `tomllib` on Python 3.11 and `tomli` on lower Python versions

**Describe alternatives you've considered**
Get uiri to pick up development on toml? Probably no alternatives.


Discussion / hints:

Fix patch:
diff --git a/docs/introduction.rst b/docs/introduction.rst
index 4be46d51..98466cd7 100644
--- a/docs/introduction.rst
+++ b/docs/introduction.rst
@@ -279,12 +279,17 @@ enable the features.
    :widths: 20, 10, 25
 
    YAML load/dump, ruamel.yaml or PyYAML, ruamel.yaml will be used instead of PyYAML if it's available to support the YAML 1.2 specification.
-   TOML load/dump, toml, none
+   TOML load/dump, toml [#]_ or tomllib [#]_ and tomli-w [#]_ or tomli [#]_ and tomli-w or tomlkit [#]_ , tomllib is in the standard lib since python 3.11.0
    BSON load/dump, bson, bson from pymongo package may work and bson [#]_ does not
    Template config, Jinja2 [#]_ , none
    Validation with JSON schema, jsonschema [#]_ , Not required to generate JSON schema.
    Query with JMESPath expression, jmespath [#]_ , none
 
+.. [#] https://pypi.python.org/pypi/toml/
+.. [#] https://docs.python.org/3/library/tomllib.html
+.. [#] https://pypi.python.org/pypi/tomli-w/
+.. [#] https://pypi.python.org/pypi/tomli/
+.. [#] https://pypi.python.org/pypi/tomlkit/
 .. [#] https://pypi.python.org/pypi/bson/
 .. [#] https://pypi.python.org/pypi/Jinja2/
 .. [#] https://pypi.python.org/pypi/jsonschema/
diff --git a/src/anyconfig/backend/__init__.py b/src/anyconfig/backend/__init__.py
index 8e42fdca..d3961f7f 100644
--- a/src/anyconfig/backend/__init__.py
+++ b/src/anyconfig/backend/__init__.py
@@ -13,6 +13,7 @@
     pickle,
     properties,
     shellvars,
+    toml,
     yaml,
     xml
 )
@@ -39,10 +40,9 @@ def warn(name: str, feature: str):
 else:
     warn('yaml', 'YAML')
 
-try:
-    from . import toml
-    PARSERS.append(toml.Parser)
-except ImportError:
+if toml.PARSERS:
+    PARSERS.extend(toml.PARSERS)
+else:
     warn('toml', 'TOML')
 
 
diff --git a/src/anyconfig/backend/toml/__init__.py b/src/anyconfig/backend/toml/__init__.py
new file mode 100644
index 00000000..2e4fd4bb
--- /dev/null
+++ b/src/anyconfig/backend/toml/__init__.py
@@ -0,0 +1,40 @@
+#
+# Copyright (C) 2015 - 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
+#
+"""Backend modules to load and dump TOML data files.
+
+- toml: https://github.com/uiri/toml
+- tomli: https://github.com/hukkin/tomli
+- tomllib and tomli-w:
+  - tomlib: https://docs.python.org/3/library/tomllib.html
+  - toml-w: https://github.com/hukkin/tomli-w
+- tomlkit: https://tomlkit.readthedocs.io/
+
+Changelog:
+
+.. versionchanged:: 0.9.8
+
+   - Added tomli, tomllib and tomli-w, and tomlkit support
+"""
+from ..base import ParserClssT
+
+PARSERS: ParserClssT = []  # type: ignore
+
+try:
+    from . import tomllib
+    PARSERS: ParserClssT = [tomllib.Parser]
+except ImportError:
+    pass
+
+try:
+    from . import toml
+    PARSERS.append(toml.Parser)
+except ImportError:
+    pass
+
+try:
+    from . import tomlkit
+    PARSERS.append(tomlkit.Parser)
+except ImportError:
+    pass
diff --git a/src/anyconfig/backend/toml.py b/src/anyconfig/backend/toml/toml.py
similarity index 89%
rename from src/anyconfig/backend/toml.py
rename to src/anyconfig/backend/toml/toml.py
index 327ddfc4..ab8b8ecb 100644
--- a/src/anyconfig/backend/toml.py
+++ b/src/anyconfig/backend/toml/toml.py
@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2015 - 2021 Satoru SATOH <satoru.satoh@gmail.com>
+# Copyright (C) 2015 - 2023 Satoru SATOH <satoru.satoh @ gmail.com>
 # SPDX-License-Identifier: MIT
 #
 # Ref. python -c "import toml; help(toml); ..."
@@ -21,13 +21,13 @@
 """
 import toml
 
-from . import base
+from .. import base
 
 
 class Parser(base.StringStreamFnParser):
     """TOML parser."""
 
-    _cid = 'toml'
+    _cid = 'toml.toml'
     _type = 'toml'
     _extensions = ['toml']
     _ordered = True
diff --git a/src/anyconfig/backend/toml/tomlkit.py b/src/anyconfig/backend/toml/tomlkit.py
new file mode 100644
index 00000000..d5623fd9
--- /dev/null
+++ b/src/anyconfig/backend/toml/tomlkit.py
@@ -0,0 +1,40 @@
+#
+# Copyright (C) 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
+#
+# Ref. python -c "import tomlkit; help(tomlkit); ..."
+#
+r"""A backend module to load and dump TOML files.
+
+- Format to support: TOML, https://github.com/toml-lang/toml
+- Requirements: tomlkit, https://tomlkit.readthedocs.io
+- Development Status :: 4 - Beta
+- Limitations: None obvious
+- Special options:
+
+  - tomlkit.dump[s]: sort_keys [false] to sort keys.
+
+Changelog:
+
+    .. versionadded:: 0.13.1
+"""
+import tomlkit
+
+from .. import base
+
+
+class Parser(base.StringStreamFnParser):
+    """TOML parser."""
+
+    _cid = 'toml.tomlkit'
+    _type = 'toml'
+    _extensions = ['toml']
+    _ordered = True
+    _dump_opts = ['sort_keys']
+
+    _load_from_string_fn = base.to_method(tomlkit.loads)
+    _load_from_stream_fn = base.to_method(tomlkit.load)
+    _dump_to_string_fn = base.to_method(tomlkit.dumps)
+    _dump_to_stream_fn = base.to_method(tomlkit.dump)
+
+# vim:sw=4:ts=4:et:
diff --git a/src/anyconfig/backend/toml/tomllib.py b/src/anyconfig/backend/toml/tomllib.py
new file mode 100644
index 00000000..484aec3d
--- /dev/null
+++ b/src/anyconfig/backend/toml/tomllib.py
@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
+#
+# Ref. python -c "import toml; help(toml); ..."
+#
+r"""A backend module to load and dump TOML files.
+
+- Format to support: TOML, https://github.com/toml-lang/toml
+- Requirements: tomli or tomllib and tomli-w
+  - tomllib: https://docs.python.org/3/library/tomllib.html
+  - tomli: https://github.com/hukkin/tomli
+  - tomli-w: https://github.com/hukkin/tomli-w
+- Development Status :: 4 - Beta
+- Limitations: None obvious
+- Special options:
+  - tomllib.load{s,} only accept 'parse_float' keyword option.
+
+Changelog:
+
+    .. versionadded:: 0.13.1
+"""
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+import tomli_w
+
+from .. import base
+
+
+class Parser(base.StringStreamFnParser):
+    """TOML parser using tomlib and tomli-w."""
+
+    _cid = 'toml.tomllib'
+    _type = 'toml'
+    _extensions = ['toml']
+    _ordered = True
+    _load_opts = ['parse_float']
+
+    _load_from_string_fn = base.to_method(tomllib.loads)
+    _load_from_stream_fn = base.to_method(tomllib.load)
+    _dump_to_string_fn = base.to_method(tomli_w.dumps)
+    _dump_to_stream_fn = base.to_method(tomli_w.dump)


Test patch:
diff --git a/tests/backend/toml/__init__.py b/tests/backend/toml/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/backend/test_toml.py b/tests/backend/toml/common.py
similarity index 52%
rename from tests/backend/test_toml.py
rename to tests/backend/toml/common.py
index 747f6f8d..b2edc3d2 100644
--- a/tests/backend/test_toml.py
+++ b/tests/backend/toml/common.py
@@ -1,27 +1,16 @@
 #
-# Copyright (C) 2015 - 2019 Satoru SATOH <satoru.satoh@gmail.com>
-# Copyright (C) 2017 Red Hat, Inc.
-# License: MIT
+# Copyright (C) 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
 #
 # pylint: disable=missing-docstring,invalid-name,too-few-public-methods
-# pylint: disable=ungrouped-imports
+# pylint: disable=raise-missing-from
 from collections import OrderedDict as ODict
 
-try:
-    import anyconfig.backend.toml as TT
-except ImportError:
-    import unittest
-    raise unittest.SkipTest
-
-import tests.backend.common as TBC
-
-
-_DOB = TT.toml.loads("dob = 1979-05-27T07:32:00Z")['dob']
 
 CNF = ODict((('title', 'TOML Example'),
              ('owner',
               ODict((('name', 'Tom Preston-Werner'),
-                     ('dob', _DOB)))),
+                     ('dob', '1979-05-27T07:32:00Z')))),
              ('database',
               ODict((('server', '192.168.1.1'),
                      ('ports', [8001, 8001, 8002]),
@@ -35,22 +24,3 @@
              ('clients',
               ODict((('data', [['gamma', 'delta'], [1, 2]]),
                      ('hosts', ['alpha', 'omega']))))))
-
-
-class HasParserTrait(TBC.HasParserTrait):
-
-    psr = TT.Parser()
-    cnf = CNF
-    cnf_s = TBC.read_from_res("20-00-cnf.toml")
-
-
-class Test_10(TBC.Test_10_dumps_and_loads, HasParserTrait):
-
-    load_options = dump_options = dict(dummy="this_will_be_ignored")
-
-
-class Test_20(TBC.Test_20_dump_and_load, HasParserTrait):
-
-    pass
-
-# vim:sw=4:ts=4:et:
diff --git a/tests/backend/toml/test_toml.py b/tests/backend/toml/test_toml.py
new file mode 100644
index 00000000..d0bcc326
--- /dev/null
+++ b/tests/backend/toml/test_toml.py
@@ -0,0 +1,47 @@
+#
+# Copyright (C) 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
+#
+# pylint: disable=missing-docstring,invalid-name,too-few-public-methods
+# pylint: disable=raise-missing-from
+import tests.backend.common as TBC
+try:
+    import anyconfig.backend.toml.toml as TT
+except ImportError:
+    import unittest
+    raise unittest.SkipTest
+
+from .common import CNF
+
+
+def odict_to_dict(obj):
+    try:
+        if isinstance(obj, str):
+            return obj
+        if isinstance(obj, dict):
+            return {k: odict_to_dict(v) for k, v in obj.items()}
+        # other iterables.
+        return [odict_to_dict(e) for e in obj]
+    except TypeError:
+        return obj
+
+
+CNF["owner"]["dob"] = TT.toml.loads("dob = 1979-05-27T07:32:00Z")['dob']
+
+
+class HasParserTrait(TBC.HasParserTrait):
+    psr = TT.Parser()
+
+    # convert collections.OrderedDict object to dict object.
+    cnf = odict_to_dict(CNF)
+    cnf_s = TBC.read_from_res("20-00-cnf.toml")
+
+
+class Test_10(TBC.Test_10_dumps_and_loads, HasParserTrait):
+    load_options = dump_options = {"dummy": "this_will_be_ignored"}
+
+
+class Test_20(TBC.Test_20_dump_and_load, HasParserTrait):
+    pass
+
+# vim:sw=4:ts=4:et:
diff --git a/tests/backend/toml/test_tomlkit.py b/tests/backend/toml/test_tomlkit.py
new file mode 100644
index 00000000..36e55f68
--- /dev/null
+++ b/tests/backend/toml/test_tomlkit.py
@@ -0,0 +1,32 @@
+#
+# Copyright (C) 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
+#
+# pylint: disable=missing-docstring,invalid-name,too-few-public-methods
+# pylint: disable=raise-missing-from
+import tests.backend.common as TBC
+try:
+    import anyconfig.backend.toml.tomlkit as TT
+except ImportError:
+    import unittest
+    raise unittest.SkipTest
+
+from .common import CNF
+
+
+CNF["owner"]["dob"] = TT.tomlkit.loads("dob = 1979-05-27T07:32:00Z")['dob']
+
+class HasParserTrait(TBC.HasParserTrait):
+    psr = TT.Parser()
+    cnf = CNF
+    cnf_s = TBC.read_from_res("20-00-cnf.toml")
+
+
+class Test_10(TBC.Test_10_dumps_and_loads, HasParserTrait):
+    load_options = dump_options = {"dummy": "this_will_be_ignored"}
+
+
+class Test_20(TBC.Test_20_dump_and_load, HasParserTrait):
+    pass
+
+# vim:sw=4:ts=4:et:
diff --git a/tests/backend/toml/test_tomllib.py b/tests/backend/toml/test_tomllib.py
new file mode 100644
index 00000000..0bf53763
--- /dev/null
+++ b/tests/backend/toml/test_tomllib.py
@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2023 Satoru SATOH <satoru.satoh @ gmail.com>
+# SPDX-License-Identifier: MIT
+#
+# pylint: disable=missing-docstring,invalid-name,too-few-public-methods
+# pylint: disable=raise-missing-from
+import tests.backend.common as TBC
+try:
+    import anyconfig.backend.toml.tomllib as TT
+except ImportError:
+    import unittest
+    raise unittest.SkipTest
+
+from .common import CNF
+
+
+CNF["owner"]["dob"] = TT.tomllib.loads("dob = 1979-05-27T07:32:00Z")['dob']
+
+
+class HasParserTrait(TBC.HasParserTrait):
+    psr = TT.Parser()
+    cnf = CNF
+    cnf_s = TBC.read_from_res("20-00-cnf.toml")
+
+
+class Test_10(TBC.Test_10_dumps_and_loads, HasParserTrait):
+    load_options = dump_options = {"dummy": "this_will_be_ignored"}
+
+
+class Test_20(TBC.Test_20_dump_and_load, HasParserTrait):
+    pass
+
+# vim:sw=4:ts=4:et: