Repository: osbuild/osbuild

Issue:
Ensure every supported distribution can use stages with toml
We have been doing the toml import dance for a while now to support RHEL 8 (`pytoml`) and RHEL 9 + Fedora (`toml`, which is ummaintained, and `tomli`).  With RHEL 10 and newer Fedora versions, we also have [`tomllib`](https://docs.python.org/3/library/tomllib.html) (stdlib, 3.11+), which can only read toml.  We should clarify our import conditions and make sure we're doing the same (and correct) thing everywhere.

We should also differentiate places where toml is imported by a stage, running in the build root, vs cases where osbuild itself, or a source, are importing toml.  Stages will need to have conditional imports (catch `ModuleNotFoundError`) while modules running on the host only need to use the library that's a dependency of osbuild itself (currently `tomli` for < RHEL 10 and `tomllib` for everyting else).

Current requirements:
- `osbuild/util/containers.py`
    - Utility functions that can be imported by both stages and host modules (sources, inputs).
    - There is only one function that requires `toml`, `get_host_storage()`, which gets imported by host modules.  It does a local import inside the function to avoid issues with stages importing the module to use other functions.
    - **Move to a separate module and import tomllib -> tomli**
- `stages/org.osbuild.containers.storage.conf`:
    - Runs in the build root.
    - **Needs to write files, so it can't use `tomllib` on newer distros.**
    - **import tomli -> toml -> pytoml**.

- `stages/org.osbuild.bootc.install.config`:
    - Runs in the build root, but only for newer distros (RHEL 9+), so it wont require RHEL 8 support.
    - **Needs to write files, so it can't use `tomllib` on newer distros.**
    - **import tomli -> toml**.

- `stages/test/test_containers_storage_conf.py`:
    - Test code.  Should run on all currently supported distros and any distro on which we run unit tests.
    - **import tomllib -> tomli -> toml -> pytoml**

Discussion:
achilleas-k: FWIW, all test configs built successfully with this PR in osbuild/images: https://github.com/osbuild/images/pull/863
achilleas-k: > Great Job! Not sure why schutzbot is unhappy, I restarted the failed tests.

I'm looking into the rpm build failures now.
elkoniu: Branch rebased on top of rpm build failure fix


Patch:
diff --git a/Schutzfile b/Schutzfile
index a30f4df0..028dfbd0 100644
--- a/Schutzfile
+++ b/Schutzfile
@@ -19,7 +19,7 @@
           {
             "title": "fedora-updates",
             "name": "fedora-updates",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-updates-released-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-updates-released-20240819"
           }
         ],
         "aarch64": [
@@ -31,7 +31,7 @@
           {
             "title": "fedora-updates",
             "name": "fedora-updates",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-aarch64-updates-released-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-aarch64-updates-released-20240819"
           }
         ]
       }
@@ -50,7 +50,7 @@
           {
             "title": "fedora-updates",
             "name": "fedora-updates",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f40/f40-x86_64-updates-released-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f40/f40-x86_64-updates-released-20240819"
           }
         ],
         "aarch64": [
@@ -62,7 +62,7 @@
           {
             "title": "fedora-updates",
             "name": "fedora-updates",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f40/f40-aarch64-updates-released-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f40/f40-aarch64-updates-released-20240819"
           }
         ]
       }
@@ -76,14 +76,14 @@
           {
             "title": "fedora",
             "name": "fedora",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f41/f41-x86_64-rawhide-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f41/f41-x86_64-rawhide-20240819"
           }
         ],
         "aarch64": [
           {
             "title": "fedora",
             "name": "fedora",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f41/f41-aarch64-rawhide-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f41/f41-aarch64-rawhide-20240819"
           }
         ]
       }
@@ -97,34 +97,34 @@
           {
             "title": "baseos",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-x86_64-baseos-20240514"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-x86_64-baseos-20240819"
           },
           {
             "title": "appstream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-x86_64-appstream-20240514"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-x86_64-appstream-20240819"
           },
           {
             "title": "crb",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-x86_64-crb-20240514"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-x86_64-crb-20240819"
           }
         ],
         "aarch64": [
           {
             "title": "baseos",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-aarch64-baseos-20240514"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-aarch64-baseos-20240819"
           },
           {
             "title": "appstream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-aarch64-appstream-20240514"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-aarch64-appstream-20240819"
           },
           {
             "title": "crb",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-aarch64-crb-20240514"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el9/cs9-aarch64-crb-20240819"
           }
         ]
       }
@@ -138,34 +138,34 @@
           {
             "title": "baseos",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-x86_64-baseos-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-x86_64-baseos-20240819"
           },
           {
             "title": "appstream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-x86_64-appstream-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-x86_64-appstream-20240819"
           },
           {
             "title": "crb",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-x86_64-crb-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-x86_64-crb-20240819"
           }
         ],
         "aarch64": [
           {
             "title": "baseos",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-aarch64-baseos-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-aarch64-baseos-20240819"
           },
           {
             "title": "appstream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-aarch64-appstream-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-aarch64-appstream-20240819"
           },
           {
             "title": "crb",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-aarch64-crb-20240701"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/el10/cs10-aarch64-crb-20240819"
           }
         ]
       }
@@ -179,34 +179,34 @@
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-baseos-n9.5-20240601"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-baseos-n9.5-20240819"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-appstream-n9.5-20240601"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-appstream-n9.5-20240819"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-crb-n9.5-20240601"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-crb-n9.5-20240819"
           }
         ],
         "aarch64": [
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-baseos-n9.5-20240601"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-baseos-n9.5-20240819"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-appstream-n9.5-20240601"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-appstream-n9.5-20240819"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-crb-n9.5-20240601"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-crb-n9.5-20240819"
           }
         ]
       }
@@ -220,37 +220,37 @@
           {
             "title": "RHEL-10-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-x86_64-baseos-n10.0-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-x86_64-baseos-n10.0-20240819"
           },
           {
             "title": "RHEL-10-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-x86_64-appstream-n10.0-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-x86_64-appstream-n10.0-20240819"
           },
           {
             "title": "RHEL-10-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-x86_64-crb-n10.0-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-x86_64-crb-n10.0-20240819"
           }
         ],
         "aarch64": [
           {
             "title": "RHEL-10-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-aarch64-baseos-n10.0-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-aarch64-baseos-n10.0-20240819"
           },
           {
             "title": "RHEL-10-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-aarch64-appstream-n10.0-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-aarch64-appstream-n10.0-20240819"
           },
           {
             "title": "RHEL-10-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-aarch64-crb-n10.0-20240715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el10/el10-aarch64-crb-n10.0-20240819"
           }
         ]
       }
     ]
   }
-}
+}
\ No newline at end of file
diff --git a/inputs/org.osbuild.containers-storage b/inputs/org.osbuild.containers-storage
index 49b60328..0ba6f0bb 100755
--- a/inputs/org.osbuild.containers-storage
+++ b/inputs/org.osbuild.containers-storage
@@ -15,7 +15,7 @@ import os
 import sys
 
 from osbuild import inputs
-from osbuild.util import containers
+from osbuild.util import host
 from osbuild.util.mnt import MountGuard
 
 SCHEMA = r"""
@@ -89,7 +89,7 @@ class ContainersStorageInput(inputs.InputService):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.mg = MountGuard()
-        self.storage_conf = containers.get_host_storage()
+        self.storage_conf = host.get_container_storage()
 
     def bind_mount_local_storage(self, target):
         source = self.storage_conf["storage"]["graphroot"]
diff --git a/osbuild.spec b/osbuild.spec
index 5f0017e9..99cbdc86 100644
--- a/osbuild.spec
+++ b/osbuild.spec
@@ -1,7 +1,7 @@
 %global         forgeurl https://github.com/osbuild/osbuild
 %global         selinuxtype targeted
 
-Version:        126
+Version:        127
 
 %forgemeta
 
@@ -144,6 +144,15 @@ Requires: python3-libdnf5 >= 5.2.1
 Requires: python3-libdnf
 %endif
 
+# osbuild 125 added a new "solver" field and osbuild-composer only
+# supports this since 116
+Conflicts: osbuild-composer <= 115
+
+# This version needs to get bumped every time the osbuild-dnf-json
+# version changes in an incompatible way. Packages like osbuild-composer
+# can depend on the exact API version this way
+Provides: osbuild-dnf-json-api = 7
+
 %description    depsolve-dnf
 Contains depsolving capabilities for package managers.
 
diff --git a/osbuild/__init__.py b/osbuild/__init__.py
index 9c0abc1d..4e809843 100644
--- a/osbuild/__init__.py
+++ b/osbuild/__init__.py
@@ -10,7 +10,7 @@ independent of osbuild but used across the osbuild codebase.
 
 from .pipeline import Manifest, Pipeline, Stage
 
-__version__ = "126"
+__version__ = "127"
 
 __all__ = [
     "Manifest",
diff --git a/osbuild/util/chroot.py b/osbuild/util/chroot.py
new file mode 100644
index 00000000..661a7a8f
--- /dev/null
+++ b/osbuild/util/chroot.py
@@ -0,0 +1,41 @@
+import os
+import subprocess
+
+
+class ChrootProcDevSys:
+    """
+    Sets up mounts for the virtual filesystems inside a root tree, preparing it for running commands using chroot. This
+    should be used whenever a stage needs to run a command against the root tree but doesn't support a --root option or
+    similar.
+    Cleans up mounts when done.
+
+    This mounts /proc, /dev, and /sys.
+    """
+
+    def __init__(self, root: str):
+        self.root = root
+
+    def __enter__(self):
+        for d in ["/proc", "/dev", "/sys"]:
+            if not os.path.exists(self.root + d):
+                print(f"Making missing chroot directory: {d}")
+                os.makedirs(self.root + d)
+
+        subprocess.check_call(["/usr/bin/mount",
+                               "-t", "proc",
+                               "-o", "nosuid,noexec,nodev",
+                               "proc", f"{self.root}/proc"])
+        subprocess.check_call(["/usr/bin/mount",
+                               "-t", "devtmpfs",
+                               "-o", "mode=0755,noexec,nosuid,strictatime",
+                               "devtmpfs", f"{self.root}/dev"])
+        subprocess.check_call(["/usr/bin/mount",
+                               "-t", "sysfs",
+                               "-o", "nosuid,noexec,nodev",
+                               "sysfs", f"{self.root}/sys"])
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, tracebk):
+        for d in ["/proc", "/dev", "/sys"]:
+            subprocess.check_call(["/usr/bin/umount", self.root + d])
diff --git a/osbuild/util/containers.py b/osbuild/util/containers.py
index 6f6a76d0..b3b99f77 100644
--- a/osbuild/util/containers.py
+++ b/osbuild/util/containers.py
@@ -180,26 +180,3 @@ def container_source(image):
 
     with container_source_fn(image, image_filepath, container_format) as image_source:
         yield image_name, image_source
-
-
-def get_host_storage():
-    """
-    Read the host storage configuration.
-    """
-    # pylint: disable=import-outside-toplevel
-    # importing this at the top level will break the buildroot
-    try:
-        import tomllib as toml
-    except ImportError:
-        import tomli as toml
-
-    config_paths = ("/etc/containers/storage.conf", "/usr/share/containers/storage.conf")
-    for conf_path in config_paths:
-        try:
-            with open(conf_path, "rb") as conf_file:
-                storage_conf = toml.load(conf_file)
-            return storage_conf
-        except FileNotFoundError:
-            pass
-
-    raise FileNotFoundError(f"could not find container storage configuration in any of {config_paths}")
diff --git a/osbuild/util/host.py b/osbuild/util/host.py
new file mode 100644
index 00000000..8b956622
--- /dev/null
+++ b/osbuild/util/host.py
@@ -0,0 +1,20 @@
+"""
+Utility functions that only run on the host (osbuild internals or host modules like sources).
+
+These should not be used by stages or code that runs in the build root.
+"""
+from osbuild.util import toml
+
+
+def get_container_storage():
+    """
+    Read the host storage configuration.
+    """
+    config_paths = ("/etc/containers/storage.conf", "/usr/share/containers/storage.conf")
+    for conf_path in config_paths:
+        try:
+            return toml.load_from_file(conf_path)
+        except FileNotFoundError:
+            pass
+
+    raise FileNotFoundError(f"could not find container storage configuration in any of {config_paths}")
diff --git a/osbuild/util/toml.py b/osbuild/util/toml.py
new file mode 100644
index 00000000..389c0d39
--- /dev/null
+++ b/osbuild/util/toml.py
@@ -0,0 +1,78 @@
+"""
+Utility functions for reading and writing toml files.
+
+Handles module imports for all supported versions (in a build root or on a host).
+"""
+import importlib
+from types import ModuleType
+from typing import Optional
+
+# Different modules require different file mode (text vs binary)
+_toml_modules = {
+    "tomllib": {"mode": "rb"},  # stdlib since 3.11 (read-only)
+    "tomli": {"mode": "rb"},  # EL9+
+    "toml": {"mode": "r", "encoding": "utf-8"},  # older unmaintained lib, needed for backwards compatibility
+    "pytoml": {"mode": "r", "encoding": "utf-8"},  # deprecated, needed for backwards compatibility (EL8 manifests)
+}
+
+
+_toml: Optional[ModuleType] = None
+_rargs: dict = {}
+for module, args in _toml_modules.items():
+    try:
+        _toml = importlib.import_module(module)
+        _rargs = args
+        break
+    except ModuleNotFoundError:
+        pass
+else:
+    raise ModuleNotFoundError("No toml module found: " + ", ".join(_toml_modules))
+
+# Different modules require different file mode (text vs binary)
+_tomlw_modules = {
+    "tomli_w": {"mode": "wb"},  # EL9+
+    "toml": {"mode": "w", "encoding": "utf-8"},  # older unmaintained lib, needed for backwards compatibility
+    "pytoml": {"mode": "w", "encoding": "utf-8"},  # deprecated, needed for backwards compatibility (EL8 manifests)
+}
+
+
+_tomlw: Optional[ModuleType] = None
+_wargs: dict = {}
+for module, args in _tomlw_modules.items():
+    try:
+        _tomlw = importlib.import_module(module)
+        _wargs = args
+        break
+    except ModuleNotFoundError:
+        # allow importing without write support
+        pass
+
+
+def load_from_file(path):
+    if _toml is None:
+        raise RuntimeError("no toml module available")
+
+    with open(path, **_rargs) as tomlfile:  # pylint: disable=unspecified-encoding
+        return _toml.load(tomlfile)
+
+
+def dump_to_file(data, path, header=""):
+    if _tomlw is None:
+        raise RuntimeError("no toml module available with write support")
+
+    with open(path, **_wargs) as tomlfile:  # pylint: disable=unspecified-encoding
+        if header:
+            _write_comment(tomlfile, header)
+
+        _tomlw.dump(data, tomlfile)
+
+
+def _write_comment(f, comment: list):
+    if not comment:
+        return
+
+    data = "\n".join(map(lambda c: f"# {c}", comment)) + "\n\n"
+    if "b" in f.mode:
+        f.write(data.encode())
+    else:
+        f.write(data)
diff --git a/setup.py b/setup.py
index 3b715618..fa42bfa8 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ import setuptools
 
 setuptools.setup(
     name="osbuild",
-    version="126",
+    version="127",
     description="A build system for OS images",
     packages=["osbuild", "osbuild.formats", "osbuild.util","osbuild.solver"],
     license='Apache-2.0',
diff --git a/sources/org.osbuild.containers-storage b/sources/org.osbuild.containers-storage
index 4be3c652..01898d9a 100755
--- a/sources/org.osbuild.containers-storage
+++ b/sources/org.osbuild.containers-storage
@@ -18,7 +18,7 @@ import subprocess as sp
 import sys
 
 from osbuild import sources
-from osbuild.util import containers
+from osbuild.util import host
 
 SCHEMA = """
 "additionalProperties": false,
@@ -49,7 +49,7 @@ class ContainersStorageSource(sources.SourceService):
         Construct the full image name that references an image with a given checksum in the local storage.
         """
         if self.storage_conf is None:
-            conf = containers.get_host_storage()
+            conf = host.get_container_storage()
         driver = conf["storage"]["driver"]
         graphroot = conf["storage"]["graphroot"]
         runroot = conf["storage"]["runroot"]
diff --git a/stages/org.osbuild.bootc.install.config b/stages/org.osbuild.bootc.install.config
index 10a2664c..3390b43e 100755
--- a/stages/org.osbuild.bootc.install.config
+++ b/stages/org.osbuild.bootc.install.config
@@ -2,12 +2,8 @@
 import pathlib
 import sys
 
-try:
-    import toml
-except ModuleNotFoundError:
-    import pytoml as toml
-
 import osbuild.api
+from osbuild.util import toml
 
 
 def main(tree, options):
@@ -17,8 +13,7 @@ def main(tree, options):
     path = pathlib.Path(tree) / "usr/lib/bootc/install" / filename
     path.parent.mkdir(parents=True, exist_ok=True)
 
-    with open(path, "w", encoding="utf8") as config_file:
-        toml.dump(config, config_file)
+    toml.dump_to_file(config, path)
 
     return 0
 
diff --git a/stages/org.osbuild.containers.storage.conf b/stages/org.osbuild.containers.storage.conf
index f1c7ae7c..5827f06d 100755
--- a/stages/org.osbuild.containers.storage.conf
+++ b/stages/org.osbuild.containers.storage.conf
@@ -4,12 +4,8 @@ import os
 import sys
 from typing import Dict
 
-try:
-    import toml
-except ModuleNotFoundError:
-    import pytoml as toml
-
 import osbuild.api
+from osbuild.util import toml
 
 DEFAULT_LOCATION = "/etc/containers/storage.conf"
 
@@ -50,14 +46,6 @@ def merge_config(section: str, data: Dict, config: Dict):
     have.update(want)
 
 
-def write_comment(f, comment: list):
-    if not comment:
-        return
-
-    data = "\n".join(map(lambda c: f"# {c}", comment))
-    f.write(data + "\n\n")
-
-
 def main(tree, options):
     location = options.get("filename", DEFAULT_LOCATION)
     config = options["config"]
@@ -69,24 +57,17 @@ def main(tree, options):
 
     # if a filebase was specified, we use it as base
     if filebase:
-        with open(filebase, "r", encoding="utf8") as f:
-            data = toml.load(f)
+        data = toml.load_from_file(filebase)
 
     # if the target exists, we merge it
     with contextlib.suppress(FileNotFoundError):
-        with open(path, "r", encoding="utf8") as f:
-            have = toml.load(f)
-
-            merge_config("storage", data, have)
+        have = toml.load_from_file(path)
+        merge_config("storage", data, have)
 
     # now merge our configuration into data
     merge_config("storage", data, config)
 
-    with open(path, "w", encoding="utf8") as f:
-        write_comment(f, HEADER)
-        write_comment(f, comment)
-
-        toml.dump(data, f)
+    toml.dump_to_file(data, path, header=HEADER + ["\n"] + comment)
 
     return 0
 
diff --git a/stages/org.osbuild.dracut b/stages/org.osbuild.dracut
index 36f853fe..44e3c9ca 100755
--- a/stages/org.osbuild.dracut
+++ b/stages/org.osbuild.dracut
@@ -1,9 +1,9 @@
 #!/usr/bin/python3
-import os
 import subprocess
 import sys
 
 import osbuild.api
+from osbuild.util.chroot import ChrootProcDevSys
 
 
 def yesno(name: str, value: bool) -> str:
@@ -11,36 +11,6 @@ def yesno(name: str, value: bool) -> str:
     return f"--{prefix}{name}"
 
 
-class DracutMounts:
-    """
-    Setup the mounts inside a chroot root directory, which will be used
-    for running dracut inside it. Cleanup mounts when done.
-    This mounts /proc, /dev, and /sys.
-    """
-
-    def __init__(self, root: str):
-        self.root = root
-
-    def __enter__(self):
-        for d in ["/proc", "/dev", "/sys"]:
-            if not os.path.exists(self.root + d):
-                print(f"Making missing dracut chroot directory: {d}")
-                os.makedirs(self.root + d)
-
-        subprocess.check_call(["/usr/bin/mount", "-t", "proc", "-o",
-                               "nosuid,noexec,nodev", "proc", f"{self.root}/proc"])
-        subprocess.check_call(["/usr/bin/mount", "-t", "devtmpfs", "-o",
-                               "mode=0755,noexec,nosuid,strictatime", "devtmpfs", f"{self.root}/dev"])
-        subprocess.check_call(["/usr/bin/mount", "-t", "sysfs", "-o",
-                               "nosuid,noexec,nodev", "sysfs", f"{self.root}/sys"])
-
-        return self
-
-    def __exit__(self, exc_type, exc_value, tracebk):
-        for d in ["/proc", "/dev", "/sys"]:
-            subprocess.check_call(["/usr/bin/umount", self.root + d])
-
-
 # pylint: disable=too-many-branches
 def main(tree, options):
     kernels = options["kernel"]
@@ -113,7 +83,7 @@ def main(tree, options):
         if initoverlayfs:
             initfs_bin = "/usr/bin/initoverlayfs-install"
 
-        with DracutMounts(tree):
+        with ChrootProcDevSys(tree):
             subprocess.run(["/usr/sbin/chroot", tree, initfs_bin,
                             "--no-hostonly",
                             "--kver", kver]
diff --git a/stages/org.osbuild.update-crypto-policies b/stages/org.osbuild.update-crypto-policies
index 035bb7af..b174497b 100755
--- a/stages/org.osbuild.update-crypto-policies
+++ b/stages/org.osbuild.update-crypto-policies
@@ -3,15 +3,21 @@ import subprocess
 import sys
 
 from osbuild import api
+from osbuild.util.chroot import ChrootProcDevSys
 
 
 def main(tree, options):
     policy = options["policy"]
 
-    cmd = ["/usr/sbin/chroot", tree,
-           "/usr/bin/update-crypto-policies", "--set", policy]
+    with ChrootProcDevSys(tree):
+        # update-crypto-polciies uses /proc/self/mountinfo to find and verify that fips paths have been mounted to their
+        # expected locations by searching for the following path suffixes:
+        #   /crypto-policies/default-fips-config
+        #   /crypto-policies/back-ends/FIPS
+        cmd = ["/usr/sbin/chroot", tree,
+               "/usr/bin/update-crypto-policies", "--set", policy]
 
-    subprocess.run(cmd, check=True)
+        subprocess.run(cmd, check=True)
 
     return 0
 
diff --git a/tox.ini b/tox.ini
index 0acde8a2..2de0b44a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -74,3 +74,44 @@ deps =
 
 commands =
     bash -c 'python -m mypy --follow-imports=skip --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs {env:TYPEABLES_STRICT}'
+
+[testenv:tomllib]
+description = "test osbuild.util.toml with tomllib"
+deps =
+    pytest
+    mako
+    tomli-w
+
+commands =
+    python -m pytest -s -vvv -k test_util_toml
+
+[testenv:tomli]
+description = "test osbuild.util.toml with tomli"
+deps =
+    pytest
+    tomli
+    tomli-w
+    mako
+
+commands =
+    python -m pytest -s -vvv -k test_util_toml
+
+[testenv:toml]
+description = "test osbuild.util.toml with toml"
+deps =
+    pytest
+    toml
+    mako
+
+commands =
+    python -m pytest -s -vvv -k test_util_toml
+
+[testenv:pytoml]
+description = "test osbuild.util.toml with pytoml"
+deps =
+    pytest
+    pytoml
+    mako
+
+commands =
+    python -m pytest -s -vvv -k test_util_toml


Test patch:
diff --git a/stages/test/test_bootc_install_config.py b/stages/test/test_bootc_install_config.py
index 853eb553..c839f6f4 100644
--- a/stages/test/test_bootc_install_config.py
+++ b/stages/test/test_bootc_install_config.py
@@ -4,6 +4,7 @@ import re
 import pytest
 
 from osbuild import testutil
+from osbuild.util import toml
 
 STAGE_NAME = "org.osbuild.bootc.install.config"
 
@@ -24,7 +25,15 @@ STAGE_NAME = "org.osbuild.bootc.install.config"
                     }
                 }
             },
-            '[install.filesystem.root]\ntype = "xfs"\n',
+            {
+                "install": {
+                    "filesystem": {
+                        "root": {
+                            "type": "xfs"
+                        }
+                    }
+                }
+            },
         ),
         (
             {
@@ -39,7 +48,11 @@ STAGE_NAME = "org.osbuild.bootc.install.config"
                     }
                 }
             },
-            '[install]\nkargs = [ "ro", "biosdevname=0", "net.ifnames=0",]\n',
+            {
+                "install": {
+                    "kargs": ["ro", "biosdevname=0", "net.ifnames=0"],
+                }
+            },
         ),
         (
             {
@@ -52,14 +65,20 @@ STAGE_NAME = "org.osbuild.bootc.install.config"
                     }
                 },
             },
-            '[install]\nblock = [ "tmp2-luks",]\n',
+            {
+                "install": {
+                    "block": ["tmp2-luks"]
+                }
+            }
         )
     ]
 )
 def test_bootc_install_config(tmp_path, stage_module, options, expected_config):
     stage_module.main(tmp_path, options)
-    with open(os.path.join(tmp_path, "usr/lib/bootc/install", options["filename"]), encoding="utf-8") as config_file:
-        assert config_file.read() == expected_config
+    # different toml libraries write out arrays in different ways (one line vs many), so we need to parse the result
+    # to compare
+    config_data = toml.load_from_file(os.path.join(tmp_path, "usr/lib/bootc/install", options["filename"]))
+    assert config_data == expected_config
 
 
 @pytest.mark.parametrize(
diff --git a/stages/test/test_containers_storage_conf.py b/stages/test/test_containers_storage_conf.py
index 2aed0a5e..d98409e6 100644
--- a/stages/test/test_containers_storage_conf.py
+++ b/stages/test/test_containers_storage_conf.py
@@ -5,12 +5,8 @@ import re
 
 import pytest
 
-try:
-    import toml
-except ModuleNotFoundError:
-    import pytoml as toml
-
 from osbuild import testutil
+from osbuild.util import toml
 
 TEST_INPUT = [
     ({}, {"additionalimagestores": ["/path/to/store"]}, [("storage.options.additionalimagestores", ["/path/to/store"])]),
@@ -49,8 +45,7 @@ def test_containers_storage_conf_integration(tmp_path, stage_module, test_filena
     assert os.path.exists(confpath)
 
     conf = None
-    with open(confpath, 'r', encoding="utf-8") as f:
-        conf = toml.load(f)
+    conf = toml.load_from_file(confpath)
 
     assert conf is not None
 
diff --git a/test/data/stages/update-crypto-policies/diff.json b/test/data/stages/update-crypto-policies/diff.json
index 594c53cd..63507673 100644
--- a/test/data/stages/update-crypto-policies/diff.json
+++ b/test/data/stages/update-crypto-policies/diff.json
@@ -1,7 +1,5 @@
 {
-  "added_files": [
-    "/dev/null"
-  ],
+  "added_files": [],
   "deleted_files": [],
   "differences": {
     "/etc/crypto-policies/back-ends/bind.config": {
diff --git a/test/mod/test_util_toml.py b/test/mod/test_util_toml.py
new file mode 100644
index 00000000..9bcb022f
--- /dev/null
+++ b/test/mod/test_util_toml.py
@@ -0,0 +1,43 @@
+#
+# Tests for the 'osbuild.util.toml' module
+#
+import os
+from tempfile import TemporaryDirectory
+
+from osbuild.util import toml
+
+data_obj = {
+    "top": {
+        "t2-1": {
+            "list": ["a", "b", "c"]
+        },
+        "t2-2": {
+            "str": "test"
+        }
+    }
+}
+
+data_str = """
+[top.t2-1]
+list = ["a", "b", "c"]
+
+[top.t2-2]
+str = "test"
+"""
+
+
+def test_write_read():
+    with TemporaryDirectory() as tmpdir:
+        path = os.path.join(tmpdir, "test.toml")
+        toml.dump_to_file(data_obj, path)
+        rdata = toml.load_from_file(path)
+        assert data_obj == rdata
+
+
+def test_read():
+    with TemporaryDirectory() as tmpdir:
+        path = os.path.join(tmpdir, "test.toml")
+        with open(path, "w", encoding="utf-8") as test_file:
+            test_file.write(data_str)
+        rdata = toml.load_from_file(path)
+        assert rdata == data_obj
diff --git a/tools/test/test_depsolve.py b/tools/test/test_depsolve.py
index 9423a4e4..287b0d4f 100644
--- a/tools/test/test_depsolve.py
+++ b/tools/test/test_depsolve.py
@@ -1250,7 +1250,7 @@ def test_depsolve(tmp_path, repo_servers, dnf_config, detect_fn, test_case):
     try:
         detect_fn()
     except RuntimeError as e:
-        pytest.skip(e)
+        pytest.skip(str(e))
 
     # pylint: disable=fixme
     # TODO: remove this once dnf5 implementation is fixed
@@ -1314,7 +1314,7 @@ def test_dump(tmp_path, repo_servers, dnf_config, detect_fn, test_case):
     try:
         detect_fn()
     except RuntimeError as e:
-        pytest.skip(e)
+        pytest.skip(str(e))
 
     repo_servers_copy = repo_servers.copy()
     if "additional_servers" in test_case:
@@ -1361,7 +1361,7 @@ def test_search(tmp_path, repo_servers, dnf_config, detect_fn, test_case):
     try:
         detect_fn()
     except RuntimeError as e:
-        pytest.skip(e)
+        pytest.skip(str(e))
 
     repo_servers_copy = repo_servers.copy()
     if "additional_servers" in test_case: