Repository: osbuild/osbuild
Pull request #225 (2020-02-19 16:48:08)

No automatic commits for the last stage
This series of commits does away with the automatic commit at the end of the last stage of the normal pipeline (for now the auto-commit of the last stage of the _build_ pipeline is kept).
It does that by channelling all read and write operations to the store via the object `Object` (renamed from `TreeObject`). This `Object` can have a _base_ (i.e. another `Object`, realisticly a tree) and implements Copy-On-Write semantics: read operations will be done directly from the _base_ directory, if no `write` operation has been done to the tree yet. On such a `write` op, the contents of _base_ will be coped to a temporary directory. When an `Object` is committed to the store, the contents of the temporary directory will be moved inside the store. The `Object` will then be reset (a new temporary tree created) and the _base_ of `Object` will be set to the newly created entry in the store. This ensures that even with two pipeline runs are racing and thus two `Objects` with the same content created only one will win, but both `Objects` will from then on use the same _base_ (tree) for all further stages.

TODO:
 - [x] commit messages
 - [x] tests


Patch:
diff --git a/.gitignore b/.gitignore
index a4d4ca6c..469eb009 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,8 +10,6 @@ __pycache__
 
 /test/.vagrant
 
-/runners/org.osbuild.host
-
 /.vscode
 /.idea
 /.gdb_history
diff --git a/osbuild.spec b/osbuild.spec
index 23eb2be9..4f21f09c 100644
--- a/osbuild.spec
+++ b/osbuild.spec
@@ -70,23 +70,6 @@ install -p -m 0755 $(find sources -type f) %{buildroot}%{pkgdir}/sources
 mkdir -p %{buildroot}%{pkgdir}/stages/osbuild
 mkdir -p %{buildroot}%{pkgdir}/assemblers/osbuild
 
-# install host runner
-%if 0%{?fc30}
-ln -s org.osbuild.fedora30 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?fc31}
-ln -s org.osbuild.fedora31 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?fc32}
-ln -s org.osbuild.fedora32 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?fc33}
-ln -s org.osbuild.fedora33 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?el8}
-ln -s org.osbuild.rhel82 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-
 %check
 exit 0
 # We have some integration tests, but those require running a VM, so that would
diff --git a/osbuild/objectstore.py b/osbuild/objectstore.py
index 1a640d9c..4335f417 100644
--- a/osbuild/objectstore.py
+++ b/osbuild/objectstore.py
@@ -5,6 +5,7 @@ import hashlib
 import os
 import subprocess
 import tempfile
+from typing import Optional
 from . import treesum
 
 
@@ -26,44 +27,159 @@ def suppress_oserror(*errnos):
             raise e
 
 
-class TreeObject:
-    def __init__(self, path: str):
-        os.makedirs(path, mode=0o755, exist_ok=True)
-        self.path = path
+def mount(source, target, bind=True, ro=True, private=True, mode="0755"):
+    options = []
+    if bind:
+        options += ["bind"]
+    if ro:
+        options += ["ro"]
+    if mode:
+        options += [mode]
+
+    args = []
+    if private:
+        args += ["--make-private"]
+    if options:
+        args += ["-o", ",".join(options)]
+    subprocess.run(["mount"] + args + [source, target], check=True)
+
+
+def umount(target, lazy=True):
+    args = []
+    if lazy:
+        args += ["--lazy"]
+    subprocess.run(["umount"] + args + [target], check=True)
+
+
+class Object:
+    def __init__(self, store: "ObjectStore"):
+        self._init = True
+        self._readers = 0
+        self._base = None
+        self._workdir = None
+        self._tree = None
+        self.store = store
+        self.reset()
+
+    def init(self) -> None:
+        """Initialize the object with content of its base"""
+        self._check_writable()
+        self._check_readers()
+        if self._init:
+            return
 
-    def init(self, source: str) -> None:
-        """Initialize the tree with a source tree"""
+        source = self.store.resolve_ref(self._base)
         subprocess.run(["cp", "--reflink=auto", "-a",
-                        f"{source}/.", self.path],
+                        f"{source}/.", self._tree],
                        check=True)
+        self._init = True
+
+    @property
+    def base(self) -> Optional[str]:
+        return self._base
+
+    @base.setter
+    def base(self, base_id: Optional[str]):
+        self._init = not base_id
+        self._base = base_id
 
     @property
     def treesum(self) -> str:
-        """Calculate the treesum of the tree"""
-        with self.open() as fd:
+        """Calculate the treesum of the object"""
+        with self._open() as fd:
             m = hashlib.sha256()
             treesum.treesum(m, fd)
             treesum_hash = m.hexdigest()
             return treesum_hash
 
+    @property
+    def _path(self) -> str:
+        if self._base and not self._init:
+            path = self.store.resolve_ref(self._base)
+        else:
+            path = self._tree
+        return path
+
+    def write(self) -> str:
+        """Return a path that can be written to"""
+        self._check_writable()
+        self._check_readers()
+        self.init()
+        return self._tree
+
+    @contextlib.contextmanager
+    def read(self) -> str:
+        self._check_writable()
+        with self.tempdir("mount") as target:
+            mount(self._path, target)
+            try:
+                self._readers += 1
+                yield target
+            finally:
+                umount(target)
+                self._readers -= 1
+
+    def store_tree(self, destination: str):
+        """Store the tree at destination and reset itself
+
+        Moves the tree atomically by using rename(2). If the
+        target already exist, does nothing. Afterwards it
+        resets itself and can be used as if it was new.
+        """
+        self._check_writable()
+        self._check_readers()
+        self.init()
+        with suppress_oserror(errno.ENOTEMPTY, errno.EEXIST):
+            os.rename(self._tree, destination)
+        self.reset()
+
+    def reset(self):
+        self.cleanup()
+        self._workdir = self.store.tempdir(suffix="object")
+        self._tree = os.path.join(self._workdir.name, "tree")
+        os.makedirs(self._tree, mode=0o755, exist_ok=True)
+        self._init = not self._base
+
+    def cleanup(self):
+        self._check_readers()
+        if self._workdir:
+            self._workdir.cleanup()
+            self._workdir = None
+
+    def _check_readers(self):
+        """Internal: Raise a ValueError if there are readers"""
+        if self._readers:
+            raise ValueError("Read operation is ongoing")
+
+    def _check_writable(self):
+        """Internal: Raise a ValueError if not writable"""
+        if not self._workdir:
+            raise ValueError("Object is not writable")
+
     @contextlib.contextmanager
-    def open(self):
+    def _open(self):
         """Open the directory and return the file descriptor"""
-        try:
-            fd = os.open(self.path, os.O_DIRECTORY)
-            yield fd
-        finally:
-            os.close(fd)
+        with self.read() as path:
+            fd = os.open(path, os.O_DIRECTORY)
+            try:
+                yield fd
+            finally:
+                os.close(fd)
 
-    def move(self, destination: str):
-        """Move the tree to destination
+    def tempdir(self, suffix=None):
+        workdir = self._workdir.name
+        if suffix:
+            suffix = "-" + suffix
+        return tempfile.TemporaryDirectory(dir=workdir,
+                                           suffix=suffix)
 
-        Does so atomically by using rename(2). If the
-        target already exist, use that instead
-        """
-        with suppress_oserror(errno.ENOTEMPTY, errno.EEXIST):
-            os.rename(self.path, destination)
-        self.path = destination
+    def __enter__(self):
+        self._check_writable()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.cleanup()
+        return exc_type is None
 
 
 class ObjectStore:
@@ -78,94 +194,86 @@ class ObjectStore:
     def contains(self, object_id):
         if not object_id:
             return False
-        return os.access(f"{self.refs}/{object_id}", os.F_OK)
+        return os.access(self.resolve_ref(object_id), os.F_OK)
 
-    @contextlib.contextmanager
-    def get(self, object_id):
-        with tempfile.TemporaryDirectory(dir=self.store) as tmp:
-            if object_id:
-                subprocess.run(["mount", "-o", "bind,ro,mode=0755", f"{self.refs}/{object_id}", tmp], check=True)
-                try:
-                    yield tmp
-                finally:
-                    subprocess.run(["umount", "--lazy", tmp], check=True)
-            else:
-                # None was given as object_id, just return an empty directory
-                yield tmp
+    def resolve_ref(self, object_id: Optional[str]) -> Optional[str]:
+        """Returns the path to the given object_id"""
+        if not object_id:
+            return None
+        return f"{self.refs}/{object_id}"
 
+    def tempdir(self, prefix=None, suffix=None):
+        """Return a tempfile.TemporaryDirectory within the store"""
+        return tempfile.TemporaryDirectory(dir=self.store,
+                                           prefix=prefix,
+                                           suffix=suffix)
 
     @contextlib.contextmanager
-    def new(self, object_id, base_id=None):
-        """Creates a new directory for `object_id`.
+    def get(self, object_id):
+        with Object(self) as obj:
+            obj.base = object_id
+            with obj.read() as path:
+                yield path
 
-        This method must be used as a context manager. It returns a path to a
-        temporary directory and only commits it when the context completes
-        without raising an exception.
+    @contextlib.contextmanager
+    def new(self, base_id=None):
+        """Creates a new temporary `Object`.
+
+        This method must be used as a context manager. It returns
+        a temporary instance of `Object`, which can then be used
+        for interaction with the store.
+        If changes to the object's content were made (by calling
+        `Object.write`), these must manually be committed to the
+        store via `commit()`.
         """
-        with tempfile.TemporaryDirectory(dir=self.store) as tmp:
-            # the tree that is yielded will be added to the content store
-            # on success as object_id
-            tree = TreeObject(f"{tmp}/tree")
-
-            if base_id:
-                # the base, the working tree and the output tree are all on
-                # the same fs, so attempt a lightweight copy if the fs
-                # supports it
-                tree.init(f"{self.refs}/{base_id}")
 
-            yield tree.path
+        with Object(self) as obj:
 
-            # if the yield above raises an exception, the working tree
-            # is cleaned up by tempfile, otherwise, the it the content
-            # of it was created or modified by the caller. All that is
-            # left to do is to commit it to the object store
-            self.commit(tree, object_id)
+            if base_id:
+                # if we were given a base id then this is the base
+                # content for the new object
+                # NB: `Object` has copy-on-write semantics, so no
+                # copying of the data takes places at this point
+                obj.base = base_id
 
-    def snapshot(self, tree_path: str, object_id: str) -> str:
-        """Commit `tree_path` to store and ref it as `object_id`
+            yield obj
 
-        Create a snapshot of `tree_path` and store it via its
-        content hash in the object directory; additionally
-        create a new reference to it via `object_id` in the
-        reference directory.
+    def commit(self, obj: Object, object_id: str) -> str:
+        """Commits a Object to the object store
 
-        Returns: The treesum of the snapshot
-        """
-        # Make a new temporary directory and TreeObject; initialize
-        # the latter with the contents of `tree_path` and commit
-        # it to the store
-        with tempfile.TemporaryDirectory(dir=self.store) as tmp:
-            tree = TreeObject(f"{tmp}/tree")
-            tree.init(tree_path)
-            return self.commit(tree, object_id)
-
-    def commit(self, tree: TreeObject, object_id: str) -> str:
-        """Commits a TreeObject to the object store
-
-        Move the contents of the tree (TreeObject) to object directory
-        of the store with the content hash (tree.treesum) as its name.
+        Move the contents of the obj (Object) to object directory
+        of the store with the content hash (obj.treesum) as its name.
         Creates a symlink to that ('objects/{hash}') in the references
         directory with the object_id as the name ('refs/{object_id}).
         If the link already exists, it will be atomically replaced.
 
-        Returns: The treesum of the tree
+        Returns: The treesum of the object
         """
-        treesum_hash = tree.treesum
+        treesum_hash = obj.treesum
 
-        # the tree is stored in the objects directory using its content
+        # the object is stored in the objects directory using its content
         # hash as its name, ideally a given object_id (i.e., given config)
         # will always produce the same content hash, but that is not
-        # guaranteed. If a tree with the same treesum already exist, us
+        # guaranteed. If an object with the same treesum already exist, us
         # the existing one instead
-        tree.move(f"{self.objects}/{treesum_hash}")
+        obj.store_tree(f"{self.objects}/{treesum_hash}")
 
         # symlink the object_id (config hash) in the refs directory to the
         # treesum (content hash) in the objects directory. If a symlink by
         # that name alreday exists, atomically replace it, but leave the
         # backing object in place (it may be in use).
-        with tempfile.TemporaryDirectory(dir=self.store) as tmp:
+        with self.tempdir() as tmp:
             link = f"{tmp}/link"
             os.symlink(f"../objects/{treesum_hash}", link)
-            os.replace(link, f"{self.refs}/{object_id}")
+            os.replace(link, self.resolve_ref(object_id))
+
+        # the reference that is pointing to `treesum_hash` is now the base
+        # of `obj`. It is not actively initialized but any subsequent calls
+        # to `obj.write()` will initialize it again
+        # NB: in the case that an object with the same treesum as `obj`
+        # already existed in the store obj.store_tree() will not actually
+        # have written anything to the store. In this case `obj` will then
+        # be initialized with the content of the already existing object.
+        obj.base = object_id
 
         return treesum_hash
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index a435141c..58eb36a2 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -228,61 +228,66 @@ class Pipeline:
     @contextlib.contextmanager
     def get_buildtree(self, object_store):
         if self.build:
-            with object_store.get(self.build.tree_id) as tree:
-                yield tree
+            with object_store.new() as tree:
+                tree.base = self.build.tree_id
+                with tree.read() as path:
+                    yield path
         else:
-            with tempfile.TemporaryDirectory(dir=object_store.store) as tmp:
-                subprocess.run(["mount", "--make-private", "-o", "bind,ro,mode=0755", "/", tmp], check=True)
+            with object_store.tempdir() as tmp:
+                objectstore.mount("/", tmp)
                 try:
                     yield tmp
                 finally:
-                    subprocess.run(["umount", "--lazy", tmp], check=True)
+                    objectstore.umount(tmp)
 
     def run(self, store, interactive=False, libdir=None, secrets=None):
         os.makedirs("/run/osbuild", exist_ok=True)
         object_store = objectstore.ObjectStore(store)
         results = {}
 
-        if self.build:
+        if self.build and self.build.stages:
+            # For now, the last build stage is always committed to the object store
+            self.build.stages[-1].checkpoint = True
+
             r = self.build.run(store, interactive, libdir, secrets)
             results["build"] = r
             if not r["success"]:
                 results["success"] = False
                 return results
 
-        with self.get_buildtree(object_store) as build_tree:
+        with self.get_buildtree(object_store) as build_tree, \
+             object_store.new(base_id=self.tree_id) as tree:
+
             if self.stages:
                 if not object_store.contains(self.tree_id):
                     # Find the last stage that already exists in the object store, and use
                     # that as the base.
-                    base = None
                     base_idx = -1
+                    tree.base = None
                     for i in reversed(range(len(self.stages))):
                         if object_store.contains(self.stages[i].id):
-                            base = self.stages[i].id
+                            tree.base = self.stages[i].id
                             base_idx = i
                             break
 
-                    # The tree does not exist. Create it and save it to the object store. If
-                    # two run() calls race each-other, two trees may be generated, and it
+                    # If two run() calls race each-other, two trees may be generated  and it
                     # is nondeterministic which of them will end up referenced by the tree_id
-                    # in the content store. However, we guarantee that all tree_id's and all
-                    # generated trees remain valid.
+                    # in the content store if they are both committed. However, after the call
+                    # to commit all the trees will be based on the winner.
                     results["stages"] = []
                     try:
-                        with object_store.new(self.tree_id, base_id=base) as tree:
-                            for stage in self.stages[base_idx + 1:]:
-                                r = stage.run(tree,
-                                              self.runner,
-                                              build_tree,
-                                              store,
-                                              interactive=interactive,
-                                              libdir=libdir,
-                                              var=store,
-                                              secrets=secrets)
-                                if stage.checkpoint:
-                                    object_store.snapshot(tree, stage.id)
-                                results["stages"].append(r.as_dict())
+                        for stage in self.stages[base_idx + 1:]:
+                            r = stage.run(tree.write(),
+                                          self.runner,
+                                          build_tree,
+                                          store,
+                                          interactive=interactive,
+                                          libdir=libdir,
+                                          var=store,
+                                          secrets=secrets)
+                            if stage.checkpoint:
+                                object_store.commit(tree, stage.id)
+                            results["stages"].append(r.as_dict())
                     except BuildError as err:
                         results["stages"].append(err.as_dict())
                         results["success"] = False
@@ -293,16 +298,17 @@ class Pipeline:
             if self.assembler:
                 if not object_store.contains(self.output_id):
                     try:
-                        with object_store.get(self.tree_id) as tree, \
-                             object_store.new(self.output_id) as output_dir:
-                            r = self.assembler.run(tree,
+                        with tree.read() as input_tree, \
+                             object_store.new() as output_dir:
+                            r = self.assembler.run(input_tree,
                                                    self.runner,
                                                    build_tree,
-                                                   output_dir=output_dir,
+                                                   output_dir=output_dir.write(),
                                                    interactive=interactive,
                                                    libdir=libdir,
                                                    var=store)
                             results["assembler"] = r.as_dict()
+                            object_store.commit(output_dir, self.output_id)
                     except BuildError as err:
                         results["assembler"] = err.as_dict()
                         results["success"] = False
@@ -314,6 +320,33 @@ class Pipeline:
         return results
 
 
+def detect_os(*paths):
+    """Detect the os from an os-release file.
+
+    The first file in 'paths' is used and interpreted like a os-release(5)
+    file.
+
+    Returns ID + VERSION_ID (without dots), which is the same format that
+    runners are named as.
+    """
+    path = next((p for p in paths if os.path.exists(p)), None)
+    if not path:
+        raise FileNotFoundError("none of the specified os-release files exist")
+
+    with open(path) as f:
+        osrelease = {}
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            if line[0] == "#":
+                continue
+            key, value = line.split("=", 1)
+            osrelease[key] = value.strip('"')
+
+    return osrelease["ID"] + osrelease["VERSION_ID"].replace(".", "")
+
+
 def load_build(description, sources_options):
     pipeline = description.get("pipeline")
     if pipeline:
@@ -329,7 +362,7 @@ def load(description, sources_options):
     if build:
         build_pipeline, runner = load_build(build, sources_options)
     else:
-        build_pipeline, runner = None, "org.osbuild.host"
+        build_pipeline, runner = None, "org.osbuild." + detect_os("/etc/os-release", "/usr/lib/os-release")
 
     pipeline = Pipeline(runner, build_pipeline)
 
diff --git a/runners/org.osbuild.rhel81 b/runners/org.osbuild.rhel81
new file mode 100755
index 00000000..ae00cf34
--- /dev/null
+++ b/runners/org.osbuild.rhel81
@@ -0,0 +1,147 @@
+#!/usr/bin/python3.6
+
+import array
+import json
+import shutil
+import os
+import socket
+import subprocess
+import sys
+
+
+# copied from remoteloop.py
+def load_fds(sock, msglen):
+    fds = array.array("i")   # Array of ints
+    msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
+    for cmsg_level, cmsg_type, cmsg_data in ancdata:
+        if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
+            # Append data, ignoring any truncated integers at the end.
+            fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+    return json.loads(msg), list(fds), addr
+
+def ldconfig():
+    # ld.so.conf must exist, or `ldconfig` throws a warning
+    subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
+    subprocess.run(["ldconfig"], check=True)
+
+
+def sysusers():
+    try:
+        subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    except subprocess.CalledProcessError as error:
+        sys.stderr.write(error.stdout)
+        sys.exit(1)
+
+
+def update_ca_trust():
+    if not shutil.which("update-ca-trust"):
+        return
+
+    # generate /etc/pki/tls/certs/ca-bundle.crt
+    os.makedirs("/etc/pki/ca-trust/extracted/pem")
+    os.makedirs("/etc/pki/tls/certs")
+    os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
+
+    # allow to fail, because it sometimes mysteriously does
+    subprocess.run(["update-ca-trust", "extract"], check=False)
+
+
+def append_certs(cert_conf, dir_fd, parents=b""):
+    for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
+        if entry.is_file():
+            line = os.path.join(parents, entry.name)
+            cert_conf.write(line)
+            cert_conf.write(b"\n")
+        elif entry.is_dir():
+            append_certs(cert_conf,
+                         os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
+                         os.path.join(parents, entry.name))
+
+
+def update_ca_certificates():
+    if not shutil.which("update-ca-certificates"):
+        return
+
+    # generate /etc/ssl/certs/ca-certificates.crt
+    os.makedirs("/etc/ssl/certs")
+    with open("/etc/ca-certificates.conf", "wb") as f:
+        append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
+    subprocess.run(["update-ca-certificates"], check=True)
+
+
+def tmpfiles():
+    # Allow systemd-tmpfiles to return non-0. Some packages want to create
+    # directories owned by users that are not set up with systemd-sysusers.
+    subprocess.run(["systemd-tmpfiles", "--create"], check=False)
+
+
+def nsswitch():
+    # the default behavior is fine, but using nss-resolve does not
+    # necessarily work in a non-booted container, so make sure that
+    # is not configured.
+    try:
+        os.remove("/etc/nsswitch.conf")
+    except FileNotFoundError:
+        pass
+
+
+def setup_stdio():
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+        sock.connect("/run/osbuild/api/osbuild")
+        req = {'method': 'setup-stdio'}
+        sock.send(json.dumps(req).encode('utf-8'))
+        msg, fds, _ = load_fds(sock, 1024)
+        for io in ['stdin', 'stdout', 'stderr']:
+            target = getattr(sys, io)
+            source = fds[msg[io]]
+            os.dup2(source, target.fileno())
+            os.close(source)
+
+def os_release():
+    """/usr/lib/os-release doesn't exist. The `redhat-release` package
+    generates `/etc/os-release directly. To work around this, do the same here.
+
+    https://bugzilla.redhat.com/show_bug.cgi?id=1766754
+    """
+
+    # remove the symlink that systemd-nspawn creates
+    os.remove("/etc/os-release")
+    with open("/etc/os-release", "w") as f:
+        f.write('NAME="Red Hat Enterprise Linux"\n')
+        f.write('VERSION="8.1 (Ootpa)"\n')
+        f.write('ID="rhel"\n')
+        f.write('ID_LIKE="fedora"\n')
+        f.write('VERSION_ID="8.1"\n')
+        f.write('PLATFORM_ID="platform:el8"\n')
+        f.write('PRETTY_NAME="Red Hat Enterprise Linux 8.1 (Ootpa)"\n')
+        f.write('ANSI_COLOR="0;31"\n')
+        f.write('CPE_NAME="cpe:/o:redhat:enterprise_linux:8.1:GA"\n')
+        f.write('HOME_URL="https://www.redhat.com/"\n')
+        f.write('BUG_REPORT_URL="https://bugzilla.redhat.com/"\n')
+
+
+def python_alternatives():
+    """/usr/bin/python3 is a symlink to /etc/alternatives/python3, which points
+    to /usr/bin/python3.6 by default. Recreate the link in /etc, so that
+    shebang lines in stages and assemblers work.
+    """
+    os.makedirs("/etc/alternatives", exist_ok=True)
+    try:
+        os.symlink("/usr/bin/python3.6", "/etc/alternatives/python3")
+    except FileExistsError:
+        pass
+
+if __name__ == "__main__":
+    setup_stdio()
+    ldconfig()
+    sysusers()
+    update_ca_trust()
+    update_ca_certificates()
+    tmpfiles()
+    nsswitch()
+    os_release()
+    python_alternatives()
+
+    r = subprocess.run(sys.argv[1:], check=False)
+    sys.exit(r.returncode)


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 83a7e7bd..5946754b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -35,6 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: docker.io/library/python:3.7
+      options: --privileged  # Needed for bind mounts in unit tests
     steps:
       - name: Clone repository
         uses: actions/checkout@v2
@@ -51,6 +52,11 @@ jobs:
           cd osbuild
           python3 -m unittest -v test.test_objectstore
 
+      - name: Run test_osrelease
+        run: |
+          cd osbuild
+          python3 -m unittest -v test.test_osrelease
+
   rpm_build:
     name: "RPM build"
     needs:
diff --git a/test/os-release/fedora30 b/test/os-release/fedora30
new file mode 100644
index 00000000..9f0d60b3
--- /dev/null
+++ b/test/os-release/fedora30
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="30 (Workstation Edition)"
+ID=fedora
+VERSION_ID=30
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f30"
+PRETTY_NAME="Fedora 30 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:30"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=30
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=30
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/fedora31 b/test/os-release/fedora31
new file mode 100644
index 00000000..72050278
--- /dev/null
+++ b/test/os-release/fedora31
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="31 (Workstation Edition)"
+ID=fedora
+VERSION_ID=31
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f31"
+PRETTY_NAME="Fedora 31 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:31"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f31/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=31
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=31
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/fedora32 b/test/os-release/fedora32
new file mode 100644
index 00000000..e8f9713d
--- /dev/null
+++ b/test/os-release/fedora32
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="32 (Workstation Edition)"
+ID=fedora
+VERSION_ID=32
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f32"
+PRETTY_NAME="Fedora 32 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:32"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/%{doc_version}/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=32
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=32
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/fedora33 b/test/os-release/fedora33
new file mode 100644
index 00000000..eb3d8d50
--- /dev/null
+++ b/test/os-release/fedora33
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="33 (Workstation Edition)"
+ID=fedora
+VERSION_ID=33
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f30"
+PRETTY_NAME="Fedora 33 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:33"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=33
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=33
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/rhel81 b/test/os-release/rhel81
new file mode 100644
index 00000000..216c1d22
--- /dev/null
+++ b/test/os-release/rhel81
@@ -0,0 +1,16 @@
+NAME="Red Hat Enterprise Linux"
+VERSION="8.1 (Ootpa)"
+ID="rhel"
+ID_LIKE="fedora"
+VERSION_ID="8.1"
+PLATFORM_ID="platform:el8"
+PRETTY_NAME="Red Hat Enterprise Linux 8.1 (Ootpa)"
+ANSI_COLOR="0;31"
+CPE_NAME="cpe:/o:redhat:enterprise_linux:8.1:GA"
+HOME_URL="https://www.redhat.com/"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
+REDHAT_BUGZILLA_PRODUCT_VERSION=8.1
+REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+REDHAT_SUPPORT_PRODUCT_VERSION="8.1"
diff --git a/test/os-release/rhel82 b/test/os-release/rhel82
new file mode 100644
index 00000000..30542561
--- /dev/null
+++ b/test/os-release/rhel82
@@ -0,0 +1,16 @@
+NAME="Red Hat Enterprise Linux"
+VERSION="8.2 (Ootpa)"
+ID="rhel"
+ID_LIKE="fedora"
+VERSION_ID="8.2"
+PLATFORM_ID="platform:el8"
+PRETTY_NAME="Red Hat Enterprise Linux 8.2 Beta (Ootpa)"
+ANSI_COLOR="0;31"
+CPE_NAME="cpe:/o:redhat:enterprise_linux:8.2:beta"
+HOME_URL="https://www.redhat.com/"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
+REDHAT_BUGZILLA_PRODUCT_VERSION=8.2
+REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+REDHAT_SUPPORT_PRODUCT_VERSION="8.2 Beta"
diff --git a/test/os-release/ubuntu1804 b/test/os-release/ubuntu1804
new file mode 100644
index 00000000..dc5a5c2a
--- /dev/null
+++ b/test/os-release/ubuntu1804
@@ -0,0 +1,12 @@
+NAME="Ubuntu"
+VERSION="18.04.3 LTS (Bionic Beaver)"
+ID=ubuntu
+ID_LIKE=debian
+PRETTY_NAME="Ubuntu 18.04.3 LTS"
+VERSION_ID="18.04"
+HOME_URL="https://www.ubuntu.com/"
+SUPPORT_URL="https://help.ubuntu.com/"
+BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
+PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
+VERSION_CODENAME=bionic
+UBUNTU_CODENAME=bionic
diff --git a/test/osbuildtest.py b/test/osbuildtest.py
index e465a5ea..9b22588a 100644
--- a/test/osbuildtest.py
+++ b/test/osbuildtest.py
@@ -8,6 +8,8 @@ import subprocess
 import tempfile
 import unittest
 
+import osbuild
+
 
 class TestCase(unittest.TestCase):
     """A TestCase to test running the osbuild program.
@@ -44,6 +46,20 @@ class TestCase(unittest.TestCase):
             osbuild_cmd.append("--sources")
             osbuild_cmd.append(sources)
 
+        # Create a checkpoint at the last stage, i.e.
+        # commit the final tree to the store, so that
+        # tests can use it to compare against
+        if input:
+            manifest = json.loads(input)
+        else:
+            with open(pipeline, "r") as f:
+                manifest = json.load(f)
+
+        parsed = osbuild.load(manifest, {})
+        if parsed.tree_id:
+            osbuild_cmd.append("--checkpoint")
+            osbuild_cmd.append(parsed.tree_id)
+
         stdin = subprocess.PIPE if input else None
 
         p = subprocess.Popen(osbuild_cmd, encoding="utf-8", stdin=stdin, stdout=subprocess.PIPE)
diff --git a/test/test_objectstore.py b/test/test_objectstore.py
index c39dec8c..4e9dfa33 100644
--- a/test/test_objectstore.py
+++ b/test/test_objectstore.py
@@ -24,22 +24,32 @@ class TestObjectStore(unittest.TestCase):
         # always use a temporary store so item counting works
         with tempfile.TemporaryDirectory(dir="/var/tmp") as tmp:
             object_store = objectstore.ObjectStore(tmp)
-            with object_store.new("a") as tree:
-                p = Path(f"{tree}/A")
+            # No objects or references should be in the store
+            assert len(os.listdir(object_store.refs)) == 0
+            assert len(os.listdir(object_store.objects)) == 0
+
+            with object_store.new() as tree:
+                path = tree.write()
+                p = Path(f"{path}/A")
                 p.touch()
+                object_store.commit(tree, "a")
 
+            assert object_store.contains("a")
             assert os.path.exists(f"{object_store.refs}/a")
             assert os.path.exists(f"{object_store.refs}/a/A")
             assert len(os.listdir(object_store.refs)) == 1
             assert len(os.listdir(object_store.objects)) == 1
             assert len(os.listdir(f"{object_store.refs}/a/")) == 1
 
-            with object_store.new("b") as tree:
-                p = Path(f"{tree}/A")
+            with object_store.new() as tree:
+                path = tree.write()
+                p = Path(f"{path}/A")
                 p.touch()
-                p = Path(f"{tree}/B")
+                p = Path(f"{path}/B")
                 p.touch()
+                object_store.commit(tree, "b")
 
+            assert object_store.contains("b")
             assert os.path.exists(f"{object_store.refs}/b")
             assert os.path.exists(f"{object_store.refs}/b/B")
 
@@ -47,16 +57,24 @@ class TestObjectStore(unittest.TestCase):
             assert len(os.listdir(object_store.objects)) == 2
             assert len(os.listdir(f"{object_store.refs}/b/")) == 2
 
+            self.assertEqual(object_store.resolve_ref(None), None)
+            self.assertEqual(object_store.resolve_ref("a"),
+                             f"{object_store.refs}/a")
+
     def test_duplicate(self):
         with tempfile.TemporaryDirectory(dir="/var/tmp") as tmp:
             object_store = objectstore.ObjectStore(tmp)
-            with object_store.new("a") as tree:
-                p = Path(f"{tree}/A")
+            with object_store.new() as tree:
+                path = tree.write()
+                p = Path(f"{path}/A")
                 p.touch()
+                object_store.commit(tree, "a")
 
-            with object_store.new("b") as tree:
+            with object_store.new() as tree:
+                path = tree.write()
                 shutil.copy2(f"{object_store.refs}/a/A",
-                             f"{tree}/A")
+                             f"{path}/A")
+                object_store.commit(tree, "b")
 
             assert os.path.exists(f"{object_store.refs}/a")
             assert os.path.exists(f"{object_store.refs}/a/A")
@@ -67,14 +85,138 @@ class TestObjectStore(unittest.TestCase):
             assert len(os.listdir(f"{object_store.refs}/a/")) == 1
             assert len(os.listdir(f"{object_store.refs}/b/")) == 1
 
+    def test_object_base(self):
+        # operate with a clean object store
+        with tempfile.TemporaryDirectory(dir="/var/tmp") as tmp:
+            object_store = objectstore.ObjectStore(tmp)
+            with object_store.new() as tree:
+                path = tree.write()
+                p = Path(f"{path}/A")
+                p.touch()
+                object_store.commit(tree, "a")
+
+            with object_store.new() as tree:
+                tree.base = "a"
+                object_store.commit(tree, "b")
+
+            with object_store.new() as tree:
+                tree.base = "b"
+                path = tree.write()
+                p = Path(f"{path}/C")
+                p.touch()
+                object_store.commit(tree, "c")
+
+            assert os.path.exists(f"{object_store.refs}/a/A")
+            assert os.path.exists(f"{object_store.refs}/b/A")
+            assert os.path.exists(f"{object_store.refs}/c/A")
+            assert os.path.exists(f"{object_store.refs}/c/C")
+
+            assert len(os.listdir(object_store.refs)) == 3
+            assert len(os.listdir(object_store.objects)) == 2
+
+    def test_object_copy_on_write(self):
+        # operate with a clean object store
+        with tempfile.TemporaryDirectory(dir="/var/tmp") as tmp:
+            # sample data to be used for read, write checks
+            data = "23"
+
+            object_store = objectstore.ObjectStore(tmp)
+            assert len(os.listdir(object_store.refs)) == 0
+
+            with object_store.new() as tree:
+                path = tree.write()
+                with open(f"{path}/data", "w") as f:
+                    f.write(data)
+                    st = os.fstat(f.fileno())
+                    data_inode = st.st_ino
+                # commit the object as "x"
+                x_hash = object_store.commit(tree, "x")
+                # after the commit, "x" is now the base
+                # of "tree"
+                self.assertEqual(tree.base, "x")
+                # check that "data" is still the very
+                # same file after committing
+                with tree.read() as path:
+                    with open(f"{path}/data", "r") as f:
+                        st = os.fstat(f.fileno())
+                        self.assertEqual(st.st_ino, data_inode)
+                        data_read = f.read()
+                        self.assertEqual(data, data_read)
+
+            # the object referenced by "x" should act as
+            # the base of a new object. As long as the
+            # new one is not modified, it should have
+            # the very same content
+            with object_store.new(base_id="x") as tree:
+                self.assertEqual(tree.base, "x")
+                self.assertEqual(tree.treesum, x_hash)
+                with tree.read() as path:
+                    with open(f"{path}/data", "r") as f:
+                        # copy-on-write: since we have not written
+                        # to the tree yet, "data" should be the
+                        # very same file as that one of object "x"
+                        st = os.fstat(f.fileno())
+                        self.assertEqual(st.st_ino, data_inode)
+                        data_read = f.read()
+                        self.assertEqual(data, data_read)
+                path = tree.write()
+                # "data" must of course still be present
+                assert os.path.exists(f"{path}/data")
+                # but since it is a copy, have a different inode
+                st = os.stat(f"{path}/data")
+                self.assertNotEqual(st.st_ino, data_inode)
+                p = Path(f"{path}/other_data")
+                p.touch()
+                # now that we have written, the treesum
+                # should have changed
+                self.assertNotEqual(tree.treesum, x_hash)
+
+    def test_object_mode(self):
+        object_store = objectstore.ObjectStore(self.store)
+        with object_store.new() as tree:
+            # check that trying to write to a tree that is
+            # currently being read from fails
+            with tree.read() as _:
+                with self.assertRaises(ValueError):
+                    tree.write()
+
+                # check multiple readers are ok
+                with tree.read() as _:
+                    # calculating the treesum also is reading,
+                    # so this is 3 nested readers
+                    _ = tree.treesum
+
+                # writing should still fail
+                with self.assertRaises(ValueError):
+                    tree.write()
+
+            # Now that all readers are gone, writing should
+            # work
+            tree.write()
+
+            # and back to reading, one last time
+            with tree.read() as _:
+                with self.assertRaises(ValueError):
+                    tree.write()
+
+        # tree has exited the context, it should NOT be
+        # writable anymore
+        with self.assertRaises(ValueError):
+            tree.write()
+
     def test_snapshot(self):
         object_store = objectstore.ObjectStore(self.store)
-        with object_store.new("b") as tree:
-            p = Path(f"{tree}/A")
+        with object_store.new() as tree:
+            path = tree.write()
+            p = Path(f"{path}/A")
             p.touch()
-            object_store.snapshot(tree, "a")
-            p = Path(f"{tree}/B")
+            assert not object_store.contains("a")
+            object_store.commit(tree, "a")
+            assert object_store.contains("a")
+            path = tree.write()
+            p = Path(f"{path}/B")
             p.touch()
+            object_store.commit(tree, "b")
 
         # check the references exist
         assert os.path.exists(f"{object_store.refs}/a")
diff --git a/test/test_osrelease.py b/test/test_osrelease.py
new file mode 100644
index 00000000..1c8578cd
--- /dev/null
+++ b/test/test_osrelease.py
@@ -0,0 +1,17 @@
+import os
+import unittest
+
+import osbuild
+
+class TestOSRelease(unittest.TestCase):
+    def test_non_existant(self):
+        self.assertRaises(FileNotFoundError, osbuild.pipeline.detect_os, "💩")
+
+    def test_detect_os(self):
+        """Test host os detection. test/os-release contains the os-release files
+        for all supported runners.
+        """
+        for entry in os.scandir("test/os-release"):
+            with self.subTest(entry.name):
+                self.assertEqual(osbuild.pipeline.detect_os(entry.path), entry.name)
+