Repository: systemd/mkosi

Issue:
OCI as output format
mkosi is a nice tool for building (in addition to other things) nspawn images.
While nspawn can use a large number of different formats, OCI is a very convenient one since there is existing infrastructure for distribution (container registries).
Should mkosi learn how to generate the [on disk OCI format](https://github.com/opencontainers/image-spec/blob/main/image-layout.md)?
Additionally, should mkosi be able to reuse existing directory trees from OCI images / layers as inputs?
Not sure if this makes sense and if this belongs here (or should be implemented in systemd-repart instead).


Discussion:
If there's a tool that makes this easy and is packaged everywhere, we can support oci.
Currently the easiest workaround is likely to use a directory output and invoke buildah (or docker build with a dockerfile) to put it in a scratch image.
> If there's a tool that makes this easy and is packaged everywhere, we can support oci.

[`skopeo-copy(1)`](https://manpages.debian.org/bookworm/skopeo/skopeo-copy.1.en.html) is widely packaged by distros and can do parts of it but it seems like the tar and a manifest.json still have to be created manually. So unless mkosi would be able to utilize some advantages of creating those images directly like multiple layers to have smaller diffs for upgrades I still stand by my statement that `FROM scratch\nCOPY mkosi.output/...` is way simpler
isnt generating oci trivial? i.e. you just have to add some JSON to a tarball, and you are done?
Yes. You could use the generated sysroot as the only layer and create the necessary files to conform to this spec: https://github.com/opencontainers/image-spec/blob/main/image-layout.md

The result can either just be a plain directory or could be packaged as a tar.


Resolving patch:
diff --git a/mkosi/__init__.py b/mkosi/__init__.py
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -20,7 +20,7 @@
 import uuid
 from collections.abc import Iterator, Mapping, Sequence
 from pathlib import Path
-from typing import Optional, TextIO, Union, cast
+from typing import Optional, Union, cast
 
 from mkosi.archive import extract_tar, make_cpio, make_tar
 from mkosi.burn import run_burn
@@ -75,6 +75,7 @@
     flock,
     flock_or_die,
     format_rlimit,
+    hash_file,
     make_executable,
     one_zero,
     parents_below,
@@ -2457,17 +2458,6 @@ def copy_initrd(context: Context) -> None:
         break
 
 
-def hash_file(of: TextIO, path: Path) -> None:
-    bs = 16 * 1024**2
-    h = hashlib.sha256()
-
-    with path.open("rb") as sf:
-        while (buf := sf.read(bs)):
-            h.update(buf)
-
-    of.write(h.hexdigest() + " *" + path.name + "\n")
-
-
 def calculate_sha256sum(context: Context) -> None:
     if not context.config.checksum:
         return
@@ -2478,7 +2468,7 @@ def calculate_sha256sum(context: Context) -> None:
     with complete_step("Calculating SHA256SUMS…"):
         with open(context.workspace / context.config.output_checksum, "w") as f:
             for p in context.staging.iterdir():
-                hash_file(f, p)
+                print(hash_file(p) + " *" + p.name, file=f)
 
         (context.workspace / context.config.output_checksum).rename(context.staging / context.config.output_checksum)
 
@@ -3270,6 +3260,103 @@ def make_disk(
     return make_image(context, msg=msg, skip=skip, split=split, tabs=tabs, root=context.root, definitions=definitions)
 
 
+def make_oci(context: Context, root_layer: Path, dst: Path) -> None:
+    ca_store = dst / "blobs" / "sha256"
+    with umask(~0o755):
+        ca_store.mkdir(parents=True)
+
+    layer_diff_digest = hash_file(root_layer)
+    maybe_compress(
+        context,
+        context.config.compress_output,
+        context.staging / "rootfs.layer",
+        # Pass explicit destination to suppress adding an extension
+        context.staging / "rootfs.layer",
+    )
+    layer_digest = hash_file(root_layer)
+    root_layer.rename(ca_store / layer_digest)
+
+    creation_time = (
+        datetime.datetime.fromtimestamp(context.config.source_date_epoch, tz=datetime.timezone.utc)
+        if context.config.source_date_epoch is not None
+        else datetime.datetime.now(tz=datetime.timezone.utc)
+    ).isoformat()
+
+    oci_config = {
+        "created": creation_time,
+        "architecture": context.config.architecture.to_oci(),
+        # Name of the operating system which the image is built to run on as defined by
+        # https://github.com/opencontainers/image-spec/blob/v1.0.2/config.md#properties.
+        "os": "linux",
+        "rootfs": {
+            "type": "layers",
+            "diff_ids": [f"sha256:{layer_diff_digest}"],
+        },
+        "config": {
+            "Cmd": [
+                "/sbin/init",
+                *context.config.kernel_command_line,
+            ],
+        },
+        "history": [
+            {
+                "created": creation_time,
+                "comment": "Created by mkosi",
+            },
+        ],
+    }
+    oci_config_blob = json.dumps(oci_config)
+    oci_config_digest = hashlib.sha256(oci_config_blob.encode()).hexdigest()
+    with umask(~0o644):
+        (ca_store / oci_config_digest).write_text(oci_config_blob)
+
+    layer_suffix = context.config.compress_output.oci_media_type_suffix()
+    oci_manifest = {
+        "schemaVersion": 2,
+        "mediaType": "application/vnd.oci.image.manifest.v1+json",
+        "config": {
+            "mediaType": "application/vnd.oci.image.config.v1+json",
+            "digest": f"sha256:{oci_config_digest}",
+            "size": (ca_store / oci_config_digest).stat().st_size,
+        },
+        "layers": [
+            {
+                "mediaType": f"application/vnd.oci.image.layer.v1.tar{layer_suffix}",
+                "digest": f"sha256:{layer_digest}",
+                "size": (ca_store / layer_digest).stat().st_size,
+            }
+        ],
+        "annotations": {
+            "io.systemd.mkosi.version": __version__,
+            **({
+                "org.opencontainers.image.version": context.config.image_version,
+            } if context.config.image_version else {}),
+        }
+    }
+    oci_manifest_blob = json.dumps(oci_manifest)
+    oci_manifest_digest = hashlib.sha256(oci_manifest_blob.encode()).hexdigest()
+    with umask(~0o644):
+        (ca_store / oci_manifest_digest).write_text(oci_manifest_blob)
+
+        (dst / "index.json").write_text(
+            json.dumps(
+                {
+                    "schemaVersion": 2,
+                    "mediaType": "application/vnd.oci.image.index.v1+json",
+                    "manifests": [
+                        {
+                            "mediaType": "application/vnd.oci.image.manifest.v1+json",
+                            "digest": f"sha256:{oci_manifest_digest}",
+                            "size": (ca_store / oci_manifest_digest).stat().st_size,
+                        }
+                    ],
+                }
+            )
+        )
+
+        (dst / "oci-layout").write_text(json.dumps({"imageLayoutVersion": "1.0.0"}))
+
+
 def make_esp(context: Context, uki: Path) -> list[Partition]:
     if not (arch := context.config.architecture.to_efi()):
         die(f"Architecture {context.config.architecture} does not support UEFI")
@@ -3571,6 +3658,17 @@ def build_image(context: Context) -> None:
             tools=context.config.tools(),
             sandbox=context.sandbox,
         )
+    elif context.config.output_format == OutputFormat.oci:
+        make_tar(
+            context.root, context.staging / "rootfs.layer",
+            tools=context.config.tools(),
+            sandbox=context.sandbox,
+        )
+        make_oci(
+            context,
+            context.staging / "rootfs.layer",
+            context.staging / context.config.output_with_format,
+        )
     elif context.config.output_format == OutputFormat.cpio:
         make_cpio(
             context.root, context.staging / context.config.output_with_format,
diff --git a/mkosi/config.py b/mkosi/config.py
--- a/mkosi/config.py
+++ b/mkosi/config.py
@@ -173,6 +173,7 @@ class OutputFormat(StrEnum):
     sysext    = enum.auto()
     tar       = enum.auto()
     uki       = enum.auto()
+    oci       = enum.auto()
 
     def extension(self) -> str:
         return {
@@ -205,6 +206,7 @@ class Compression(StrEnum):
     xz   = enum.auto()
     bz2  = enum.auto()
     gz   = enum.auto()
+    gzip = "gz"
     lz4  = enum.auto()
     lzma = enum.auto()
 
@@ -216,6 +218,18 @@ def extension(self) -> str:
             Compression.zstd: ".zst"
         }.get(self, f".{self}")
 
+    def oci_media_type_suffix(self) -> str:
+        suffix = {
+            Compression.none: "",
+            Compression.gz:   "+gzip",
+            Compression.zstd: "+zstd",
+        }.get(self)
+
+        if not suffix:
+            die(f"Compression {self} not supported for OCI layers")
+
+        return suffix
+
 
 class DocFormat(StrEnum):
     auto     = enum.auto()
@@ -379,6 +393,28 @@ def to_qemu(self) -> str:
 
         return a
 
+    def to_oci(self) -> str:
+        a = {
+            Architecture.arm         : "arm",
+            Architecture.arm64       : "arm64",
+            Architecture.loongarch64 : "loong64",
+            Architecture.mips64_le   : "mips64le",
+            Architecture.mips_le     : "mipsle",
+            Architecture.ppc         : "ppc",
+            Architecture.ppc64       : "ppc64",
+            Architecture.ppc64_le    : "ppc64le",
+            Architecture.riscv32     : "riscv",
+            Architecture.riscv64     : "riscv64",
+            Architecture.s390x       : "s390x",
+            Architecture.x86         : "386",
+            Architecture.x86_64      : "amd64",
+        }.get(self)
+
+        if not a:
+            die(f"Architecture {self} not supported by OCI")
+
+        return a
+
     def default_serial_tty(self) -> str:
         return {
             Architecture.arm      : "ttyAMA0",
@@ -634,6 +670,8 @@ def config_default_compression(namespace: argparse.Namespace) -> Compression:
             return Compression.xz
         else:
             return Compression.zstd
+    elif namespace.output_format == OutputFormat.oci:
+        return Compression.gz
     else:
         return Compression.none
 
diff --git a/mkosi/util.py b/mkosi/util.py
--- a/mkosi/util.py
+++ b/mkosi/util.py
@@ -7,6 +7,7 @@
 import errno
 import fcntl
 import functools
+import hashlib
 import importlib
 import importlib.resources
 import itertools
@@ -298,3 +299,15 @@ def _write_contents(target, source):
             p.parent.chmod(0o755)
 
         yield p
+
+
+def hash_file(path: Path) -> str:
+    # TODO Replace with hashlib.file_digest after dropping support for Python 3.10.
+    bs = 16 * 1024**2
+    h = hashlib.sha256()
+
+    with path.open("rb") as sf:
+        while (buf := sf.read(bs)):
+            h.update(buf)
+
+    return h.hexdigest()


Test patch:
diff --git a/tests/test_boot.py b/tests/test_boot.py
--- a/tests/test_boot.py
+++ b/tests/test_boot.py
@@ -57,7 +57,7 @@ def test_format(config: Image.Config, format: OutputFormat) -> None:
         if image.config.distribution == Distribution.rhel_ubi:
             return
 
-        if format in (OutputFormat.tar, OutputFormat.none) or format.is_extension_image():
+        if format in (OutputFormat.tar, OutputFormat.oci, OutputFormat.none) or format.is_extension_image():
             return
 
         if format == OutputFormat.directory and not find_virtiofsd():