Repository: osbuild/osbuild
Pull request #1759 (2024-04-25 20:13:30)

workflow: bump osbuild-ci container to include podman, et al.
This will include the latest osbuild-ci container changes from https://github.com/osbuild/containers/pull/73

This includes a bunch of binaries that were missing and lead to skpped tests.


Patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1a43717d..8c3acb74 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -41,6 +41,8 @@ RPM:
           - aws/fedora-38-aarch64
           - aws/fedora-39-x86_64
           - aws/fedora-39-aarch64
+          - aws/fedora-40-x86_64
+          - aws/fedora-40-aarch64
           - aws/centos-stream-8-x86_64
           - aws/centos-stream-8-aarch64
           - aws/centos-stream-9-x86_64
diff --git a/osbuild.spec b/osbuild.spec
index 8e9c4a35..c94bb8a9 100644
--- a/osbuild.spec
+++ b/osbuild.spec
@@ -137,12 +137,7 @@ manifests and osbuild.
 Summary:        Dependency solving support for DNF
 Requires:       %{name} = %{version}-%{release}
 
-# Fedora 40 and later use libdnf5, RHEL and Fedora < 40 use libdnf
-%if 0%{?fedora} >= 40
-Requires: python3-libdnf5 >= 5.1.1
-%else
 Requires: python3-libdnf
-%endif
 
 %description    depsolve-dnf
 Contains depsolving capabilities for package managers.
@@ -213,12 +208,7 @@ install -p -m 0755 data/10-osbuild-inhibitor.rules %{buildroot}%{_udevrulesdir}
 
 # Install `osbuild-depsolve-dnf` into libexec
 mkdir -p %{buildroot}%{_libexecdir}
-# Fedora 40 and later use dnf5-json, RHEL and Fedora < 40 use dnf-json
-%if 0%{?fedora} >= 40
-install -p -m 0755 tools/osbuild-depsolve-dnf5 %{buildroot}%{_libexecdir}/osbuild-depsolve-dnf
-%else
 install -p -m 0755 tools/osbuild-depsolve-dnf %{buildroot}%{_libexecdir}/osbuild-depsolve-dnf
-%endif
 
 %check
 exit 0
diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl
index 670717a4..6a8293f2 100755
--- a/sources/org.osbuild.curl
+++ b/sources/org.osbuild.curl
@@ -20,6 +20,7 @@ up the download.
 
 import concurrent.futures
 import os
+import platform
 import subprocess
 import sys
 import tempfile
@@ -153,6 +154,7 @@ class CurlSource(sources.SourceService):
             # some mirrors are sometimes broken. retry manually, because we could be
             # redirected to a different, working, one on retry.
             return_code = 0
+            arch = platform.machine()
             for _ in range(10):
                 curl_command = [
                     "curl",
@@ -161,6 +163,7 @@ class CurlSource(sources.SourceService):
                     "--connect-timeout", "30",
                     "--fail",
                     "--location",
+                    "--header", f"User-Agent: osbuild (Linux.{arch}; https://osbuild.org/)",
                     "--output", checksum,
                 ]
                 if proxy:
diff --git a/stages/org.osbuild.skopeo b/stages/org.osbuild.skopeo
index f50935d5..45bb98db 100755
--- a/stages/org.osbuild.skopeo
+++ b/stages/org.osbuild.skopeo
@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 import os
+import pathlib
 import subprocess
 import sys
 
@@ -12,6 +13,7 @@ def main(inputs, output, options):
 
     destination = options["destination"]
     dest_type = destination["type"]
+    remove_signatures = options.get("remove-signatures")
 
     for image in images.values():
         with containers.container_source(image) as (image_name, image_source):
@@ -19,13 +21,19 @@ def main(inputs, output, options):
                 storage_root = destination.get("storage-path", "/var/lib/containers/storage")
                 storage_driver = destination.get("storage-driver", "overlay")
                 dest = f"containers-storage:[{storage_driver}@{output}{storage_root}+/run/containers/storage]{image_name}"
-            elif dest_type == "oci":
+            elif dest_type in ("oci", "oci-archive", "dir"):
                 path = destination["path"]
-                dest = f"oci:{output}{path}"
+                dest = f"{dest_type}:{output}{path}"
+                pathlib.Path(f"{output}{path}").parent.mkdir(parents=True, exist_ok=True)
             else:
                 raise ValueError(f"Unknown destination type '{dest_type}'")
 
-            subprocess.run(["skopeo", "copy", image_source, dest], check=True)
+            cmd = ["skopeo", "copy"]
+            if remove_signatures:
+                cmd.append("--remove-signatures")
+            cmd.extend([image_source, dest])
+
+            subprocess.run(cmd, check=True)
 
     if dest_type == "containers-storage" and storage_driver == "overlay":
         # Each time the overlay backend runs on an xfs fs it creates this file:
diff --git a/stages/org.osbuild.skopeo.meta.json b/stages/org.osbuild.skopeo.meta.json
index 777ebe0a..95812a90 100644
--- a/stages/org.osbuild.skopeo.meta.json
+++ b/stages/org.osbuild.skopeo.meta.json
@@ -47,6 +47,44 @@
             "type": "string"
           }
         }
+      },
+      "destination-oci-archive": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "type",
+          "path"
+        ],
+        "properties": {
+          "type": {
+            "enum": [
+              "oci-archive"
+            ]
+          },
+          "path": {
+            "description": "Location of a tar archive compliant with 'Open Container Image Layout Specification'",
+            "type": "string"
+          }
+        }
+      },
+      "destination-dir": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "type",
+          "path"
+        ],
+        "properties": {
+          "type": {
+            "enum": [
+              "dir"
+            ]
+          },
+          "path": {
+            "description": "Location of a directory storing the manifest, layer tarballs and signatures as individual files. This is a non-standardized format, primarily useful for debugging or noninvasive container inspection.",
+            "type": "string"
+          }
+        }
       }
     },
     "inputs": {
@@ -80,8 +118,19 @@
             },
             {
               "$ref": "#/definitions/destination-oci"
+            },
+            {
+              "$ref": "#/definitions/destination-oci-archive"
+            },
+            {
+              "$ref": "#/definitions/destination-dir"
             }
           ]
+        },
+        "remove-signatures": {
+          "type": "boolean",
+          "default": false,
+          "description": "Do not copy signatures, if any, from source-image. Necessary when copying a signed image to a destination which does not support signatures."
         }
       }
     }
diff --git a/tools/osbuild-depsolve-dnf5 b/tools/osbuild-depsolve-dnf5
index 64afd199..029d6033 100755
--- a/tools/osbuild-depsolve-dnf5
+++ b/tools/osbuild-depsolve-dnf5
@@ -223,7 +223,7 @@ class Solver():
         else:
             raise ValueError("missing either `baseurl`, `metalink`, or `mirrorlist` in repo")
 
-        repo.sslverify = desc.get("sslverify", True)
+        conf.sslverify = desc.get("sslverify", True)
         if "sslcacert" in desc:
             conf.sslcacert = desc["sslcacert"]
         if "sslclientkey" in desc:


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5c9f0f67..6ab1b51a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,7 +30,7 @@ jobs:
     - name: "Run"
       uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
       with:
-        image: ghcr.io/osbuild/osbuild-ci:latest-202404161303
+        image: ghcr.io/osbuild/osbuild-ci:latest-202404252004
         run: |
           # Hacky replacement of container storage driver:
           # The default overlayfs doesn't work in the runner, so let's change
@@ -60,7 +60,7 @@ jobs:
           # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
           TEST_WORKERS: "-n 4"
         with:
-          image: ghcr.io/osbuild/osbuild-ci:latest-202404161303
+          image: ghcr.io/osbuild/osbuild-ci:latest-202404252004
           run: |
             OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
             tox -e "py36" -- ${{ env.TEST_WORKERS }} test.run.test_assemblers
diff --git a/osbuild/testutil/__init__.py b/osbuild/testutil/__init__.py
index 322944de..0681ff9b 100644
--- a/osbuild/testutil/__init__.py
+++ b/osbuild/testutil/__init__.py
@@ -183,3 +183,21 @@ def find_one_subclass_in_module(module: ModuleType, subclass: Type) -> object:
             raise ValueError(f"already have {cls}, also found {name}:{memb}")
         cls = memb
     return cls
+
+
+def make_fake_images_inputs(fake_oci_path, name):
+    fname = fake_oci_path.name
+    dirname = fake_oci_path.parent
+    return {
+        "images": {
+            "path": dirname,
+            "data": {
+                "archives": {
+                    fname: {
+                        "format": "oci-archive",
+                        "name": name,
+                    },
+                },
+            },
+        },
+    }
diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py
index 49813ad0..47c7849c 100644
--- a/sources/test/test_curl_source.py
+++ b/sources/test/test_curl_source.py
@@ -208,3 +208,20 @@ def test_curl_download_proxy(mocked_run, tmp_path, monkeypatch, sources_service,
             assert args[0][idx:idx + 2] == ["--proxy", "http://my-proxy"]
         else:
             assert "--proxy" not in args[0]
+
+
+@patch("subprocess.run")
+def test_curl_user_agent(mocked_run, tmp_path, sources_service):
+    test_sources = make_test_sources(tmp_path, 80, 2,)
+    fake_curl_downloader = FakeCurlDownloader(test_sources)
+    mocked_run.side_effect = fake_curl_downloader.faked_run
+
+    sources_service.cache = tmp_path / "curl-cache"
+    sources_service.cache.mkdir()
+    sources_service.fetch_all(test_sources)
+
+    for call_args in mocked_run.call_args_list:
+        args, _kwargs = call_args
+        idx = args[0].index("--header")
+        assert "User-Agent: osbuild" in args[0][idx + 1]
+        assert "https://osbuild.org/" in args[0][idx + 1]
diff --git a/stages/test/test_bootc_install_to_fs.py b/stages/test/test_bootc_install_to_fs.py
index f31dc03e..7f1fe9a0 100644
--- a/stages/test/test_bootc_install_to_fs.py
+++ b/stages/test/test_bootc_install_to_fs.py
@@ -31,7 +31,7 @@ FAKE_INPUTS = {
         "path": "/input/images/path",
         "data": {
             "archives": {
-                "/input/images/path": {
+                "filename": {
                     "format": "oci-archive",
                     "name": "some-img-name",
                 },
@@ -69,7 +69,7 @@ def test_bootc_install_to_fs(mock_run, mocked_named_tmp, mocked_temp_dir, stage_
             "path": "/input/images/path",
             "data": {
                 "archives": {
-                    "/input/images/path": {
+                    "filename": {
                         "format": "oci-archive",
                         "name": "some-img-name",
                     },
diff --git a/stages/test/test_container_deploy.py b/stages/test/test_container_deploy.py
index 02a11795..748369af 100644
--- a/stages/test/test_container_deploy.py
+++ b/stages/test/test_container_deploy.py
@@ -9,7 +9,7 @@ from unittest.mock import call, patch
 import pytest
 
 import osbuild.testutil
-from osbuild.testutil import has_executable, make_container
+from osbuild.testutil import has_executable, make_container, make_fake_images_inputs
 
 STAGE_NAME = "org.osbuild.container-deploy"
 
@@ -22,28 +22,15 @@ def test_container_deploy_integration(tmp_path, stage_module):
     with make_container(tmp_path, {"file1": "file1 from base"}) as base_tag:
         with make_container(tmp_path, {"file1": "file1 from final layer"}, base_tag) as cont_tag:
             # export for the container-deploy stage
-            fake_container_dst = tmp_path / "fake-container"
+            fake_oci_path = tmp_path / "fake-container"
             subprocess.check_call([
                 "podman", "save",
                 "--format=oci-archive",
-                f"--output={fake_container_dst}",
+                f"--output={fake_oci_path}",
                 cont_tag,
             ])
 
-    inputs = {
-        "images": {
-            # seems to be unused with fake_container_path?
-            "path": fake_container_dst,
-            "data": {
-                "archives": {
-                    fake_container_dst: {
-                        "format": "oci-archive",
-                        "name": cont_tag,
-                    },
-                },
-            },
-        },
-    }
+    inputs = make_fake_images_inputs(fake_oci_path, "some-name")
     output_dir = tmp_path / "output"
     options = {}
 
@@ -66,28 +53,15 @@ def test_container_deploy_exclude(tmp_path, stage_module):
         "dir2/file4": "dir2/file4 content",
     }) as base_tag:
         # export for the container-deploy stage
-        fake_container_dst = tmp_path / "fake-container"
+        fake_oci_path = tmp_path / "fake-container"
         subprocess.check_call([
             "podman", "save",
             "--format=oci-archive",
-            f"--output={fake_container_dst}",
+            f"--output={fake_oci_path}",
             base_tag,
         ])
 
-    inputs = {
-        "images": {
-            # seems to be unused with fake_container_path?
-            "path": fake_container_dst,
-            "data": {
-                "archives": {
-                    fake_container_dst: {
-                        "format": "oci-archive",
-                        "name": base_tag,
-                    },
-                },
-            },
-        },
-    }
+    inputs = make_fake_images_inputs(fake_oci_path, "some-name")
     options = {
         "exclude": [
             "file2",
diff --git a/stages/test/test_skopeo.py b/stages/test/test_skopeo.py
index 2b591e72..13f897c0 100644
--- a/stages/test/test_skopeo.py
+++ b/stages/test/test_skopeo.py
@@ -1,8 +1,13 @@
 #!/usr/bin/python3
+import json
+import os
+import subprocess
+import tarfile
 
 import pytest
 
 from osbuild import testutil
+from osbuild.testutil import has_executable, make_container, make_fake_images_inputs
 
 STAGE_NAME = "org.osbuild.skopeo"
 
@@ -13,8 +18,15 @@ STAGE_NAME = "org.osbuild.skopeo"
     ({"destination": {}}, "is not valid under any of the given schemas"),
     ({"destination": {"type": "foo"}}, "is not valid under any of the given schemas"),
     ({"destination": {"type": "oci"}}, "is not valid under any of the given schemas"),
+    ({"destination": {"type": "dir"}}, "is not valid under any of the given schemas"),
+    ({"destination": {"type": "os-archive"}}, "is not valid under any of the given schemas"),
+    ({"destination": {"type": "dir", "path": "/foo"}, "remove-signatures": "YesPlease"},
+     "'YesPlease' is not of type 'boolean'"),
     # good
     ({"destination": {"type": "oci", "path": "/foo"}}, ""),
+    ({"destination": {"type": "oci-archive", "path": "/foo"}}, ""),
+    ({"destination": {"type": "dir", "path": "/foo"}}, ""),
+    ({"destination": {"type": "dir", "path": "/foo"}, "remove-signatures": True}, ""),
 
     # this one might not be expected but it's valid because we don't require any
     # *inputs* and it'll be a no-op in the stage
@@ -33,3 +45,101 @@ def test_schema_validation_skopeo(stage_schema, test_data, expected_err):
     else:
         assert res.valid is False
         testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
+
+
+def make_fake_oci_archive(tmp_path):
+    with make_container(tmp_path, {"file1": "file1 from final layer"}) as cont_tag:
+        # export for the container-deploy stage
+        fake_container_dst = tmp_path / "fake-container"
+        subprocess.check_call([
+            "podman", "save",
+            "--format=oci-archive",
+            f"--output={fake_container_dst}",
+            cont_tag,
+        ])
+    return fake_container_dst
+
+
+def assert_manifest_file(manifest_file):
+    assert manifest_file.exists()
+    data = json.loads(manifest_file.read_bytes())
+    assert data.get("config") is not None, \
+        "'manifest.json' seems corrupt - no 'config' section found"
+    assert data["config"].get("digest") is not None, \
+        "'manifest.json' seems corrupt - no 'config.digest' section found"
+
+
+def _test_skopeo_copy(tmp_path, stage_module, typ, dest_name):
+    fake_oci_path = make_fake_oci_archive(tmp_path)
+    inputs = make_fake_images_inputs(fake_oci_path, "some-name")
+
+    output_dir = tmp_path / "output"
+    local_path = f"/some/{dest_name}"
+    options = {
+        "destination": {
+            "type": typ,
+            "path": local_path,
+        }
+    }
+    stage_module.main(inputs, output_dir, options)
+    result = output_dir / local_path.lstrip("/")
+    assert result.exists()
+    return result
+
+
+@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
+@pytest.mark.skipif(not has_executable("podman"), reason="no podman executable")
+def test_skopeo_copy_dir(tmp_path, stage_module):
+    result_path = _test_skopeo_copy(tmp_path, stage_module, "dir", "skopeo-dir")
+    assert (result_path / "version").exists()
+    assert_manifest_file(result_path / "manifest.json")
+
+
+@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
+@pytest.mark.skipif(not has_executable("podman"), reason="no podman executable")
+def test_skopeo_copy_oci_archive(tmp_path, stage_module):
+    result_path = _test_skopeo_copy(tmp_path, stage_module, "oci-archive", "skopeo-archive.tar")
+    assert result_path.exists()
+    with tarfile.open(result_path) as tf:
+        assert tf
+
+
+@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
+@pytest.mark.skipif(not has_executable("podman"), reason="no podman executable")
+def test_skopeo_copy_oci(tmp_path, stage_module):
+    result_path = _test_skopeo_copy(tmp_path, stage_module, "oci", "skopeo-archive-oci")
+    index_file = (result_path / "index.json")
+    assert index_file.exists()
+
+    data = json.loads(index_file.read_bytes())
+    assert data.get("manifests") is not None, "'index.json' seems corrupt - no 'manifests' section found"
+    assert data["manifests"][0].get("digest") is not None, \
+        "'manifest.json' seems corrupt - no 'manifests[0].digest' section found"
+    data_digest = data["manifests"][0]["digest"].split(':')
+    manifest_file = result_path / "blobs" / data_digest[0] / data_digest[1]
+    assert_manifest_file(manifest_file)
+
+
+@pytest.mark.parametrize("remove_signatures", [True, False])
+def test_skopeo_copy_remove_signatures(tmp_path, stage_module, remove_signatures):
+    # We are completely faking everything here, because we mock out skopeo
+    fake_oci_path = tmp_path / "fake-container"
+    inputs = make_fake_images_inputs(fake_oci_path, "some-name")
+
+    with testutil.mock_command("skopeo", "") as args:
+        options = {
+            "destination": {
+                "type": "oci",
+                "path": "/tmp/test-output-skopeo-dir",
+            },
+            "remove-signatures": remove_signatures,
+        }
+        output_dir = tmp_path / "output"
+        stage_module.main(inputs, output_dir, options)
+
+        if remove_signatures:
+            # Check that skopeo has --remove-signatures right after the copy subcommand
+            assert args.call_args_list[0][0:2] == ["copy", "--remove-signatures"]
+        else:
+            # Check that --remove-signatures is not present in the skopeo command
+            assert "--remove-signatures" not in args.call_args_list[0]
diff --git a/test/mod/test_util_mnt.py b/test/mod/test_util_mnt.py
index 51038774..259ac026 100644
--- a/test/mod/test_util_mnt.py
+++ b/test/mod/test_util_mnt.py
@@ -57,4 +57,4 @@ def test_osbuild_mount_failure_msg(tmp_path):
             "options": [],
         }
         mnt_service.mount(args)
-    assert "special device /dev/invalid-src does not exist" in str(e.value)
+    assert re.search(r"special device /dev/invalid-src does not exist|Can't open blockdev.", str(e.value))
diff --git a/test/mod/test_util_path.py b/test/mod/test_util_path.py
index cfeb1656..017671b2 100644
--- a/test/mod/test_util_path.py
+++ b/test/mod/test_util_path.py
@@ -3,6 +3,7 @@
 #
 import os
 import time
+from datetime import datetime
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -18,7 +19,10 @@ def tempdir_fixture():
 
 
 def test_clamp_mtime(tempdir):
-    start = int(time.time())
+    # This does not need to be precise, anything created within the last 1½h
+    # should be "clamped". By adding a bit of slack chrony, random
+    # RTC jumps and DST/timezone changes will not affect the test
+    clamp_start = int(time.time() - 90 * 60)
 
     tree = Path(tempdir, "tree")
     tree.mkdir()
@@ -33,12 +37,13 @@ def test_clamp_mtime(tempdir):
     link = Path(tree, "link")
     link.symlink_to(folder, target_is_directory=True)
 
-    timepoint = 1644758228
-    path.clamp_mtime(tree, start, timepoint)
+    # Some time way in the past
+    clamp_to = int(datetime(2022, 2, 13, 14, 17, 8).timestamp())
+    path.clamp_mtime(tree, clamp_start, clamp_to)
 
     def ensure_mtime(target, dfd):
         stat = os.stat(target, dir_fd=dfd, follow_symlinks=False)
-        assert stat.st_mtime <= timepoint, f"failed for {target} ({dfd})"
+        assert stat.st_mtime <= clamp_to, f"failed for {target} ({dfd})"
 
     for _, dirs, files, dfd in os.fwalk(tree):
         ensure_mtime(".", dfd)
diff --git a/tools/test/test_depsolve.py b/tools/test/test_depsolve.py
index 89b3507d..fb92cac0 100644
--- a/tools/test/test_depsolve.py
+++ b/tools/test/test_depsolve.py
@@ -225,7 +225,7 @@ def config_combos(tmp_path, servers):
                 "name": server["name"],
                 "baseurl": server["address"],
                 "check_gpg": False,
-                "ignoressl": True,
+                "sslverify": False,
                 "rhsm": False,
                 "gpgkeys": [TEST_KEY + server["name"]],
             })
@@ -277,3 +277,4 @@ def test_depsolve(tmp_path, cache_dir, repo_servers, dnf_cmd, detect_fn, test_ca
         assert res["repos"].keys() == test_case["results"]["reponames"]
         for repo in res["repos"].values():
             assert repo["gpgkeys"] == [TEST_KEY + repo["id"]]
+            assert repo["sslverify"] is False