Repository: osbuild/osbuild
Pull request #1655 (2024-03-11 08:40:36)

workflow: run all unit tests in parallel
There were special cases in the workflow for only running specific tests in parallel. However how that the test_assemblers parallel runs are fixed [0] there is really no need for special cases anymore and we can just run them all in parallel.

[0] https://github.com/osbuild/osbuild/pull/1641


Patch:
diff --git a/osbuild/buildroot.py b/osbuild/buildroot.py
index 85c3e192..9e16053b 100644
--- a/osbuild/buildroot.py
+++ b/osbuild/buildroot.py
@@ -138,6 +138,12 @@ class BuildRoot(contextlib.AbstractContextManager):
 
             self.var = os.path.join(self.tmp, "var")
             os.makedirs(self.var, exist_ok=True)
+            # Ensure /var/tmp is available, see
+            #  https://github.com/osbuild/bootc-image-builder/issues/223
+            try:
+                os.symlink("/tmp", os.path.join(self.var, "tmp"))
+            except FileExistsError:
+                pass
 
             proc = os.path.join(self.tmp, "proc")
             os.makedirs(proc)
diff --git a/osbuild/remoteloop.py b/osbuild/remoteloop.py
index 86e9c823..0544e0be 100644
--- a/osbuild/remoteloop.py
+++ b/osbuild/remoteloop.py
@@ -1,5 +1,4 @@
 import contextlib
-import errno
 import os
 
 from . import api, loop
@@ -43,25 +42,7 @@ class LoopServer(api.BaseAPI):
         self.ctl = loop.LoopControl()
 
     def _create_device(self, fd, dir_fd, offset=None, sizelimit=None):
-        while True:
-            # Getting an unbound loopback device and attaching a backing
-            # file descriptor to it is racy, so we must use a retry loop
-            lo = loop.Loop(self.ctl.get_unbound())
-            try:
-                lo.configure(fd, offset=offset, sizelimit=sizelimit, autoclear=True)
-            except BlockingIOError:
-                lo.clear_fd()
-                lo.close()
-                continue
-            except OSError as e:
-                lo.close()
-                # `configure` returns EBUSY when the pages from the previously
-                # bound file have not been fully cleared yet.
-                if e.errno == errno.EBUSY:
-                    continue
-                raise e
-            break
-
+        lo = self.ctl.loop_for_fd(fd, offset=offset, sizelimit=sizelimit, autoclear=True)
         lo.mknod(dir_fd)
         # Pin the Loop objects so they are only released when the LoopServer
         # is destroyed.
diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl
index 086c97ba..5967867b 100755
--- a/sources/org.osbuild.curl
+++ b/sources/org.osbuild.curl
@@ -2,13 +2,19 @@
 """
 Source for downloading files from URLs.
 
-The files are indexed by their content hash. Can download files
-that require secrets. The only secret provider currently supported
-is `org.osbuild.rhsm` for downloading Red Hat content that requires
-a subscriptions.
-
-Internally use curl to download the files; the files are cached in
-an internal cache. Multiple parallel connections are used to speed
+The files are indexed by their content hash. It can download files
+that require secrets. The secret providers currently supported are:
+
+- `org.osbuild.rhsm` for downloading Red Hat content that requires
+  a subscriptions.
+- `org.osbuild.mtls` for downloading content that requires client
+  certificats. The paths to the key and cert should be set in the
+  environment in OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY,
+  OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT, and optionally
+  OSBUILD_SOURCES_CURL_SSL_CA_CERT.
+
+It uses curl to download the files; the files are cached in an
+internal cache. Multiple parallel connections are used to speed
 up the download.
 """
 
@@ -107,6 +113,16 @@ class CurlSource(sources.SourceService):
             if self.subscriptions is None:
                 self.subscriptions = Subscriptions.from_host_system()
             url["secrets"] = self.subscriptions.get_secrets(url.get("url"))
+        elif url.get("secrets", {}).get("name") == "org.osbuild.mtls":
+            key = os.getenv("OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY")
+            cert = os.getenv("OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT")
+            if not (key and cert):
+                raise RuntimeError(f"mtls secrets required but key ({key}) or cert ({cert}) not defined")
+            url["secrets"] = {
+                'ssl_ca_cert': os.getenv("OSBUILD_SOURCES_CURL_SSL_CA_CERT"),
+                'ssl_client_cert': cert,
+                'ssl_client_key': key,
+            }
 
         return checksum, url
 
diff --git a/stages/org.osbuild.kickstart b/stages/org.osbuild.kickstart
index 2153af72..ad3ac02c 100755
--- a/stages/org.osbuild.kickstart
+++ b/stages/org.osbuild.kickstart
@@ -402,6 +402,10 @@ SCHEMA = r"""
       "nohome": {
         "description": "Disables automatic creation of the /home partition",
         "type": "boolean"
+      },
+      "noswap": {
+        "description": "Disables automatic creation of a swap partition",
+        "type": "boolean"
       }
     }
   },
@@ -536,7 +540,8 @@ def make_autopart(options: Dict) -> str:
     cmd = "autopart"
     for key in ["type", "fstype", "nolvm", "encrypted", "passphrase",
                 "escrowcert", "backuppassphrase", "cipher", "luks-version",
-                "pbkdf", "pbkdf-memory", "pbkdf-time", "pbkdf-iterations", "nohome"]:
+                "pbkdf", "pbkdf-memory", "pbkdf-time", "pbkdf-iterations", "nohome",
+                "noswap"]:
         if key not in autopart:
             continue
         val = autopart[key]


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fce74e8f..c2537e5a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,6 +6,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
+  TEST_WORKERS: "-n 4"
+  # Share the store between the workers speeds things up further
+  OSBUILD_TEST_STORE: /var/tmp/osbuild-test-store
+
 jobs:
   test_suite:
     name: "Unittest"
@@ -35,18 +41,6 @@ jobs:
       with:
         image: ghcr.io/osbuild/osbuild-ci:latest-202308241910
         run: |
-          # Note that only "test.run.test_stages" runs in parallel because
-          # the other tests are not sufficiently isolated and will cause
-          # random failures. But test_stages is the long running one with
-          # almost 2h.
-          if [ "${{ matrix.test }}" = "test.run.test_stages" ]; then
-              # Using 4 workers is a bit arbitrary, "auto" is probably too
-              # aggressive.
-              export TEST_WORKERS="-n 4"
-              # Share the store between the workers speeds things up further
-              export OSBUILD_TEST_STORE=/var/tmp/osbuild-test-store
-          fi
-
           # Hacky replacement of container storage driver:
           # The default overlayfs doesn't work in the runner, so let's change
           # it to vfs for the local storage skopeo stage test.
@@ -66,9 +60,5 @@ jobs:
         with:
           image: ghcr.io/osbuild/osbuild-ci:latest-202308241910
           run: |
-            # Using 4 workers is a bit arbitrary, "auto" is probably too
-            # aggressive.
-            export TEST_WORKERS="-n 4"
-            export OSBUILD_TEST_STORE=/var/tmp/osbuild-test-store
             TEST_CATEGORY="test.run.test_assemblers" \
             tox -e "py36"
diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py
new file mode 100644
index 00000000..8ac4c8fd
--- /dev/null
+++ b/sources/test/test_curl_source.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python3
+
+import contextlib
+import os
+import pathlib
+import socket
+import tempfile
+
+import pytest
+
+SOURCES_NAME = "org.osbuild.curl"
+
+
+def test_curl_source_not_exists(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+    }
+    checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+    assert not curl_source.exists(checksum, desc)
+
+
+def test_curl_source_exists(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+    }
+    checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+    pathlib.Path(os.path.join(tmpdir.name, checksum)).touch()
+    assert curl_source.exists(checksum, desc)
+
+
+def test_curl_source_transform(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+        "secrets": {
+            "name": "org.osbuild.mtls",
+        },
+    }
+
+    with contextlib.ExitStack() as cm:
+        os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY"] = "key"
+        os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT"] = "cert"
+
+        def cb():
+            del os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY"]
+            del os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT"]
+        cm.callback(cb)
+        checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+        pathlib.Path(os.path.join(tmpdir.name, checksum)).touch()
+        new_desc = curl_source.transform(checksum, desc)
+        assert new_desc[1]["secrets"]["ssl_client_key"] == "key"
+        assert new_desc[1]["secrets"]["ssl_client_cert"] == "cert"
+        assert new_desc[1]["secrets"]["ssl_ca_cert"] is None
+
+
+def test_curl_source_transform_fail(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+        "secrets": {
+            "name": "org.osbuild.mtls",
+        },
+    }
+    checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+    pathlib.Path(os.path.join(tmpdir.name, checksum)).touch()
+    with pytest.raises(RuntimeError) as exc:
+        curl_source.transform(checksum, desc)
+    assert "mtls secrets required" in str(exc)
diff --git a/stages/test/test_kickstart.py b/stages/test/test_kickstart.py
index 03ee9ee7..e62f4433 100644
--- a/stages/test/test_kickstart.py
+++ b/stages/test/test_kickstart.py
@@ -140,7 +140,10 @@ TEST_INPUT = [
     ({"autopart": {"pbkdf-time": 128}}, "autopart --pbkdf-time=128"),
     ({"autopart": {"pbkdf-iterations": 256}}, "autopart --pbkdf-iterations=256"),
     ({"autopart": {"nohome": True}}, "autopart --nohome"),
+    ({"autopart": {"noswap": True}}, "autopart --noswap"),
     ({"autopart": {"type": "plain", "fstype": "xfs", "nohome": True}}, "autopart --type=plain --fstype=xfs --nohome"),
+    ({"autopart": {"type": "plain", "fstype": "xfs", "nohome": True, "noswap": True}},
+     "autopart --type=plain --fstype=xfs --nohome --noswap"),
     ({
         "lang": "en_US.UTF-8",
         "keyboard": "us",
diff --git a/test/mod/test_buildroot.py b/test/mod/test_buildroot.py
index 8197caae..d7146338 100644
--- a/test/mod/test_buildroot.py
+++ b/test/mod/test_buildroot.py
@@ -44,11 +44,19 @@ def test_basic(tempdir, runner):
 
         # Test we can use `.run` multiple times
         r = root.run(["/usr/bin/true"], monitor)
-        assert r.returncode == 0
+        assert r.returncode == 0, f"{r.stdout} {r.stderr}"
 
         r = root.run(["/usr/bin/false"], monitor)
         assert r.returncode != 0
 
+        # Test that fs setup looks correct
+        r = root.run(["readlink", "-f", "/var/tmp"], monitor)
+        assert r.returncode == 0
+        assert r.stdout.strip().split("\n")[-1] == "/tmp"
+        r = root.run(["stat", "-L", "--format=%a", "/var/tmp"], monitor)
+        assert r.returncode == 0
+        assert r.stdout.strip().split("\n")[-1] == "1777"
+
 
 @pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
 def test_runner_fail(tempdir):