Repository: osbuild/osbuild
Pull request #1649 (2024-03-07 09:56:21)

osbuild: ensure a usable /var/tmp is available inside the buildroot
Colin asked for this in
https://github.com/osbuild/bootc-image-builder/issues/223 and it's easy enough.


Patch:
diff --git a/osbuild/buildroot.py b/osbuild/buildroot.py
index 85c3e192..9e16053b 100644
--- a/osbuild/buildroot.py
+++ b/osbuild/buildroot.py
@@ -138,6 +138,12 @@ class BuildRoot(contextlib.AbstractContextManager):
 
             self.var = os.path.join(self.tmp, "var")
             os.makedirs(self.var, exist_ok=True)
+            # Ensure /var/tmp is available, see
+            #  https://github.com/osbuild/bootc-image-builder/issues/223
+            try:
+                os.symlink("/tmp", os.path.join(self.var, "tmp"))
+            except FileExistsError:
+                pass
 
             proc = os.path.join(self.tmp, "proc")
             os.makedirs(proc)
diff --git a/osbuild/remoteloop.py b/osbuild/remoteloop.py
index 86e9c823..0544e0be 100644
--- a/osbuild/remoteloop.py
+++ b/osbuild/remoteloop.py
@@ -1,5 +1,4 @@
 import contextlib
-import errno
 import os
 
 from . import api, loop
@@ -43,25 +42,7 @@ class LoopServer(api.BaseAPI):
         self.ctl = loop.LoopControl()
 
     def _create_device(self, fd, dir_fd, offset=None, sizelimit=None):
-        while True:
-            # Getting an unbound loopback device and attaching a backing
-            # file descriptor to it is racy, so we must use a retry loop
-            lo = loop.Loop(self.ctl.get_unbound())
-            try:
-                lo.configure(fd, offset=offset, sizelimit=sizelimit, autoclear=True)
-            except BlockingIOError:
-                lo.clear_fd()
-                lo.close()
-                continue
-            except OSError as e:
-                lo.close()
-                # `configure` returns EBUSY when the pages from the previously
-                # bound file have not been fully cleared yet.
-                if e.errno == errno.EBUSY:
-                    continue
-                raise e
-            break
-
+        lo = self.ctl.loop_for_fd(fd, offset=offset, sizelimit=sizelimit, autoclear=True)
         lo.mknod(dir_fd)
         # Pin the Loop objects so they are only released when the LoopServer
         # is destroyed.
diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl
index 086c97ba..5967867b 100755
--- a/sources/org.osbuild.curl
+++ b/sources/org.osbuild.curl
@@ -2,13 +2,19 @@
 """
 Source for downloading files from URLs.
 
-The files are indexed by their content hash. Can download files
-that require secrets. The only secret provider currently supported
-is `org.osbuild.rhsm` for downloading Red Hat content that requires
-a subscriptions.
-
-Internally use curl to download the files; the files are cached in
-an internal cache. Multiple parallel connections are used to speed
+The files are indexed by their content hash. It can download files
+that require secrets. The secret providers currently supported are:
+
+- `org.osbuild.rhsm` for downloading Red Hat content that requires
+  a subscriptions.
+- `org.osbuild.mtls` for downloading content that requires client
+  certificats. The paths to the key and cert should be set in the
+  environment in OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY,
+  OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT, and optionally
+  OSBUILD_SOURCES_CURL_SSL_CA_CERT.
+
+It uses curl to download the files; the files are cached in an
+internal cache. Multiple parallel connections are used to speed
 up the download.
 """
 
@@ -107,6 +113,16 @@ class CurlSource(sources.SourceService):
             if self.subscriptions is None:
                 self.subscriptions = Subscriptions.from_host_system()
             url["secrets"] = self.subscriptions.get_secrets(url.get("url"))
+        elif url.get("secrets", {}).get("name") == "org.osbuild.mtls":
+            key = os.getenv("OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY")
+            cert = os.getenv("OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT")
+            if not (key and cert):
+                raise RuntimeError(f"mtls secrets required but key ({key}) or cert ({cert}) not defined")
+            url["secrets"] = {
+                'ssl_ca_cert': os.getenv("OSBUILD_SOURCES_CURL_SSL_CA_CERT"),
+                'ssl_client_cert': cert,
+                'ssl_client_key': key,
+            }
 
         return checksum, url
 
diff --git a/stages/org.osbuild.kickstart b/stages/org.osbuild.kickstart
index 2153af72..ad3ac02c 100755
--- a/stages/org.osbuild.kickstart
+++ b/stages/org.osbuild.kickstart
@@ -402,6 +402,10 @@ SCHEMA = r"""
       "nohome": {
         "description": "Disables automatic creation of the /home partition",
         "type": "boolean"
+      },
+      "noswap": {
+        "description": "Disables automatic creation of a swap partition",
+        "type": "boolean"
       }
     }
   },
@@ -536,7 +540,8 @@ def make_autopart(options: Dict) -> str:
     cmd = "autopart"
     for key in ["type", "fstype", "nolvm", "encrypted", "passphrase",
                 "escrowcert", "backuppassphrase", "cipher", "luks-version",
-                "pbkdf", "pbkdf-memory", "pbkdf-time", "pbkdf-iterations", "nohome"]:
+                "pbkdf", "pbkdf-memory", "pbkdf-time", "pbkdf-iterations", "nohome",
+                "noswap"]:
         if key not in autopart:
             continue
         val = autopart[key]


Test patch:
diff --git a/sources/test/test_curl_source.py b/sources/test/test_curl_source.py
new file mode 100644
index 00000000..8ac4c8fd
--- /dev/null
+++ b/sources/test/test_curl_source.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python3
+
+import contextlib
+import os
+import pathlib
+import socket
+import tempfile
+
+import pytest
+
+SOURCES_NAME = "org.osbuild.curl"
+
+
+def test_curl_source_not_exists(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+    }
+    checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+    assert not curl_source.exists(checksum, desc)
+
+
+def test_curl_source_exists(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+    }
+    checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+    pathlib.Path(os.path.join(tmpdir.name, checksum)).touch()
+    assert curl_source.exists(checksum, desc)
+
+
+def test_curl_source_transform(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+        "secrets": {
+            "name": "org.osbuild.mtls",
+        },
+    }
+
+    with contextlib.ExitStack() as cm:
+        os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY"] = "key"
+        os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT"] = "cert"
+
+        def cb():
+            del os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_KEY"]
+            del os.environ["OSBUILD_SOURCES_CURL_SSL_CLIENT_CERT"]
+        cm.callback(cb)
+        checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+        pathlib.Path(os.path.join(tmpdir.name, checksum)).touch()
+        new_desc = curl_source.transform(checksum, desc)
+        assert new_desc[1]["secrets"]["ssl_client_key"] == "key"
+        assert new_desc[1]["secrets"]["ssl_client_cert"] == "cert"
+        assert new_desc[1]["secrets"]["ssl_ca_cert"] is None
+
+
+def test_curl_source_transform_fail(sources_module):
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    curl_source = sources_module.CurlSource.from_args(["--service-fd", str(sock.fileno())])
+    tmpdir = tempfile.TemporaryDirectory()
+    curl_source.cache = tmpdir.name
+    desc = {
+        "url": "http://localhost:80/a",
+        "secrets": {
+            "name": "org.osbuild.mtls",
+        },
+    }
+    checksum = "sha256:1234567890123456789012345678901234567890909b14ffb032aa20fa23d9ad6"
+    pathlib.Path(os.path.join(tmpdir.name, checksum)).touch()
+    with pytest.raises(RuntimeError) as exc:
+        curl_source.transform(checksum, desc)
+    assert "mtls secrets required" in str(exc)
diff --git a/stages/test/test_kickstart.py b/stages/test/test_kickstart.py
index 03ee9ee7..e62f4433 100644
--- a/stages/test/test_kickstart.py
+++ b/stages/test/test_kickstart.py
@@ -140,7 +140,10 @@ TEST_INPUT = [
     ({"autopart": {"pbkdf-time": 128}}, "autopart --pbkdf-time=128"),
     ({"autopart": {"pbkdf-iterations": 256}}, "autopart --pbkdf-iterations=256"),
     ({"autopart": {"nohome": True}}, "autopart --nohome"),
+    ({"autopart": {"noswap": True}}, "autopart --noswap"),
     ({"autopart": {"type": "plain", "fstype": "xfs", "nohome": True}}, "autopart --type=plain --fstype=xfs --nohome"),
+    ({"autopart": {"type": "plain", "fstype": "xfs", "nohome": True, "noswap": True}},
+     "autopart --type=plain --fstype=xfs --nohome --noswap"),
     ({
         "lang": "en_US.UTF-8",
         "keyboard": "us",
diff --git a/stages/test/test_systemd_unit_create.py b/stages/test/test_systemd_unit_create.py
index 74e05f48..b0f251e0 100644
--- a/stages/test/test_systemd_unit_create.py
+++ b/stages/test/test_systemd_unit_create.py
@@ -46,9 +46,17 @@ def test_schema_validation(stage_schema, test_data, expected_err):
         testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
-def test_systemd_unit_create_simple(tmp_path, stage_module):
+@pytest.mark.parametrize("unit_type,unit_path,expected_prefix", [
+    ("system", "usr", "usr/lib/systemd/system"),
+    ("system", "etc", "etc/systemd/system"),
+    ("global", "usr", "usr/lib/systemd/user"),
+    ("global", "etc", "etc/systemd/user"),
+])
+def test_systemd_unit_create(tmp_path, stage_module, unit_type, unit_path, expected_prefix):
     options = {
         "filename": "create-directory.service",
+        "unit-type": unit_type,
+        "unit-path": unit_path,
         "config": {
             "Unit": {
                 "Description": "Create directory",
@@ -75,7 +83,7 @@ def test_systemd_unit_create_simple(tmp_path, stage_module):
             }
         }
     }
-    expected_unit_path = tmp_path / "usr/lib/systemd/system/create-directory.service"
+    expected_unit_path = tmp_path / expected_prefix / "create-directory.service"
     # should the stage create the dir?
     expected_unit_path.parent.mkdir(parents=True)
 
diff --git a/test/mod/test_buildroot.py b/test/mod/test_buildroot.py
index 8197caae..d7146338 100644
--- a/test/mod/test_buildroot.py
+++ b/test/mod/test_buildroot.py
@@ -44,11 +44,19 @@ def test_basic(tempdir, runner):
 
         # Test we can use `.run` multiple times
         r = root.run(["/usr/bin/true"], monitor)
-        assert r.returncode == 0
+        assert r.returncode == 0, f"{r.stdout} {r.stderr}"
 
         r = root.run(["/usr/bin/false"], monitor)
         assert r.returncode != 0
 
+        # Test that fs setup looks correct
+        r = root.run(["readlink", "-f", "/var/tmp"], monitor)
+        assert r.returncode == 0
+        assert r.stdout.strip().split("\n")[-1] == "/tmp"
+        r = root.run(["stat", "-L", "--format=%a", "/var/tmp"], monitor)
+        assert r.returncode == 0
+        assert r.stdout.strip().split("\n")[-1] == "1777"
+
 
 @pytest.mark.skipif(not TestBase.can_bind_mount(), reason="root only")
 def test_runner_fail(tempdir):