Repository: osbuild/osbuild
Pull request #244 (2020-02-28 10:25:47)

pipeline: detect host instead of using org.osbuild.host
Detect the host dynamically from `/etc/os-release` instead of relying on
the `org.osbuild.host` symlink.

It is awkward to install a symlink that tells osbuild which distro is is
running on, when there is a standard way to detect this.

This makes it easier to run osbuild from sources and removes the need to
include every host in the spec file. The latter became hard to do,
because there's no obvious way to distinguish RHEL minor releases.

teg: Do we also need to check alternate locations for os-release?
larskarlitski: > Do we also need to check alternate locations for os-release?

Yes. Added.

Patch:
diff --git a/.gitignore b/.gitignore
index a4d4ca6c..469eb009 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,8 +10,6 @@ __pycache__
 
 /test/.vagrant
 
-/runners/org.osbuild.host
-
 /.vscode
 /.idea
 /.gdb_history
diff --git a/osbuild.spec b/osbuild.spec
index 23eb2be9..4f21f09c 100644
--- a/osbuild.spec
+++ b/osbuild.spec
@@ -70,23 +70,6 @@ install -p -m 0755 $(find sources -type f) %{buildroot}%{pkgdir}/sources
 mkdir -p %{buildroot}%{pkgdir}/stages/osbuild
 mkdir -p %{buildroot}%{pkgdir}/assemblers/osbuild
 
-# install host runner
-%if 0%{?fc30}
-ln -s org.osbuild.fedora30 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?fc31}
-ln -s org.osbuild.fedora31 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?fc32}
-ln -s org.osbuild.fedora32 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?fc33}
-ln -s org.osbuild.fedora33 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-%if 0%{?el8}
-ln -s org.osbuild.rhel82 %{buildroot}%{pkgdir}/runners/org.osbuild.host
-%endif
-
 %check
 exit 0
 # We have some integration tests, but those require running a VM, so that would
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index a435141c..abf3ce40 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -314,6 +314,33 @@ class Pipeline:
         return results
 
 
+def detect_os(*paths):
+    """Detect the os from an os-release file.
+
+    The first file in 'paths' is used and interpreted like a os-release(5)
+    file.
+
+    Returns ID + VERSION_ID (without dots), which is the same format that
+    runners are named as.
+    """
+    path = next((p for p in paths if os.path.exists(p)), None)
+    if not path:
+        raise FileNotFoundError("none of the specified os-release files exist")
+
+    with open(path) as f:
+        osrelease = {}
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            if line[0] == "#":
+                continue
+            key, value = line.split("=", 1)
+            osrelease[key] = value.strip('"')
+
+    return osrelease["ID"] + osrelease["VERSION_ID"].replace(".", "")
+
+
 def load_build(description, sources_options):
     pipeline = description.get("pipeline")
     if pipeline:
@@ -329,7 +356,7 @@ def load(description, sources_options):
     if build:
         build_pipeline, runner = load_build(build, sources_options)
     else:
-        build_pipeline, runner = None, "org.osbuild.host"
+        build_pipeline, runner = None, "org.osbuild." + detect_os("/etc/os-release", "/usr/lib/os-release")
 
     pipeline = Pipeline(runner, build_pipeline)
 
diff --git a/runners/org.osbuild.rhel81 b/runners/org.osbuild.rhel81
new file mode 100755
index 00000000..ae00cf34
--- /dev/null
+++ b/runners/org.osbuild.rhel81
@@ -0,0 +1,147 @@
+#!/usr/bin/python3.6
+
+import array
+import json
+import shutil
+import os
+import socket
+import subprocess
+import sys
+
+
+# copied from remoteloop.py
+def load_fds(sock, msglen):
+    fds = array.array("i")   # Array of ints
+    msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
+    for cmsg_level, cmsg_type, cmsg_data in ancdata:
+        if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
+            # Append data, ignoring any truncated integers at the end.
+            fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+    return json.loads(msg), list(fds), addr
+
+def ldconfig():
+    # ld.so.conf must exist, or `ldconfig` throws a warning
+    subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
+    subprocess.run(["ldconfig"], check=True)
+
+
+def sysusers():
+    try:
+        subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    except subprocess.CalledProcessError as error:
+        sys.stderr.write(error.stdout)
+        sys.exit(1)
+
+
+def update_ca_trust():
+    if not shutil.which("update-ca-trust"):
+        return
+
+    # generate /etc/pki/tls/certs/ca-bundle.crt
+    os.makedirs("/etc/pki/ca-trust/extracted/pem")
+    os.makedirs("/etc/pki/tls/certs")
+    os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
+
+    # allow to fail, because it sometimes mysteriously does
+    subprocess.run(["update-ca-trust", "extract"], check=False)
+
+
+def append_certs(cert_conf, dir_fd, parents=b""):
+    for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
+        if entry.is_file():
+            line = os.path.join(parents, entry.name)
+            cert_conf.write(line)
+            cert_conf.write(b"\n")
+        elif entry.is_dir():
+            append_certs(cert_conf,
+                         os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
+                         os.path.join(parents, entry.name))
+
+
+def update_ca_certificates():
+    if not shutil.which("update-ca-certificates"):
+        return
+
+    # generate /etc/ssl/certs/ca-certificates.crt
+    os.makedirs("/etc/ssl/certs")
+    with open("/etc/ca-certificates.conf", "wb") as f:
+        append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
+    subprocess.run(["update-ca-certificates"], check=True)
+
+
+def tmpfiles():
+    # Allow systemd-tmpfiles to return non-0. Some packages want to create
+    # directories owned by users that are not set up with systemd-sysusers.
+    subprocess.run(["systemd-tmpfiles", "--create"], check=False)
+
+
+def nsswitch():
+    # the default behavior is fine, but using nss-resolve does not
+    # necessarily work in a non-booted container, so make sure that
+    # is not configured.
+    try:
+        os.remove("/etc/nsswitch.conf")
+    except FileNotFoundError:
+        pass
+
+
+def setup_stdio():
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+        sock.connect("/run/osbuild/api/osbuild")
+        req = {'method': 'setup-stdio'}
+        sock.send(json.dumps(req).encode('utf-8'))
+        msg, fds, _ = load_fds(sock, 1024)
+        for io in ['stdin', 'stdout', 'stderr']:
+            target = getattr(sys, io)
+            source = fds[msg[io]]
+            os.dup2(source, target.fileno())
+            os.close(source)
+
+def os_release():
+    """/usr/lib/os-release doesn't exist. The `redhat-release` package
+    generates `/etc/os-release directly. To work around this, do the same here.
+
+    https://bugzilla.redhat.com/show_bug.cgi?id=1766754
+    """
+
+    # remove the symlink that systemd-nspawn creates
+    os.remove("/etc/os-release")
+    with open("/etc/os-release", "w") as f:
+        f.write('NAME="Red Hat Enterprise Linux"\n')
+        f.write('VERSION="8.1 (Ootpa)"\n')
+        f.write('ID="rhel"\n')
+        f.write('ID_LIKE="fedora"\n')
+        f.write('VERSION_ID="8.1"\n')
+        f.write('PLATFORM_ID="platform:el8"\n')
+        f.write('PRETTY_NAME="Red Hat Enterprise Linux 8.1 (Ootpa)"\n')
+        f.write('ANSI_COLOR="0;31"\n')
+        f.write('CPE_NAME="cpe:/o:redhat:enterprise_linux:8.1:GA"\n')
+        f.write('HOME_URL="https://www.redhat.com/"\n')
+        f.write('BUG_REPORT_URL="https://bugzilla.redhat.com/"\n')
+
+
+def python_alternatives():
+    """/usr/bin/python3 is a symlink to /etc/alternatives/python3, which points
+    to /usr/bin/python3.6 by default. Recreate the link in /etc, so that
+    shebang lines in stages and assemblers work.
+    """
+    os.makedirs("/etc/alternatives", exist_ok=True)
+    try:
+        os.symlink("/usr/bin/python3.6", "/etc/alternatives/python3")
+    except FileExistsError:
+        pass
+
+if __name__ == "__main__":
+    setup_stdio()
+    ldconfig()
+    sysusers()
+    update_ca_trust()
+    update_ca_certificates()
+    tmpfiles()
+    nsswitch()
+    os_release()
+    python_alternatives()
+
+    r = subprocess.run(sys.argv[1:], check=False)
+    sys.exit(r.returncode)


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 83a7e7bd..9a5edc6f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -51,6 +51,11 @@ jobs:
           cd osbuild
           python3 -m unittest -v test.test_objectstore
 
+      - name: Run test_osrelease
+        run: |
+          cd osbuild
+          python3 -m unittest -v test.test_osrelease
+
   rpm_build:
     name: "RPM build"
     needs:
diff --git a/test/os-release/fedora30 b/test/os-release/fedora30
new file mode 100644
index 00000000..9f0d60b3
--- /dev/null
+++ b/test/os-release/fedora30
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="30 (Workstation Edition)"
+ID=fedora
+VERSION_ID=30
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f30"
+PRETTY_NAME="Fedora 30 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:30"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=30
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=30
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/fedora31 b/test/os-release/fedora31
new file mode 100644
index 00000000..72050278
--- /dev/null
+++ b/test/os-release/fedora31
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="31 (Workstation Edition)"
+ID=fedora
+VERSION_ID=31
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f31"
+PRETTY_NAME="Fedora 31 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:31"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f31/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=31
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=31
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/fedora32 b/test/os-release/fedora32
new file mode 100644
index 00000000..e8f9713d
--- /dev/null
+++ b/test/os-release/fedora32
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="32 (Workstation Edition)"
+ID=fedora
+VERSION_ID=32
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f32"
+PRETTY_NAME="Fedora 32 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:32"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/%{doc_version}/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=32
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=32
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/fedora33 b/test/os-release/fedora33
new file mode 100644
index 00000000..eb3d8d50
--- /dev/null
+++ b/test/os-release/fedora33
@@ -0,0 +1,21 @@
+NAME=Fedora
+VERSION="33 (Workstation Edition)"
+ID=fedora
+VERSION_ID=33
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f30"
+PRETTY_NAME="Fedora 33 (Workstation Edition)"
+ANSI_COLOR="0;34"
+LOGO=fedora-logo-icon
+CPE_NAME="cpe:/o:fedoraproject:fedora:33"
+HOME_URL="https://fedoraproject.org/"
+DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f30/system-administrators-guide/"
+SUPPORT_URL="https://fedoraproject.org/wiki/Communicating_and_getting_help"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+REDHAT_BUGZILLA_PRODUCT="Fedora"
+REDHAT_BUGZILLA_PRODUCT_VERSION=33
+REDHAT_SUPPORT_PRODUCT="Fedora"
+REDHAT_SUPPORT_PRODUCT_VERSION=33
+PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation
diff --git a/test/os-release/rhel81 b/test/os-release/rhel81
new file mode 100644
index 00000000..216c1d22
--- /dev/null
+++ b/test/os-release/rhel81
@@ -0,0 +1,16 @@
+NAME="Red Hat Enterprise Linux"
+VERSION="8.1 (Ootpa)"
+ID="rhel"
+ID_LIKE="fedora"
+VERSION_ID="8.1"
+PLATFORM_ID="platform:el8"
+PRETTY_NAME="Red Hat Enterprise Linux 8.1 (Ootpa)"
+ANSI_COLOR="0;31"
+CPE_NAME="cpe:/o:redhat:enterprise_linux:8.1:GA"
+HOME_URL="https://www.redhat.com/"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
+REDHAT_BUGZILLA_PRODUCT_VERSION=8.1
+REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+REDHAT_SUPPORT_PRODUCT_VERSION="8.1"
diff --git a/test/os-release/rhel82 b/test/os-release/rhel82
new file mode 100644
index 00000000..30542561
--- /dev/null
+++ b/test/os-release/rhel82
@@ -0,0 +1,16 @@
+NAME="Red Hat Enterprise Linux"
+VERSION="8.2 (Ootpa)"
+ID="rhel"
+ID_LIKE="fedora"
+VERSION_ID="8.2"
+PLATFORM_ID="platform:el8"
+PRETTY_NAME="Red Hat Enterprise Linux 8.2 Beta (Ootpa)"
+ANSI_COLOR="0;31"
+CPE_NAME="cpe:/o:redhat:enterprise_linux:8.2:beta"
+HOME_URL="https://www.redhat.com/"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8"
+REDHAT_BUGZILLA_PRODUCT_VERSION=8.2
+REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+REDHAT_SUPPORT_PRODUCT_VERSION="8.2 Beta"
diff --git a/test/os-release/ubuntu1804 b/test/os-release/ubuntu1804
new file mode 100644
index 00000000..dc5a5c2a
--- /dev/null
+++ b/test/os-release/ubuntu1804
@@ -0,0 +1,12 @@
+NAME="Ubuntu"
+VERSION="18.04.3 LTS (Bionic Beaver)"
+ID=ubuntu
+ID_LIKE=debian
+PRETTY_NAME="Ubuntu 18.04.3 LTS"
+VERSION_ID="18.04"
+HOME_URL="https://www.ubuntu.com/"
+SUPPORT_URL="https://help.ubuntu.com/"
+BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
+PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
+VERSION_CODENAME=bionic
+UBUNTU_CODENAME=bionic
diff --git a/test/test_osrelease.py b/test/test_osrelease.py
new file mode 100644
index 00000000..1c8578cd
--- /dev/null
+++ b/test/test_osrelease.py
@@ -0,0 +1,17 @@
+import os
+import unittest
+
+import osbuild
+
+class TestOSRelease(unittest.TestCase):
+    def test_non_existant(self):
+        self.assertRaises(FileNotFoundError, osbuild.pipeline.detect_os, "💩")
+
+    def test_detect_os(self):
+        """Test host os detection. test/os-release contains the os-release files
+        for all supported runners.
+        """
+        for entry in os.scandir("test/os-release"):
+            with self.subTest(entry.name):
+                self.assertEqual(osbuild.pipeline.detect_os(entry.path), entry.name)
+