Repository: osbuild/osbuild
Pull request #727 (2021-07-15 09:34:08)

[RHEL8.4 branch] sources/curl: Implement new way of getting RHSM secrets with a fallback mechanism
This feature has been requested for backporting into RHEL8.4. As opposed to the "main" osbuild branch, this one contains a fallback to the previous behavior in case it magically worked for someone. At systems with proper subscription this should not be the case.

gicmo: I wonder if we should land the fallback in main first. Weird to have the z-stream have a feature that is not in main, no?
msehnout: > I wonder if we should land the fallback in main first. Weird to have the z-stream have a feature that is not in main, no?

The discussion around that topic is here: https://github.com/osbuild/osbuild/pull/726#issuecomment-883535913

Basically I didn't want to have the fallback in main, but seems like Brian found a usage for it in the cross-distro building.
teg: The same logic has landed upstream, we can merge this.

Patch:
diff --git a/osbuild/sources.py b/osbuild/sources.py
index 384180af..b045b9d3 100644
--- a/osbuild/sources.py
+++ b/osbuild/sources.py
@@ -79,8 +79,18 @@ class SourcesServer(api.BaseAPI):
             "libdir": self.libdir
         }
 
+        # We want the `osbuild` python package that contains this
+        # very module, which might be different from the system wide
+        # installed one, to be accessible to the Input programs so
+        # we detect our origin and set the `PYTHONPATH` accordingly
+        modorigin = importlib.util.find_spec("osbuild").origin
+        modpath = os.path.dirname(modorigin)
+        env = os.environ.copy()
+        env["PYTHONPATH"] = os.path.dirname(modpath)
+
         r = subprocess.run(
             [f"{self.libdir}/sources/{source}"],
+            env=env,
             input=json.dumps(msg),
             stdout=subprocess.PIPE,
             encoding="utf-8",
diff --git a/osbuild/util/rhsm.py b/osbuild/util/rhsm.py
new file mode 100644
index 00000000..21a2d502
--- /dev/null
+++ b/osbuild/util/rhsm.py
@@ -0,0 +1,108 @@
+"""Red Hat Subscription Manager support module
+
+This module implements utilities that help with interactions
+with the subscriptions attached to the host machine.
+"""
+
+import configparser
+import contextlib
+import glob
+import os
+import re
+
+
+class Subscriptions:
+    def __init__(self, repositories):
+        self.repositories = repositories
+        # These are used as a fallback if the repositories don't
+        # contain secrets for a requested URL.
+        self.secrets = None
+
+    def get_fallback_rhsm_secrets(self):
+        rhsm_secrets = {
+            'ssl_ca_cert': "/etc/rhsm/ca/redhat-uep.pem",
+            'ssl_client_key': "",
+            'ssl_client_cert': ""
+        }
+
+        keys = glob.glob("/etc/pki/entitlement/*-key.pem")
+        for key in keys:
+            # The key and cert have the same prefix
+            cert = key.rstrip("-key.pem") + ".pem"
+            # The key is only valid if it has a matching cert
+            if os.path.exists(cert):
+                rhsm_secrets['ssl_client_key'] = key
+                rhsm_secrets['ssl_client_cert'] = cert
+                # Once the dictionary is complete, assign it to the object
+                self.secrets = rhsm_secrets
+
+        raise RuntimeError("no matching rhsm key and cert")
+
+    @classmethod
+    def from_host_system(cls):
+        """Read redhat.repo file and process the list of repositories in there."""
+        ret = cls(None)
+        with contextlib.suppress(FileNotFoundError):
+            with open("/etc/yum.repos.d/redhat.repo", "r") as fp:
+                ret = cls.parse_repo_file(fp)
+
+        with contextlib.suppress(RuntimeError):
+            ret.get_fallback_rhsm_secrets()
+
+        if not ret.repositories and not ret.secrets:
+            raise RuntimeError("No RHSM secrets found on this host.")
+
+        return ret
+
+    @staticmethod
+    def _process_baseurl(input_url):
+        """Create a regex from a baseurl.
+
+        The osbuild manifest format does not contain information about repositories.
+        It only includes URLs of each RPM. In order to make this RHSM support work,
+        osbuild needs to find a relation between a "baseurl" in a *.repo file and the
+        URL given in the manifest. To do so, it creates a regex from all baseurls
+        found in the *.repo file and matches them against the URL.
+        """
+        # First escape meta characters that might occur in a URL
+        input_url = re.escape(input_url)
+
+        # Now replace variables with regexes (see man 5 yum.conf for the list)
+        for variable in ["\\$releasever", "\\$arch", "\\$basearch", "\\$uuid"]:
+            input_url = input_url.replace(variable, "[^/]*")
+
+        return re.compile(input_url)
+
+    @classmethod
+    def parse_repo_file(cls, fp):
+        """Take a file object and reads its content assuming it is a .repo file."""
+        parser = configparser.ConfigParser()
+        parser.read_file(fp)
+
+        repositories = dict()
+        for section in parser.sections():
+            current = {
+                "matchurl": cls._process_baseurl(parser.get(section, "baseurl"))
+            }
+            for parameter in ["sslcacert", "sslclientkey", "sslclientcert"]:
+                current[parameter] = parser.get(section, parameter)
+
+            repositories[section] = current
+
+        return cls(repositories)
+
+    def get_secrets(self, url):
+        # Try to find a matching URL from redhat.repo file first
+        for parameters in self.repositories.values():
+            if parameters["matchurl"].match(url) is not None:
+                return {
+                    "ssl_ca_cert": parameters["sslcacert"],
+                    "ssl_client_key": parameters["sslclientkey"],
+                    "ssl_client_cert": parameters["sslclientcert"]
+                }
+
+        # In case there is no matching URL, try the fallback
+        if self.secrets:
+            return self.secrets
+
+        raise RuntimeError(f"There are no RHSM secret associated with {url}")
diff --git a/sources/org.osbuild.curl b/sources/org.osbuild.curl
index 842d080c..cbfa163d 100755
--- a/sources/org.osbuild.curl
+++ b/sources/org.osbuild.curl
@@ -14,7 +14,6 @@ up the download.
 
 
 import concurrent.futures
-import glob
 import itertools
 import json
 import math
@@ -24,6 +23,8 @@ import sys
 import tempfile
 import time
 
+from osbuild.util.rhsm import Subscriptions
+
 
 SCHEMA = """
 "additionalProperties": false,
@@ -154,30 +155,10 @@ def fetch(url, checksum, directory):
             pass
 
 
-def get_rhsm_secrets():
-    rhsm_secrets = {
-        'ssl_ca_cert': "/etc/rhsm/ca/redhat-uep.pem",
-        'ssl_client_key': "",
-        'ssl_client_cert': ""
-    }
-
-    keys = glob.glob("/etc/pki/entitlement/*-key.pem")
-    for key in keys:
-        # The key and cert have the same prefix
-        cert = key.rstrip("-key.pem") + ".pem"
-        # The key is only valid if it has a matching cert
-        if os.path.exists(cert):
-            rhsm_secrets['ssl_client_key'] = key
-            rhsm_secrets['ssl_client_cert'] = cert
-            return rhsm_secrets
-
-    raise RuntimeError("no matching rhsm key and cert")
-
-
 def download(checksums, urls, cache):
     with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
         requested_urls = []
-        rhsm_secrets = None
+        subscriptions = None
 
         for checksum in checksums:
             try:
@@ -190,13 +171,13 @@ def download(checksums, urls, cache):
                 # check if url needs rhsm secrets
                 if url.get("secrets", {}).get("name") == "org.osbuild.rhsm":
                     # rhsm secrets only need to be retrieved once and can then be reused
-                    if rhsm_secrets is None:
-                        try:
-                            rhsm_secrets = get_rhsm_secrets()
-                        except RuntimeError as e:
-                            json.dump({"error": e.args[0]}, sys.stdout)
-                            return 1
-                    url["secrets"] = rhsm_secrets
+                    try:
+                        if subscriptions is None:
+                            subscriptions = Subscriptions.from_host_system()
+                        url["secrets"] = subscriptions.get_secrets(url.get("url"))
+                    except RuntimeError as e:
+                        json.dump({"error": e.args[0]}, sys.stdout)
+                        return 1
             else:
                 url = {"url": url}
 


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 691bd06d..e4b1e807 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -109,7 +109,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: codespell-project/actions-codespell@master
         with:
-          ignore_words_list: msdos
+          ignore_words_list: msdos, assmelber
           skip: ./.git,coverity,rpmbuild,samples
 
   schutzbot:
diff --git a/test/mod/test_util_rhsm.py b/test/mod/test_util_rhsm.py
new file mode 100644
index 00000000..6a1e28a9
--- /dev/null
+++ b/test/mod/test_util_rhsm.py
@@ -0,0 +1,68 @@
+#
+# Tests for the `osbuild.util.rhsm` module.
+#
+
+from io import StringIO
+
+from osbuild.util.rhsm import Subscriptions
+
+REPO_FILE = """[jpp]
+name = Red Hat JBoss Portal
+baseurl = https://cdn.redhat.com/1.0/$basearch/os
+enabled = 0
+gpgcheck = 1
+gpgkey = file://
+sslverify = 1
+sslcacert = /etc/rhsm/ca/redhat-uep.pem
+sslclientkey = /etc/pki/entitlement/1-key.pem
+sslclientcert = /etc/pki/entitlement/1.pem
+metadata_expire = 86400
+enabled_metadata = 0
+
+[jws]
+name = Red Hat JBoss Web
+baseurl = https://cdn.redhat.com/$releasever/jws/1.0/$basearch/os
+enabled = 0
+gpgcheck = 1
+gpgkey = file://
+sslverify = 1
+sslcacert = /etc/rhsm/ca/redhat-uep.pem
+sslclientkey = /etc/pki/entitlement/2-key.pem
+sslclientcert = /etc/pki/entitlement/2.pem
+metadata_expire = 86400
+enabled_metadata = 0
+"""
+
+
+def test_from_host_system():
+    subscriptions = Subscriptions.parse_repo_file(StringIO(REPO_FILE))
+    rpm_url_cases = [
+        {
+            "url": "https://cdn.redhat.com/8/jws/1.0/risc_v/os/Packages/fishy-fish-1-1.el8.risc_v.rpm",
+            "success": True,
+            "key": "2"
+        },
+        {
+            "url": "https://cdn.redhat.com/8/jws/1.0/os/Packages/fishy-fish-1-1.el8.risc_v.rpm",
+            "success": False,
+            "key": ""
+        },
+        {
+            "url": "https://cdn.redhat.com/1.0/x86_64/os/Packages/aaa.rpm",
+            "success": True,
+            "key": "1"
+        },
+    ]
+    for test_case in rpm_url_cases:
+        try:
+            secrets = subscriptions.get_secrets(test_case["url"])
+        except RuntimeError as e:
+            if not test_case["success"]:
+                continue
+
+            raise e
+
+        assert test_case["success"] # Verify this test case should pass
+        assert secrets["ssl_ca_cert"] == "/etc/rhsm/ca/redhat-uep.pem"
+        assert secrets["ssl_client_key"] == f'/etc/pki/entitlement/{test_case["key"]}-key.pem'
+        assert secrets["ssl_client_cert"] == f'/etc/pki/entitlement/{test_case["key"]}.pem'