Repository: osbuild/osbuild
Pull request #427 (2020-05-29 14:00:20)

Source verification
Add the necessary bits and pieces to do source verification and validation. 

teg: Nice!

Patch:
diff --git a/osbuild/meta.py b/osbuild/meta.py
index b836f2b3..81e37e73 100644
--- a/osbuild/meta.py
+++ b/osbuild/meta.py
@@ -1,10 +1,10 @@
 """Introspection and validation for osbuild
 
 This module contains utilities that help to introspect parts
-that constitute the inner parts of osbuild, i.e. its stages
-and assembler (which is also considered a type of stage in
-this context). Additionally, it provides classes and functions
-to do schema validation of OSBuild manifests and stage options.
+that constitute the inner parts of osbuild, i.e. its stages,
+assemblers and sources. Additionally, it provides classes and
+functions to do schema validation of OSBuild manifests and
+module options.
 
 A central `Index` class can be used to obtain stage and schema
 information. For the former a `ModuleInfo` class is returned via
@@ -264,7 +264,7 @@ class ModuleInfo:
     """Meta information about a stage
 
     Represents the information about a osbuild pipeline
-    modules, like a stage or an assembler.
+    modules, like a stage, assembler or source.
     Contains the short description (`desc`), a longer
     description (`info`) and the JSON schema of valid options
     (`opts`). The `validate` method will check a the options
@@ -288,15 +288,19 @@ class ModuleInfo:
             "title": f"Pipeline {self.type}",
             "type": "object",
             "additionalProperties": False,
-            "properties": {
+        }
+
+        if self.type in ("Stage", "Assembler"):
+            schema["properties"] = {
                 "name": {"type": "string"},
                 "options": {
                     "type": "object",
                     **self.opts
                 }
-            },
-            "required": ["name"]
-        }
+            }
+            schema["required"] = ["name"]
+        else:
+            schema.update(self.opts)
 
         # if there are is a definitions node, it needs to be at
         # the top level schema node, since the schema inside the
@@ -355,7 +359,8 @@ class ModuleInfo:
     def module_class_to_directory(klass: str) -> str:
         mapping = {
             "Stage": "stages",
-            "Assembler": "assemblers"
+            "Assembler": "assemblers",
+            "Source": "sources"
         }
 
         return mapping.get(klass)
@@ -413,7 +418,7 @@ class Index:
             with contextlib.suppress(FileNotFoundError):
                 with open(path, "r") as f:
                     schema = json.load(f)
-        elif klass in ["Stage", "Assembler"]:
+        elif klass in ["Stage", "Assembler", "Source"]:
             info = self.get_module_info(klass, name)
             if info:
                 schema = info.schema
@@ -468,4 +473,11 @@ def validate(manifest: Dict, index: Index) -> ValidationResult:
         res = schema.validate(asm)
         result.merge(res, path=["pipeline", "assembler"])
 
+    # sources
+    sources = manifest.get("sources", {})
+    for name, source in sources.items():
+        schema = index.get_schema("Source", name)
+        res = schema.validate(source)
+        result.merge(res, path=["sources", name])
+
     return result
diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile
index 54d513bf..d85732ad 100644
--- a/schutzbot/Jenkinsfile
+++ b/schutzbot/Jenkinsfile
@@ -28,6 +28,10 @@ pipeline {
                     }
                     steps {
                         sh "schutzbot/mockbuild.sh"
+                        stash (
+                            includes: 'osbuild-mock.repo',
+                            name: 'fedora31'
+                        )
                     }
                 }
                 stage('Fedora 32') {
@@ -37,6 +41,10 @@ pipeline {
                     }
                     steps {
                         sh "schutzbot/mockbuild.sh"
+                        stash (
+                            includes: 'osbuild-mock.repo',
+                            name: 'fedora32'
+                        )
                     }
                 }
                 stage('RHEL 8 CDN') {
@@ -46,6 +54,23 @@ pipeline {
                     }
                     steps {
                         sh "schutzbot/mockbuild.sh"
+                        stash (
+                            includes: 'osbuild-mock.repo',
+                            name: 'rhel8cdn'
+                        )
+                    }
+                }
+                stage('RHEL 8.3 Nightly') {
+                    agent { label "rhel83" }
+                    environment {
+                        OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
+                    }
+                    steps {
+                        sh "schutzbot/mockbuild.sh"
+                        stash (
+                            includes: 'osbuild-mock.repo',
+                            name: 'rhel83'
+                        )
                     }
                 }
             }
@@ -59,6 +84,7 @@ pipeline {
                     agent { label "fedora31" }
                     environment { TEST_TYPE = "image" }
                     steps {
+                        unstash 'fedora31'
                         run_tests()
                     }
                     post {
@@ -71,6 +97,7 @@ pipeline {
                     agent { label "fedora32" }
                     environment { TEST_TYPE = "image" }
                     steps {
+                        unstash 'fedora32'
                         run_tests()
                     }
                     post {
@@ -83,6 +110,7 @@ pipeline {
                     agent { label "rhel8" }
                     environment { TEST_TYPE = "image" }
                     steps {
+                        unstash 'rhel8cdn'
                         run_tests()
                     }
                     post {
diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh
index 54d821ae..65875a58 100755
--- a/schutzbot/mockbuild.sh
+++ b/schutzbot/mockbuild.sh
@@ -1,22 +1,26 @@
 #!/bin/bash
 set -euxo pipefail
 
+# Colorful output.
+function greenprint {
+  echo -e "\033[1;32m${1}\033[0m"
+}
+
 # Get OS details.
 source /etc/os-release
 
-# Install packages.
-sudo dnf -qy install createrepo_c mock
-if [[ $ID == 'fedora' ]]; then
-    sudo dnf -qy install python3-openstackclient
-else
-    sudo pip3 -qq install python-openstackclient
-fi
-
 # Set variables.
 CONTAINER=osbuildci-artifacts
 WORKSPACE=${WORKSPACE:-$(pwd)}
 MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)"
-REPO_DIR=repo/${BUILD_TAG}/${ID}${VERSION_ID//./}
+POST_MERGE_SHA=$(git rev-parse --short HEAD)
+REPO_DIR=repo/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
+REPO_URL=${MOCK_REPO_BASE_URL}/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
+
+# Print some data.
+greenprint "🧬 Using mock config: ${MOCK_CONFIG}"
+greenprint "📦 Post merge SHA: ${POST_MERGE_SHA}"
+greenprint "📤 RPMS will be uploaded to: ${REPO_URL}"
 
 # Clone osbuild-composer.
 # TODO(mhayden): After the next osbuild-composer release, use the latest tag
@@ -26,27 +30,52 @@ REPO_DIR=repo/${BUILD_TAG}/${ID}${VERSION_ID//./}
 git clone https://github.com/osbuild/osbuild-composer
 
 # Build source RPMs.
+greenprint "🔧 Building source RPMs."
 make srpm
 make -C osbuild-composer srpm
 
+# Fix RHEL 8 mock template for non-subscribed images.
+if [[ $NODE_NAME == *rhel8[23]* ]]; then
+    greenprint "📋 Updating RHEL 8 mock template for unsubscribed image"
+    sudo curl --retry 5 -Lsko /etc/mock/templates/rhel-8.tpl \
+        https://gitlab.cee.redhat.com/snippets/2208/raw
+fi
+
 # Compile RPMs in a mock chroot
+greenprint "🎁 Building RPMs with mock"
 sudo mock -r $MOCK_CONFIG --no-bootstrap-chroot \
     --resultdir $REPO_DIR --with=tests \
     rpmbuild/SRPMS/*.src.rpm osbuild-composer/rpmbuild/SRPMS/*.src.rpm
 sudo chown -R $USER ${REPO_DIR}
 
 # Move the logs out of the way.
+greenprint "🧹 Retaining logs from mock build"
 mv ${REPO_DIR}/*.log $WORKSPACE
 
 # Create a repo of the built RPMs.
+greenprint "⛓️ Creating dnf repository"
 createrepo_c ${REPO_DIR}
 
 # Prepare to upload to swift.
+greenprint "🛂 Setting up OpenStack authentication credentials"
 mkdir -p ~/.config/openstack
 cp $OPENSTACK_CREDS ~/.config/openstack/clouds.yml
 export OS_CLOUD=psi
 
 # Upload repository to swift.
+greenprint "☁ Uploading RPMs to OpenStack object storage"
 pushd repo
     find * -type f -print | xargs openstack object create -f value $CONTAINER
-popd
\ No newline at end of file
+popd
+
+# Create a repository file.
+greenprint "📜 Generating dnf repository file"
+tee osbuild-mock.repo << EOF
+[osbuild-mock]
+name=osbuild mock ${JOB_NAME}-${POST_MERGE_SHA} ${ID}${VERSION_ID//./}
+baseurl=${REPO_URL}
+enabled=1
+gpgcheck=0
+# Default dnf repo priority is 99. Lower number means higher priority.
+priority=5
+EOF
diff --git a/sources/org.osbuild.dnf b/sources/org.osbuild.dnf
index d67e4dbb..5529a3c9 100755
--- a/sources/org.osbuild.dnf
+++ b/sources/org.osbuild.dnf
@@ -1,9 +1,18 @@
 #!/usr/bin/python3
+"""
+Download packages via dnf
+"""
+
 
 import json
 import sys
 
 
+SCHEMA = """
+"additionalProperties": true
+"""
+
+
 def main(options, sources, secrets):
     repos = options.get("repos", {})
     repo_secrets = secrets.get("repos", {})
diff --git a/sources/org.osbuild.files b/sources/org.osbuild.files
index 99c5dbd5..cf9eec57 100755
--- a/sources/org.osbuild.files
+++ b/sources/org.osbuild.files
@@ -1,4 +1,17 @@
 #!/usr/bin/python3
+"""
+Source for downloading files from URLs.
+
+The files are indexed by their content hash. Can download files
+that require secrets. The only secret provider currently supported
+is `org.osbuild.rhsm` for downloading Red Hat content that requires
+a subscriptions.
+
+Internally use curl to download the files; the files are cached in
+an internal cache. Multiple parallel connections are used to speed
+up the download.
+"""
+
 
 import concurrent.futures
 import glob
@@ -10,6 +23,47 @@ import sys
 import tempfile
 
 
+SCHEMA = """
+"additionalProperties": false,
+"properties": {
+  "urls": {
+    "description": "The files to fetch indexed their content checksum",
+    "type": "object",
+    "additionalProperties": false,
+    "patternProperties": {
+      "(md5|sha1|sha256|sha384|sha512):[0-9a-f]{5,64}": {
+        "oneOf": [{
+          "type": "string",
+          "description": "URL to download the file from."
+        }, {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["url"],
+          "properties": {
+            "url": {
+              "type": "string",
+              "description": "URL to download the file from."
+            },
+            "secrets": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["name"],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Name of the secrets provider."
+                }
+              }
+            }
+          }
+        }]
+      }
+    }
+  }
+}
+"""
+
+
 def verify_checksum(filename, checksum):
     algorithm, checksum = checksum.split(":", 1)
     if algorithm not in ("md5", "sha1", "sha256", "sha384", "sha512"):
diff --git a/sources/org.osbuild.ostree b/sources/org.osbuild.ostree
index 6026d9a1..48024a16 100755
--- a/sources/org.osbuild.ostree
+++ b/sources/org.osbuild.ostree
@@ -1,4 +1,11 @@
 #!/usr/bin/python3
+"""Fetch OSTree commits from an repository
+
+Uses ostree to pull specific commits from (remote) repositories
+at the provided `url`. Can verify the commit, if one or more
+gpg keys are provided via `gpgkeys`.
+"""
+
 
 import json
 import os
@@ -7,6 +14,45 @@ import subprocess
 import uuid
 
 
+SCHEMA = """
+"additionalProperties": false,
+"properties": {
+  "commits": {
+    "description": "The commits to fetch indexed their checksum",
+    "type": "object",
+    "additionalProperties": false,
+    "patternProperties": {
+      "[0-9a-f]{5,64}": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["remote"],
+        "properties": {
+          "remote": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["url"],
+            "properties": {
+              "url": {
+                "type": "string",
+                "description": "URL of the repository."
+              },
+              "gpgkeys": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "description": "GPG keys to verify the commits"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+"""
+
+
 def ostree(*args, _input=None, **kwargs):
     args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()]
     print("ostree " + " ".join(args), file=sys.stderr)


Test patch:
diff --git a/schutzbot/run_tests.sh b/schutzbot/run_tests.sh
index d494c25a..16631071 100755
--- a/schutzbot/run_tests.sh
+++ b/schutzbot/run_tests.sh
@@ -5,18 +5,8 @@ set -euxo pipefail
 source /etc/os-release
 
 # Set up a dnf repository for the RPMs we built via mock.
-sudo tee /etc/yum.repos.d/osbuild-mock.repo > /dev/null << EOF
-[osbuild-mock]
-name=osbuild mock ${BUILD_TAG} ${ID}${VERSION_ID//./}
-baseurl=${MOCK_REPO_BASE_URL}/${BUILD_TAG}/${ID}${VERSION_ID//./}
-enabled=1
-gpgcheck=0
-# Default dnf repo priority is 99. Lower number means higher priority.
-priority=5
-EOF
-
-# Verify that the repository we added is working properly.
-sudo dnf list all | grep osbuild-mock
+sudo cp osbuild-mock.repo /etc/yum.repos.d/osbuild-mock.repo
+dnf repository-packages osbuild-mock list
 
 # Create temporary directories for Ansible.
 sudo mkdir -vp /opt/ansible_{local,remote}
@@ -38,9 +28,6 @@ trap "preserve_journal" ERR
 # Write a simple hosts file for Ansible.
 echo -e "[test_instances]\nlocalhost ansible_connection=local" > hosts.ini
 
-# Get the SHA of osbuild which Jenkins checked out for us.
-OSBUILD_VERSION=$(git rev-parse HEAD)
-
 # Deploy osbuild-composer and osbuild using RPMs built in a mock chroot.
 # NOTE(mhayden): Jenkins clones the repository and then merges the code from
 # the pull request into the repo. This creates a new SHA that exists only in
diff --git a/test/test_osbuild.py b/test/test_osbuild.py
index 0edd6675..56a5fb01 100644
--- a/test/test_osbuild.py
+++ b/test/test_osbuild.py
@@ -81,7 +81,7 @@ class TestDescriptions(unittest.TestCase):
         index = osbuild.meta.Index(os.curdir)
 
         modules = []
-        for klass in ("Stage", "Assembler"):
+        for klass in ("Stage", "Assembler", "Source"):
             mods = index.list_modules_for_class(klass)
             modules += [(klass, module) for module in mods]