Repository: osbuild/osbuild-composer
Pull request #2252 (2022-02-03 16:04:29)

service-maintenance: Skip db cleanup
Let's enable the cloud cleanup first, and then move on the db.





Patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ba0ddce9..db9d7a4db 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,7 @@ stages:
     - schutzbot/update_github_status.sh update
     - schutzbot/save_journal.sh
     - schutzbot/run_cloud_cleaner.sh
+  interruptible: true
   tags:
     - terraform
   artifacts:
@@ -36,7 +37,15 @@ init:
     - shell
   script:
     - schutzbot/update_github_status.sh start
-  interruptible: true
+
+
+.RPM_RUNNERS_RHEL: &RPM_RUNNERS_RHEL
+  RUNNER:
+    - aws/rhel-8.6-nightly-x86_64
+    - aws/rhel-8.6-nightly-aarch64
+    - aws/rhel-9.0-nightly-x86_64
+    - aws/rhel-9.0-nightly-aarch64
+  INTERNAL_NETWORK: ["true"]
 
 RPM:
   stage: rpmbuild
@@ -45,7 +54,6 @@ RPM:
     - if: '$CI_PIPELINE_SOURCE != "schedule"'
   script:
     - sh "schutzbot/mockbuild.sh"
-  interruptible: true
   after_script:
     - schutzbot/update_github_status.sh update
     - schutzbot/save_journal.sh
@@ -64,12 +72,26 @@ RPM:
           - aws/rhel-8.5-ga-aarch64
           - aws/centos-stream-8-x86_64
           - aws/centos-stream-8-aarch64
-      - RUNNER:
-          - aws/rhel-8.6-nightly-x86_64
-          - aws/rhel-8.6-nightly-aarch64
-          - aws/rhel-9.0-nightly-x86_64
-          - aws/rhel-9.0-nightly-aarch64
-        INTERNAL_NETWORK: ["true"]
+      - <<: *RPM_RUNNERS_RHEL
+
+Build -tests RPM for RHEL:
+  stage: rpmbuild
+  extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
+  script:
+    - sh "schutzbot/mockbuild.sh"
+  interruptible: true
+  after_script:
+    - schutzbot/update_github_status.sh update
+    - schutzbot/save_journal.sh
+  artifacts:
+    paths:
+      - journal-log.gpg
+  parallel:
+    matrix:
+      - <<: *RPM_RUNNERS_RHEL
 
 Container:
   stage: rpmbuild
@@ -78,7 +100,6 @@ Container:
     - if: '$CI_PIPELINE_SOURCE != "schedule"'
   script:
     - sh "schutzbot/containerbuild.sh"
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -102,7 +123,6 @@ Prepare-rhel-internal:
     - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
   script:
     - schutzbot/prepare-rhel-internal.sh
-  interruptible: true
   artifacts:
     paths:
       - rhel-${RHEL_MAJOR}.json
@@ -128,7 +148,6 @@ Base:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/base_tests.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -183,7 +202,6 @@ Regression:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/regression.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -217,7 +235,6 @@ Image Tests:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/image_tests.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -242,7 +259,6 @@ Test Case Generation:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/generation.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -264,7 +280,6 @@ OSTree:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/ostree.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -334,15 +349,11 @@ Integration:
     - /usr/libexec/tests/osbuild-composer/${SCRIPT}
   parallel:
     matrix:
-      - SCRIPT:
-          - koji.sh
-          - aws.sh
+      - <<: *INTEGRATION_TESTS
         RUNNER:
           - aws/fedora-34-x86_64
           - aws/fedora-35-x86_64
           - aws/centos-stream-8-x86_64
-      - <<: *INTEGRATION_TESTS
-        RUNNER:
           - aws/rhel-8.5-ga-x86_64
           - aws/rhel-8.6-nightly-x86_64
           - aws/rhel-9.0-nightly-x86_64
@@ -365,8 +376,8 @@ API:
   extends: .terraform
   rules:
     - if: '$CI_PIPELINE_SOURCE != "schedule"'
-    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
-    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
+    # note: cloud API is not supported for on-prem installations so
+    # don't run this test case for nightly trees
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/api.sh ${IMAGE_TYPE}
@@ -389,7 +400,6 @@ libvirt:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/libvirt.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -408,7 +418,6 @@ RHEL 9 on 8:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/koji.sh
-  interruptible: true
   variables:
     RUNNER: aws/rhel-8.5-ga-x86_64
     INTERNAL_NETWORK: "true"
@@ -463,7 +472,6 @@ Installer:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/installers.sh
-  interruptible: true
   parallel:
     matrix:
       - RUNNER:
@@ -486,6 +494,8 @@ SCHEDULED_CLOUD_CLEANER:
 SonarQube:
   stage: test
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
   script:
     - schutzbot/sonarqube.sh
   variables:
diff --git a/cmd/osbuild-service-maintenance/main.go b/cmd/osbuild-service-maintenance/main.go
index 1e375c9a9..8f2e6b25d 100644
--- a/cmd/osbuild-service-maintenance/main.go
+++ b/cmd/osbuild-service-maintenance/main.go
@@ -39,19 +39,6 @@ func main() {
 		logrus.Info("Dry run, no state will be changed")
 	}
 
-	dbURL := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s",
-		conf.PGUser,
-		conf.PGPassword,
-		conf.PGHost,
-		conf.PGPort,
-		conf.PGDatabase,
-		conf.PGSSLMode,
-	)
-	jobs, err := dbjobqueue.New(dbURL)
-	if err != nil {
-		panic(err)
-	}
-
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
@@ -94,6 +81,24 @@ func main() {
 	wg.Wait()
 	logrus.Info("🦀🦀🦀 cloud cleanup done 🦀🦀🦀")
 
+	if conf.PGHost == "" {
+		logrus.Info("🦀🦀🦀 db host not defined, skipping db cleanup 🦀🦀🦀")
+		return
+	}
+
+	dbURL := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s",
+		conf.PGUser,
+		conf.PGPassword,
+		conf.PGHost,
+		conf.PGPort,
+		conf.PGDatabase,
+		conf.PGSSLMode,
+	)
+	jobs, err := dbjobqueue.New(dbURL)
+	if err != nil {
+		panic(err)
+	}
+
 	var jobTypes []string
 	for _, a := range archs {
 		jobTypes = append(jobTypes, fmt.Sprintf("%s:%s", jobType, a))
diff --git a/cmd/osbuild-worker/jobimpl-osbuild-koji.go b/cmd/osbuild-worker/jobimpl-osbuild-koji.go
index ba87a904c..24dee508c 100644
--- a/cmd/osbuild-worker/jobimpl-osbuild-koji.go
+++ b/cmd/osbuild-worker/jobimpl-osbuild-koji.go
@@ -56,12 +56,36 @@ func (impl *OSBuildKojiJobImpl) kojiUpload(file *os.File, server, directory, fil
 	return k.Upload(file, directory, filename)
 }
 
+func validateKojiResult(result *worker.OSBuildKojiJobResult, jobID string) {
+	logWithId := logrus.WithField("jobId", jobID)
+	if result.JobError != nil {
+		logWithId.Errorf("osbuild job failed: %s", result.JobError.Reason)
+		return
+	}
+	// if the job failed, but the JobError is
+	// nil, we still need to handle this as an error
+	if result.OSBuildOutput == nil || !result.OSBuildOutput.Success {
+		reason := "osbuild job was unsuccessful"
+		logWithId.Errorf("osbuild job failed: %s", reason)
+		result.JobError = clienterrors.WorkerClientError(clienterrors.ErrorBuildJob, reason)
+	}
+}
+
 func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
+	var result worker.OSBuildKojiJobResult
 	outputDirectory, err := ioutil.TempDir(impl.Output, job.Id().String()+"-*")
 	if err != nil {
 		return fmt.Errorf("error creating temporary output directory: %v", err)
 	}
 	defer func() {
+		validateKojiResult(&result, job.Id().String())
+
+		// this is necessary for early return errors
+		err = job.Update(&result)
+		if err != nil {
+			logrus.Warnf("Error reporting job result: %v", err)
+		}
+
 		err := os.RemoveAll(outputDirectory)
 		if err != nil {
 			logrus.Warnf("Error removing temporary output directory (%s): %v", outputDirectory, err)
@@ -80,7 +104,6 @@ func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
 		return err
 	}
 
-	var result worker.OSBuildKojiJobResult
 	result.Arch = common.CurrentArch()
 	result.HostOS, err = distro.GetRedHatRelease()
 	if err != nil {
@@ -93,6 +116,7 @@ func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
 			var manifestJR worker.ManifestJobByIDResult
 			err = job.DynamicArgs(1, &manifestJR)
 			if err != nil {
+				result.JobError = clienterrors.WorkerClientError(clienterrors.ErrorParsingDynamicArgs, "Error parsing dynamic args")
 				return err
 			}
 
@@ -103,10 +127,14 @@ func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
 			}
 			args.Manifest = manifestJR.Manifest
 			if len(args.Manifest) == 0 {
-				return fmt.Errorf("received empty manifest")
+				err := fmt.Errorf("Received empty manifest")
+				result.JobError = clienterrors.WorkerClientError(clienterrors.ErrorManifestDependency, err.Error())
+				return err
 			}
 		} else {
-			return fmt.Errorf("job has no manifest")
+			err := fmt.Errorf("Job has no manifest")
+			result.JobError = clienterrors.WorkerClientError(clienterrors.ErrorManifestDependency, err.Error())
+			return err
 		}
 	}
 
@@ -118,7 +146,9 @@ func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
 			exports = []string{"assembler"}
 		} else if len(exports) > 1 {
 			// this worker only supports returning one (1) export
-			return fmt.Errorf("at most one build artifact can be exported")
+			err = fmt.Errorf("at most one build artifact can be exported")
+			result.JobError = clienterrors.WorkerClientError(clienterrors.ErrorBuildJob, err.Error())
+			return err
 		}
 		result.OSBuildOutput, err = RunOSBuild(args.Manifest, impl.Store, outputDirectory, exports, os.Stderr)
 		if err != nil {
diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go
index 5f0dae4c2..bfaba5078 100644
--- a/cmd/osbuild-worker/jobimpl-osbuild.go
+++ b/cmd/osbuild-worker/jobimpl-osbuild.go
@@ -57,7 +57,7 @@ func validateResult(result *worker.OSBuildJobResult, jobID string) {
 	}
 	// if the job failed, but the JobError is
 	// nil, we still need to handle this as an error
-	if !result.OSBuildOutput.Success {
+	if result.OSBuildOutput == nil || !result.OSBuildOutput.Success {
 		reason := "osbuild job was unsuccessful"
 		logWithId.Errorf("osbuild job failed: %s", reason)
 		result.JobError = clienterrors.WorkerClientError(clienterrors.ErrorBuildJob, reason)
@@ -111,6 +111,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 		var manifestJR worker.ManifestJobByIDResult
 		err = job.DynamicArgs(0, &manifestJR)
 		if err != nil {
+			osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorParsingDynamicArgs, "Error parsing dynamic args")
 			return err
 		}
 
diff --git a/go.mod b/go.mod
index c1085d922..363f6bcd9 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	cloud.google.com/go v0.100.0
-	cloud.google.com/go/cloudbuild v0.2.0
+	cloud.google.com/go/cloudbuild v1.0.0
 	cloud.google.com/go/storage v1.18.2
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-sdk-for-go v57.4.0+incompatible
diff --git a/go.sum b/go.sum
index 78212a671..2d7a76123 100644
--- a/go.sum
+++ b/go.sum
@@ -34,8 +34,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/cloudbuild v0.2.0 h1:Wt51/IZhA0wRnTwICmr8hkxF3pAjVIvdXfMEkkdv2YY=
-cloud.google.com/go/cloudbuild v0.2.0/go.mod h1:nW1pCgRvDTXeKEO7Ao58gnX8QNceUXukeT7EoQBWOYM=
+cloud.google.com/go/cloudbuild v1.0.0 h1:NLh+EHhYuvCfowjelkra3TOa6yRoAvClDZISdRzmyjc=
+cloud.google.com/go/cloudbuild v1.0.0/go.mod h1:Lt6rTNAQmYXacQs1X1Q+bV8obdb9qJREzHn0n+3Qu/w=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@@ -1098,12 +1098,12 @@ google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKr
 google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
 google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
 google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210827211047-25e5f791fe06/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
diff --git a/internal/distro/rhel90/distro.go b/internal/distro/rhel90/distro.go
index c4d4519fd..86b416acb 100644
--- a/internal/distro/rhel90/distro.go
+++ b/internal/distro/rhel90/distro.go
@@ -718,7 +718,7 @@ func newDistro(distroName string) distro.Distro {
 		name:          "qcow2",
 		filename:      "disk.qcow2",
 		mimeType:      "application/x-qemu-disk",
-		kernelOptions: "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+		kernelOptions: "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
 		packageSets: map[string]packageSetFunc{
 			buildPkgsKey: distroBuildPackageSet,
 			osPkgsKey:    qcow2CommonPackageSet,
@@ -1009,7 +1009,7 @@ func newDistro(distroName string) distro.Distro {
 			osPkgsKey:    ec2CommonPackageSet,
 		},
 		defaultImageConfig:  defaultAMIImageConfigX86_64,
-		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
 		bootable:            true,
 		bootType:            distro.LegacyBootType,
 		defaultSize:         10 * GigaByte,
@@ -1029,7 +1029,7 @@ func newDistro(distroName string) distro.Distro {
 			osPkgsKey:    ec2CommonPackageSet,
 		},
 		defaultImageConfig:  defaultAMIImageConfig,
-		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
 		bootable:            true,
 		defaultSize:         10 * GigaByte,
 		pipelines:           ec2Pipelines,
@@ -1048,7 +1048,7 @@ func newDistro(distroName string) distro.Distro {
 			osPkgsKey:    rhelEc2PackageSet,
 		},
 		defaultImageConfig:  defaultEc2ImageConfigX86_64,
-		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
 		bootable:            true,
 		bootType:            distro.LegacyBootType,
 		defaultSize:         10 * GigaByte,
@@ -1068,7 +1068,7 @@ func newDistro(distroName string) distro.Distro {
 			osPkgsKey:    rhelEc2PackageSet,
 		},
 		defaultImageConfig:  defaultEc2ImageConfig,
-		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
 		bootable:            true,
 		defaultSize:         10 * GigaByte,
 		pipelines:           rhelEc2Pipelines,
@@ -1087,7 +1087,7 @@ func newDistro(distroName string) distro.Distro {
 			osPkgsKey:    rhelEc2HaPackageSet,
 		},
 		defaultImageConfig:  defaultEc2ImageConfigX86_64,
-		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
 		bootable:            true,
 		bootType:            distro.LegacyBootType,
 		defaultSize:         10 * GigaByte,
@@ -1218,7 +1218,7 @@ func newDistro(distroName string) distro.Distro {
 			osPkgsKey:    rhelEc2SapPackageSet,
 		},
 		defaultImageConfig:  defaultEc2SapImageConfigX86_64,
-		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto processor.max_cstate=1 intel_idle.max_cstate=1",
+		kernelOptions:       "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 processor.max_cstate=1 intel_idle.max_cstate=1",
 		bootable:            true,
 		bootType:            distro.LegacyBootType,
 		defaultSize:         10 * GigaByte,
diff --git a/internal/jobqueue/dbjobqueue/dbjobqueue.go b/internal/jobqueue/dbjobqueue/dbjobqueue.go
index 246798bec..84f289a03 100644
--- a/internal/jobqueue/dbjobqueue/dbjobqueue.go
+++ b/internal/jobqueue/dbjobqueue/dbjobqueue.go
@@ -21,7 +21,6 @@ import (
 	logrus "github.com/sirupsen/logrus"
 
 	"github.com/osbuild/osbuild-composer/internal/jobqueue"
-	"github.com/osbuild/osbuild-composer/internal/prometheus"
 )
 
 const (
@@ -183,7 +182,6 @@ func (q *DBJobQueue) Enqueue(jobType string, args interface{}, dependencies []uu
 		return uuid.Nil, fmt.Errorf("unable to commit database transaction: %v", err)
 	}
 
-	prometheus.PendingJobs.WithLabelValues(jobType).Inc()
 	logrus.Infof("Enqueued job of type %s with ID %s(dependencies %v)", jobType, id, dependencies)
 
 	return id, nil
@@ -234,11 +232,6 @@ func (q *DBJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID,
 		}
 	}
 
-	diff := started.Sub(*queued).Seconds()
-	prometheus.JobWaitDuration.WithLabelValues(jobType).Observe(diff)
-	prometheus.PendingJobs.WithLabelValues(jobType).Dec()
-	prometheus.RunningJobs.WithLabelValues(jobType).Inc()
-
 	// insert heartbeat
 	_, err = conn.Exec(ctx, sqlInsertHeartbeat, token, id)
 	if err != nil {
@@ -290,10 +283,6 @@ func (q *DBJobQueue) DequeueByID(ctx context.Context, id uuid.UUID) (uuid.UUID,
 	}
 
 	logrus.Infof("Dequeued job of type %v with ID %s", jobType, id)
-	diff := started.Sub(*queued).Seconds()
-	prometheus.JobWaitDuration.WithLabelValues(jobType).Observe(diff)
-	prometheus.PendingJobs.WithLabelValues(jobType).Dec()
-	prometheus.RunningJobs.WithLabelValues(jobType).Inc()
 
 	return token, dependencies, jobType, args, nil
 }
@@ -362,9 +351,6 @@ func (q *DBJobQueue) FinishJob(id uuid.UUID, result interface{}) error {
 	}
 
 	logrus.Infof("Finished job with ID %s", id)
-	diff := finished.Sub(*started).Seconds()
-	prometheus.JobDuration.WithLabelValues(jobType).Observe(diff)
-	prometheus.RunningJobs.WithLabelValues(jobType).Dec()
 
 	return nil
 }
@@ -387,11 +373,6 @@ func (q *DBJobQueue) CancelJob(id uuid.UUID) error {
 	}
 
 	logrus.Infof("Cancelled job with ID %s", id)
-	if started != nil {
-		prometheus.RunningJobs.WithLabelValues(jobType).Dec()
-	} else {
-		prometheus.PendingJobs.WithLabelValues(jobType).Dec()
-	}
 
 	return nil
 }
diff --git a/internal/prometheus/job_metrics.go b/internal/prometheus/job_metrics.go
index 9c6e17b5e..eda3dba98 100644
--- a/internal/prometheus/job_metrics.go
+++ b/internal/prometheus/job_metrics.go
@@ -1,12 +1,24 @@
 package prometheus
 
 import (
+	"time"
+
+	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 const workerSubsystem = "composer_worker"
 
+var (
+	TotalJobs = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name:      "total_jobs",
+		Namespace: namespace,
+		Subsystem: workerSubsystem,
+		Help:      "Total jobs",
+	}, []string{"type", "status"})
+)
+
 var (
 	PendingJobs = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Name:      "pending_jobs",
@@ -32,7 +44,7 @@ var (
 		Subsystem: workerSubsystem,
 		Help:      "Duration spent by workers on a job.",
 		Buckets:   []float64{.1, .2, .5, 1, 2, 4, 8, 16, 32, 40, 48, 64, 96, 128, 160, 192, 224, 256, 320, 382, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2049},
-	}, []string{"type"})
+	}, []string{"type", "status"})
 )
 
 var (
@@ -44,3 +56,33 @@ var (
 		Buckets:   []float64{.1, .2, .5, 1, 2, 4, 8, 16, 32, 40, 48, 64, 96, 128, 160, 192, 224, 256, 320, 382, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2049},
 	}, []string{"type"})
 )
+
+func EnqueueJobMetrics(jobType string) {
+	PendingJobs.WithLabelValues(jobType).Inc()
+}
+
+func DequeueJobMetrics(queued time.Time, started time.Time, jobType string) {
+	if !started.IsZero() && !queued.IsZero() {
+		diff := started.Sub(queued).Seconds()
+		JobWaitDuration.WithLabelValues(jobType).Observe(diff)
+		PendingJobs.WithLabelValues(jobType).Dec()
+		RunningJobs.WithLabelValues(jobType).Inc()
+	}
+}
+
+func CancelJobMetrics(started time.Time, jobType string) {
+	if !started.IsZero() {
+		RunningJobs.WithLabelValues(jobType).Dec()
+	} else {
+		PendingJobs.WithLabelValues(jobType).Dec()
+	}
+}
+
+func FinishJobMetrics(started time.Time, finished time.Time, canceled bool, jobType string, status clienterrors.StatusCode) {
+	if !finished.IsZero() && !canceled {
+		diff := finished.Sub(started).Seconds()
+		JobDuration.WithLabelValues(jobType, status.ToString()).Observe(diff)
+		TotalJobs.WithLabelValues(jobType, status.ToString()).Inc()
+		RunningJobs.WithLabelValues(jobType).Dec()
+	}
+}
diff --git a/internal/worker/clienterrors/errors.go b/internal/worker/clienterrors/errors.go
index b5f584cc5..69648faed 100644
--- a/internal/worker/clienterrors/errors.go
+++ b/internal/worker/clienterrors/errors.go
@@ -34,6 +34,40 @@ type Error struct {
 	Details interface{}     `json:"details"`
 }
 
+const (
+	JobStatusSuccess        = "2xx"
+	JobStatusUserInputError = "4xx"
+	JobStatusInternalError  = "5xx"
+)
+
+type StatusCode string
+
+func (s *StatusCode) ToString() string {
+	return string(*s)
+}
+
+func GetStatusCode(err *Error) StatusCode {
+	if err == nil {
+		return JobStatusSuccess
+	}
+	switch err.ID {
+	case ErrorDNFDepsolveError:
+		return JobStatusUserInputError
+	case ErrorDNFMarkingError:
+		return JobStatusUserInputError
+	case ErrorNoDynamicArgs:
+		return JobStatusUserInputError
+	case ErrorInvalidTargetConfig:
+		return JobStatusUserInputError
+	case ErrorSharingTarget:
+		return JobStatusUserInputError
+	case ErrorInvalidTarget:
+		return JobStatusUserInputError
+	default:
+		return JobStatusInternalError
+	}
+}
+
 func WorkerClientError(code ClientErrorCode, reason string, details ...interface{}) *Error {
 	return &Error{
 		ID:      code,
diff --git a/internal/worker/server.go b/internal/worker/server.go
index aa9a84ce4..4ba38579a 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/jobqueue"
+	"github.com/osbuild/osbuild-composer/internal/prometheus"
 	"github.com/osbuild/osbuild-composer/internal/worker/api"
 	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
 )
@@ -93,35 +94,40 @@ func (s *Server) WatchHeartbeats() {
 }
 
 func (s *Server) EnqueueOSBuild(arch string, job *OSBuildJob) (uuid.UUID, error) {
-	return s.jobs.Enqueue("osbuild:"+arch, job, nil)
+	return s.enqueue("osbuild:"+arch, job, nil)
 }
 
 func (s *Server) EnqueueOSBuildAsDependency(arch string, job *OSBuildJob, manifestID uuid.UUID) (uuid.UUID, error) {
-	return s.jobs.Enqueue("osbuild:"+arch, job, []uuid.UUID{manifestID})
+	return s.enqueue("osbuild:"+arch, job, []uuid.UUID{manifestID})
 }
 
 func (s *Server) EnqueueOSBuildKoji(arch string, job *OSBuildKojiJob, initID uuid.UUID) (uuid.UUID, error) {
-	return s.jobs.Enqueue("osbuild-koji:"+arch, job, []uuid.UUID{initID})
+	return s.enqueue("osbuild-koji:"+arch, job, []uuid.UUID{initID})
 }
 
 func (s *Server) EnqueueOSBuildKojiAsDependency(arch string, job *OSBuildKojiJob, manifestID, initID uuid.UUID) (uuid.UUID, error) {
-	return s.jobs.Enqueue("osbuild-koji:"+arch, job, []uuid.UUID{initID, manifestID})
+	return s.enqueue("osbuild-koji:"+arch, job, []uuid.UUID{initID, manifestID})
 }
 
 func (s *Server) EnqueueKojiInit(job *KojiInitJob) (uuid.UUID, error) {
-	return s.jobs.Enqueue("koji-init", job, nil)
+	return s.enqueue("koji-init", job, nil)
 }
 
 func (s *Server) EnqueueKojiFinalize(job *KojiFinalizeJob, initID uuid.UUID, buildIDs []uuid.UUID) (uuid.UUID, error) {
-	return s.jobs.Enqueue("koji-finalize", job, append([]uuid.UUID{initID}, buildIDs...))
+	return s.enqueue("koji-finalize", job, append([]uuid.UUID{initID}, buildIDs...))
 }
 
 func (s *Server) EnqueueDepsolve(job *DepsolveJob) (uuid.UUID, error) {
-	return s.jobs.Enqueue("depsolve", job, nil)
+	return s.enqueue("depsolve", job, nil)
 }
 
 func (s *Server) EnqueueManifestJobByID(job *ManifestJobByID, parent uuid.UUID) (uuid.UUID, error) {
-	return s.jobs.Enqueue("manifest-id-only", job, []uuid.UUID{parent})
+	return s.enqueue("manifest-id-only", job, []uuid.UUID{parent})
+}
+
+func (s *Server) enqueue(jobType string, job interface{}, dependencies []uuid.UUID) (uuid.UUID, error) {
+	prometheus.EnqueueJobMetrics(jobType)
+	return s.jobs.Enqueue(jobType, job, dependencies)
 }
 
 func (s *Server) OSBuildJobStatus(id uuid.UUID, result *OSBuildJobResult) (*JobStatus, []uuid.UUID, error) {
@@ -326,6 +332,12 @@ func (s *Server) JobType(id uuid.UUID) (string, error) {
 }
 
 func (s *Server) Cancel(id uuid.UUID) error {
+	jobType, status, _, err := s.jobStatus(id, nil)
+	if err != nil {
+		logrus.Errorf("error getting job status: %v", err)
+	} else {
+		prometheus.CancelJobMetrics(status.Started, jobType)
+	}
 	return s.jobs.CancelJob(id)
 }
 
@@ -419,6 +431,13 @@ func (s *Server) requestJob(ctx context.Context, arch string, jobTypes []string,
 		return
 	}
 
+	jobType, status, _, err := s.jobStatus(jobId, nil)
+	if err != nil {
+		logrus.Errorf("error retrieving job status: %v", err)
+	} else {
+		prometheus.DequeueJobMetrics(status.Queued, status.Started, jobType)
+	}
+
 	for _, depID := range depIDs {
 		// TODO: include type of arguments
 		_, result, _, _, _, _, _, _ := s.jobs.JobStatus(depID)
@@ -462,6 +481,15 @@ func (s *Server) FinishJob(token uuid.UUID, result json.RawMessage) error {
 		}
 	}
 
+	var jobResult JobResult
+	jobType, status, _, err := s.jobStatus(jobId, &jobResult)
+	if err != nil {
+		logrus.Errorf("error finding job status: %v", err)
+	} else {
+		statusCode := clienterrors.GetStatusCode(jobResult.JobError)
+		prometheus.FinishJobMetrics(status.Started, status.Finished, status.Canceled, jobType, statusCode)
+	}
+
 	// Move artifacts from the temporary location to the final job
 	// location. Log any errors, but do not treat them as fatal. The job is
 	// already finished.
diff --git a/osbuild-composer.spec b/osbuild-composer.spec
index 06bf44a01..a76eced7d 100644
--- a/osbuild-composer.spec
+++ b/osbuild-composer.spec
@@ -2,6 +2,11 @@
 # Pass --with tests to rpmbuild to override
 %bcond_with tests
 
+# When --with relax_requires is specified osbuild-composer-tests
+# will require osbuild-composer only by name, excluding version/release
+# This is used internally during nightly pipeline testing!
+%bcond_with relax_requires
+
 %global goipath         github.com/osbuild/osbuild-composer
 
 Version:        44
@@ -371,7 +376,11 @@ The dnf-json binary used by osbuild-composer and the workers.
 
 %package tests
 Summary:    Integration tests
+%if %{with relax_requires}
+Requires:   %{name}
+%else
 Requires:   %{name} = %{version}-%{release}
+%endif
 Requires:   composer-cli
 Requires:   createrepo_c
 Requires:   xorriso
@@ -399,6 +408,7 @@ Requires:   libvirt-daemon-driver-storage-disk
 Requires:   libvirt-daemon-kvm
 Requires:   qemu-img
 Requires:   qemu-kvm
+Requires:   rpmdevtools
 Requires:   virt-install
 Requires:   expect
 Requires:   python3-lxml
diff --git a/schutzbot/deploy.sh b/schutzbot/deploy.sh
index d9b332668..5fc98d544 100755
--- a/schutzbot/deploy.sh
+++ b/schutzbot/deploy.sh
@@ -39,11 +39,17 @@ function setup_repo {
   local project=$1
   local commit=$2
   local priority=${3:-10}
+
+  local REPO_PATH=${project}/${DISTRO_VERSION}/${ARCH}/${commit}
+  if [[ "${NIGHTLY:=false}" == "true" && "${project}" == "osbuild-composer" ]]; then
+    REPO_PATH=nightly/${REPO_PATH}
+  fi
+
   greenprint "Setting up dnf repository for ${project} ${commit}"
   sudo tee "/etc/yum.repos.d/${project}.repo" << EOF
 [${project}]
 name=${project} ${commit}
-baseurl=http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com/${project}/${DISTRO_VERSION}/${ARCH}/${commit}
+baseurl=http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com/${REPO_PATH}
 enabled=1
 gpgcheck=0
 priority=${priority}
@@ -110,13 +116,6 @@ fi
 if [ -f "rhel${VERSION_ID%.*}internal.repo" ]; then
     greenprint "Preparing repos for internal build testing"
     sudo mv rhel"${VERSION_ID%.*}"internal.repo /etc/yum.repos.d/
-    # Use osbuild from schutzfile if desired for testing custom osbuild-composer packages
-    # specified by $REPO_URL in ENV and used in prepare-rhel-internal.sh
-    if [ "${SCHUTZ_OSBUILD:=false}" == true ]; then
-        sudo rm -f /etc/yum.repos.d/osbuild-composer.repo
-    else
-        sudo rm -f /etc/yum.repos.d/osbuild*.repo
-    fi
 fi
 
 greenprint "Installing test packages for ${PROJECT}"
@@ -127,6 +126,31 @@ retry sudo dnf -y upgrade selinux-policy
 # Note: installing only -tests to catch missing dependencies
 retry sudo dnf -y install "${PROJECT}-tests"
 
+if [ "${NIGHTLY:=false}" == "true" ]; then
+    # check if we've installed the osbuild-composer RPM from the nightly tree
+    # under test or happen to install a newer version from one of the S3 repositories
+    rpm -qi osbuild-composer
+    if ! rpm -qi osbuild-composer | grep "Build Host" | grep "redhat.com"; then
+        echo "ERROR: Installed osbuild-composer RPM is not the official one"
+        exit 2
+    else
+        echo "INFO: Installed osbuild-composer RPM seems to be official"
+    fi
+
+    # cross-check the installed RPM against the one under COMPOSE_URL
+    source tools/define-compose-url.sh
+
+    INSTALLED=$(rpm -q --qf "%{name}-%{version}-%{release}.%{arch}.rpm" osbuild-composer)
+    RPM_URL="${COMPOSE_URL}/compose/AppStream/${ARCH}/os/Packages/${INSTALLED}"
+    RETURN_CODE=$(curl --silent -o -I -L -s -w "%{http_code}" "${RPM_URL}")
+    if [ "$RETURN_CODE" != 200 ]; then
+        echo "ERROR: Installed ${INSTALLED} not found at ${RPM_URL}. Response was ${RETURN_CODE}"
+        exit 3
+    else
+        echo "INFO: Installed ${INSTALLED} found at ${RPM_URL}, which matches SUT!"
+    fi
+fi
+
 if [ -n "${CI}" ]; then
     # copy repo files b/c GitLab can't upload artifacts
     # which are outside the build directory
diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh
index 1770d6eb5..eba6d4ba6 100755
--- a/schutzbot/mockbuild.sh
+++ b/schutzbot/mockbuild.sh
@@ -38,6 +38,9 @@ fi
 # Relative path of the repository – used for constructing both the local and
 # remote paths below, so that they're consistent.
 REPO_PATH=osbuild-composer/${DISTRO_VERSION}/${ARCH}/${COMMIT}
+if [ "${NIGHTLY:=false}" == "true" ]; then
+    REPO_PATH=nightly/${REPO_PATH}
+fi
 
 # Directory to hold the RPMs temporarily before we upload them.
 REPO_DIR=repo/${REPO_PATH}
@@ -99,10 +102,15 @@ sudo mock -r "$MOCK_CONFIG" --buildsrpm \
   --sources "./osbuild-composer-${COMMIT}.tar.gz" \
   --resultdir ./srpm
 
+if [ "${NIGHTLY:=false}" == "true" ]; then
+    RELAX_REQUIRES="--with=relax_requires"
+fi
+
 greenprint "🎁 Building RPMs"
 sudo mock -r "$MOCK_CONFIG" \
     --define "commit ${COMMIT}" \
     --with=tests \
+    ${RELAX_REQUIRES:+"$RELAX_REQUIRES"} \
     --resultdir "$REPO_DIR" \
     srpm/*.src.rpm
 
@@ -112,6 +120,12 @@ sudo chown -R "$USER" "${REPO_DIR%%/*}"
 greenprint "🧹 Remove logs from mock build"
 rm "${REPO_DIR}"/*.log
 
+# leave only -tests RPM to minimize interference when installing
+# osbuild-composer.rpm from distro repositories
+if [ "${NIGHTLY:=false}" == "true" ]; then
+    find "${REPO_DIR}" -type f -not -name "osbuild-composer-tests*.rpm" -exec rm -f "{}" \;
+fi
+
 # Create a repo of the built RPMs.
 greenprint "⛓️ Creating dnf repository"
 createrepo_c "${REPO_DIR}"
diff --git a/schutzbot/prepare-rhel-internal.sh b/schutzbot/prepare-rhel-internal.sh
index 2947324d4..9099bcb29 100755
--- a/schutzbot/prepare-rhel-internal.sh
+++ b/schutzbot/prepare-rhel-internal.sh
@@ -90,62 +90,14 @@ EOF
 
 cp rhel-"${VERSION_ID%.*}".json rhel-"${VERSION_ID%.*}"-beta.json
 
-JOB_NAME="${JOB_NAME:-${CI_JOB_ID}}"
-# Do not create tests repo if it's provided from ENV
-if [ -z "${REPO_URL+x}" ]; then
-    greenprint "📦 Installing requirements"
-    sudo dnf -y install createrepo_c wget python3-pip
-
-    # Install s3cmd if it is not present.
-    if ! s3cmd --version > /dev/null 2>&1; then
-        greenprint "📦 Installing s3cmd"
-        sudo pip3 -q install s3cmd
-    fi
-
-    REPO_DIR_LATEST="repo/${JOB_NAME}/latest/internal"
-    mkdir -p "$REPO_DIR_LATEST"
-
-    greenprint "Discover latest osbuild-composer NVR"
-    # Download only osbuild-composer-worker-*.rpm and use it in rpm --qf below
-    wget --quiet --recursive --no-parent --no-directories --accept "osbuild-composer-worker-*.rpm" \
-        "${COMPOSE_URL}/compose/AppStream/x86_64/os/Packages/"
-
-    # Download osbuild-composer-tests from Brew b/c it is not available in the distro
-    # version matches osbuild-composer from the internal tree
-    greenprint "Downloading osbuild-composer-tests RPMs from Brew"
-    for ARCH in $ALL_ARCHES; do
-        TESTS_RPM_URL=$(rpm -qp ./osbuild-composer-worker-*.rpm --qf "http://download.devel.redhat.com/brewroot/vol/rhel-${VERSION_ID%.*}/packages/osbuild-composer/%{version}/%{release}/$ARCH/osbuild-composer-tests-%{version}-%{release}.$ARCH.rpm")
-        wget --directory-prefix "$REPO_DIR_LATEST" "$TESTS_RPM_URL"
-    done
-
-    greenprint "⛓ Creating dnf repository"
-    createrepo_c "${REPO_DIR_LATEST}"
-
-    # Bucket in S3 where our artifacts are uploaded
-    REPO_BUCKET=osbuild-composer-repos
-
-    # Remove the previous latest repo for this job.
-    # Don't fail if the path is missing.
-    AWS_ACCESS_KEY_ID="$V2_AWS_ACCESS_KEY_ID" \
-    AWS_SECRET_ACCESS_KEY="$V2_AWS_SECRET_ACCESS_KEY" \
-    s3cmd --recursive rm "s3://${REPO_BUCKET}/${REPO_DIR_LATEST}" || true
-
-    # Upload repository to S3.
-    greenprint "☁ Uploading RPMs to S3"
-    AWS_ACCESS_KEY_ID="$V2_AWS_ACCESS_KEY_ID" \
-    AWS_SECRET_ACCESS_KEY="$V2_AWS_SECRET_ACCESS_KEY" \
-    s3cmd --acl-public put --recursive repo/ s3://${REPO_BUCKET}/repo/
-
-    # Public URL for the S3 bucket with our artifacts.
-    MOCK_REPO_BASE_URL="http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com"
-
-    # Full URL to the RPM repository after they are uploaded.
-    REPO_URL=${MOCK_REPO_BASE_URL}/${REPO_DIR_LATEST}
-fi
+# Create tests .repo file if REPO_URL is provided from ENV
+# Otherwise osbuild-composer-tests.rpm will be downloaded from
+# existing repositories
+if [ -n "${REPO_URL+x}" ]; then
+    JOB_NAME="${JOB_NAME:-${CI_JOB_ID}}"
 
-# amend repository file.
-greenprint "📜 Amend dnf repository file"
-tee -a rhel"${VERSION_ID%.*}"internal.repo << EOF
+    greenprint "📜 Amend dnf repository file"
+    tee -a rhel"${VERSION_ID%.*}"internal.repo << EOF
 
 [osbuild-composer-tests-multi-arch]
 name=Tests ${JOB_NAME}
@@ -155,3 +107,5 @@ gpgcheck=0
 # osbuild-composer repo priority is 5
 priority=1
 EOF
+
+fi
diff --git a/templates/composer.yml b/templates/composer.yml
index 3f3d0761a..ae27b8502 100644
--- a/templates/composer.yml
+++ b/templates/composer.yml
@@ -248,33 +248,6 @@ objects:
                   cpu: "${CPU_LIMIT}"
                   memory: "${MEMORY_LIMIT}"
               env:
-              - name: PGHOST
-                valueFrom:
-                  secretKeyRef:
-                    name: composer-db
-                    key: db.host
-              - name: PGPORT
-                valueFrom:
-                  secretKeyRef:
-                    name: composer-db
-                    key: db.port
-              - name: PGDATABASE
-                valueFrom:
-                  secretKeyRef:
-                    name: composer-db
-                    key: db.name
-              - name: PGUSER
-                valueFrom:
-                  secretKeyRef:
-                    name: composer-db
-                    key: db.user
-              - name: PGPASSWORD
-                valueFrom:
-                  secretKeyRef:
-                    name: composer-db
-                    key: db.password
-              - name: PGSSLMODE
-                value: "${PGSSLMODE}"
               - name: GCP_AUTH_PROVIDER_X509_CERT_URL
                 valueFrom:
                   secretKeyRef:
diff --git a/templates/dashboards/grafana-dashboard-image-builder-worker-general.configmap.yml b/templates/dashboards/grafana-dashboard-image-builder-worker-general.configmap.yml
index fbf67075c..dfd5d78c0 100644
--- a/templates/dashboards/grafana-dashboard-image-builder-worker-general.configmap.yml
+++ b/templates/dashboards/grafana-dashboard-image-builder-worker-general.configmap.yml
@@ -31,8 +31,7 @@ data:
       "editable": true,
       "gnetId": null,
       "graphTooltip": 0,
-      "id": 3,
-      "iteration": 1639488117492,
+      "iteration": 1644236264867,
       "links": [],
       "panels": [
         {
@@ -44,14 +43,1091 @@ data:
             "x": 0,
             "y": 0
           },
-          "id": 128,
+          "id": 212,
+          "panels": [],
+          "title": "Build Job Success Rate",
+          "type": "row"
+        },
+        {
+          "datasource": "${datasource}",
+          "description": "The percentage of successful osbuild jobs for the selected time range",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "decimals": 0,
+              "mappings": [
+                {
+                  "options": {
+                    "match": "null",
+                    "result": {
+                      "index": 0,
+                      "text": "N/A"
+                    }
+                  },
+                  "type": "special"
+                }
+              ],
+              "noValue": "0",
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": "0.95"
+                  },
+                  {
+                    "color": "green",
+                    "value": "0.955"
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 5,
+            "x": 0,
+            "y": 1
+          },
+          "id": 197,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "text": {},
+            "textMode": "auto"
+          },
+          "pluginVersion": "8.1.5",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(increase(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"2xx\"}[$__range]))/sum(increase(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$__range]))",
+              "interval": "",
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "title": "Build Job Success Rate",
+          "type": "stat"
+        },
+        {
+          "datasource": "${datasource}",
+          "description": "The throughput rate of osbuild job errors and non-errors over time for the selected time range",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 11,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "success/sec"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 11,
+            "x": 5,
+            "y": 1
+          },
+          "id": 202,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single"
+            }
+          },
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$interval])) - sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval]))",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "success/sec",
+              "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval]))",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "errors/sec",
+              "refId": "B"
+            }
+          ],
+          "title": "Build Job Throughput Rate",
+          "type": "timeseries"
+        },
+        {
+          "datasource": "${datasource}",
+          "description": "The number of osbuild job errors (as a percentage) over time for the selected time range",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "bars",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "max": 1,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 1
+          },
+          "id": 194,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "hidden",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single"
+            }
+          },
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval]))/sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$interval]))",
+              "interval": "",
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "title": "Build Job Error Rate",
+          "type": "timeseries"
+        },
+        {
+          "cacheTimeout": 1,
+          "datasource": "${datasource}",
+          "description": "How long will it take to consume all our budget if our error consumption remains at the current rate for the selected date range.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "decimals": 2,
+              "mappings": [
+                {
+                  "options": {
+                    "match": "null",
+                    "result": {
+                      "index": 0,
+                      "text": "1.40 days"
+                    }
+                  },
+                  "type": "special"
+                },
+                {
+                  "options": {
+                    "from": 672,
+                    "result": {
+                      "index": 1,
+                      "text": "∞"
+                    },
+                    "to": 3360100
+                  },
+                  "type": "range"
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": 40
+                  },
+                  {
+                    "color": "green",
+                    "value": 50
+                  }
+                ]
+              },
+              "unit": "h"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 4,
+            "x": 0,
+            "y": 9
+          },
+          "id": 238,
+          "interval": null,
+          "links": [],
+          "maxDataPoints": 100,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "text": {
+              "valueSize": 80
+            },
+            "textMode": "auto"
+          },
+          "pluginVersion": "8.1.5",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "28 * 24 * (1 - $stability_slo) / ((sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[28d]))/ sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[28d]))) + 0.001)",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Error Budget Remaining",
+          "type": "stat"
+        },
+        {
+          "cacheTimeout": null,
+          "datasource": "${datasource}",
+          "description": "The percentage of error budget consumed for the selected time range. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 100,
+                "gradientMode": "scheme",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 0,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": true,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "max": 1,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": 0.95
+                  },
+                  {
+                    "color": "red",
+                    "value": 1
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 20,
+            "x": 4,
+            "y": 9
+          },
+          "id": 239,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "hidden",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single"
+            }
+          },
+          "pluginVersion": "8.1.5",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "1 - ((1 - sum(increase(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[28d]))/sum(increase(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[28d]))) - $stability_slo)/ (1 - $stability_slo)",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 10,
+              "legendFormat": "errorbudget",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Error Budget Consumed",
+          "type": "timeseries"
+        },
+        {
+          "collapsed": false,
+          "datasource": null,
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 17
+          },
+          "id": 129,
+          "panels": [],
+          "title": "Build Job Duration",
+          "type": "row"
+        },
+        {
+          "datasource": "${datasource}",
+          "description": "The duration of 90% of osbuild jobs",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "decimals": 0,
+              "mappings": [
+                {
+                  "options": {
+                    "match": "null",
+                    "result": {
+                      "index": 0,
+                      "text": "N/A"
+                    }
+                  },
+                  "type": "special"
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": "1228"
+                  },
+                  {
+                    "color": "red",
+                    "value": "1536"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 5,
+            "x": 0,
+            "y": 18
+          },
+          "id": 200,
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "text": {},
+            "textMode": "auto"
+          },
+          "pluginVersion": "8.1.5",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$__range])) by (le))",
+              "interval": "",
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "title": "Build Job Duration",
+          "type": "stat"
+        },
+        {
+          "datasource": "${datasource}",
+          "description": "The request Duration for osbuild jobs over the selected date range",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 35,
+                "gradientMode": "scheme",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 3,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": true,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": "175"
+                  },
+                  {
+                    "color": "red",
+                    "value": "200"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "99p"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "95p"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "blue",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "90p"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "yellow",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "50p"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "blue",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "p90"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "yellow",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "p99"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "p50"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "blue",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 11,
+            "x": 5,
+            "y": 18
+          },
+          "id": 201,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single"
+            }
+          },
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "histogram_quantile(0.5, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))",
+              "interval": "",
+              "legendFormat": "p50",
+              "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "p90",
+              "refId": "B"
+            },
+            {
+              "exemplar": true,
+              "expr": "histogram_quantile(0.99, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "p99",
+              "refId": "C"
+            }
+          ],
+          "title": "Build Job Duration",
+          "type": "timeseries"
+        },
+        {
+          "datasource": "${datasource}",
+          "description": "Percent of requests exceeding Duration allowed by SLO",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "max": 1,
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 18
+          },
+          "id": 210,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "hidden",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single"
+            }
+          },
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[$interval]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[$interval]))",
+              "interval": "",
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "title": "Slow Request Rate",
+          "type": "timeseries"
+        },
+        {
+          "cacheTimeout": 1,
+          "datasource": "${datasource}",
+          "description": "How long will it take to consume all our budget if our error consumption remains at the current rate for the selected date range.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "decimals": 2,
+              "mappings": [
+                {
+                  "options": {
+                    "match": "null",
+                    "result": {
+                      "index": 0,
+                      "text": "1.40 days"
+                    }
+                  },
+                  "type": "special"
+                },
+                {
+                  "options": {
+                    "from": 672,
+                    "result": {
+                      "index": 1,
+                      "text": "∞"
+                    },
+                    "to": 3360100
+                  },
+                  "type": "range"
+                }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": 40
+                  },
+                  {
+                    "color": "green",
+                    "value": 50
+                  }
+                ]
+              },
+              "unit": "h"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 4,
+            "x": 0,
+            "y": 26
+          },
+          "id": 198,
+          "interval": null,
+          "links": [],
+          "maxDataPoints": 100,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "text": {
+              "valueSize": 80
+            },
+            "textMode": "auto"
+          },
+          "pluginVersion": "8.1.5",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "28 * 24 * (1 - $latency_slo) / (1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[$__range]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[$__range])))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Error Budget Remaining",
+          "type": "stat"
+        },
+        {
+          "cacheTimeout": null,
+          "datasource": "${datasource}",
+          "description": "The percentage of error budget consumed for the selected time range. ",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 100,
+                "gradientMode": "scheme",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 0,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": true,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": 0.95
+                  },
+                  {
+                    "color": "red",
+                    "value": 1
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 20,
+            "x": 4,
+            "y": 26
+          },
+          "id": 199,
+          "links": [],
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "hidden",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single"
+            }
+          },
+          "pluginVersion": "8.1.5",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "1 - ((sum(increase(image_builder_composer_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[28d]))/sum(increase(image_builder_composer_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[28d]))) - $latency_slo)/ (1 - $latency_slo)",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 10,
+              "legendFormat": "errorbudget",
+              "refId": "A"
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Error Budget Consumed",
+          "type": "timeseries"
+        },
+        {
+          "collapsed": false,
+          "datasource": null,
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 34
+          },
+          "id": 214,
           "panels": [],
-          "title": "Depsolve Duration",
+          "title": "Depsolve Job Success Rate",
           "type": "row"
         },
         {
           "datasource": "${datasource}",
-          "description": "The duration of 90% of depsolve jobs",
+          "description": "The percentage of successful depsolve jobs for the selected time range",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -74,20 +1150,20 @@ data:
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
+                    "color": "red",
                     "value": null
                   },
                   {
                     "color": "#EAB839",
-                    "value": "25"
+                    "value": "0.95"
                   },
                   {
-                    "color": "red",
-                    "value": "30"
+                    "color": "green",
+                    "value": "0.955"
                   }
                 ]
               },
-              "unit": "s"
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -95,9 +1171,9 @@ data:
             "h": 8,
             "w": 5,
             "x": 0,
-            "y": 1
+            "y": 35
           },
-          "id": 197,
+          "id": 225,
           "mappings": [
             {
               "options": {
@@ -128,43 +1204,43 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$__range])) by (le))",
+              "expr": "sum(increase(image_builder_composer_worker_total_jobs{type=\"depsolve\", status=\"2xx\"}[$__range]))/sum(increase(image_builder_composer_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[$__range]))",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
             }
           ],
-          "title": "Depsolve Job Duration",
+          "title": "Depsolve Job Success Rate",
           "type": "stat"
         },
         {
           "datasource": "${datasource}",
-          "description": "The request Duration for depsolve jobs over the selected date range",
+          "description": "The throughput rate of osbuild job errors and non-errors over time for the selected time range",
           "fieldConfig": {
             "defaults": {
               "color": {
                 "mode": "thresholds"
               },
               "custom": {
-                "axisLabel": "seconds",
+                "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
                 "drawStyle": "line",
-                "fillOpacity": 35,
-                "gradientMode": "scheme",
+                "fillOpacity": 11,
+                "gradientMode": "none",
                 "hideFrom": {
                   "legend": false,
                   "tooltip": false,
                   "viz": false
                 },
                 "lineInterpolation": "linear",
-                "lineWidth": 3,
+                "lineWidth": 1,
                 "pointSize": 5,
                 "scaleDistribution": {
                   "type": "linear"
                 },
-                "showPoints": "never",
-                "spanNulls": true,
+                "showPoints": "auto",
+                "spanNulls": false,
                 "stacking": {
                   "group": "A",
                   "mode": "none"
@@ -174,127 +1250,28 @@ data:
                 }
               },
               "mappings": [],
-              "min": 0,
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  },
-                  {
-                    "color": "#EAB839",
-                    "value": "175"
-                  },
                   {
                     "color": "red",
-                    "value": "200"
+                    "value": null
                   }
                 ]
               },
-              "unit": "s"
+              "unit": "none"
             },
             "overrides": [
               {
                 "matcher": {
                   "id": "byName",
-                  "options": "99p"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "red",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "95p"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "blue",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "90p"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "yellow",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "50p"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "blue",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "p90"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "yellow",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "p99"
-                },
-                "properties": [
-                  {
-                    "id": "color",
-                    "value": {
-                      "fixedColor": "red",
-                      "mode": "fixed"
-                    }
-                  }
-                ]
-              },
-              {
-                "matcher": {
-                  "id": "byName",
-                  "options": "p50"
+                  "options": "success/sec"
                 },
                 "properties": [
                   {
                     "id": "color",
                     "value": {
-                      "fixedColor": "blue",
+                      "fixedColor": "green",
                       "mode": "fixed"
                     }
                   }
@@ -306,9 +1283,9 @@ data:
             "h": 8,
             "w": 11,
             "x": 5,
-            "y": 1
+            "y": 35
           },
-          "id": 205,
+          "id": 227,
           "options": {
             "legend": {
               "calcs": [],
@@ -322,34 +1299,27 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.5, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))",
+              "expr": "sum(rate(image_builder_composer_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[$interval])) - sum(rate(image_builder_composer_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[$interval]))",
+              "hide": false,
               "interval": "",
-              "legendFormat": "p50",
+              "legendFormat": "success/sec",
               "refId": "A"
             },
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))",
+              "expr": "sum(rate(image_builder_composer_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[$interval]))",
               "hide": false,
               "interval": "",
-              "legendFormat": "p90",
+              "legendFormat": "errors/sec",
               "refId": "B"
-            },
-            {
-              "exemplar": true,
-              "expr": "histogram_quantile(0.99, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))",
-              "hide": false,
-              "interval": "",
-              "legendFormat": "p99",
-              "refId": "C"
             }
           ],
-          "title": "Depsolve Job Duration",
+          "title": "Build Job Throughput Rate",
           "type": "timeseries"
         },
         {
           "datasource": "${datasource}",
-          "description": "Percent of requests exceeding Duration allowed by SLO",
+          "description": "The number of osbuild job errors (as a percentage) over time for the selected time range",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -388,13 +1358,9 @@ data:
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  },
                   {
                     "color": "red",
-                    "value": 80
+                    "value": null
                   }
                 ]
               },
@@ -406,9 +1372,9 @@ data:
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 1
+            "y": 35
           },
-          "id": 194,
+          "id": 228,
           "options": {
             "legend": {
               "calcs": [],
@@ -422,13 +1388,13 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"32\", type=\"depsolve\"}[$interval]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=\"depsolve\"}[$interval]))",
+              "expr": "sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status=\"5xx\"}[$interval]))/sum(rate(image_builder_composer_worker_total_jobs{type=~\"osbuild:.*\", status!=\"4xx\"}[$interval]))",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
             }
           ],
-          "title": "Slow Request Rate",
+          "title": "Build Job Error Rate",
           "type": "timeseries"
         },
         {
@@ -489,9 +1455,9 @@ data:
             "h": 8,
             "w": 4,
             "x": 0,
-            "y": 9
+            "y": 43
           },
-          "id": 115,
+          "id": 223,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -516,7 +1482,7 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "28 * 24 * (1 - $latency_slo) / (1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"32\",type=\"depsolve\"}[$__range]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=\"depsolve\"}[$__range])))",
+              "expr": "28 * 24 * (1 - $stability_slo) / ((sum(rate(image_builder_composer_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[28d]))/ sum(rate(image_builder_composer_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[28d]))) + 0.001)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -597,9 +1563,9 @@ data:
             "h": 8,
             "w": 20,
             "x": 4,
-            "y": 9
+            "y": 43
           },
-          "id": 119,
+          "id": 224,
           "links": [],
           "options": {
             "legend": {
@@ -615,7 +1581,7 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "1 - ((sum(increase(image_builder_composer_worker_job_duration_seconds_bucket{le=\"32\", type=\"depsolve\"}[28d]))/sum(increase(image_builder_composer_worker_job_duration_seconds_count{type=\"depsolve\"}[28d]))) - $latency_slo)/ (1 - $latency_slo)",
+              "expr": "1 - ((1 - sum(increase(image_builder_composer_worker_total_jobs{type=\"depsolve\", status=\"5xx\"}[28d]))/sum(increase(image_builder_composer_worker_total_jobs{type=\"depsolve\", status!=\"4xx\"}[28d]))) - $stability_slo)/ (1 - $stability_slo)",
               "instant": false,
               "interval": "",
               "intervalFactor": 10,
@@ -635,16 +1601,16 @@ data:
             "h": 1,
             "w": 24,
             "x": 0,
-            "y": 17
+            "y": 51
           },
-          "id": 129,
+          "id": 128,
           "panels": [],
-          "title": "Osbuild Job Duration",
+          "title": "Depsolve Duration",
           "type": "row"
         },
         {
           "datasource": "${datasource}",
-          "description": "The duration of 90% of osbuild jobs",
+          "description": "The duration of 90% of depsolve jobs",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -672,11 +1638,11 @@ data:
                   },
                   {
                     "color": "#EAB839",
-                    "value": "1228"
+                    "value": "25"
                   },
                   {
                     "color": "red",
-                    "value": "1536"
+                    "value": "30"
                   }
                 ]
               },
@@ -688,9 +1654,9 @@ data:
             "h": 8,
             "w": 5,
             "x": 0,
-            "y": 18
+            "y": 52
           },
-          "id": 200,
+          "id": 237,
           "mappings": [
             {
               "options": {
@@ -721,18 +1687,18 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$__range])) by (le))",
+              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$__range])) by (le))",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
             }
           ],
-          "title": "Osbuild Job Duration",
+          "title": "Depsolve Job Duration",
           "type": "stat"
         },
         {
           "datasource": "${datasource}",
-          "description": "The request Duration for osbuild jobs over the selected date range",
+          "description": "The request Duration for depsolve jobs over the selected date range",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -899,9 +1865,9 @@ data:
             "h": 8,
             "w": 11,
             "x": 5,
-            "y": 18
+            "y": 52
           },
-          "id": 201,
+          "id": 205,
           "options": {
             "legend": {
               "calcs": [],
@@ -915,14 +1881,14 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.5, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))",
+              "expr": "histogram_quantile(0.5, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))",
               "interval": "",
               "legendFormat": "p50",
               "refId": "A"
             },
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))",
+              "expr": "histogram_quantile(0.9, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))",
               "hide": false,
               "interval": "",
               "legendFormat": "p90",
@@ -930,14 +1896,14 @@ data:
             },
             {
               "exemplar": true,
-              "expr": "histogram_quantile(0.99, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=~\"osbuild:.*\"}[$interval])) by (le))",
+              "expr": "histogram_quantile(0.99, sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{type=\"depsolve\"}[$interval])) by (le))",
               "hide": false,
               "interval": "",
               "legendFormat": "p99",
               "refId": "C"
             }
           ],
-          "title": "Osbuild Job Duration",
+          "title": "Depsolve Job Duration",
           "type": "timeseries"
         },
         {
@@ -946,13 +1912,13 @@ data:
           "fieldConfig": {
             "defaults": {
               "color": {
-                "mode": "palette-classic"
+                "mode": "thresholds"
               },
               "custom": {
                 "axisLabel": "",
                 "axisPlacement": "auto",
                 "barAlignment": 0,
-                "drawStyle": "line",
+                "drawStyle": "bars",
                 "fillOpacity": 0,
                 "gradientMode": "none",
                 "hideFrom": {
@@ -978,7 +1944,6 @@ data:
               },
               "mappings": [],
               "max": 1,
-              "min": 0,
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
@@ -1000,9 +1965,9 @@ data:
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 18
+            "y": 52
           },
-          "id": 210,
+          "id": 234,
           "options": {
             "legend": {
               "calcs": [],
@@ -1016,7 +1981,7 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[$interval]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[$interval]))",
+              "expr": "1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"32\", type=\"depsolve\"}[$interval]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=\"depsolve\"}[$interval]))",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1083,9 +2048,9 @@ data:
             "h": 8,
             "w": 4,
             "x": 0,
-            "y": 26
+            "y": 60
           },
-          "id": 198,
+          "id": 115,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -1110,7 +2075,7 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "28 * 24 * (1 - $latency_slo) / (1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[$__range]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[$__range])))",
+              "expr": "28 * 24 * (1 - $latency_slo) / (1 - sum(rate(image_builder_composer_worker_job_duration_seconds_bucket{le=\"32\",type=\"depsolve\"}[$__range]))/sum(rate(image_builder_composer_worker_job_duration_seconds_count{type=\"depsolve\"}[$__range])))",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -1165,6 +2130,7 @@ data:
               },
               "links": [],
               "mappings": [],
+              "max": 1,
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
@@ -1190,9 +2156,9 @@ data:
             "h": 8,
             "w": 20,
             "x": 4,
-            "y": 26
+            "y": 60
           },
-          "id": 199,
+          "id": 119,
           "links": [],
           "options": {
             "legend": {
@@ -1208,7 +2174,7 @@ data:
           "targets": [
             {
               "exemplar": true,
-              "expr": "1 - ((sum(increase(image_builder_composer_worker_job_duration_seconds_bucket{le=\"1536\",type=~\"osbuild:.*\"}[28d]))/sum(increase(image_builder_composer_worker_job_duration_seconds_count{type=~\"osbuild:.*\"}[28d]))) - $latency_slo)/ (1 - $latency_slo)",
+              "expr": "1 - ((sum(increase(image_builder_composer_worker_job_duration_seconds_bucket{le=\"32\", type=\"depsolve\"}[28d]))/sum(increase(image_builder_composer_worker_job_duration_seconds_count{type=\"depsolve\"}[28d]))) - $latency_slo)/ (1 - $latency_slo)",
               "instant": false,
               "interval": "",
               "intervalFactor": 10,
@@ -1228,7 +2194,7 @@ data:
             "h": 1,
             "w": 24,
             "x": 0,
-            "y": 34
+            "y": 68
           },
           "id": 207,
           "panels": [],
@@ -1281,7 +2247,7 @@ data:
             "h": 8,
             "w": 5,
             "x": 0,
-            "y": 35
+            "y": 69
           },
           "id": 208,
           "mappings": [
@@ -1492,7 +2458,7 @@ data:
             "h": 8,
             "w": 11,
             "x": 5,
-            "y": 35
+            "y": 69
           },
           "id": 209,
           "options": {
@@ -1593,7 +2559,7 @@ data:
             "h": 8,
             "w": 8,
             "x": 16,
-            "y": 35
+            "y": 69
           },
           "id": 204,
           "options": {
@@ -1772,5 +2738,5 @@ data:
       "timezone": "",
       "title": "Image Builder Worker",
       "uid": "image-builder-worker",
-      "version": 1
+      "version": 2
     }
diff --git a/templates/dnf-json.yml b/templates/dnf-json.yml
new file mode 100644
index 000000000..6b825adb0
--- /dev/null
+++ b/templates/dnf-json.yml
@@ -0,0 +1,108 @@
+apiVersion: v1
+kind: Template
+labels:
+  app: dnf-json
+  template: dnf-json
+metadata:
+  annotations:
+    description: |
+      Internal depsolving service meant as a supporting service for image-builder in crc
+  name: dnf-json
+objects:
+
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      service: dnf-json
+    name: dnf-json
+  spec:
+    replicas: 9
+    selector:
+      matchLabels:
+        name: dnf-json
+    template:
+      metadata:
+        labels:
+          name: dnf-json
+      spec:
+        serviceAccountName: dnf-json
+        affinity:
+          podAntiAffinity:
+            preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    name: dnf-json
+                topologyKey: kubernetes.io/hostname
+        containers:
+        - image: "${IMAGE_NAME}:${IMAGE_TAG}"
+          name: dnf-json
+          command: [ "python3", "/opt/entrypoint.py", "--dnf-json", "--dnf-json-port", "8080" ]
+          env:
+            - name: OVERWRITE_CACHE_DIR
+              value: "${CACHE_DIR}"
+          resources:
+            requests:
+              cpu: "${CPU_REQUEST}"
+              memory: "${MEMORY_REQUEST}"
+            limits:
+              cpu: "${CPU_LIMIT}"
+              memory: "${MEMORY_LIMIT}"
+          ports:
+          - name: api
+            containerPort: 8080
+            protocol: TCP
+          volumeMounts:
+          - name: cache
+            mountPath: "${CACHE_DIR}"
+        volumes:
+        - name: cache
+          emptyDir: {}
+
+- apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    name: dnf-json
+  imagePullSecrets:
+  - name: quay-cloudservices-pull
+
+- apiVersion: v1
+  kind: Service
+  metadata:
+    labels:
+      service: dnf-json
+    name: dnf-json
+  spec:
+    ports:
+      - name: dnf-json
+        protocol: TCP
+        port: 8080
+        targetPort: 8080
+    selector:
+      name: dnf-json
+
+parameters:
+- description: composer image name
+  name: IMAGE_NAME
+  value: quay.io/app-sre/composer
+  required: true
+- description: composer image tag
+  name: IMAGE_TAG
+  required: true
+# dnf-json can handle 1 depsolve at a time, therefore it benefits from multiple small containers
+- name: CPU_REQUEST
+  description: CPU request per container
+  value: "125m"
+- name: CPU_LIMIT
+  description: CPU limit per container
+  value: "125m"
+- name: MEMORY_REQUEST
+  description: Memory request per container
+  value: "128Mi"
+- name: MEMORY_LIMIT
+  description: Memory limit per container
+  value: "256Mi"
+- name: CACHE_DIR
+  value: "/var/cache/dnf-json"
diff --git a/vendor/modules.txt b/vendor/modules.txt
index def061f37..977b3bfa9 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -10,7 +10,7 @@ cloud.google.com/go/internal/trace
 cloud.google.com/go/internal/version
 cloud.google.com/go/longrunning
 cloud.google.com/go/longrunning/autogen
-# cloud.google.com/go/cloudbuild v0.2.0
+# cloud.google.com/go/cloudbuild v1.0.0
 ## explicit
 cloud.google.com/go/cloudbuild/apiv1
 # cloud.google.com/go/storage v1.18.2


Test patch:
diff --git a/test/README.md b/test/README.md
index 50239222f..9ac7020f3 100644
--- a/test/README.md
+++ b/test/README.md
@@ -250,6 +250,16 @@ an issue where the first line in the credentials file gets lost resulting in
 incomplete credentials. The work-around is to define a dummy ENV variable on
 the first line!
 
+### Cloud cleaner
+
+Some tests deploy images to different clouds. After the tests run, these images
+get removed by cloud cleaner (`cmd/cloud-cleaner`). But if something fails during the tests, or the
+pipeline gets canceled, that image could get left behind in the cloud, wasting
+resources.
+
+To deal with this problem, we have scheduled cloud cleaner (`schutzbot/scheduled_cloud_cleaner.sh`), that goes through all
+the clouds looking for testing resources that were not removed.
+
 ## Integration testing
 
 Since `osbuild-composer` externally provides two types of API, there also
diff --git a/test/cases/api.sh b/test/cases/api.sh
index cb24e41d1..879b84d78 100755
--- a/test/cases/api.sh
+++ b/test/cases/api.sh
@@ -238,6 +238,7 @@ function cleanupAzure() {
   # since this function can be called at any time, ensure that we don't expand unbound variables
   AZURE_CMD="${AZURE_CMD:-}"
   AZURE_IMAGE_NAME="${AZURE_IMAGE_NAME:-}"
+  AZURE_INSTANCE_NAME="${AZURE_INSTANCE_NAME:-}"
 
   # do not run clean-up if the image name is not yet defined
   if [[ -n "$AZURE_CMD" && -n "$AZURE_IMAGE_NAME" ]]; then
diff --git a/test/cases/base_tests.sh b/test/cases/base_tests.sh
index ede5e1099..c364f7e2c 100755
--- a/test/cases/base_tests.sh
+++ b/test/cases/base_tests.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -euo pipefail
 
+source /usr/libexec/tests/osbuild-composer/shared_lib.sh
+
 WORKING_DIRECTORY=/usr/libexec/osbuild-composer
 TESTS_PATH=/usr/libexec/osbuild-composer-test
 mkdir --parents /tmp/logs
@@ -11,12 +13,19 @@ FAILED_TESTS=()
 
 TEST_CASES=(
   "osbuild-weldr-tests"
-  "osbuild-dnf-json-tests"
   "osbuild-composer-cli-tests"
   "osbuild-auth-tests"
   "osbuild-composer-dbjobqueue-tests"
 )
 
+if nvrGreaterOrEqual "osbuild-composer" "41"; then
+    # 0 - osbuild-composer == v41
+    # 11 - osbuild-composer > v41
+    # 12 - osbuild-composer < v41
+    echo "INFO: enabling osbuild-dnf-json-tests"
+    TEST_CASES+=("osbuild-dnf-json-tests")
+fi
+
 # Print out a nice test divider so we know when tests stop and start.
 test_divider () {
     printf "%0.s-" {1..78} && echo
diff --git a/test/cases/cross-distro.sh b/test/cases/cross-distro.sh
index 44def3f0c..20612c384 100755
--- a/test/cases/cross-distro.sh
+++ b/test/cases/cross-distro.sh
@@ -3,7 +3,6 @@
 #
 # Test the available distributions. Only allow releases for the current distro.
 #
-
 APISOCKET=/run/weldr/api.socket
 
 source /etc/os-release
@@ -75,17 +74,81 @@ if ! sudo curl -s --unix-socket $APISOCKET http:///localhost/api/status > /dev/n
     exit 1
 fi
 
-if ! DISTROS=$(sudo curl -s --unix-socket $APISOCKET http:///localhost/api/v1/distros/list); then
+if ! RECOGNIZED_DISTROS=$(sudo curl -s --unix-socket $APISOCKET http:///localhost/api/v1/distros/list | jq -r '.distros[]'); then
     echo "osbuild-composer server error getting distros list. 😢"
     exit 1
 fi
 
-REMAINDER=$(echo "$DISTROS" | jq -r '.distros' | grep -v -E "$PATTERN")
-if [ -n "$REMAINDER" ]; then
-    echo "🔥 Unexpected distros installed:"
-    echo "$REMAINDER"
+# Get a list of all installed distros and compare it with a pattern matching host distribution
+# Filter out beta and centos-stream, see GH issue #2257
+INSTALLED_DISTROS=$(find "/usr/share/osbuild-composer/repositories" -name '*.json' -printf '%P\n' | awk -F "." '{ print $1 }' | grep -Ev 'beta|stream' | sort)
+INSTALLED_REMAINDER=$(echo "$INSTALLED_DISTROS" | grep -v -E "$PATTERN")
+# Check if there are any extra distros that match the host pattern but are not recognized
+UNRECOGNIZED_DISTROS=$(echo "${INSTALLED_DISTROS}" | grep -v "${RECOGNIZED_DISTROS}")
+if [ -n "$INSTALLED_REMAINDER" ] || [ -n "$UNRECOGNIZED_DISTROS" ];then
+    echo "Unexpected distros detected:"
+    echo "$INSTALLED_REMAINDER"
+    echo "$UNRECOGNIZED_DISTROS"
     exit 1
 else
-    echo "🎉 All tests passed."
-    exit 0
+    echo "All installed distros are recognized by composer."
+fi
+
+# set path to all composer repositories
+if [ ! -d "repositories/" ]; then
+    git clone --depth 1 http://github.com/osbuild/osbuild-composer
+    REPO_PATH="osbuild-composer/repositories/"
+else
+    REPO_PATH="repositories/"
 fi
+
+# ALL_DISTROS - all possible distros from upstream repository
+# ALL_EXPECTED_DISTROS - all distros matching host pattern
+# ALL_REMAINDERS - all the unrecognized distros
+# Filter out beta and centos-stream, see GH issue #2257
+ALL_DISTROS=$(find "$REPO_PATH" -name '*.json' -printf '%P\n' | awk -F "." '{ print $1 }')
+ALL_EXPECTED_DISTROS=$(echo "$ALL_DISTROS" | grep -E "$PATTERN" | grep -Ev 'beta|stream' | sort)
+ALL_REMAINDERS=$(echo "$ALL_DISTROS" | grep -v "$RECOGNIZED_DISTROS")
+
+# Check for any missing distros based on the expected host pattern
+if [ "$ALL_EXPECTED_DISTROS" != "$INSTALLED_DISTROS" ];then
+    echo "Some distros are missing!"
+    echo "Missing distros:"
+    diff <(echo "${ALL_EXPECTED_DISTROS}") <(echo "${INSTALLED_DISTROS}") | grep "<" | sed 's/^<\ //g'
+    exit 1
+fi
+
+# Push a blueprint with unsupported distro to see if composer fails gracefuly
+for REMAINING_DISTRO in $ALL_REMAINDERS; do
+    TEST_BP=blueprint.toml
+    tee "$TEST_BP" > /dev/null << EOF
+name = "bash"
+description = "A base system with bash"
+version = "0.0.1"
+distro= "$REMAINING_DISTRO"
+
+[[packages]]
+name = "bash"
+EOF
+    RESPONSE=$(sudo composer-cli blueprints push $TEST_BP 2>&1)
+
+    # there is a different reponse if legacy composer-cli is used
+    if rpm -q --quiet weldr-client; then
+        EXPECTED_RESPONSE="ERROR: BlueprintsError: '$REMAINING_DISTRO' is not a valid distribution"
+    else
+        EXPECTED_RESPONSE="'$REMAINING_DISTRO' is not a valid distribution"
+        RESPONSE=${RESPONSE#*: }
+    fi
+
+    if [ "$RESPONSE" == "$EXPECTED_RESPONSE" ];then
+            echo "Blueprint push with $REMAINING_DISTRO distro failed as expected."
+    else
+            echo "Something went wrong during blueprint push test."
+            echo "RESPONSE=$RESPONSE"
+            echo "EXPECTED_RESPONSE=$EXPECTED_RESPONSE"
+            exit 1
+    fi
+done
+
+echo "🎉 All tests passed."
+exit 0
diff --git a/test/cases/shared_lib.sh b/test/cases/shared_lib.sh
new file mode 100755
index 000000000..4d4112d15
--- /dev/null
+++ b/test/cases/shared_lib.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/bash
+
+function nvrGreaterOrEqual {
+    local rpm_name=$1
+    local min_version=$2
+
+    set +e
+
+    rpm_version=$(rpm -q --qf "%{version}" "${rpm_name}")
+    rpmdev-vercmp "${rpm_version}" "${min_version}"
+    if [ "$?" != "12" ]; then
+        # 0 - rpm_version == min_version
+        # 11 - rpm_version > min_version
+        # 12 - rpm_version < min_version
+        echo "DEBUG: ${rpm_version} >= ${min_version}"
+        set -e
+        return
+    fi
+
+    set -e
+    false
+}
diff --git a/test/data/manifests/rhel_85-aarch64-ami-boot.json b/test/data/manifests/rhel_85-aarch64-ami-boot.json
index 8e10a8b7b..4e04ea3e8 100644
--- a/test/data/manifests/rhel_85-aarch64-ami-boot.json
+++ b/test/data/manifests/rhel_85-aarch64-ami-boot.json
@@ -10871,6 +10871,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/proc": ".M.......",
         "/sys": ".M.......",
         "/var/log/lastlog": ".M....G..",
diff --git a/test/data/manifests/rhel_85-aarch64-ec2-boot.json b/test/data/manifests/rhel_85-aarch64-ec2-boot.json
index e56617807..0ac815aac 100644
--- a/test/data/manifests/rhel_85-aarch64-ec2-boot.json
+++ b/test/data/manifests/rhel_85-aarch64-ec2-boot.json
@@ -10921,6 +10921,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-beta.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_85-x86_64-ami-boot.json b/test/data/manifests/rhel_85-x86_64-ami-boot.json
index 00fa7b197..accae8d37 100644
--- a/test/data/manifests/rhel_85-x86_64-ami-boot.json
+++ b/test/data/manifests/rhel_85-x86_64-ami-boot.json
@@ -10454,6 +10454,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/proc": ".M.......",
         "/sys": ".M.......",
         "/var/log/lastlog": ".M....G..",
diff --git a/test/data/manifests/rhel_85-x86_64-ec2-boot.json b/test/data/manifests/rhel_85-x86_64-ec2-boot.json
index e9e1fb179..92f78858c 100644
--- a/test/data/manifests/rhel_85-x86_64-ec2-boot.json
+++ b/test/data/manifests/rhel_85-x86_64-ec2-boot.json
@@ -10505,6 +10505,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-beta.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_85-x86_64-ec2_ha-boot.json b/test/data/manifests/rhel_85-x86_64-ec2_ha-boot.json
index 09d88c85c..887c0f847 100644
--- a/test/data/manifests/rhel_85-x86_64-ec2_ha-boot.json
+++ b/test/data/manifests/rhel_85-x86_64-ec2_ha-boot.json
@@ -12949,6 +12949,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config-ha.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-ha.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_86-aarch64-ami-boot.json b/test/data/manifests/rhel_86-aarch64-ami-boot.json
index 5b4f38830..79b465311 100644
--- a/test/data/manifests/rhel_86-aarch64-ami-boot.json
+++ b/test/data/manifests/rhel_86-aarch64-ami-boot.json
@@ -10737,8 +10737,8 @@
         "fstype": "xfs",
         "label": "root",
         "partuuid": "6264D520-3FB9-423F-8AB8-7A0A8E3D3562",
-        "size": 9989681152,
-        "start": 747685888,
+        "size": 9989732352,
+        "start": 747634688,
         "type": "0FC63DAF-8483-4772-8E79-3D69D8477DE4",
         "uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75"
       },
@@ -10757,7 +10757,7 @@
         "fstype": "xfs",
         "label": null,
         "partuuid": "CB07C243-BC44-4717-853E-28852021225B",
-        "size": 536922112,
+        "size": 536870912,
         "start": 210763776,
         "type": "0FC63DAF-8483-4772-8E79-3D69D8477DE4",
         "uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"
@@ -10851,6 +10851,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/proc": ".M.......",
         "/sys": ".M.......",
         "/var/log/lastlog": ".M....G..",
diff --git a/test/data/manifests/rhel_86-aarch64-ec2-boot.json b/test/data/manifests/rhel_86-aarch64-ec2-boot.json
index 2c4e04902..f5486ae19 100644
--- a/test/data/manifests/rhel_86-aarch64-ec2-boot.json
+++ b/test/data/manifests/rhel_86-aarch64-ec2-boot.json
@@ -10787,8 +10787,8 @@
         "fstype": "xfs",
         "label": "root",
         "partuuid": "6264D520-3FB9-423F-8AB8-7A0A8E3D3562",
-        "size": 9989681152,
-        "start": 747685888,
+        "size": 9989732352,
+        "start": 747634688,
         "type": "0FC63DAF-8483-4772-8E79-3D69D8477DE4",
         "uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75"
       },
@@ -10807,7 +10807,7 @@
         "fstype": "xfs",
         "label": null,
         "partuuid": "CB07C243-BC44-4717-853E-28852021225B",
-        "size": 536922112,
+        "size": 536870912,
         "start": 210763776,
         "type": "0FC63DAF-8483-4772-8E79-3D69D8477DE4",
         "uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"
@@ -10901,6 +10901,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-beta.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_86-x86_64-ami-boot.json b/test/data/manifests/rhel_86-x86_64-ami-boot.json
index 0d1968968..e3b4aa83e 100644
--- a/test/data/manifests/rhel_86-x86_64-ami-boot.json
+++ b/test/data/manifests/rhel_86-x86_64-ami-boot.json
@@ -10434,6 +10434,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/proc": ".M.......",
         "/sys": ".M.......",
         "/var/log/lastlog": ".M....G..",
diff --git a/test/data/manifests/rhel_86-x86_64-ec2-boot.json b/test/data/manifests/rhel_86-x86_64-ec2-boot.json
index 5e153a7cc..5e4b56b81 100644
--- a/test/data/manifests/rhel_86-x86_64-ec2-boot.json
+++ b/test/data/manifests/rhel_86-x86_64-ec2-boot.json
@@ -10486,6 +10486,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-beta.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_86-x86_64-ec2_ha-boot.json b/test/data/manifests/rhel_86-x86_64-ec2_ha-boot.json
index 8b050a8ab..bdc1d44db 100644
--- a/test/data/manifests/rhel_86-x86_64-ec2_ha-boot.json
+++ b/test/data/manifests/rhel_86-x86_64-ec2_ha-boot.json
@@ -12959,6 +12959,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config-ha.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-ha.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_86-x86_64-ec2_sap-boot.json b/test/data/manifests/rhel_86-x86_64-ec2_sap-boot.json
index df426b2ae..fd507d44e 100644
--- a/test/data/manifests/rhel_86-x86_64-ec2_sap-boot.json
+++ b/test/data/manifests/rhel_86-x86_64-ec2_sap-boot.json
@@ -13410,6 +13410,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config-sap-bundle.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-sap-bundle-e4s.repo": "..5....T.",
         "/proc": ".M.......",
diff --git a/test/data/manifests/rhel_90-aarch64-ami-boot.json b/test/data/manifests/rhel_90-aarch64-ami-boot.json
index a83c38825..22182ae6e 100644
--- a/test/data/manifests/rhel_90-aarch64-ami-boot.json
+++ b/test/data/manifests/rhel_90-aarch64-ami-boot.json
@@ -278,7 +278,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto"
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0"
             }
           },
           {
@@ -1016,7 +1016,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
               "uefi": {
                 "vendor": "redhat",
                 "unified": true
@@ -9094,7 +9094,7 @@
       "profile-id": "sssd"
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.aarch64"
     },
     "bootloader": "unknown",
@@ -9105,7 +9105,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -9115,7 +9115,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.aarch64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.aarch64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.aarch64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.aarch64"
       }
@@ -9957,6 +9957,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/udev/hwdb.bin": ".M.......",
         "/proc": ".M.......",
         "/sys": ".M.......",
diff --git a/test/data/manifests/rhel_90-aarch64-ec2-boot.json b/test/data/manifests/rhel_90-aarch64-ec2-boot.json
index fac186795..639d642e7 100644
--- a/test/data/manifests/rhel_90-aarch64-ec2-boot.json
+++ b/test/data/manifests/rhel_90-aarch64-ec2-boot.json
@@ -287,7 +287,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto"
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0"
             }
           },
           {
@@ -1030,7 +1030,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
               "uefi": {
                 "vendor": "redhat",
                 "unified": true
@@ -9143,7 +9143,7 @@
       "profile-id": "sssd"
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.aarch64"
     },
     "bootloader": "unknown",
@@ -9154,7 +9154,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -9164,7 +9164,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.aarch64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.aarch64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 iommu.strict=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.aarch64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.aarch64"
       }
@@ -10007,6 +10007,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/udev/hwdb.bin": ".M.......",
         "/etc/yum.repos.d/redhat-rhui-beta.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config.repo": "..5....T.",
diff --git a/test/data/manifests/rhel_90-aarch64-qcow2-boot.json b/test/data/manifests/rhel_90-aarch64-qcow2-boot.json
index d3cb78480..184ade182 100644
--- a/test/data/manifests/rhel_90-aarch64-qcow2-boot.json
+++ b/test/data/manifests/rhel_90-aarch64-qcow2-boot.json
@@ -291,7 +291,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -932,7 +932,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
               "uefi": {
                 "vendor": "redhat",
                 "unified": true
@@ -9189,7 +9189,7 @@
   "image-info": {
     "/etc/resolv.conf": [],
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.aarch64"
     },
     "bootloader": "unknown",
@@ -9200,7 +9200,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.aarch64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.aarch64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.aarch64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.aarch64"
       }
diff --git a/test/data/manifests/rhel_90-aarch64-qcow2_customize-boot.json b/test/data/manifests/rhel_90-aarch64-qcow2_customize-boot.json
index ce58a997b..c3e5ac1c5 100644
--- a/test/data/manifests/rhel_90-aarch64-qcow2_customize-boot.json
+++ b/test/data/manifests/rhel_90-aarch64-qcow2_customize-boot.json
@@ -349,7 +349,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -1272,7 +1272,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 debug",
               "uefi": {
                 "vendor": "redhat",
                 "unified": true
@@ -11623,7 +11623,7 @@
   "image-info": {
     "/etc/resolv.conf": [],
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 debug",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.aarch64"
     },
     "bootloader": "unknown",
@@ -11634,7 +11634,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -11644,7 +11644,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.aarch64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.aarch64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.aarch64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.aarch64"
       }
diff --git a/test/data/manifests/rhel_90-ppc64le-qcow2-boot.json b/test/data/manifests/rhel_90-ppc64le-qcow2-boot.json
index ef0d7912b..c8134503b 100644
--- a/test/data/manifests/rhel_90-ppc64le-qcow2-boot.json
+++ b/test/data/manifests/rhel_90-ppc64le-qcow2-boot.json
@@ -302,7 +302,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -989,7 +989,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
               "legacy": "powerpc-ieee1275",
               "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.ppc64le"
             }
@@ -9985,7 +9985,7 @@
       ]
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.ppc64le"
     },
     "bootloader": "unknown",
@@ -9996,7 +9996,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.ppc64le.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.ppc64le",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.ppc64le) 9.0 (Plow)",
         "version": "5.14.0-4.el9.ppc64le"
       }
@@ -10776,7 +10776,7 @@
         "partuuid": "14fc63d2-02",
         "size": 524288000,
         "start": 5242880,
-        "type": "0",
+        "type": "83",
         "uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"
       },
       {
diff --git a/test/data/manifests/rhel_90-ppc64le-qcow2_customize-boot.json b/test/data/manifests/rhel_90-ppc64le-qcow2_customize-boot.json
index 50d81457e..f0fed00e4 100644
--- a/test/data/manifests/rhel_90-ppc64le-qcow2_customize-boot.json
+++ b/test/data/manifests/rhel_90-ppc64le-qcow2_customize-boot.json
@@ -360,7 +360,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -1335,7 +1335,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 debug",
               "legacy": "powerpc-ieee1275",
               "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.ppc64le"
             }
@@ -12479,7 +12479,7 @@
       ]
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 debug",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.ppc64le"
     },
     "bootloader": "unknown",
@@ -12490,7 +12490,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -12500,7 +12500,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.ppc64le.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.ppc64le",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.ppc64le) 9.0 (Plow)",
         "version": "5.14.0-4.el9.ppc64le"
       }
@@ -13307,7 +13307,7 @@
         "partuuid": "14fc63d2-02",
         "size": 524288000,
         "start": 5242880,
-        "type": "0",
+        "type": "83",
         "uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"
       },
       {
diff --git a/test/data/manifests/rhel_90-s390x-qcow2-boot.json b/test/data/manifests/rhel_90-s390x-qcow2-boot.json
index 0381adac8..ad98816e3 100644
--- a/test/data/manifests/rhel_90-s390x-qcow2-boot.json
+++ b/test/data/manifests/rhel_90-s390x-qcow2-boot.json
@@ -406,7 +406,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -10927,10 +10927,10 @@
         "grub_arg": "--unrestricted",
         "grub_class": "kernel",
         "grub_users": "$grub_users",
-        "id": "-20211207101433-5.14.0-4.el9.s390x",
+        "id": "-20220204144212-5.14.0-4.el9.s390x",
         "initrd": "/boot/initramfs-5.14.0-4.el9.s390x.img",
         "linux": "/boot/vmlinuz-5.14.0-4.el9.s390x",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
         "title": " (5.14.0-4.el9.s390x)",
         "version": "5.14.0-4.el9.s390x"
       }
@@ -11684,7 +11684,7 @@
         "partuuid": "14fc63d2-01",
         "size": 524288000,
         "start": 1048576,
-        "type": "0",
+        "type": "83",
         "uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"
       },
       {
diff --git a/test/data/manifests/rhel_90-s390x-qcow2_customize-boot.json b/test/data/manifests/rhel_90-s390x-qcow2_customize-boot.json
index dcadb0266..f000cecea 100644
--- a/test/data/manifests/rhel_90-s390x-qcow2_customize-boot.json
+++ b/test/data/manifests/rhel_90-s390x-qcow2_customize-boot.json
@@ -464,7 +464,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -13261,10 +13261,10 @@
         "grub_arg": "--unrestricted",
         "grub_class": "kernel",
         "grub_users": "$grub_users",
-        "id": "-20211207101636-0-rescue-ffffffffffffffffffffffffffffffff",
+        "id": "-20220204144421-0-rescue-ffffffffffffffffffffffffffffffff",
         "initrd": "/boot/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/boot/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
         "title": " (0-rescue-ffffffffffffffffffffffffffffffff)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -13272,10 +13272,10 @@
         "grub_arg": "--unrestricted",
         "grub_class": "kernel",
         "grub_users": "$grub_users",
-        "id": "-20211207101636-5.14.0-4.el9.s390x",
+        "id": "-20220204144421-5.14.0-4.el9.s390x",
         "initrd": "/boot/initramfs-5.14.0-4.el9.s390x.img",
         "linux": "/boot/vmlinuz-5.14.0-4.el9.s390x",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
         "title": " (5.14.0-4.el9.s390x)",
         "version": "5.14.0-4.el9.s390x"
       }
@@ -14056,7 +14056,7 @@
         "partuuid": "14fc63d2-01",
         "size": 524288000,
         "start": 1048576,
-        "type": "0",
+        "type": "83",
         "uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"
       },
       {
diff --git a/test/data/manifests/rhel_90-x86_64-ami-boot.json b/test/data/manifests/rhel_90-x86_64-ami-boot.json
index bb00085cf..b14c99c39 100644
--- a/test/data/manifests/rhel_90-x86_64-ami-boot.json
+++ b/test/data/manifests/rhel_90-x86_64-ami-boot.json
@@ -292,7 +292,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto"
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295"
             }
           },
           {
@@ -1011,7 +1011,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
               "legacy": "i386-pc",
               "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
             }
@@ -8878,7 +8878,7 @@
       "profile-id": "sssd"
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
     },
     "bootloader": "grub",
@@ -8889,7 +8889,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -8899,7 +8899,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.x86_64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.x86_64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.x86_64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.x86_64"
       }
@@ -9721,6 +9721,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/udev/hwdb.bin": ".M.......",
         "/proc": ".M.......",
         "/sys": ".M.......",
diff --git a/test/data/manifests/rhel_90-x86_64-ec2-boot.json b/test/data/manifests/rhel_90-x86_64-ec2-boot.json
index 240e0ca77..e288a3102 100644
--- a/test/data/manifests/rhel_90-x86_64-ec2-boot.json
+++ b/test/data/manifests/rhel_90-x86_64-ec2-boot.json
@@ -303,7 +303,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto"
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295"
             }
           },
           {
@@ -1027,7 +1027,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
               "legacy": "i386-pc",
               "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
             }
@@ -8929,7 +8929,7 @@
       "profile-id": "sssd"
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
     },
     "bootloader": "grub",
@@ -8940,7 +8940,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -8950,7 +8950,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.x86_64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.x86_64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.x86_64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.x86_64"
       }
@@ -9773,6 +9773,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/udev/hwdb.bin": ".M.......",
         "/etc/yum.repos.d/redhat-rhui-beta.repo": "..5....T.",
         "/etc/yum.repos.d/redhat-rhui-client-config.repo": "..5....T.",
diff --git a/test/data/manifests/rhel_90-x86_64-ec2_ha-boot.json b/test/data/manifests/rhel_90-x86_64-ec2_ha-boot.json
index 7371f3820..7ac8e170e 100644
--- a/test/data/manifests/rhel_90-x86_64-ec2_ha-boot.json
+++ b/test/data/manifests/rhel_90-x86_64-ec2_ha-boot.json
@@ -313,7 +313,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto"
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295"
             }
           },
           {
@@ -1221,7 +1221,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
               "legacy": "i386-pc",
               "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
             }
@@ -11316,7 +11316,7 @@
       "profile-id": "sssd"
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
     },
     "bootloader": "grub",
@@ -11327,7 +11327,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -11337,7 +11337,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.x86_64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.x86_64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.x86_64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.x86_64"
       }
@@ -12353,6 +12353,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/udev/hwdb.bin": ".M.......",
         "/proc": ".M.......",
         "/sys": ".M.......",
diff --git a/test/data/manifests/rhel_90-x86_64-ec2_sap-boot.json b/test/data/manifests/rhel_90-x86_64-ec2_sap-boot.json
index f2b683ed8..ed2460dd6 100644
--- a/test/data/manifests/rhel_90-x86_64-ec2_sap-boot.json
+++ b/test/data/manifests/rhel_90-x86_64-ec2_sap-boot.json
@@ -331,7 +331,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto processor.max_cstate=1 intel_idle.max_cstate=1"
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 processor.max_cstate=1 intel_idle.max_cstate=1"
             }
           },
           {
@@ -1392,7 +1392,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto processor.max_cstate=1 intel_idle.max_cstate=1",
+              "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 processor.max_cstate=1 intel_idle.max_cstate=1",
               "legacy": "i386-pc",
               "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
             }
@@ -11895,7 +11895,7 @@
       "profile-id": "sssd"
     },
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto processor.max_cstate=1 intel_idle.max_cstate=1",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 processor.max_cstate=1 intel_idle.max_cstate=1",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
     },
     "bootloader": "grub",
@@ -11906,7 +11906,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto processor.max_cstate=1 intel_idle.max_cstate=1",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 processor.max_cstate=1 intel_idle.max_cstate=1",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -11916,7 +11916,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.x86_64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.x86_64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto processor.max_cstate=1 intel_idle.max_cstate=1 $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 processor.max_cstate=1 intel_idle.max_cstate=1 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.x86_64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.x86_64"
       }
@@ -12967,6 +12967,7 @@
         "/etc/pam.d/smartcard-auth": "....L....",
         "/etc/pam.d/system-auth": "....L....",
         "/etc/rhsm/rhsm.conf": "..5....T.",
+        "/etc/ssh/sshd_config": "S.5....T.",
         "/etc/udev/hwdb.bin": ".M.......",
         "/proc": ".M.......",
         "/sys": ".M.......",
diff --git a/test/data/manifests/rhel_90-x86_64-qcow2-boot.json b/test/data/manifests/rhel_90-x86_64-qcow2-boot.json
index 94e584e4f..0be30df97 100644
--- a/test/data/manifests/rhel_90-x86_64-qcow2-boot.json
+++ b/test/data/manifests/rhel_90-x86_64-qcow2-boot.json
@@ -305,7 +305,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -952,7 +952,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
               "legacy": "i386-pc",
               "uefi": {
                 "vendor": "redhat"
@@ -9442,7 +9442,7 @@
   "image-info": {
     "/etc/resolv.conf": [],
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
     },
     "bootloader": "grub",
@@ -9453,7 +9453,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.x86_64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.x86_64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.x86_64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.x86_64"
       }
diff --git a/test/data/manifests/rhel_90-x86_64-qcow2_customize-boot.json b/test/data/manifests/rhel_90-x86_64-qcow2_customize-boot.json
index 6be6f2e8f..8ee5e6ca5 100644
--- a/test/data/manifests/rhel_90-x86_64-qcow2_customize-boot.json
+++ b/test/data/manifests/rhel_90-x86_64-qcow2_customize-boot.json
@@ -363,7 +363,7 @@
             "type": "org.osbuild.kernel-cmdline",
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0"
             }
           },
           {
@@ -1309,7 +1309,7 @@
             "options": {
               "root_fs_uuid": "6e4ff95f-f662-45ee-a82a-bdf44a2d0b75",
               "boot_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
-              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+              "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 debug",
               "legacy": "i386-pc",
               "uefi": {
                 "vendor": "redhat"
@@ -12082,7 +12082,7 @@
   "image-info": {
     "/etc/resolv.conf": [],
     "boot-environment": {
-      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+      "kernelopts": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 debug",
       "saved_entry": "ffffffffffffffffffffffffffffffff-5.14.0-4.el9.x86_64"
     },
     "bootloader": "grub",
@@ -12093,7 +12093,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-0-rescue-ffffffffffffffffffffffffffffffff.img",
         "linux": "/vmlinuz-0-rescue-ffffffffffffffffffffffffffffffff",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0",
         "title": "Red Hat Enterprise Linux (0-rescue-ffffffffffffffffffffffffffffffff) 9.0 (Plow)",
         "version": "0-rescue-ffffffffffffffffffffffffffffffff"
       },
@@ -12103,7 +12103,7 @@
         "grub_users": "$grub_users",
         "initrd": "/initramfs-5.14.0-4.el9.x86_64.img $tuned_initrd",
         "linux": "/vmlinuz-5.14.0-4.el9.x86_64",
-        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto $tuned_params",
+        "options": "root=UUID=6e4ff95f-f662-45ee-a82a-bdf44a2d0b75 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 $tuned_params",
         "title": "Red Hat Enterprise Linux (5.14.0-4.el9.x86_64) 9.0 (Plow)",
         "version": "5.14.0-4.el9.x86_64"
       }