Repository: osbuild/osbuild-composer
Pull request #1240 (2021-02-19 13:43:33)

tools: move to new utility-container builds
Use the new tags of the `osbuild/containers` repository. The
infrastructure was simplified a lot and the new tags are much easier to
handle without any conflicts when building other images.

Note that this change uses the `latest` tag of all images. As an
alternative, we can also switch to the `latest-<date>` tags, which would
use a pinned immutable image tagged by the given date. However, these
tags are burried somewhere deep down in the ./tools/ directory and are
easy to forget upgrading. So for now use `latest` until we have better
synchronization infrastructure in place.

However, if these images ever break and we want to get back stable CI
behavior, we can always switch back to older builds by picking those
immutable tags instead of `latest`.


Patch:
diff --git a/.gitignore b/.gitignore
index 534086142..3efe28a99 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
 __pycache__
 
 .vscode
+.DS_Store
 
+/containers/config/
 /bin
 /rpmbuild
diff --git a/NEWS.md b/NEWS.md
index 2db7b5491..f6af8f1a0 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,37 @@
 # OSBuild Composer - Operating System Image Composition Services
 
+## CHANGES WITH 28:
+
+  * OSBuild Composer can now build CentOS Stream 8 images!
+
+  * OSBuild Composer can now upload images to VMWare!
+
+  * Cloudapi contains new /openapi.json and /version endpoints. The already
+    existing `POST /compose` endpoint now supports specification of additional
+    packages.
+
+  * Blueprints now support definition of alternative kernels, such as kernel-rt,
+    using the `customizations.kernel.name` key.
+
+  * Attempt to use the `customization.kernel.append` key in a blueprint for the
+    OSTree image type now results in error, because the customization was ignored.
+
+  * OSBuild dependency now requires version 24 because of its new sysconfig and
+    RHSM stages.
+
+  * RHEL qcow2 images now come with disabled product-id and subscription-manager
+    plugins in dnf to make them consistent with images produced by imagefactory.
+
+  * OSTree `parent` can now be replaced with `url` pointing to a repository
+    that contains the parent commit.
+
+Contributions from: Achilleas Koutsou, Alexander Todorov, Antonio Murdaca,
+                    Brian C. Lane, Chloe Kaubisch, Jacob Kozol, Jozef Mikovic,
+                    Major Hayden, Martin Sehnoutka, Ondřej Budai, Sanne Raymaekers,
+                    Tomas Hozza, Tom Gundersen
+
+— Brno, 2021-02-20
+
 ## CHANGES WITH 27:
 
   * Starting from this release, we are putting longer news posts in
diff --git a/README.md b/README.md
index 1eb7630dc..03da623cf 100644
--- a/README.md
+++ b/README.md
@@ -45,7 +45,7 @@ provided infrastructure and services.
 
 The requirements for this project are:
 
- * `osbuild >= 24`
+ * `osbuild >= 26`
  * `systemd >= 244`
 
 At build-time, the following software is required:
diff --git a/Schutzfile b/Schutzfile
index 250979881..22972185d 100644
--- a/Schutzfile
+++ b/Schutzfile
@@ -1,5 +1,10 @@
 {
   "fedora-32": {
+    "dependencies": {
+      "osbuild": {
+        "commit": "6f1350d72d210767062a8e346e1472305703686b"
+      }
+    },
     "dependants": {
       "koji-osbuild": {
         "commit": "4fdc457745e1147475ea3ac1e3b073e592d7b174"
@@ -9,7 +14,7 @@
   "rhel-8.3": {
     "dependencies": {
       "osbuild": {
-        "commit": "755c07a142ce2e8e88420590be553d8a255f3326"
+        "commit": "6f1350d72d210767062a8e346e1472305703686b"
       }
     },
     "dependants": {
@@ -20,5 +25,26 @@
         ]
       }
     }
+  },
+  "fedora-33": {
+    "dependencies": {
+      "osbuild": {
+        "commit": "6f1350d72d210767062a8e346e1472305703686b"
+      }
+    }
+  },
+  "centos-8": {
+    "dependencies": {
+      "osbuild": {
+        "commit": "6f1350d72d210767062a8e346e1472305703686b"
+      }
+    }
+  },
+  "rhel-8.4": {
+    "dependencies": {
+      "osbuild": {
+        "commit": "6f1350d72d210767062a8e346e1472305703686b"
+      }
+    }
   }
 }
diff --git a/cmd/osbuild-worker/jobimpl-osbuild-koji.go b/cmd/osbuild-worker/jobimpl-osbuild-koji.go
index f7d2d26fb..545546bfa 100644
--- a/cmd/osbuild-worker/jobimpl-osbuild-koji.go
+++ b/cmd/osbuild-worker/jobimpl-osbuild-koji.go
@@ -18,6 +18,7 @@ import (
 
 type OSBuildKojiJobImpl struct {
 	Store       string
+	Output      string
 	KojiServers map[string]koji.GSSAPICredentials
 }
 
@@ -54,7 +55,7 @@ func (impl *OSBuildKojiJobImpl) kojiUpload(file *os.File, server, directory, fil
 }
 
 func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
-	outputDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-worker-*")
+	outputDirectory, err := ioutil.TempDir(impl.Output, job.Id().String()+"-*")
 	if err != nil {
 		return fmt.Errorf("error creating temporary output directory: %v", err)
 	}
diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go
index 512e559df..c6f0e1e85 100644
--- a/cmd/osbuild-worker/jobimpl-osbuild.go
+++ b/cmd/osbuild-worker/jobimpl-osbuild.go
@@ -25,6 +25,7 @@ import (
 
 type OSBuildJobImpl struct {
 	Store       string
+	Output      string
 	KojiServers map[string]koji.GSSAPICredentials
 }
 
@@ -62,7 +63,7 @@ func osbuildStagesToRPMs(stages []osbuild.StageResult) []koji.RPM {
 }
 
 func (impl *OSBuildJobImpl) Run(job worker.Job) error {
-	outputDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-worker-*")
+	outputDirectory, err := ioutil.TempDir(impl.Output, job.Id().String()+"-*")
 	if err != nil {
 		return fmt.Errorf("error creating temporary output directory: %v", err)
 	}
@@ -154,7 +155,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 				Datastore:  options.Datastore,
 			}
 
-			tempDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-worker-vmware-*")
+			tempDirectory, err := ioutil.TempDir(impl.Output, job.Id().String()+"-vmware-*")
 			if err != nil {
 				r = append(r, err)
 				continue
diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go
index 578f77c7c..62907ce81 100644
--- a/cmd/osbuild-worker/main.go
+++ b/cmd/osbuild-worker/main.go
@@ -117,6 +117,8 @@ func main() {
 		log.Fatal("CACHE_DIRECTORY is not set. Is the service file missing CacheDirectory=?")
 	}
 	store := path.Join(cacheDirectory, "osbuild-store")
+	output := path.Join(cacheDirectory, "output")
+	_ = os.Mkdir(output, os.ModeDir)
 
 	kojiServers := make(map[string]koji.GSSAPICredentials)
 	for server, creds := range config.KojiServers {
@@ -152,10 +154,12 @@ func main() {
 	jobImpls := map[string]JobImplementation{
 		"osbuild": &OSBuildJobImpl{
 			Store:       store,
+			Output:      output,
 			KojiServers: kojiServers,
 		},
 		"osbuild-koji": &OSBuildKojiJobImpl{
 			Store:       store,
+			Output:      output,
 			KojiServers: kojiServers,
 		},
 		"koji-init": &KojiInitJobImpl{
diff --git a/docs/news/unreleased/centos-stream.md b/docs/news/28/centos-stream.md
similarity index 100%
rename from docs/news/unreleased/centos-stream.md
rename to docs/news/28/centos-stream.md
diff --git a/docs/news/unreleased/cloudapi-packages.md b/docs/news/28/cloudapi-packages.md
similarity index 100%
rename from docs/news/unreleased/cloudapi-packages.md
rename to docs/news/28/cloudapi-packages.md
diff --git a/docs/news/unreleased/kernel-customizations-name.md b/docs/news/28/kernel-customizations-name.md
similarity index 100%
rename from docs/news/unreleased/kernel-customizations-name.md
rename to docs/news/28/kernel-customizations-name.md
diff --git a/docs/news/unreleased/openapi-endpoint-in-cloudapi.md b/docs/news/28/openapi-endpoint-in-cloudapi.md
similarity index 100%
rename from docs/news/unreleased/openapi-endpoint-in-cloudapi.md
rename to docs/news/28/openapi-endpoint-in-cloudapi.md
diff --git a/docs/news/unreleased/osbuild-24.md b/docs/news/28/osbuild-24.md
similarity index 100%
rename from docs/news/unreleased/osbuild-24.md
rename to docs/news/28/osbuild-24.md
diff --git a/docs/news/unreleased/osbuild-rhsm-stage.md b/docs/news/28/osbuild-rhsm-stage.md
similarity index 100%
rename from docs/news/unreleased/osbuild-rhsm-stage.md
rename to docs/news/28/osbuild-rhsm-stage.md
diff --git a/docs/news/unreleased/ostree-kernel-append.md b/docs/news/28/ostree-kernel-append.md
similarity index 100%
rename from docs/news/unreleased/ostree-kernel-append.md
rename to docs/news/28/ostree-kernel-append.md
diff --git a/docs/news/unreleased/sysconfig.md b/docs/news/28/sysconfig.md
similarity index 100%
rename from docs/news/unreleased/sysconfig.md
rename to docs/news/28/sysconfig.md
diff --git a/docs/news/unreleased/vmware-upload-target.md b/docs/news/28/vmware-upload-target.md
similarity index 100%
rename from docs/news/unreleased/vmware-upload-target.md
rename to docs/news/28/vmware-upload-target.md
diff --git a/docs/news/28/weldr-ostree-url.md b/docs/news/28/weldr-ostree-url.md
new file mode 100644
index 000000000..5bbc7ad50
--- /dev/null
+++ b/docs/news/28/weldr-ostree-url.md
@@ -0,0 +1,40 @@
+# Weldr API: Allow parent OSTree commit to be read from repository
+
+The weldr API for building OSTree based images is extended to optionally take an `url` parameter instead of the current `parent`.
+
+The `parent` parameter contains the OSTree commit SHA of the parent commit when building an update commit. Obtaining this is cumbersome, so instead the `url` of the repository containing the desired parent commit can be specified. In this case, composer will take the current `HEAD` of the given `ref` as the parent.
+
+At most one of `parent` and `url` can be specified in a given compose request.
+
+Before:
+
+    curl --silent \
+        --header "Content-Type: application/json" \
+        --unix-socket /run/weldr/api.socket \
+        http://localhost/api/v1/compose \
+        --data "{ \
+            \"blueprint_name\": \"foo\", \
+            \"compose_type\": \"rhel-edge-commit\", \
+            \"ostree\": {\ \
+                \"parent\": \"b8a69e5c79be5830bb272356809a52b1660d2013c26f6973d549d0a312a8d21a\", \
+                \"ref\": \"fedora/stable/x86_64/iot\" \
+            } \
+        }"
+
+After:
+
+    curl --silent \
+        --header "Content-Type: application/json" \
+        --unix-socket /run/weldr/api.socket \
+        http://localhost/api/v1/compose \
+        --data "{ \
+            \"blueprint_name\": \"foo\", \
+            \"compose_type\": \"rhel-edge-commit\", \
+            \"ostree\": {\ \
+                \"url\": \"https://d2ju0wfl996cmc.cloudfront.net/\", \
+                \"ref\": \"fedora/stable/x86_64/iot\" \
+            } \
+        }"
+
+Relevant PRs:
+https://github.com/osbuild/osbuild-composer/pull/1235
diff --git a/docs/news/unreleased/.gitkeep b/docs/news/unreleased/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/news/unreleased/rhel84-grub2-saved-entry.md b/docs/news/unreleased/rhel84-grub2-saved-entry.md
new file mode 100644
index 000000000..ed3f52b26
--- /dev/null
+++ b/docs/news/unreleased/rhel84-grub2-saved-entry.md
@@ -0,0 +1,5 @@
+# RHEL8.4: Fix grub2 kernel selection
+
+By marking the kernel we install as the `saved_entry`, we make sure that installing additional/subsequent kernels do not unintentienally change the default kernel to be booted into.
+
+Relevant PR: https://github.com/osbuild/osbuild-composer/pull/1241
diff --git a/internal/blueprint/blueprint.go b/internal/blueprint/blueprint.go
index bb4d60e2a..c30a16a46 100644
--- a/internal/blueprint/blueprint.go
+++ b/internal/blueprint/blueprint.go
@@ -104,13 +104,10 @@ func (b *Blueprint) GetPackages() []string {
 		packages = append(packages, "@"+group.Name)
 	}
 
-	if kc := b.Customizations.GetKernel(); kc != nil && kc.Name != "" {
-		kpkg := Package{Name: b.Customizations.Kernel.Name}
-		packages = append(packages, kpkg.ToNameVersion())
-	} else { // no Kernel specified; add default
-		kpkg := Package{Name: "kernel"}
-		packages = append(packages, kpkg.ToNameVersion())
-	}
+	kc := b.Customizations.GetKernel()
+	kpkg := Package{Name: kc.Name}
+	packages = append(packages, kpkg.ToNameVersion())
+
 	return packages
 }
 
diff --git a/internal/blueprint/customizations.go b/internal/blueprint/customizations.go
index f6867d7e0..ad88c51f9 100644
--- a/internal/blueprint/customizations.go
+++ b/internal/blueprint/customizations.go
@@ -155,11 +155,21 @@ func (c *Customizations) GetGroups() []GroupCustomization {
 }
 
 func (c *Customizations) GetKernel() *KernelCustomization {
-	if c == nil {
-		return nil
+	var name string
+	var append string
+	if c != nil && c.Kernel != nil {
+		name = c.Kernel.Name
+		append = c.Kernel.Append
 	}
 
-	return c.Kernel
+	if name == "" {
+		name = "kernel"
+	}
+
+	return &KernelCustomization{
+		Name:   name,
+		Append: append,
+	}
 }
 
 func (c *Customizations) GetFirewall() *FirewallCustomization {
diff --git a/internal/distro/distro.go b/internal/distro/distro.go
index 606f29e19..3946ac97f 100644
--- a/internal/distro/distro.go
+++ b/internal/distro/distro.go
@@ -65,6 +65,9 @@ type ImageType interface {
 	// Retrns the MIME-type for the image type.
 	MIMEType() string
 
+	// Returns the default OSTree ref for the image type.
+	OSTreeRef() string
+
 	// Returns the proper image size for a given output format. If the input size
 	// is 0 the default value for the format will be returned.
 	Size(size uint64) uint64
diff --git a/internal/distro/fedora32/distro.go b/internal/distro/fedora32/distro.go
index 64ef7a396..69618b029 100644
--- a/internal/distro/fedora32/distro.go
+++ b/internal/distro/fedora32/distro.go
@@ -18,6 +18,7 @@ import (
 
 const name = "fedora-32"
 const modulePlatformID = "platform:f32"
+const ostreeRef = "fedora/32/%s/iot"
 
 type distribution struct {
 	arches        map[string]architecture
@@ -146,6 +147,13 @@ func (t *imageType) MIMEType() string {
 	return t.mimeType
 }
 
+func (t *imageType) OSTreeRef() string {
+	if t.rpmOstree {
+		return fmt.Sprintf(ostreeRef, t.arch.name)
+	}
+	return ""
+}
+
 func (t *imageType) Size(size uint64) uint64 {
 	const MegaByte = 1024 * 1024
 	// Microsoft Azure requires vhd images to be rounded up to the nearest MB
@@ -426,7 +434,7 @@ func (t *imageType) fsTabStageOptions(uefi bool) *osbuild.FSTabStageOptions {
 func (t *imageType) grub2StageOptions(kernelOptions string, kernel *blueprint.KernelCustomization, uefi bool) *osbuild.GRUB2StageOptions {
 	id := uuid.MustParse("76a22bf4-f153-4541-b6c7-0332c0dfaeac")
 
-	if kernel != nil {
+	if kernel != nil && kernel.Append != "" {
 		kernelOptions += " " + kernel.Append
 	}
 
@@ -513,13 +521,9 @@ func qemuAssembler(format string, filename string, uefi bool, imageOptions distr
 }
 
 func ostreeCommitAssembler(options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
-	ref := options.OSTree.Ref
-	if ref == "" {
-		ref = fmt.Sprintf("fedora/32/%s/iot", arch.Name())
-	}
 	return osbuild.NewOSTreeCommitAssembler(
 		&osbuild.OSTreeCommitAssemblerOptions{
-			Ref:    ref,
+			Ref:    options.OSTree.Ref,
 			Parent: options.OSTree.Parent,
 			Tar: osbuild.OSTreeCommitAssemblerTarOptions{
 				Filename: "commit.tar",
diff --git a/internal/distro/fedora33/distro.go b/internal/distro/fedora33/distro.go
index c37e66f57..a1867daaa 100644
--- a/internal/distro/fedora33/distro.go
+++ b/internal/distro/fedora33/distro.go
@@ -18,6 +18,7 @@ import (
 
 const name = "fedora-33"
 const modulePlatformID = "platform:f33"
+const ostreeRef = "fedora/33/%s/iot"
 
 type distribution struct {
 	arches        map[string]architecture
@@ -146,6 +147,13 @@ func (t *imageType) MIMEType() string {
 	return t.mimeType
 }
 
+func (t *imageType) OSTreeRef() string {
+	if t.rpmOstree {
+		return fmt.Sprintf(ostreeRef, t.arch.name)
+	}
+	return ""
+}
+
 func (t *imageType) Size(size uint64) uint64 {
 	const MegaByte = 1024 * 1024
 	// Microsoft Azure requires vhd images to be rounded up to the nearest MB
@@ -438,7 +446,7 @@ func (t *imageType) fsTabStageOptions(uefi bool) *osbuild.FSTabStageOptions {
 func (t *imageType) grub2StageOptions(kernelOptions string, kernel *blueprint.KernelCustomization, uefi bool) *osbuild.GRUB2StageOptions {
 	id := uuid.MustParse("76a22bf4-f153-4541-b6c7-0332c0dfaeac")
 
-	if kernel != nil {
+	if kernel != nil && kernel.Append != "" {
 		kernelOptions += " " + kernel.Append
 	}
 
@@ -525,13 +533,9 @@ func qemuAssembler(format string, filename string, uefi bool, imageOptions distr
 }
 
 func ostreeCommitAssembler(options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
-	ref := options.OSTree.Ref
-	if ref == "" {
-		ref = fmt.Sprintf("fedora/33/%s/iot", arch.Name())
-	}
 	return osbuild.NewOSTreeCommitAssembler(
 		&osbuild.OSTreeCommitAssemblerOptions{
-			Ref:    ref,
+			Ref:    options.OSTree.Ref,
 			Parent: options.OSTree.Parent,
 			Tar: osbuild.OSTreeCommitAssemblerTarOptions{
 				Filename: "commit.tar",
diff --git a/internal/distro/rhel8/distro.go b/internal/distro/rhel8/distro.go
index 1b91312ee..c1a154cd5 100644
--- a/internal/distro/rhel8/distro.go
+++ b/internal/distro/rhel8/distro.go
@@ -18,6 +18,7 @@ import (
 
 const name = "rhel-8"
 const modulePlatformID = "platform:el8"
+const ostreeRef = "rhel/8/%s/edge"
 
 type distribution struct {
 	arches        map[string]architecture
@@ -148,6 +149,13 @@ func (t *imageType) MIMEType() string {
 	return t.mimeType
 }
 
+func (t *imageType) OSTreeRef() string {
+	if t.rpmOstree {
+		return fmt.Sprintf(ostreeRef, t.arch.name)
+	}
+	return ""
+}
+
 func (t *imageType) Size(size uint64) uint64 {
 	const MegaByte = 1024 * 1024
 	// Microsoft Azure requires vhd images to be rounded up to the nearest MB
@@ -467,7 +475,7 @@ func (t *imageType) fsTabStageOptions(uefi bool) *osbuild.FSTabStageOptions {
 func (t *imageType) grub2StageOptions(kernelOptions string, kernel *blueprint.KernelCustomization, uefi bool) *osbuild.GRUB2StageOptions {
 	id := uuid.MustParse("0bd700f8-090f-4556-b797-b340297ea1bd")
 
-	if kernel != nil {
+	if kernel != nil && kernel.Append != "" {
 		kernelOptions += " " + kernel.Append
 	}
 
@@ -611,13 +619,9 @@ func tarAssembler(filename, compression string) *osbuild.Assembler {
 }
 
 func ostreeCommitAssembler(options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
-	ref := options.OSTree.Ref
-	if ref == "" {
-		ref = fmt.Sprintf("rhel/8/%s/edge", arch.Name())
-	}
 	return osbuild.NewOSTreeCommitAssembler(
 		&osbuild.OSTreeCommitAssemblerOptions{
-			Ref:    ref,
+			Ref:    options.OSTree.Ref,
 			Parent: options.OSTree.Parent,
 			Tar: osbuild.OSTreeCommitAssemblerTarOptions{
 				Filename: "commit.tar",
diff --git a/internal/distro/rhel84/distro.go b/internal/distro/rhel84/distro.go
index b13b2823e..e4acbaa9e 100644
--- a/internal/distro/rhel84/distro.go
+++ b/internal/distro/rhel84/distro.go
@@ -22,6 +22,7 @@ import (
 const name = "rhel-84"
 const centosName = "centos-8"
 const modulePlatformID = "platform:el8"
+const ostreeRef = "rhel/8/%s/edge"
 
 type distribution struct {
 	arches        map[string]architecture
@@ -161,6 +162,13 @@ func (t *imageType) MIMEType() string {
 	return t.mimeType
 }
 
+func (t *imageType) OSTreeRef() string {
+	if t.rpmOstree {
+		return fmt.Sprintf(ostreeRef, t.arch.name)
+	}
+	return ""
+}
+
 func (t *imageType) Size(size uint64) uint64 {
 	const MegaByte = 1024 * 1024
 	// Microsoft Azure requires vhd images to be rounded up to the nearest MB
@@ -295,7 +303,7 @@ func (t *imageType) pipeline(c *blueprint.Customizations, options distro.ImageOp
 
 	if t.bootable {
 		if t.arch.Name() != "s390x" {
-			p.AddStage(osbuild.NewGRUB2Stage(t.grub2StageOptions(pt, t.kernelOptions, c.GetKernel(), t.arch.uefi, t.arch.legacy)))
+			p.AddStage(osbuild.NewGRUB2Stage(t.grub2StageOptions(pt, t.kernelOptions, c.GetKernel(), packageSpecs, t.arch.uefi, t.arch.legacy)))
 		}
 	}
 
@@ -515,7 +523,7 @@ func (t *imageType) systemdStageOptions(enabledServices, disabledServices []stri
 	}
 }
 
-func (t *imageType) grub2StageOptions(pt *disk.PartitionTable, kernelOptions string, kernel *blueprint.KernelCustomization, uefi bool, legacy string) *osbuild.GRUB2StageOptions {
+func (t *imageType) grub2StageOptions(pt *disk.PartitionTable, kernelOptions string, kernel *blueprint.KernelCustomization, packages []rpmmd.PackageSpec, uefi bool, legacy string) *osbuild.GRUB2StageOptions {
 	if pt == nil {
 		panic("partition table must be defined for grub2 stage, this is a programming error")
 	}
@@ -524,13 +532,12 @@ func (t *imageType) grub2StageOptions(pt *disk.PartitionTable, kernelOptions str
 		panic("root partition must be defined for grub2 stage, this is a programming error")
 	}
 
-	id := uuid.MustParse(rootPartition.Filesystem.UUID)
-
-	if kernel != nil {
-		kernelOptions += " " + kernel.Append
+	stageOptions := osbuild.GRUB2StageOptions{
+		RootFilesystemUUID: uuid.MustParse(rootPartition.Filesystem.UUID),
+		KernelOptions:      kernelOptions,
+		Legacy:             legacy,
 	}
 
-	var uefiOptions *osbuild.GRUB2UEFI
 	if uefi {
 		var vendor string
 		if t.arch.distro.isCentos {
@@ -538,21 +545,28 @@ func (t *imageType) grub2StageOptions(pt *disk.PartitionTable, kernelOptions str
 		} else {
 			vendor = "redhat"
 		}
-		uefiOptions = &osbuild.GRUB2UEFI{
+		stageOptions.UEFI = &osbuild.GRUB2UEFI{
 			Vendor: vendor,
 		}
 	}
 
 	if !uefi {
-		legacy = t.arch.legacy
+		stageOptions.Legacy = t.arch.legacy
 	}
 
-	return &osbuild.GRUB2StageOptions{
-		RootFilesystemUUID: id,
-		KernelOptions:      kernelOptions,
-		Legacy:             legacy,
-		UEFI:               uefiOptions,
+	if kernel != nil {
+		if kernel.Append != "" {
+			stageOptions.KernelOptions += " " + kernel.Append
+		}
+		for _, pkg := range packages {
+			if pkg.Name == kernel.Name {
+				stageOptions.SavedEntry = "ffffffffffffffffffffffffffffffff-" + pkg.Version + "-" + pkg.Release + "." + pkg.Arch
+				break
+			}
+		}
 	}
+
+	return &stageOptions
 }
 
 func (t *imageType) selinuxStageOptions() *osbuild.SELinuxStageOptions {
@@ -722,13 +736,9 @@ func tarAssembler(filename, compression string) *osbuild.Assembler {
 }
 
 func ostreeCommitAssembler(options distro.ImageOptions, arch distro.Arch) *osbuild.Assembler {
-	ref := options.OSTree.Ref
-	if ref == "" {
-		ref = fmt.Sprintf("rhel/8/%s/edge", arch.Name())
-	}
 	return osbuild.NewOSTreeCommitAssembler(
 		&osbuild.OSTreeCommitAssemblerOptions{
-			Ref:    ref,
+			Ref:    options.OSTree.Ref,
 			Parent: options.OSTree.Parent,
 			Tar: osbuild.OSTreeCommitAssemblerTarOptions{
 				Filename: "commit.tar",
diff --git a/internal/osbuild/grub2_stage.go b/internal/osbuild/grub2_stage.go
index c52297381..1998b519e 100644
--- a/internal/osbuild/grub2_stage.go
+++ b/internal/osbuild/grub2_stage.go
@@ -16,6 +16,7 @@ type GRUB2StageOptions struct {
 	KernelOptions      string     `json:"kernel_opts,omitempty"`
 	Legacy             string     `json:"legacy,omitempty"`
 	UEFI               *GRUB2UEFI `json:"uefi,omitempty"`
+	SavedEntry         string     `json:"saved_entry,omitempty"`
 }
 
 type GRUB2UEFI struct {
diff --git a/internal/weldr/api.go b/internal/weldr/api.go
index 9bcbf18e0..dd17dfcab 100644
--- a/internal/weldr/api.go
+++ b/internal/weldr/api.go
@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"crypto/rand"
+	"encoding/hex"
 	"encoding/json"
 	errors_package "errors"
 	"fmt"
@@ -1798,6 +1799,36 @@ func (api *API) blueprintsTagHandler(writer http.ResponseWriter, request *http.R
 	statusResponseOK(writer)
 }
 
+func ostreeResolveRef(location, ref string) (string, error) {
+	u, err := url.Parse(location)
+	if err != nil {
+		return "", err
+	}
+	u, err = u.Parse("refs/heads/")
+	if err != nil {
+		return "", err
+	}
+	u, err = u.Parse(ref)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.Get(u.String())
+	if err != nil {
+		return "", err
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	parent := strings.TrimSpace(string(body))
+	// Check that this is at least a hex string.
+	_, err = hex.DecodeString(parent)
+	if err != nil {
+		return "", err
+	}
+	return parent, nil
+}
+
 // Schedule new compose by first translating the appropriate blueprint into a pipeline and then
 // pushing it into the channel for waiting builds.
 func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
@@ -1806,6 +1837,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
 	}
 
 	type OSTreeRequest struct {
+		URL    string `json:"url"`
 		Ref    string `json:"ref"`
 		Parent string `json:"parent"`
 	}
@@ -1856,6 +1888,11 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
 		return
 	}
 
+	// set default ostree ref, if one not provided
+	if cr.OSTree.Ref == "" {
+		cr.OSTree.Ref = imageType.OSTreeRef()
+	}
+
 	if !verifyStringsWithRegex(writer, []string{cr.BlueprintName}, ValidBlueprintName) {
 		return
 	}
@@ -1878,6 +1915,31 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
 		return
 	}
 
+	// Fetch parent ostree commit from ref + url if commit is not
+	// provided. The parameter name "parent" is perhaps slightly misleading
+	// as it represent whatever commit sha the image type requires, not
+	// strictly speaking just the parent commit.
+	if cr.OSTree.Ref != "" && cr.OSTree.URL != "" {
+		if cr.OSTree.Parent != "" {
+			errors := responseError{
+				ID:  "OSTreeOptionsError",
+				Msg: "Supply at most one of Parent and URL",
+			}
+			statusResponseError(writer, http.StatusBadRequest, errors)
+			return
+		}
+		parent, err := ostreeResolveRef(cr.OSTree.URL, cr.OSTree.Ref)
+		if err != nil {
+			errors := responseError{
+				ID:  "OSTreeCommitError",
+				Msg: err.Error(),
+			}
+			statusResponseError(writer, http.StatusBadRequest, errors)
+			return
+		}
+		cr.OSTree.Parent = parent
+	}
+
 	packages, buildPackages, err := api.depsolveBlueprint(bp, imageType)
 	if err != nil {
 		errors := responseError{
diff --git a/osbuild-composer.spec b/osbuild-composer.spec
index b1aed56c8..492d578f1 100644
--- a/osbuild-composer.spec
+++ b/osbuild-composer.spec
@@ -4,7 +4,7 @@
 
 %global goipath         github.com/osbuild/osbuild-composer
 
-Version:        27
+Version:        28
 
 %gometa
 
@@ -273,8 +273,8 @@ The core osbuild-composer binary. This is suitable both for spawning in containe
 Summary:    The worker for osbuild-composer
 Requires:   systemd
 Requires:   qemu-img
-Requires:   osbuild >= 24
-Requires:   osbuild-ostree >= 24
+Requires:   osbuild >= 26
+Requires:   osbuild-ostree >= 26
 
 # remove in F34
 Obsoletes: golang-github-osbuild-composer-worker < %{version}-%{release}
diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile
index e332ad7ce..7c7ca7bef 100644
--- a/schutzbot/Jenkinsfile
+++ b/schutzbot/Jenkinsfile
@@ -301,6 +301,23 @@ pipeline {
                         }
                     }
                 }
+                stage('F32 New OSTree') {
+                    when {
+                        expression {
+                            return env.BUILD_CAUSE != 'cron';
+                        }
+                    }
+
+                    agent { label "f32cloudbase && psi && x86_64" }
+                    steps {
+                        run_tests('ostree-ng')
+                    }
+                    post {
+                        always {
+                            preserve_logs('fedora32-ostree-ng')
+                        }
+                    }
+                }
                 stage('F32: koji-osbuild') {
                     when {
                         expression {
@@ -406,6 +423,23 @@ pipeline {
                         }
                     }
                 }
+                stage('F33 New OSTree') {
+                    when {
+                        expression {
+                            return env.BUILD_CAUSE != 'cron';
+                        }
+                    }
+
+                    agent { label "f33cloudbase && psi && x86_64" }
+                    steps {
+                        run_tests('ostree-ng')
+                    }
+                    post {
+                        always {
+                            preserve_logs('fedora33-ostree-ng')
+                        }
+                    }
+                }
                 stage('F33 aarch64 Base') {
                     when {
                         expression {
@@ -547,6 +581,23 @@ pipeline {
                         }
                     }
                 }
+                stage('EL8 New OSTree') {
+                    when {
+                        expression {
+                            return env.BUILD_CAUSE != 'cron';
+                        }
+                    }
+
+                    agent { label "rhel8cloudbase && psi && x86_64" }
+                    steps {
+                        run_tests('ostree-ng')
+                    }
+                    post {
+                        always {
+                            preserve_logs('rhel8-ostree-ng')
+                        }
+                    }
+                }
                 stage('EL8: koji-osbuild') {
                     when {
                         expression {
@@ -631,6 +682,17 @@ pipeline {
                         }
                     }
                 }
+                stage('EL8.4 New OSTree') {
+                    agent { label "rhel84cloudbase && psi && x86_64" }
+                    steps {
+                        run_tests('ostree-ng')
+                    }
+                    post {
+                        always {
+                            preserve_logs('rhel84-ostree-ng')
+                        }
+                    }
+                }
                 stage('CS8 Base') {
                     when {
                         expression {
@@ -792,6 +854,13 @@ void run_tests(test_type) {
         )
     }
 
+    if (test_type == 'ostree-ng') {
+        sh (
+            label: "New OSTree tests",
+            script: "/usr/libexec/tests/osbuild-composer/ostree-ng.sh"
+        )
+    }
+
     if (test_type == 'integration') {
         // Run Koji tests.
         sh (
diff --git a/tools/run-koji-container.sh b/tools/run-koji-container.sh
index 22cdbafa2..ffcd418c3 100755
--- a/tools/run-koji-container.sh
+++ b/tools/run-koji-container.sh
@@ -57,13 +57,13 @@ koji_start() {
     -e POSTGRES_USER=koji \
     -e POSTGRES_PASSWORD=kojipass \
     -e POSTGRES_DB=koji \
-    quay.io/osbuild/postgres:v1
+    quay.io/osbuild/postgres:13-alpine
 
   ${CONTAINER_RUNTIME} run -d --name org.osbuild.koji.kdc \
     --network org.osbuild.koji \
     -v "${SHARE_DIR}:/share:z" \
     -p 88:88/udp \
-    quay.io/osbuild/kdc:v1
+    quay.io/osbuild/kdc:latest
 
   # initialize krb pricipals and create keytabs for them
   # HTTP/localhost@LOCAL for kojihub
@@ -84,7 +84,7 @@ koji_start() {
     -e POSTGRES_PASSWORD=kojipass \
     -e POSTGRES_DB=koji \
     -e POSTGRES_HOST=org.osbuild.koji.postgres \
-    quay.io/osbuild/koji:v1
+    quay.io/osbuild/koji:latest
 
   # TODO: we need to wait for the database to be initialized here. A better method should be used.
   sleep 10


Test patch:
diff --git a/internal/blueprint/customizations_test.go b/internal/blueprint/customizations_test.go
index 1ebe21c3c..8415dc473 100644
--- a/internal/blueprint/customizations_test.go
+++ b/internal/blueprint/customizations_test.go
@@ -1,8 +1,9 @@
 package blueprint
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGetHostname(t *testing.T) {
@@ -22,6 +23,7 @@ func TestGetKernel(t *testing.T) {
 
 	expectedKernel := KernelCustomization{
 		Append: "--test",
+		Name:   "kernel",
 	}
 
 	TestCustomizations := Customizations{
@@ -214,7 +216,7 @@ func TestNoCustomizationsInBlueprint(t *testing.T) {
 	assert.Nil(t, TestBP.Customizations.GetHostname())
 	assert.Nil(t, TestBP.Customizations.GetUsers())
 	assert.Nil(t, TestBP.Customizations.GetGroups())
-	assert.Nil(t, TestBP.Customizations.GetKernel())
+	assert.Equal(t, &KernelCustomization{Name: "kernel"}, TestBP.Customizations.GetKernel())
 	assert.Nil(t, TestBP.Customizations.GetFirewall())
 	assert.Nil(t, TestBP.Customizations.GetServices())
 
diff --git a/internal/distro/distro_test_common/distro_test_common.go b/internal/distro/distro_test_common/distro_test_common.go
index 81c904a76..5dc932839 100644
--- a/internal/distro/distro_test_common/distro_test_common.go
+++ b/internal/distro/distro_test_common/distro_test_common.go
@@ -88,6 +88,9 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr
 			got, err := imageType.Manifest(tt.ComposeRequest.Blueprint.Customizations,
 				distro.ImageOptions{
 					Size: imageType.Size(0),
+					OSTree: distro.OSTreeImageOptions{
+						Ref: imageType.OSTreeRef(),
+					},
 				},
 				repos,
 				tt.RpmMD.Packages,
diff --git a/internal/distro/fedoratest/distro.go b/internal/distro/fedoratest/distro.go
index b262aacd3..293852e43 100644
--- a/internal/distro/fedoratest/distro.go
+++ b/internal/distro/fedoratest/distro.go
@@ -79,6 +79,10 @@ func (t *imageType) MIMEType() string {
 	return "application/x-test"
 }
 
+func (t *imageType) OSTreeRef() string {
+	return ""
+}
+
 func (t *imageType) Size(size uint64) uint64 {
 	return size
 }
diff --git a/internal/distro/test_distro/distro.go b/internal/distro/test_distro/distro.go
index a0c610944..3221aa5b4 100644
--- a/internal/distro/test_distro/distro.go
+++ b/internal/distro/test_distro/distro.go
@@ -63,6 +63,10 @@ func (t *TestImageType) MIMEType() string {
 	return "application/x-test"
 }
 
+func (t *TestImageType) OSTreeRef() string {
+	return ""
+}
+
 func (t *TestImageType) Size(size uint64) uint64 {
 	return 0
 }
diff --git a/test/cases/ostree-ng.sh b/test/cases/ostree-ng.sh
new file mode 100755
index 000000000..018abbf48
--- /dev/null
+++ b/test/cases/ostree-ng.sh
@@ -0,0 +1,477 @@
+#!/bin/bash
+set -euo pipefail
+
+OSBUILD_COMPOSER_TEST_DATA=/usr/share/tests/osbuild-composer/
+URL=http://192.168.100.1/repo/
+
+# Get OS data.
+source /etc/os-release
+ARCH=$(uname -m)
+
+# Provision the software under tet.
+/usr/libexec/osbuild-composer-test/provision.sh
+
+# Set os-variant and boot location used by virt-install.
+case "${ID}-${VERSION_ID}" in
+    "fedora-32")
+        IMAGE_TYPE=fedora-iot-commit
+        OSTREE_REF="fedora/32/${ARCH}/iot"
+        OS_VARIANT="fedora32"
+        BOOT_LOCATION="https://mirrors.rit.edu/fedora/fedora/linux/releases/32/Everything/x86_64/os/";;
+    "fedora-33")
+        IMAGE_TYPE=fedora-iot-commit
+        OSTREE_REF="fedora/33/${ARCH}/iot"
+        OS_VARIANT="fedora33"
+        BOOT_LOCATION="https://mirrors.rit.edu/fedora/fedora/linux/releases/33/Everything/x86_64/os/";;
+    "rhel-8.3")
+        IMAGE_TYPE=rhel-edge-commit
+        OSTREE_REF="rhel/8/${ARCH}/edge"
+        OS_VARIANT="rhel8.3"
+        # When 8.3 was released, it wasn't available on all RH internal
+        # mirrors, therefore the Boston mirror is hardcoded.
+        BOOT_LOCATION="http://download.eng.bos.redhat.com/released/rhel-8/RHEL-8/8.3.0/BaseOS/x86_64/os/";;
+    "rhel-8.4")
+        IMAGE_TYPE=rhel-edge-commit
+        OSTREE_REF="rhel/8/${ARCH}/edge"
+        OS_VARIANT="rhel8-unknown"
+        BOOT_LOCATION="http://download.devel.redhat.com/nightly/rhel-8/RHEL-8/latest-RHEL-8.4/compose/BaseOS/x86_64/os/";;
+    *)
+        echo "unsupported distro: ${ID}-${VERSION_ID}"
+        exit 1;;
+esac
+
+
+# Colorful output.
+function greenprint {
+    echo -e "\033[1;32m${1}\033[0m"
+}
+
+# Start libvirtd and test it.
+greenprint "🚀 Starting libvirt daemon"
+sudo systemctl start libvirtd
+sudo virsh list --all > /dev/null
+
+# Set a customized dnsmasq configuration for libvirt so we always get the
+# same address on bootup.
+sudo tee /tmp/integration.xml > /dev/null << EOF
+<network>
+  <name>integration</name>
+  <uuid>1c8fe98c-b53a-4ca4-bbdb-deb0f26b3579</uuid>
+  <forward mode='nat'>
+    <nat>
+      <port start='1024' end='65535'/>
+    </nat>
+  </forward>
+  <bridge name='integration' stp='on' delay='0'/>
+  <mac address='52:54:00:36:46:ef'/>
+  <ip address='192.168.100.1' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='192.168.100.2' end='192.168.100.254'/>
+      <host mac='34:49:22:B0:83:30' name='vm' ip='192.168.100.50'/>
+    </dhcp>
+  </ip>
+</network>
+EOF
+if ! sudo virsh net-info integration > /dev/null 2>&1; then
+    sudo virsh net-define /tmp/integration.xml
+    sudo virsh net-start integration
+fi
+
+# Allow anyone in the wheel group to talk to libvirt.
+greenprint "🚪 Allowing users in wheel group to talk to libvirt"
+WHEEL_GROUP=wheel
+if [[ $ID == rhel ]]; then
+    WHEEL_GROUP=adm
+fi
+sudo tee /etc/polkit-1/rules.d/50-libvirt.rules > /dev/null << EOF
+polkit.addRule(function(action, subject) {
+    if (action.id == "org.libvirt.unix.manage" &&
+        subject.isInGroup("${WHEEL_GROUP}")) {
+            return polkit.Result.YES;
+    }
+});
+EOF
+
+# Set up variables.
+TEST_UUID=$(uuidgen)
+IMAGE_KEY="osbuild-composer-ostree-test-${TEST_UUID}"
+GUEST_ADDRESS=192.168.100.50
+
+# Set up temporary files.
+TEMPDIR=$(mktemp -d)
+BLUEPRINT_FILE=${TEMPDIR}/blueprint.toml
+KS_FILE=${TEMPDIR}/ks.cfg
+COMPOSE_START=${TEMPDIR}/compose-start-${IMAGE_KEY}.json
+COMPOSE_INFO=${TEMPDIR}/compose-info-${IMAGE_KEY}.json
+
+# SSH setup.
+SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5)
+SSH_KEY=${OSBUILD_COMPOSER_TEST_DATA}keyring/id_rsa
+
+# Get the compose log.
+get_compose_log () {
+    COMPOSE_ID=$1
+    LOG_FILE=${WORKSPACE}/osbuild-${ID}-${VERSION_ID}-${COMPOSE_ID}.log
+
+    # Download the logs.
+    sudo composer-cli compose log "$COMPOSE_ID" | tee "$LOG_FILE" > /dev/null
+}
+
+# Get the compose metadata.
+get_compose_metadata () {
+    COMPOSE_ID=$1
+    METADATA_FILE=${WORKSPACE}/osbuild-${ID}-${VERSION_ID}-${COMPOSE_ID}.json
+
+    # Download the metadata.
+    sudo composer-cli compose metadata "$COMPOSE_ID" > /dev/null
+
+    # Find the tarball and extract it.
+    TARBALL=$(basename "$(find . -maxdepth 1 -type f -name "*-metadata.tar")")
+    tar -xf "$TARBALL" -C "${TEMPDIR}"
+    rm -f "$TARBALL"
+
+    # Move the JSON file into place.
+    cat "${TEMPDIR}"/"${COMPOSE_ID}".json | jq -M '.' | tee "$METADATA_FILE" > /dev/null
+}
+
+# Build ostree image.
+build_image() {
+    blueprint_file=$1
+    blueprint_name=$2
+
+    # Prepare the blueprint for the compose.
+    greenprint "📋 Preparing blueprint"
+    sudo composer-cli blueprints push "$blueprint_file"
+    sudo composer-cli blueprints depsolve "$blueprint_name"
+
+    # Get worker unit file so we can watch the journal.
+    WORKER_UNIT=$(sudo systemctl list-units | grep -o -E "osbuild.*worker.*\.service")
+    sudo journalctl -af -n 1 -u "${WORKER_UNIT}" &
+    WORKER_JOURNAL_PID=$!
+
+    # Start the compose.
+    greenprint "🚀 Starting compose"
+    if [[ $blueprint_name == upgrade ]]; then
+        sudo curl --silent --header "Content-Type: application/json" --unix-socket /run/weldr/api.socket http://localhost/api/v1/compose --data "{
+            \"blueprint_name\": \"$blueprint_name\",
+            \"compose_type\": \"$IMAGE_TYPE\",
+            \"ostree\": {
+                \"url\": \"$URL\",
+                \"ref\": \"$OSTREE_REF\"
+            }
+        }" | tee "$COMPOSE_START"
+    else
+        sudo curl --silent --header "Content-Type: application/json" --unix-socket /run/weldr/api.socket http://localhost/api/v1/compose --data "{
+            \"blueprint_name\": \"$blueprint_name\",
+            \"compose_type\": \"$IMAGE_TYPE\"
+        }" | tee "$COMPOSE_START"
+    fi
+    COMPOSE_ID=$(jq -r '.build_id' "$COMPOSE_START")
+
+    # Wait for the compose to finish.
+    greenprint "⏱ Waiting for compose to finish: ${COMPOSE_ID}"
+    while true; do
+        sudo composer-cli --json compose info "${COMPOSE_ID}" | tee "$COMPOSE_INFO" > /dev/null
+        COMPOSE_STATUS=$(jq -r '.queue_status' "$COMPOSE_INFO")
+
+        # Is the compose finished?
+        if [[ $COMPOSE_STATUS != RUNNING ]] && [[ $COMPOSE_STATUS != WAITING ]]; then
+            break
+        fi
+
+        # Wait 30 seconds and try again.
+        sleep 5
+    done
+
+    # Capture the compose logs from osbuild.
+    greenprint "💬 Getting compose log and metadata"
+    get_compose_log "$COMPOSE_ID"
+    get_compose_metadata "$COMPOSE_ID"
+
+    # Did the compose finish with success?
+    if [[ $COMPOSE_STATUS != FINISHED ]]; then
+        echo "Something went wrong with the compose. 😢"
+        exit 1
+    fi
+
+    # Stop watching the worker journal.
+    sudo kill ${WORKER_JOURNAL_PID}
+}
+
+# Wait for the ssh server up to be.
+wait_for_ssh_up () {
+    SSH_STATUS=$(sudo ssh "${SSH_OPTIONS[@]}" -i "${SSH_KEY}" admin@"${1}" '/bin/bash -c "echo -n READY"')
+    if [[ $SSH_STATUS == READY ]]; then
+        echo 1
+    else
+        echo 0
+    fi
+}
+
+# Clean up our mess.
+clean_up () {
+    greenprint "🧼 Cleaning up"
+    sudo virsh destroy "${IMAGE_KEY}"
+    if [[ $ARCH == aarch64 ]]; then
+        sudo virsh undefine "${IMAGE_KEY}" --nvram
+    else
+        sudo virsh undefine "${IMAGE_KEY}"
+    fi
+    # Remove qcow2 file.
+    sudo rm -f "$LIBVIRT_IMAGE_PATH"
+    # Remove extracted upgrade image-tar.
+    sudo rm -rf "$UPGRADE_PATH"
+    # Remove "remote" repo.
+    sudo rm -rf "${HTTPD_PATH}"/{repo,compose.json}
+    # Remomve tmp dir.
+    sudo rm -rf "$TEMPDIR"
+    # Stop httpd
+    sudo systemctl disable httpd --now
+}
+
+# Test result checking
+check_result () {
+    greenprint "Checking for test result"
+    if [[ $RESULTS == 1 ]]; then
+        greenprint "💚 Success"
+    else
+        greenprint "❌ Failed"
+        clean_up
+        exit 1
+    fi
+}
+
+##################################################
+##
+## ostree image/commit installation
+##
+##################################################
+
+# Write a blueprint for ostree image.
+tee "$BLUEPRINT_FILE" > /dev/null << EOF
+name = "ostree"
+description = "A base ostree image"
+version = "0.0.1"
+modules = []
+groups = []
+
+[[packages]]
+name = "python36"
+version = "*"
+EOF
+
+# Build installation image.
+build_image "$BLUEPRINT_FILE" ostree
+
+# Start httpd to serve ostree repo.
+greenprint "🚀 Starting httpd daemon"
+sudo systemctl start httpd
+
+# Download the image and extract tar into web server root folder.
+greenprint "📥 Downloading and extracting the image"
+sudo composer-cli compose image "${COMPOSE_ID}" > /dev/null
+IMAGE_FILENAME="${COMPOSE_ID}-commit.tar"
+HTTPD_PATH="/var/www/html"
+sudo tar -xf "${IMAGE_FILENAME}" -C ${HTTPD_PATH}
+sudo rm -f "$IMAGE_FILENAME"
+
+# Clean compose and blueprints.
+greenprint "Clean up osbuild-composer"
+sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null
+sudo composer-cli blueprints delete ostree > /dev/null
+
+# Ensure SELinux is happy with our new images.
+greenprint "👿 Running restorecon on image directory"
+sudo restorecon -Rv /var/lib/libvirt/images/
+
+# Create qcow2 file for virt install.
+greenprint "Create qcow2 file for virt install"
+LIBVIRT_IMAGE_PATH=/var/lib/libvirt/images/${IMAGE_KEY}.qcow2
+sudo qemu-img create -f qcow2 "${LIBVIRT_IMAGE_PATH}" 20G
+
+# Write kickstart file for ostree image installation.
+greenprint "Generate kickstart file"
+tee "$KS_FILE" > /dev/null << STOPHERE
+text
+lang en_US.UTF-8
+keyboard us
+timezone --utc Etc/UTC
+
+selinux --enforcing
+rootpw --lock --iscrypted locked
+user --name=admin --groups=wheel --iscrypted --password=\$6\$1LgwKw9aOoAi/Zy9\$Pn3ErY1E8/yEanJ98evqKEW.DZp24HTuqXPJl6GYCm8uuobAmwxLv7rGCvTRZhxtcYdmC0.XnYRSR9Sh6de3p0
+sshkey --username=admin "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC61wMCjOSHwbVb4VfVyl5sn497qW4PsdQ7Ty7aD6wDNZ/QjjULkDV/yW5WjDlDQ7UqFH0Sr7vywjqDizUAqK7zM5FsUKsUXWHWwg/ehKg8j9xKcMv11AkFoUoujtfAujnKODkk58XSA9whPr7qcw3vPrmog680pnMSzf9LC7J6kXfs6lkoKfBh9VnlxusCrw2yg0qI1fHAZBLPx7mW6+me71QZsS6sVz8v8KXyrXsKTdnF50FjzHcK9HXDBtSJS5wA3fkcRYymJe0o6WMWNdgSRVpoSiWaHHmFgdMUJaYoCfhXzyl7LtNb3Q+Sveg+tJK7JaRXBLMUllOlJ6ll5Hod root@localhost"
+
+bootloader --timeout=1 --append="net.ifnames=0 modprobe.blacklist=vc4"
+
+network --bootproto=dhcp --device=link --activate --onboot=on
+
+zerombr
+clearpart --all --initlabel --disklabel=msdos
+autopart --nohome --noswap --type=plain
+ostreesetup --nogpg --osname=${IMAGE_TYPE} --remote=${IMAGE_TYPE} --url=${URL} --ref=${OSTREE_REF}
+poweroff
+
+%post --log=/var/log/anaconda/post-install.log --erroronfail
+
+# no sudo password for user admin
+echo -e 'admin\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers
+
+# Remove any persistent NIC rules generated by udev
+rm -vf /etc/udev/rules.d/*persistent-net*.rules
+# And ensure that we will do DHCP on eth0 on startup
+cat > /etc/sysconfig/network-scripts/ifcfg-eth0 << EOF
+DEVICE="eth0"
+BOOTPROTO="dhcp"
+ONBOOT="yes"
+TYPE="Ethernet"
+PERSISTENT_DHCLIENT="yes"
+EOF
+
+echo "Packages within this iot or edge image:"
+echo "-----------------------------------------------------------------------"
+rpm -qa | sort
+echo "-----------------------------------------------------------------------"
+# Note that running rpm recreates the rpm db files which aren't needed/wanted
+rm -f /var/lib/rpm/__db*
+
+echo "Zeroing out empty space."
+# This forces the filesystem to reclaim space from deleted files
+dd bs=1M if=/dev/zero of=/var/tmp/zeros || :
+rm -f /var/tmp/zeros
+echo "(Don't worry -- that out-of-space error was expected.)"
+
+%end
+STOPHERE
+
+# Install ostree image via anaconda.
+greenprint "Install ostree image via anaconda"
+sudo virt-install  --initrd-inject="${KS_FILE}" \
+                   --extra-args="ks=file:/ks.cfg console=ttyS0,115200" \
+                   --name="${IMAGE_KEY}"\
+                   --disk path="${LIBVIRT_IMAGE_PATH}",format=qcow2 \
+                   --ram 3072 \
+                   --vcpus 2 \
+                   --network network=integration,mac=34:49:22:B0:83:30 \
+                   --os-type linux \
+                   --os-variant ${OS_VARIANT} \
+                   --location ${BOOT_LOCATION} \
+                   --nographics \
+                   --noautoconsole \
+                   --wait=-1 \
+                   --noreboot
+
+# Start VM.
+greenprint "Start VM"
+sudo virsh start "${IMAGE_KEY}"
+
+# Check for ssh ready to go.
+greenprint "🛃 Checking for SSH is ready to go"
+for LOOP_COUNTER in $(seq 0 30); do
+    RESULTS="$(wait_for_ssh_up $GUEST_ADDRESS)"
+    if [[ $RESULTS == 1 ]]; then
+        echo "SSH is ready now! 🥳"
+        break
+    fi
+    sleep 10
+done
+
+# Check image installation result
+check_result
+
+##################################################
+##
+## ostree image/commit upgrade
+##
+##################################################
+
+# Write a blueprint for ostree image.
+tee "$BLUEPRINT_FILE" > /dev/null << EOF
+name = "upgrade"
+description = "An upgrade ostree image"
+version = "0.0.2"
+modules = []
+groups = []
+
+[[packages]]
+name = "python36"
+version = "*"
+
+[[packages]]
+name = "wget"
+version = "*"
+EOF
+
+# Build upgrade image.
+build_image "$BLUEPRINT_FILE" upgrade
+
+# Download the image and extract tar into web server root folder.
+greenprint "📥 Downloading and extracting the image"
+sudo composer-cli compose image "${COMPOSE_ID}" > /dev/null
+IMAGE_FILENAME="${COMPOSE_ID}-commit.tar"
+UPGRADE_PATH="$(pwd)/upgrade"
+mkdir -p "$UPGRADE_PATH"
+sudo tar -xf "$IMAGE_FILENAME" -C "$UPGRADE_PATH"
+sudo rm -f "$IMAGE_FILENAME"
+
+# Clean compose and blueprints.
+greenprint "Clean up osbuild-composer again"
+sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null
+sudo composer-cli blueprints delete upgrade > /dev/null
+
+# Introduce new ostree commit into repo.
+greenprint "Introduce new ostree commit into repo"
+sudo ostree pull-local --repo "${HTTPD_PATH}/repo" "${UPGRADE_PATH}/repo" "$OSTREE_REF"
+sudo ostree summary --update --repo "${HTTPD_PATH}/repo"
+
+# Ensure SELinux is happy with all objects files.
+greenprint "👿 Running restorecon on web server root folder"
+sudo restorecon -Rv "${HTTPD_PATH}/repo" > /dev/null
+
+# Get ostree commit value.
+greenprint "Get ostree image commit value"
+UPGRADE_HASH=$(jq -r '."ostree-commit"' < "${UPGRADE_PATH}"/compose.json)
+
+# Upgrade image/commit.
+greenprint "Upgrade ostree image/commit"
+sudo ssh "${SSH_OPTIONS[@]}" -i "${SSH_KEY}" admin@${GUEST_ADDRESS} 'sudo rpm-ostree upgrade'
+sudo ssh "${SSH_OPTIONS[@]}" -i "${SSH_KEY}" admin@${GUEST_ADDRESS} 'nohup sudo systemctl reboot &>/dev/null & exit'
+
+# Sleep 10 seconds here to make sure vm restarted already
+sleep 10
+
+# Check for ssh ready to go.
+greenprint "🛃 Checking for SSH is ready to go"
+# shellcheck disable=SC2034  # Unused variables left for readability
+for LOOP_COUNTER in $(seq 0 30); do
+    RESULTS="$(wait_for_ssh_up $GUEST_ADDRESS)"
+    if [[ $RESULTS == 1 ]]; then
+        echo "SSH is ready now! 🥳"
+        break
+    fi
+    sleep 10
+done
+
+# Check ostree upgrade result
+check_result
+
+# Add instance IP address into /etc/ansible/hosts
+sudo tee "${TEMPDIR}"/inventory > /dev/null << EOF
+[ostree_guest]
+${GUEST_ADDRESS}
+
+[ostree_guest:vars]
+ansible_python_interpreter=/usr/bin/python3
+ansible_user=admin
+ansible_private_key_file=${SSH_KEY}
+ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+EOF
+
+# Test IoT/Edge OS
+sudo ansible-playbook -v -i "${TEMPDIR}"/inventory -e image_type=${IMAGE_TYPE} -e ostree_commit="${UPGRADE_HASH}" /usr/share/tests/osbuild-composer/ansible/check_ostree.yaml || RESULTS=0
+check_result
+
+# Final success clean up
+clean_up
+
+exit 0
diff --git a/test/data/manifests/centos_8-x86_64-ami-boot.json b/test/data/manifests/centos_8-x86_64-ami-boot.json
index fd918940b..69087db27 100644
--- a/test/data/manifests/centos_8-x86_64-ami-boot.json
+++ b/test/data/manifests/centos_8-x86_64-ami-boot.json
@@ -3858,7 +3858,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "centos"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
           }
         },
         {
@@ -10257,7 +10258,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/centos_8-x86_64-openstack-boot.json b/test/data/manifests/centos_8-x86_64-openstack-boot.json
index c389c6e0b..64e589920 100644
--- a/test/data/manifests/centos_8-x86_64-openstack-boot.json
+++ b/test/data/manifests/centos_8-x86_64-openstack-boot.json
@@ -4155,7 +4155,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "centos"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
           }
         },
         {
@@ -10958,7 +10959,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/centos_8-x86_64-qcow2-boot.json b/test/data/manifests/centos_8-x86_64-qcow2-boot.json
index d2304c472..e3988dd05 100644
--- a/test/data/manifests/centos_8-x86_64-qcow2-boot.json
+++ b/test/data/manifests/centos_8-x86_64-qcow2-boot.json
@@ -4132,7 +4132,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "centos"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
           }
         },
         {
@@ -10934,7 +10935,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/centos_8-x86_64-qcow2-customize.json b/test/data/manifests/centos_8-x86_64-qcow2-customize.json
index d1afa8ec3..c7062df63 100644
--- a/test/data/manifests/centos_8-x86_64-qcow2-customize.json
+++ b/test/data/manifests/centos_8-x86_64-qcow2-customize.json
@@ -4184,7 +4184,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "centos"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
           }
         },
         {
@@ -11039,7 +11040,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/centos_8-x86_64-vhd-boot.json b/test/data/manifests/centos_8-x86_64-vhd-boot.json
index b15e346ea..51e24620d 100644
--- a/test/data/manifests/centos_8-x86_64-vhd-boot.json
+++ b/test/data/manifests/centos_8-x86_64-vhd-boot.json
@@ -4111,7 +4111,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "centos"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
           }
         },
         {
@@ -10874,7 +10875,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro biosdevname=0 rootdelay=300 console=ttyS0 earlyprintk=ttyS0 net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro biosdevname=0 rootdelay=300 console=ttyS0 earlyprintk=ttyS0 net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/centos_8-x86_64-vmdk-boot.json b/test/data/manifests/centos_8-x86_64-vmdk-boot.json
index cd2d62ade..144cfacf7 100644
--- a/test/data/manifests/centos_8-x86_64-vmdk-boot.json
+++ b/test/data/manifests/centos_8-x86_64-vmdk-boot.json
@@ -3949,7 +3949,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "centos"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
           }
         },
         {
@@ -10462,7 +10463,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-277.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-aarch64-ami-boot.json b/test/data/manifests/rhel_84-aarch64-ami-boot.json
index 9861bc48f..d43729350 100644
--- a/test/data/manifests/rhel_84-aarch64-ami-boot.json
+++ b/test/data/manifests/rhel_84-aarch64-ami-boot.json
@@ -3213,7 +3213,8 @@
             "kernel_opts": "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.aarch64"
           }
         },
         {
@@ -8909,7 +8910,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.aarch64"
     },
     "bootloader": "unknown",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-aarch64-openstack-boot.json b/test/data/manifests/rhel_84-aarch64-openstack-boot.json
index 6c74a0837..47b8e5327 100644
--- a/test/data/manifests/rhel_84-aarch64-openstack-boot.json
+++ b/test/data/manifests/rhel_84-aarch64-openstack-boot.json
@@ -3446,7 +3446,8 @@
             "kernel_opts": "ro net.ifnames=0",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.aarch64"
           }
         },
         {
@@ -9470,7 +9471,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.aarch64"
     },
     "bootloader": "unknown",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-aarch64-qcow2-boot.json b/test/data/manifests/rhel_84-aarch64-qcow2-boot.json
index 8e896c3cd..498a0e1eb 100644
--- a/test/data/manifests/rhel_84-aarch64-qcow2-boot.json
+++ b/test/data/manifests/rhel_84-aarch64-qcow2-boot.json
@@ -3398,7 +3398,8 @@
             "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.aarch64"
           }
         },
         {
@@ -9378,7 +9379,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.aarch64"
     },
     "bootloader": "unknown",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-ppc64le-qcow2-boot.json b/test/data/manifests/rhel_84-ppc64le-qcow2-boot.json
index a5599f831..e5264d4b4 100644
--- a/test/data/manifests/rhel_84-ppc64le-qcow2-boot.json
+++ b/test/data/manifests/rhel_84-ppc64le-qcow2-boot.json
@@ -3677,7 +3677,8 @@
           "options": {
             "root_fs_uuid": "0194fdc2-fa2f-4cc0-81d3-ff12045b73c8",
             "kernel_opts": "console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
-            "legacy": "powerpc-ieee1275"
+            "legacy": "powerpc-ieee1275",
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.ppc64le"
           }
         },
         {
@@ -10112,7 +10113,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.ppc64le"
     },
     "bootloader": "unknown",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-x86_64-ami-boot.json b/test/data/manifests/rhel_84-x86_64-ami-boot.json
index 154a69851..64640b46d 100644
--- a/test/data/manifests/rhel_84-x86_64-ami-boot.json
+++ b/test/data/manifests/rhel_84-x86_64-ami-boot.json
@@ -3265,7 +3265,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
           }
         },
         {
@@ -9070,7 +9071,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-x86_64-openstack-boot.json b/test/data/manifests/rhel_84-x86_64-openstack-boot.json
index d7878d67e..4437d7b12 100644
--- a/test/data/manifests/rhel_84-x86_64-openstack-boot.json
+++ b/test/data/manifests/rhel_84-x86_64-openstack-boot.json
@@ -3504,7 +3504,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
           }
         },
         {
@@ -9646,7 +9647,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-x86_64-qcow2-boot.json b/test/data/manifests/rhel_84-x86_64-qcow2-boot.json
index 305100c57..6cf8a0d71 100644
--- a/test/data/manifests/rhel_84-x86_64-qcow2-boot.json
+++ b/test/data/manifests/rhel_84-x86_64-qcow2-boot.json
@@ -3438,7 +3438,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
           }
         },
         {
@@ -9509,7 +9510,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-x86_64-qcow2-customize.json b/test/data/manifests/rhel_84-x86_64-qcow2-customize.json
index 64993444a..d3a134489 100644
--- a/test/data/manifests/rhel_84-x86_64-qcow2-customize.json
+++ b/test/data/manifests/rhel_84-x86_64-qcow2-customize.json
@@ -3490,7 +3490,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
           }
         },
         {
@@ -9614,7 +9615,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 console=tty0 console=ttyS0,115200n8 no_timer_check net.ifnames=0 crashkernel=auto debug",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-x86_64-vhd-boot.json b/test/data/manifests/rhel_84-x86_64-vhd-boot.json
index 3f9cd1f6e..0dd3cd959 100644
--- a/test/data/manifests/rhel_84-x86_64-vhd-boot.json
+++ b/test/data/manifests/rhel_84-x86_64-vhd-boot.json
@@ -3465,7 +3465,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
           }
         },
         {
@@ -9572,7 +9573,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro biosdevname=0 rootdelay=300 console=ttyS0 earlyprintk=ttyS0 net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro biosdevname=0 rootdelay=300 console=ttyS0 earlyprintk=ttyS0 net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [
diff --git a/test/data/manifests/rhel_84-x86_64-vmdk-boot.json b/test/data/manifests/rhel_84-x86_64-vmdk-boot.json
index 2b8bda678..fba3a583c 100644
--- a/test/data/manifests/rhel_84-x86_64-vmdk-boot.json
+++ b/test/data/manifests/rhel_84-x86_64-vmdk-boot.json
@@ -3333,7 +3333,8 @@
             "legacy": "i386-pc",
             "uefi": {
               "vendor": "redhat"
-            }
+            },
+            "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
           }
         },
         {
@@ -9223,7 +9224,8 @@
   },
   "image-info": {
     "boot-environment": {
-      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0"
+      "kernelopts": "root=UUID=0194fdc2-fa2f-4cc0-81d3-ff12045b73c8 ro net.ifnames=0",
+      "saved_entry": "ffffffffffffffffffffffffffffffff-4.18.0-275.el8.x86_64"
     },
     "bootloader": "grub",
     "bootmenu": [