Repository: osbuild/osbuild-composer
Pull request #597 (2020-05-11 07:18:46)

worker: switch to `--output-diectory=DIR`
Since 2 releases `osbuild` accepts an `--output-directory=DIR` argument
which lets us decide where to place generated artifacts. Switch over to
it, rather than digging into the store, to make sure we will not access
the osbuild store when parallel cleanups are ongoing (which are not yet
a thing, though).

dvdhrm: The CI failure seems unrelated to me. But maybe someone with more insight can look at it?

```
                	            	- (string) (len=10) "bootloader": (string) (len=4) "grub",

                	            	+ (string) (len=10) "bootloader": (string) (len=7) "unknown",
```
dvdhrm: > Thanks! osbuild-composer currently depends on [osbuild >= 12](https://github.com/osbuild/osbuild-composer/blob/master/osbuild-composer.spec#L47). Is `--output-directory` already in 12?

Yeah, introduced with `osbuild-12`.

> > The CI failure seems unrelated to me. But maybe someone with more insight can look at it?
> 
> @gicmo has seen these before. I think they're flakes.

Right, then I will ignore it for now. Thanks!
dvdhrm: I just noticed there are 2 more tests that I could convert as well. We then no longer hard-code `refs/` anywhere. And I can rerun the CI when I repush :)
dvdhrm: `grep -r generate-test-case .` found nothing, so I am a bit unsure how to verify `generate-test-case` still works. Can someone let me know how I can run and test this tool?
ondrejbudai: `tools/test-case-generators/generate-test-case` is called by `tools/test-case-generators/generate-test-cases`. I think the following line should work:

```
tools/test-case-generators/generate-test-cases --distro fedora-32 --arch x86_64 --store . --output test/cases
```
dvdhrm: I used `mkdir store && ... --store store` and it works fine for me.

Patch:
diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go
index ce1c856cf..d80cf65f7 100644
--- a/cmd/osbuild-worker/main.go
+++ b/cmd/osbuild-worker/main.go
@@ -64,14 +64,14 @@ func (e *TargetsError) Error() string {
 }
 
 func RunJob(job *worker.Job, uploadFunc func(uuid.UUID, int, io.Reader) error) (*common.ComposeResult, error) {
-	tmpStore, err := ioutil.TempDir("/var/tmp", "osbuild-store")
+	tmpOutput, err := ioutil.TempDir("/var/tmp", "osbuild-output-*")
 	if err != nil {
-		return nil, fmt.Errorf("error setting up osbuild store: %v", err)
+		return nil, fmt.Errorf("error setting up osbuild output directory: %v", err)
 	}
 	// FIXME: how to handle errors in defer?
-	defer os.RemoveAll(tmpStore)
+	defer os.RemoveAll(tmpOutput)
 
-	result, err := RunOSBuild(job.Manifest, tmpStore, os.Stderr)
+	result, err := RunOSBuild(job.Manifest, tmpOutput, os.Stderr)
 	if err != nil {
 		return nil, err
 	}
@@ -81,7 +81,7 @@ func RunJob(job *worker.Job, uploadFunc func(uuid.UUID, int, io.Reader) error) (
 	for _, t := range job.Targets {
 		switch options := t.Options.(type) {
 		case *target.LocalTargetOptions:
-			f, err := os.Open(path.Join(tmpStore, "refs", result.OutputID, options.Filename))
+			f, err := os.Open(path.Join(tmpOutput, options.Filename))
 			if err != nil {
 				r = append(r, err)
 				continue
@@ -105,7 +105,7 @@ func RunJob(job *worker.Job, uploadFunc func(uuid.UUID, int, io.Reader) error) (
 				options.Key = job.Id.String()
 			}
 
-			_, err = a.Upload(path.Join(tmpStore, "refs", result.OutputID, options.Filename), options.Bucket, options.Key)
+			_, err = a.Upload(path.Join(tmpOutput, options.Filename), options.Bucket, options.Key)
 			if err != nil {
 				r = append(r, err)
 				continue
@@ -132,7 +132,7 @@ func RunJob(job *worker.Job, uploadFunc func(uuid.UUID, int, io.Reader) error) (
 			err := azure.UploadImage(
 				credentials,
 				metadata,
-				path.Join(tmpStore, "refs", result.OutputID, options.Filename),
+				path.Join(tmpOutput, options.Filename),
 				azureMaxUploadGoroutines,
 			)
 
diff --git a/cmd/osbuild-worker/osbuild.go b/cmd/osbuild-worker/osbuild.go
index b8a32da21..2da5dd167 100644
--- a/cmd/osbuild-worker/osbuild.go
+++ b/cmd/osbuild-worker/osbuild.go
@@ -19,10 +19,10 @@ func (e *OSBuildError) Error() string {
 	return e.Message
 }
 
-func RunOSBuild(manifest *osbuild.Manifest, store string, errorWriter io.Writer) (*common.ComposeResult, error) {
+func RunOSBuild(manifest *osbuild.Manifest, outputDirectory string, errorWriter io.Writer) (*common.ComposeResult, error) {
 	cmd := exec.Command(
 		"osbuild",
-		"--store", store,
+		"--output-directory", outputDirectory,
 		"--json", "-",
 	)
 	cmd.Stderr = errorWriter
diff --git a/codecov.yml b/codecov.yml
index 58decda6a..401467f29 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -1,4 +1,7 @@
 coverage:
+  project:
+    default:
+      treshold: 5%
   status:
     patch: no
 codecov:
diff --git a/internal/client/utils.go b/internal/client/utils.go
index e6a37be70..fee6bc311 100644
--- a/internal/client/utils.go
+++ b/internal/client/utils.go
@@ -11,7 +11,6 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
-	"time"
 )
 
 type TestState struct {
@@ -32,12 +31,10 @@ func isStringInSlice(slice []string, s string) bool {
 	return false
 }
 
-func setUpTestState(socketPath string, timeout time.Duration, unitTest bool) (*TestState, error) {
+func setUpTestState(socketPath string, unitTest bool) (*TestState, error) {
 	state := TestState{unitTest: unitTest}
 
 	state.socket = &http.Client{
-		// TODO This may be too short/simple for downloading images
-		Timeout: timeout,
 		Transport: &http.Transport{
 			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
 				return net.Dial("unix", socketPath)
diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile
new file mode 100644
index 000000000..559d67228
--- /dev/null
+++ b/schutzbot/Jenkinsfile
@@ -0,0 +1,43 @@
+pipeline {
+    agent none
+
+    options {
+        timestamps()
+        ansiColor('xterm')
+    }
+
+    stages {
+        stage("Functional Testing") {
+            parallel {
+                stage('Fedora 31 base') {
+                    agent { label "fedora31" }
+                    environment { TEST_TYPE = "base" }
+                    steps {
+                        sh "schutzbot/run_tests.sh"
+                    }
+                }
+                stage('Fedora 31 image') {
+                    agent { label "fedora31" }
+                    environment { TEST_TYPE = "image" }
+                    steps {
+                        sh "schutzbot/run_tests.sh"
+                    }
+                }
+                stage('Fedora 32 base') {
+                    agent { label "fedora32" }
+                    environment { TEST_TYPE = "base" }
+                    steps {
+                        sh "schutzbot/run_tests.sh"
+                    }
+                }
+                stage('Fedora 32 image') {
+                    agent { label "fedora32" }
+                    environment { TEST_TYPE = "image" }
+                    steps {
+                        sh "schutzbot/run_tests.sh"
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/schutzbot/vars.yml b/schutzbot/vars.yml
new file mode 100644
index 000000000..2c225cae5
--- /dev/null
+++ b/schutzbot/vars.yml
@@ -0,0 +1,41 @@
+## Global test variables
+# Tests should use this as their working directory. This part is critical
+# since dnf-json must be in $PATH.
+tests_working_directory: /usr/libexec/osbuild-composer
+
+# The test executables are here.
+tests_path: /usr/libexec/tests/osbuild-composer
+
+# Frequency to check for completed tests.
+polling_interval: 15
+
+## Non-image test variables.
+# List of base tests and their timeouts (in minutes)
+osbuild_composer_base_tests:
+  - name: osbuild-rcm-tests
+    timeout: 5
+  - name: osbuild-weldr-tests
+    timeout: 5
+  - name: osbuild-dnf-json-tests
+    timeout: 15
+  - name: osbuild-tests
+    timeout: 30
+
+## Image test variables.
+# Executable that runs image tests.
+image_test_executable: "{{ tests_path }}/osbuild-image-tests"
+
+# Timeout for image tests (in minutes).
+image_test_timeout: 45
+
+# Location of image test case files.
+image_test_case_path: /usr/share/tests/osbuild-composer/cases
+
+# List of image tests and their timeouts (in minutes).
+osbuild_composer_image_test_cases:
+  - ext4_filesystem-boot.json
+  - partitioned_disk-boot.json
+  - qcow2-boot.json
+  - tar-boot.json
+  - vhd-boot.json
+  - vmdk-boot.json


Test patch:
diff --git a/cmd/osbuild-image-tests/constants/constants-travis.go b/cmd/osbuild-image-tests/constants/constants-travis.go
index 48872e450..14601a83d 100644
--- a/cmd/osbuild-image-tests/constants/constants-travis.go
+++ b/cmd/osbuild-image-tests/constants/constants-travis.go
@@ -4,12 +4,12 @@ package constants
 
 import "os/exec"
 
-func GetOsbuildCommand(store string) *exec.Cmd {
+func GetOsbuildCommand(outputDirectory string) *exec.Cmd {
 	cmd := exec.Command(
 		"python3",
 		"-m", "osbuild",
 		"--libdir", ".",
-		"--store", store,
+		"--output-directory", outputDirectory,
 		"--json",
 		"-",
 	)
diff --git a/cmd/osbuild-image-tests/constants/constants.go b/cmd/osbuild-image-tests/constants/constants.go
index 73d986709..560cfc9b5 100644
--- a/cmd/osbuild-image-tests/constants/constants.go
+++ b/cmd/osbuild-image-tests/constants/constants.go
@@ -4,10 +4,10 @@ package constants
 
 import "os/exec"
 
-func GetOsbuildCommand(store string) *exec.Cmd {
+func GetOsbuildCommand(outputDirectory string) *exec.Cmd {
 	return exec.Command(
 		"osbuild",
-		"--store", store,
+		"--output-directory", outputDirectory,
 		"--json",
 		"-",
 	)
diff --git a/cmd/osbuild-image-tests/context-managers.go b/cmd/osbuild-image-tests/context-managers.go
index bf2727de3..39fede7c8 100644
--- a/cmd/osbuild-image-tests/context-managers.go
+++ b/cmd/osbuild-image-tests/context-managers.go
@@ -169,11 +169,10 @@ func withBootedQemuImage(image string, ns netNS, f func() error) error {
 
 // withBootedNspawnImage boots the specified image in the specified namespace
 // using nspawn. The VM is killed immediately after function returns.
-func withBootedNspawnImage(image, name string, ns netNS, f func() error) error {
+func withBootedNspawnImage(image string, ns netNS, f func() error) error {
 	cmd := exec.Command(
 		"systemd-nspawn",
 		"--boot", "--register=no",
-		"-M", name,
 		"--image", image,
 		"--network-namespace-path", ns.Path(),
 	)
@@ -195,11 +194,10 @@ func withBootedNspawnImage(image, name string, ns netNS, f func() error) error {
 
 // withBootedNspawnImage boots the specified directory in the specified namespace
 // using nspawn. The VM is killed immediately after function returns.
-func withBootedNspawnDirectory(dir, name string, ns netNS, f func() error) error {
+func withBootedNspawnDirectory(dir string, ns netNS, f func() error) error {
 	cmd := exec.Command(
 		"systemd-nspawn",
 		"--boot", "--register=no",
-		"-M", name,
 		"--directory", dir,
 		"--network-namespace-path", ns.Path(),
 	)
diff --git a/cmd/osbuild-image-tests/main_test.go b/cmd/osbuild-image-tests/main_test.go
index f8f88c46f..1228dbdbd 100644
--- a/cmd/osbuild-image-tests/main_test.go
+++ b/cmd/osbuild-image-tests/main_test.go
@@ -42,8 +42,8 @@ type testcaseStruct struct {
 // mutex to enforce only one osbuild instance run at a time, see below
 var osbuildMutex sync.Mutex
 
-// runOsbuild runs osbuild with the specified manifest and store.
-func runOsbuild(manifest []byte, store string) (string, error) {
+// runOsbuild runs osbuild with the specified manifest and output-directory.
+func runOsbuild(manifest []byte, outputDirectory string) error {
 	// Osbuild crashes when multiple instances are run at a time.
 	// This mutex enforces that there's always just one osbuild instance.
 	// This should be removed once osbuild is fixed.
@@ -51,7 +51,7 @@ func runOsbuild(manifest []byte, store string) (string, error) {
 	osbuildMutex.Lock()
 	defer osbuildMutex.Unlock()
 
-	cmd := constants.GetOsbuildCommand(store)
+	cmd := constants.GetOsbuildCommand(outputDirectory)
 
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = bytes.NewReader(manifest)
@@ -64,21 +64,12 @@ func runOsbuild(manifest []byte, store string) (string, error) {
 		if _, ok := err.(*exec.ExitError); ok {
 			var formattedOutput bytes.Buffer
 			_ = json.Indent(&formattedOutput, outBuffer.Bytes(), "", "  ")
-			return "", fmt.Errorf("running osbuild failed: %s", formattedOutput.String())
+			return fmt.Errorf("running osbuild failed: %s", formattedOutput.String())
 		}
-		return "", fmt.Errorf("running osbuild failed from an unexpected reason: %#v", err)
+		return fmt.Errorf("running osbuild failed from an unexpected reason: %#v", err)
 	}
 
-	var result struct {
-		OutputID string `json:"output_id"`
-	}
-
-	err = json.NewDecoder(&outBuffer).Decode(&result)
-	if err != nil {
-		return "", fmt.Errorf("cannot decode osbuild output: %#v", err)
-	}
-
-	return result.OutputID, nil
+	return nil
 }
 
 // testImageInfo runs image-info on image specified by imageImage and
@@ -200,9 +191,9 @@ func testBootUsingQemu(t *testing.T, imagePath string) {
 	require.NoError(t, err)
 }
 
-func testBootUsingNspawnImage(t *testing.T, imagePath string, outputID string) {
+func testBootUsingNspawnImage(t *testing.T, imagePath string) {
 	err := withNetworkNamespace(func(ns netNS) error {
-		return withBootedNspawnImage(imagePath, outputID, ns, func() error {
+		return withBootedNspawnImage(imagePath, ns, func() error {
 			testSSH(t, "localhost", constants.TestPaths.PrivateKey, &ns)
 			return nil
 		})
@@ -210,10 +201,10 @@ func testBootUsingNspawnImage(t *testing.T, imagePath string, outputID string) {
 	require.NoError(t, err)
 }
 
-func testBootUsingNspawnDirectory(t *testing.T, imagePath string, outputID string) {
+func testBootUsingNspawnDirectory(t *testing.T, imagePath string) {
 	err := withNetworkNamespace(func(ns netNS) error {
 		return withExtractedTarArchive(imagePath, func(dir string) error {
-			return withBootedNspawnDirectory(dir, outputID, ns, func() error {
+			return withBootedNspawnDirectory(dir, ns, func() error {
 				testSSH(t, "localhost", constants.TestPaths.PrivateKey, &ns)
 				return nil
 			})
@@ -305,16 +296,16 @@ func testBootUsingAzure(t *testing.T, imagePath string) {
 // The test passes if the function is able to connect to the image via ssh
 // in defined number of attempts and systemd-is-running returns running
 // or degraded status.
-func testBoot(t *testing.T, imagePath string, bootType string, outputID string) {
+func testBoot(t *testing.T, imagePath string, bootType string) {
 	switch bootType {
 	case "qemu":
 		testBootUsingQemu(t, imagePath)
 
 	case "nspawn":
-		testBootUsingNspawnImage(t, imagePath, outputID)
+		testBootUsingNspawnImage(t, imagePath)
 
 	case "nspawn-extract":
-		testBootUsingNspawnDirectory(t, imagePath, outputID)
+		testBootUsingNspawnDirectory(t, imagePath)
 
 	case "aws":
 		testBootUsingAWS(t, imagePath)
@@ -344,7 +335,7 @@ func kvmAvailable() bool {
 
 // testImage performs a series of tests specified in the testcase
 // on an image
-func testImage(t *testing.T, testcase testcaseStruct, imagePath, outputID string) {
+func testImage(t *testing.T, testcase testcaseStruct, imagePath string) {
 	if testcase.ImageInfo != nil {
 		t.Run("image info", func(t *testing.T) {
 			testImageInfo(t, imagePath, testcase.ImageInfo)
@@ -357,7 +348,7 @@ func testImage(t *testing.T, testcase testcaseStruct, imagePath, outputID string
 			return
 		}
 		t.Run("boot", func(t *testing.T) {
-			testBoot(t, imagePath, testcase.Boot.Type, outputID)
+			testBoot(t, imagePath, testcase.Boot.Type)
 		})
 	}
 }
@@ -365,22 +356,22 @@ func testImage(t *testing.T, testcase testcaseStruct, imagePath, outputID string
 // runTestcase builds the pipeline specified in the testcase and then it
 // tests the result
 func runTestcase(t *testing.T, testcase testcaseStruct) {
-	store, err := ioutil.TempDir("/var/tmp", "osbuild-image-tests-")
-	require.NoErrorf(t, err, "cannot create temporary store: %#v", err)
+	outputDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-image-tests-*")
+	require.NoErrorf(t, err, "cannot create temporary output directory: %#v", err)
 
 	defer func() {
-		err := os.RemoveAll(store)
+		err := os.RemoveAll(outputDirectory)
 		if err != nil {
-			log.Printf("cannot remove temporary store: %#v\n", err)
+			log.Printf("cannot remove temporary output directory: %#v\n", err)
 		}
 	}()
 
-	outputID, err := runOsbuild(testcase.Manifest, store)
+	err = runOsbuild(testcase.Manifest, outputDirectory)
 	require.NoError(t, err)
 
-	imagePath := fmt.Sprintf("%s/refs/%s/%s", store, outputID, testcase.ComposeRequest.Filename)
+	imagePath := fmt.Sprintf("%s/%s", outputDirectory, testcase.ComposeRequest.Filename)
 
-	testImage(t, testcase, imagePath, outputID)
+	testImage(t, testcase, imagePath)
 }
 
 // getAllCases returns paths to all testcases in the testcase directory
diff --git a/internal/client/integration_test.go b/internal/client/integration_test.go
index 20bd62b44..73a1982f4 100644
--- a/internal/client/integration_test.go
+++ b/internal/client/integration_test.go
@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"os"
 	"testing"
-	"time"
 
 	"github.com/osbuild/osbuild-composer/internal/test"
 )
@@ -21,7 +20,7 @@ var testState *TestState
 // Also makes sure there is a running server to test against
 func executeTests(m *testing.M) int {
 	var err error
-	testState, err = setUpTestState("/run/weldr/api.socket", 60*time.Second, false)
+	testState, err = setUpTestState("/run/weldr/api.socket", false)
 	if err != nil {
 		fmt.Printf("ERROR: Test setup failed: %s\n", err)
 		panic(err)
diff --git a/internal/client/unit_test.go b/internal/client/unit_test.go
index 156f70356..880a8680a 100644
--- a/internal/client/unit_test.go
+++ b/internal/client/unit_test.go
@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"os"
 	"testing"
-	"time"
 
 	"github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
 	rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
@@ -58,7 +57,7 @@ func executeTests(m *testing.M) int {
 		}
 	}()
 
-	testState, err = setUpTestState(socketPath, 60*time.Second, true)
+	testState, err = setUpTestState(socketPath, true)
 	if err != nil {
 		log.Fatalf("ERROR: Test setup failed: %s\n", err)
 	}
diff --git a/jenkins/test_runner_base.yml b/jenkins/test_runner_base.yml
index 0b9ad5736..73ca336c3 100644
--- a/jenkins/test_runner_base.yml
+++ b/jenkins/test_runner_base.yml
@@ -3,12 +3,10 @@
 - block:
 
     - name: "Run {{ test.name }} test"
-      command: "{{ tests_path }}/{{ test.name }} -test.v"
+      command: "{{ tests_path }}/{{ test.name }} -test.v -test.timeout {{ test.timeout * 60 }}s"
       args:
         chdir: "{{ tests_working_directory }}"
-      register: async_test
-      async: "{{ test.timeout * 60 }}"
-      poll: "{{ polling_interval }}"
+      register: test_cmd
 
     - name: "Mark {{ test.name }} as passed"
       set_fact:
@@ -29,8 +27,8 @@
           Logs from {{ test.name }}
           ----------------------------------------------------------------------
           stderr:
-          {{ async_test.stderr }}
+          {{ test_cmd.stderr }}
           ----------------------------------------------------------------------
           stdout:
-          {{ async_test.stdout }}
+          {{ test_cmd.stdout }}
 
diff --git a/jenkins/test_runner_image.yml b/jenkins/test_runner_image.yml
index 43d2b2d69..a22582307 100644
--- a/jenkins/test_runner_image.yml
+++ b/jenkins/test_runner_image.yml
@@ -24,7 +24,7 @@
     # upset and nobody likes it when that happens.
     - name: "Run image tests"
       command: |
-        {{ image_test_executable }} -test.v \
+        {{ image_test_executable }} -test.v -test.timeout {{image_test_timeout * 60}}s \
           {% for test_case in osbuild_composer_image_test_cases %}
             {% if loop.last %}
           {{ image_test_case_path }}/{{ test_case_prefix }}-{{ test_case }}
@@ -34,9 +34,7 @@
           {% endfor %}
       args:
         chdir: "{{ tests_working_directory }}"
-      register: async_test
-      async: "{{ image_test_timeout * 60 }}"
-      poll: "{{ polling_interval }}"
+      register: test_cmd
 
     - name: "Mark image tests as passed"
       set_fact:
@@ -57,8 +55,8 @@
           Logs from image test
           ----------------------------------------------------------------------
           stderr:
-          {{ async_test.stderr }}
+          {{ test_cmd.stderr }}
           ----------------------------------------------------------------------
           stdout:
-          {{ async_test.stdout }}
+          {{ test_cmd.stdout }}
 
diff --git a/schutzbot/run_tests.sh b/schutzbot/run_tests.sh
new file mode 100755
index 000000000..2a78253a3
--- /dev/null
+++ b/schutzbot/run_tests.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Create temporary directories for Ansible.
+sudo mkdir -vp /opt/ansible_{local,remote}
+sudo chmod -R 777 /opt/ansible_{local,remote}
+
+# Remove Fedora modular repositories to speed up dnf-json.
+sudo rm -rfv /etc/yum.repos.d/fedora*modular*
+
+# Ensure /tmp is mounted on tmpfs.
+sudo systemctl enable tmp.mount || \
+  sudo systemctl unmask tmp.mount && sudo systemctl start tmp.mount
+
+# Restart systemd to work around some Fedora issues in cloud images.
+sudo systemctl restart systemd-journald
+
+# Get the current journald cursor.
+export JOURNALD_CURSOR=$(sudo journalctl --quiet -n 1 --show-cursor | tail -n 1 | grep -oP 's\=.*$')
+
+# Add a function to preserve the system journal if something goes wrong.
+preserve_journal() {
+  sudo journalctl --after-cursor=${JOURNALD_CURSOR} > systemd-journald.log
+  exit 1
+}
+trap "preserve_journal" ERR
+
+# Ensure Ansible is installed.
+if ! rpm -q ansible; then
+  sudo dnf -y install ansible
+fi
+
+# Write a simple hosts file for Ansible.
+echo -e "[test_instances]\nlocalhost ansible_connection=local" > hosts.ini
+
+# Set Ansible's config file location.
+export ANSIBLE_CONFIG=ansible-osbuild/ansible.cfg
+
+# Deploy the software.
+git clone https://github.com/osbuild/ansible-osbuild.git ansible-osbuild
+ansible-playbook \
+  -i hosts.ini \
+  -e osbuild_composer_repo=${WORKSPACE} \
+  -e osbuild_composer_version=$(git rev-parse HEAD) \
+  ansible-osbuild/playbook.yml
+
+# Run the tests.
+ansible-playbook \
+  -e workspace=${WORKSPACE} \
+  -e journald_cursor="${JOURNALD_CURSOR}" \
+  -e test_type=${TEST_TYPE:-base} \
+  -i hosts.ini \
+  jenkins/test.yml
+
+# Collect the systemd journal anyway if we made it all the way to the end.
+sudo journalctl --after-cursor=${JOURNALD_CURSOR} > systemd-journald.log
\ No newline at end of file
diff --git a/schutzbot/test.yml b/schutzbot/test.yml
new file mode 100644
index 000000000..e0359943b
--- /dev/null
+++ b/schutzbot/test.yml
@@ -0,0 +1,40 @@
+---
+
+- hosts: localhost
+  become: yes
+  vars:
+    passed_tests: []
+    failed_tests: []
+  vars_files:
+    - vars.yml
+  tasks:
+
+    - name: Run osbuild-composer base tests
+      include_tasks: test_runner_base.yml
+      loop: "{{ osbuild_composer_base_tests }}"
+      loop_control:
+        loop_var: test
+      when:
+        - test_type == 'base'
+
+    - name: Run osbuild-composer image tests
+      include_tasks: test_runner_image.yml
+      when:
+        - test_type == 'image'
+
+    - name: Show failed and passed tests
+      debug:
+        msg: |
+          Passed tests: {{ 'None' if not passed_tests else '' }}
+          {% for test_name in passed_tests %}
+            - {{ test_name }}
+          {% endfor %}
+          Failed tests: {{ 'None' if not failed_tests else '' }}
+          {% for test_name in failed_tests %}
+            - {{ test_name }}
+          {% endfor %}
+
+    - name: Fail the test run if a test failed
+      fail:
+        msg: One or more tests failed.
+      when: failed_tests
\ No newline at end of file
diff --git a/schutzbot/test_runner_base.yml b/schutzbot/test_runner_base.yml
new file mode 100644
index 000000000..73ca336c3
--- /dev/null
+++ b/schutzbot/test_runner_base.yml
@@ -0,0 +1,34 @@
+---
+
+- block:
+
+    - name: "Run {{ test.name }} test"
+      command: "{{ tests_path }}/{{ test.name }} -test.v -test.timeout {{ test.timeout * 60 }}s"
+      args:
+        chdir: "{{ tests_working_directory }}"
+      register: test_cmd
+
+    - name: "Mark {{ test.name }} as passed"
+      set_fact:
+        passed_tests: "{{ passed_tests + [test.name] }}"
+
+  rescue:
+
+    - name: "Mark {{ test.name }} as failed"
+      set_fact:
+        failed_tests: "{{ failed_tests + [test.name] }}"
+
+  always:
+
+    - name: "Write log for {{ test.name }}"
+      copy:
+        dest: "{{ workspace }}/{{ test.name }}.log"
+        content: |
+          Logs from {{ test.name }}
+          ----------------------------------------------------------------------
+          stderr:
+          {{ test_cmd.stderr }}
+          ----------------------------------------------------------------------
+          stdout:
+          {{ test_cmd.stdout }}
+
diff --git a/schutzbot/test_runner_image.yml b/schutzbot/test_runner_image.yml
new file mode 100644
index 000000000..a22582307
--- /dev/null
+++ b/schutzbot/test_runner_image.yml
@@ -0,0 +1,62 @@
+---
+
+- name: Setup test case prefix based on distro and release
+  set_fact:
+    test_case_prefix: >-
+      {%- if ansible_distribution == "Fedora" -%}
+      fedora_{{ ansible_distribution_version }}-{{ ansible_machine }}
+      {%- else -%}
+      rhel_{{ ansible_distribution_version }}-{{ ansible_machine }}
+      {%- endif -%}
+
+- name: Show which tests will be run
+  debug:
+    msg: |
+      Running the following image test cases:
+      {% for test_case in osbuild_composer_image_test_cases %}
+        - {{ test_case | splitext | first }}
+      {% endfor %}
+
+- block:
+
+    # NOTE(mhayden): The fancy jinja2 here ensures that the last test case
+    # in the list does not have a backslash added. That would make the shell
+    # upset and nobody likes it when that happens.
+    - name: "Run image tests"
+      command: |
+        {{ image_test_executable }} -test.v -test.timeout {{image_test_timeout * 60}}s \
+          {% for test_case in osbuild_composer_image_test_cases %}
+            {% if loop.last %}
+          {{ image_test_case_path }}/{{ test_case_prefix }}-{{ test_case }}
+            {% else %}
+          {{ image_test_case_path }}/{{ test_case_prefix }}-{{ test_case }} \
+            {% endif %}
+          {% endfor %}
+      args:
+        chdir: "{{ tests_working_directory }}"
+      register: test_cmd
+
+    - name: "Mark image tests as passed"
+      set_fact:
+        passed_tests: "{{ passed_tests + ['image_tests'] }}"
+
+  rescue:
+
+    - name: "Mark image tests as failed"
+      set_fact:
+        failed_tests: "{{ failed_tests + ['image_tests'] }}"
+
+  always:
+
+    - name: "Write log for image tests"
+      copy:
+        dest: "{{ workspace }}/image_test.log"
+        content: |
+          Logs from image test
+          ----------------------------------------------------------------------
+          stderr:
+          {{ test_cmd.stderr }}
+          ----------------------------------------------------------------------
+          stdout:
+          {{ test_cmd.stdout }}
+
diff --git a/tools/test-case-generators/generate-test-case b/tools/test-case-generators/generate-test-case
index ff9647b75..a7e83b3e7 100755
--- a/tools/test-case-generators/generate-test-case
+++ b/tools/test-case-generators/generate-test-case
@@ -5,6 +5,7 @@ import subprocess
 import json
 import os
 import sys
+import tempfile
 
 '''
 This script generates a json test case. It accepts a test_case_request as input through standard input.
@@ -35,10 +36,14 @@ def get_subprocess_stdout(*args, **kwargs):
     return sp.stdout
 
 
-def run_osbuild(manifest, store):
-    osbuild_cmd = ["osbuild", "--store", store, "--json", "-"]
-    result = json.loads(get_subprocess_stdout(osbuild_cmd, encoding="utf-8", input=json.dumps(manifest)))
-    return result.get("output_id")
+def run_osbuild(manifest, store, output):
+    subprocess.run(["osbuild",
+                    "--store", store,
+                    "--output-directory", output,
+                    "--json", "-"],
+                   check=True,
+                   encoding="utf-8",
+                   input=json.dumps(manifest))
 
 
 def main(test_case, store):
@@ -52,9 +57,10 @@ def main(test_case, store):
     test_case["rpmmd"] = json.loads(get_subprocess_stdout(pipeline_command, input=compose_request, encoding="utf-8"))
 
     if boot_type != "nspawn-extract":
-        output_id = run_osbuild(test_case["manifest"], store)
-        image_file = os.path.join(store, "refs", output_id, test_case["compose-request"]["filename"])
-        test_case["image-info"] = json.loads(get_subprocess_stdout(["tools/image-info", image_file], encoding="utf-8"))
+        with tempfile.TemporaryDirectory(dir=store, prefix="test-case-output-") as output:
+            run_osbuild(test_case["manifest"], store, output)
+            image_file = os.path.join(output, test_case["compose-request"]["filename"])
+            test_case["image-info"] = json.loads(get_subprocess_stdout(["tools/image-info", image_file], encoding="utf-8"))
 
     return test_case