Repository: osbuild/osbuild-composer
Pull request #1069 (2020-11-09 14:23:42)

koji: support multi-build composes
This introduces three new job types, and splits builds for koji into an init job, potentially multiple build jobs and a finalize job. This will allow us to build different images on different architectures, all part of the same compose. And it allows us to drop kerberos credentials from the composer node, and only keep that in the workers.

This shows a sample of a compose consisting of two images being processed by one worker:

```
$ journalctl -b -u osbuild-worker\@1
osbuild-worker[19657]: Running 'koji-init' job 39f1c1ab-9d13-4e25-b995-a529230c8c43
osbuild-worker[19657]: 2020/11/10 22:42:31 Job 39f1c1ab-9d13-4e25-b995-a529230c8c43 finished
osbuild-worker[19657]: Waiting for a new job...
osbuild-worker[19657]: Running 'osbuild-koji' job f38868d7-3313-43ba-8a45-20ac9a0c4376
osbuild-worker[19657]: 2020/11/10 22:45:36 Job f38868d7-3313-43ba-8a45-20ac9a0c4376 finished
osbuild-worker[19657]: Waiting for a new job...
osbuild-worker[19657]: Running 'osbuild-koji' job 0dc55e5a-63f4-40f3-ac80-b2fa7a742294
osbuild-worker[19657]: 2020/11/10 22:49:23 Job 0dc55e5a-63f4-40f3-ac80-b2fa7a742294 finished
osbuild-worker[19657]: Waiting for a new job...
osbuild-worker[19657]: Running 'koji-finalize' job f9c049a4-db90-4bfb-a8f2-0440ddce224c
osbuild-worker[19657]: 2020/11/10 22:50:17 Job f9c049a4-db90-4bfb-a8f2-0440ddce224c finished
```

The result n koji is:
```
$ koji buildinfo 1
BUILD: name-version-release [1]
State: COMPLETE
Built by: osbuild-krb
Source: 
Volume: DEFAULT
Task: none
Finished: Tue, 10 Nov 2020 23:29:35 UTC
Tags: 
Extra: {'typeinfo': {'image': {}}}
Image archives:
/mnt/koji/packages/name/version/release/images/name-version-release.x86_64.raw
/mnt/koji/packages/name/version/release/images/name-version-release.x86_64.qcow2
```

teg: @ondrejbudai I added some more tests and addressed the issue you found (among others). I agree that indexing the dynamic args array is not ideal, happy for that to be reworked to be safer. Also hate the name ;)

If Schutzbot is happy I think that can all be addressed in follow-ups though.

Patch:
diff --git a/.gitignore b/.gitignore
index 256d19f76..534086142 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 __pycache__
 
+.vscode
+
 /bin
 /rpmbuild
diff --git a/cmd/osbuild-composer/composer.go b/cmd/osbuild-composer/composer.go
index cf3c089f6..b3bb4b769 100644
--- a/cmd/osbuild-composer/composer.go
+++ b/cmd/osbuild-composer/composer.go
@@ -18,7 +18,6 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/kojiapi"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/store"
-	"github.com/osbuild/osbuild-composer/internal/upload/koji"
 	"github.com/osbuild/osbuild-composer/internal/weldr"
 	"github.com/osbuild/osbuild-composer/internal/worker"
 
@@ -70,16 +69,15 @@ func NewComposer(config *ComposerConfigFile, stateDir, cacheDir string, logger *
 
 	c.rpm = rpmmd.NewRPMMD(path.Join(c.cacheDir, "rpmmd"), "/usr/libexec/osbuild-composer/dnf-json")
 
-	// construct job types of the form osbuild:{arch} for all arches
-	jobTypes := []string{"osbuild"}
-	jobTypesMap := map[string]bool{}
+	// construct job types of the form osbuild:{arch} and osbuild-koji:{arch} for all arches
+	jobTypes := []string{"osbuild", "koji-init", "koji-finalize"}
+	archSet := map[string]bool{}
 	for _, name := range c.distros.List() {
 		d := c.distros.GetDistro(name)
 		for _, arch := range d.ListArches() {
-			jt := "osbuild:" + arch
-			if !jobTypesMap[jt] {
-				jobTypesMap[jt] = true
-				jobTypes = append(jobTypes, jt)
+			if !archSet[arch] {
+				archSet[arch] = true
+				jobTypes = append(jobTypes, "osbuild:"+arch, "osbuild-koji:"+arch)
 			}
 		}
 	}
@@ -131,17 +129,7 @@ func (c *Composer) InitWeldr(repoPaths []string, weldrListener, localWorkerListe
 
 func (c *Composer) InitAPI(cert, key string, l net.Listener) error {
 	c.api = cloudapi.NewServer(c.workers, c.rpm, c.distros)
-
-	servers := make(map[string]koji.GSSAPICredentials)
-	for name, creds := range c.config.Koji.Servers {
-		if creds.Kerberos != nil {
-			servers[name] = koji.GSSAPICredentials{
-				Principal: creds.Kerberos.Principal,
-				KeyTab:    creds.Kerberos.KeyTab,
-			}
-		}
-	}
-	c.koji = kojiapi.NewServer(c.logger, c.workers, c.rpm, c.distros, servers)
+	c.koji = kojiapi.NewServer(c.logger, c.workers, c.rpm, c.distros)
 
 	tlsConfig, err := createTLSConfig(&connectionConfig{
 		CACertFile:     c.config.Koji.CA,
diff --git a/cmd/osbuild-composer/config.go b/cmd/osbuild-composer/config.go
index 38bec22a9..3cd6d4e7d 100644
--- a/cmd/osbuild-composer/config.go
+++ b/cmd/osbuild-composer/config.go
@@ -8,12 +8,6 @@ import (
 
 type ComposerConfigFile struct {
 	Koji struct {
-		Servers map[string]struct {
-			Kerberos *struct {
-				Principal string `toml:"principal"`
-				KeyTab    string `toml:"keytab"`
-			} `toml:"kerberos,omitempty"`
-		} `toml:"servers"`
 		AllowedDomains []string `toml:"allowed_domains"`
 		CA             string   `toml:"ca"`
 	} `toml:"koji"`
diff --git a/cmd/osbuild-worker/jobimpl-koji-finalize.go b/cmd/osbuild-worker/jobimpl-koji-finalize.go
new file mode 100644
index 000000000..ae7099975
--- /dev/null
+++ b/cmd/osbuild-worker/jobimpl-koji-finalize.go
@@ -0,0 +1,181 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/osbuild/osbuild-composer/internal/upload/koji"
+	"github.com/osbuild/osbuild-composer/internal/worker"
+)
+
+type KojiFinalizeJobImpl struct {
+	KojiServers map[string]koji.GSSAPICredentials
+}
+
+func (impl *KojiFinalizeJobImpl) kojiImport(
+	server string,
+	build koji.ImageBuild,
+	buildRoots []koji.BuildRoot,
+	images []koji.Image,
+	directory, token string) error {
+	// Koji for some reason needs TLS renegotiation enabled.
+	// Clone the default http transport and enable renegotiation.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{
+		Renegotiation: tls.RenegotiateOnceAsClient,
+	}
+
+	serverURL, err := url.Parse(server)
+	if err != nil {
+		return err
+	}
+
+	creds, exists := impl.KojiServers[serverURL.Hostname()]
+	if !exists {
+		return fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
+	}
+
+	k, err := koji.NewFromGSSAPI(server, &creds, transport)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := k.Logout()
+		if err != nil {
+			log.Printf("koji logout failed: %v", err)
+		}
+	}()
+
+	_, err = k.CGImport(build, buildRoots, images, directory, token)
+	if err != nil {
+		return fmt.Errorf("Could not import build into koji: %v", err)
+	}
+
+	return nil
+}
+
+func (impl *KojiFinalizeJobImpl) kojiFail(server string, buildID int, token string) error {
+	// Koji for some reason needs TLS renegotiation enabled.
+	// Clone the default http transport and enable renegotiation.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{
+		Renegotiation: tls.RenegotiateOnceAsClient,
+	}
+
+	serverURL, err := url.Parse(server)
+	if err != nil {
+		return err
+	}
+
+	creds, exists := impl.KojiServers[serverURL.Hostname()]
+	if !exists {
+		return fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
+	}
+
+	k, err := koji.NewFromGSSAPI(server, &creds, transport)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := k.Logout()
+		if err != nil {
+			log.Printf("koji logout failed: %v", err)
+		}
+	}()
+
+	return k.CGFailBuild(buildID, token)
+}
+
+func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
+	var args worker.KojiFinalizeJob
+	err := job.Args(&args)
+	if err != nil {
+		return err
+	}
+
+	var failure bool
+
+	var initArgs worker.KojiInitJobResult
+	err = job.DynamicArgs(0, &initArgs)
+	if err != nil {
+		return err
+	}
+
+	if initArgs.KojiError != nil {
+		failure = true
+	}
+
+	build := koji.ImageBuild{
+		BuildID:   initArgs.BuildID,
+		TaskID:    args.TaskID,
+		Name:      args.Name,
+		Version:   args.Version,
+		Release:   args.Release,
+		StartTime: int64(args.StartTime),
+		EndTime:   time.Now().Unix(),
+	}
+
+	var buildRoots []koji.BuildRoot
+	var images []koji.Image
+	for i := 1; i < job.NDynamicArgs(); i++ {
+		var buildArgs worker.OSBuildKojiJobResult
+		err = job.DynamicArgs(i, &buildArgs)
+		if err != nil {
+			return err
+		}
+		if !buildArgs.OSBuildOutput.Success || buildArgs.KojiError != nil {
+			failure = true
+			break
+		}
+		buildRoots = append(buildRoots, koji.BuildRoot{
+			ID: uint64(i),
+			Host: koji.Host{
+				Os:   buildArgs.HostOS,
+				Arch: buildArgs.Arch,
+			},
+			ContentGenerator: koji.ContentGenerator{
+				Name:    "osbuild",
+				Version: "0", // TODO: put the correct version here
+			},
+			Container: koji.Container{
+				Type: "none",
+				Arch: buildArgs.Arch,
+			},
+			Tools: []koji.Tool{},
+			RPMs:  osbuildStagesToRPMs(buildArgs.OSBuildOutput.Build.Stages),
+		})
+		images = append(images, koji.Image{
+			BuildRootID:  uint64(i),
+			Filename:     args.KojiFilenames[i-1],
+			FileSize:     buildArgs.ImageSize,
+			Arch:         buildArgs.Arch,
+			ChecksumType: "md5",
+			MD5:          buildArgs.ImageHash,
+			Type:         "image",
+			RPMs:         osbuildStagesToRPMs(buildArgs.OSBuildOutput.Stages),
+			Extra: koji.ImageExtra{
+				Info: koji.ImageExtraInfo{
+					Arch: buildArgs.Arch,
+				},
+			},
+		})
+	}
+
+	var result worker.KojiFinalizeJobResult
+	if failure {
+		result.KojiError = impl.kojiFail(args.Server, int(initArgs.BuildID), initArgs.Token)
+	} else {
+		result.KojiError = impl.kojiImport(args.Server, build, buildRoots, images, args.KojiDirectory, initArgs.Token)
+	}
+
+	err = job.Update(&result)
+	if err != nil {
+		return fmt.Errorf("Error reporting job result: %v", err)
+	}
+
+	return nil
+}
diff --git a/cmd/osbuild-worker/jobimpl-koji-init.go b/cmd/osbuild-worker/jobimpl-koji-init.go
new file mode 100644
index 000000000..ca1970db1
--- /dev/null
+++ b/cmd/osbuild-worker/jobimpl-koji-init.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+
+	"github.com/osbuild/osbuild-composer/internal/upload/koji"
+	"github.com/osbuild/osbuild-composer/internal/worker"
+)
+
+type KojiInitJobImpl struct {
+	KojiServers map[string]koji.GSSAPICredentials
+}
+
+func (impl *KojiInitJobImpl) kojiInit(server, name, version, release string) (string, uint64, error) {
+	// Koji for some reason needs TLS renegotiation enabled.
+	// Clone the default http transport and enable renegotiation.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{
+		Renegotiation: tls.RenegotiateOnceAsClient,
+	}
+
+	serverURL, err := url.Parse(server)
+	if err != nil {
+		return "", 0, err
+	}
+
+	creds, exists := impl.KojiServers[serverURL.Hostname()]
+	if !exists {
+		return "", 0, fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
+	}
+
+	k, err := koji.NewFromGSSAPI(server, &creds, transport)
+	if err != nil {
+		return "", 0, err
+	}
+	defer func() {
+		err := k.Logout()
+		if err != nil {
+			log.Printf("koji logout failed: %v", err)
+		}
+	}()
+
+	buildInfo, err := k.CGInitBuild(name, version, release)
+	if err != nil {
+		return "", 0, err
+	}
+
+	return buildInfo.Token, uint64(buildInfo.BuildID), nil
+}
+
+func (impl *KojiInitJobImpl) Run(job worker.Job) error {
+	var args worker.KojiInitJob
+	err := job.Args(&args)
+	if err != nil {
+		return err
+	}
+
+	var result worker.KojiInitJobResult
+	result.Token, result.BuildID, result.KojiError = impl.kojiInit(args.Server, args.Name, args.Version, args.Release)
+
+	err = job.Update(&result)
+	if err != nil {
+		return fmt.Errorf("Error reporting job result: %v", err)
+	}
+
+	return nil
+}
diff --git a/cmd/osbuild-worker/jobimpl-osbuild-koji.go b/cmd/osbuild-worker/jobimpl-osbuild-koji.go
new file mode 100644
index 000000000..0087a2ddc
--- /dev/null
+++ b/cmd/osbuild-worker/jobimpl-osbuild-koji.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+
+	"github.com/osbuild/osbuild-composer/internal/common"
+	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/upload/koji"
+	"github.com/osbuild/osbuild-composer/internal/worker"
+)
+
+type OSBuildKojiJobImpl struct {
+	Store       string
+	KojiServers map[string]koji.GSSAPICredentials
+}
+
+func (impl *OSBuildKojiJobImpl) kojiUpload(file *os.File, server, directory, filename string) (string, uint64, error) {
+	// Koji for some reason needs TLS renegotiation enabled.
+	// Clone the default http transport and enable renegotiation.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{
+		Renegotiation: tls.RenegotiateOnceAsClient,
+	}
+
+	serverURL, err := url.Parse(server)
+	if err != nil {
+		return "", 0, err
+	}
+
+	creds, exists := impl.KojiServers[serverURL.Hostname()]
+	if !exists {
+		return "", 0, fmt.Errorf("Koji server has not been configured: %s", serverURL.Hostname())
+	}
+
+	k, err := koji.NewFromGSSAPI(server, &creds, transport)
+	if err != nil {
+		return "", 0, err
+	}
+	defer func() {
+		err := k.Logout()
+		if err != nil {
+			log.Printf("koji logout failed: %v", err)
+		}
+	}()
+
+	return k.Upload(file, directory, filename)
+}
+
+func (impl *OSBuildKojiJobImpl) Run(job worker.Job) error {
+	outputDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-worker-*")
+	if err != nil {
+		return fmt.Errorf("error creating temporary output directory: %v", err)
+	}
+	defer func() {
+		err := os.RemoveAll(outputDirectory)
+		if err != nil {
+			log.Printf("Error removing temporary output directory (%s): %v", outputDirectory, err)
+		}
+	}()
+
+	var args worker.OSBuildKojiJob
+	err = job.Args(&args)
+	if err != nil {
+		return err
+	}
+
+	var initArgs worker.KojiInitJobResult
+	err = job.DynamicArgs(0, &initArgs)
+	if err != nil {
+		return err
+	}
+
+	var result worker.OSBuildKojiJobResult
+	result.Arch = common.CurrentArch()
+	result.HostOS, err = distro.GetRedHatRelease()
+	if err != nil {
+		return err
+	}
+
+	if initArgs.KojiError == nil {
+		result.OSBuildOutput, err = RunOSBuild(args.Manifest, impl.Store, outputDirectory, os.Stderr)
+		if err != nil {
+			return err
+		}
+
+		if result.OSBuildOutput.Success {
+			f, err := os.Open(path.Join(outputDirectory, args.ImageName))
+			if err != nil {
+				return err
+			}
+			result.ImageHash, result.ImageSize, result.KojiError = impl.kojiUpload(f, args.KojiServer, args.KojiDirectory, args.KojiFilename)
+		}
+	}
+
+	err = job.Update(&result)
+	if err != nil {
+		return fmt.Errorf("Error reporting job result: %v", err)
+	}
+
+	return nil
+}
diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go
index c06ee8ccf..578f77c7c 100644
--- a/cmd/osbuild-worker/main.go
+++ b/cmd/osbuild-worker/main.go
@@ -154,6 +154,16 @@ func main() {
 			Store:       store,
 			KojiServers: kojiServers,
 		},
+		"osbuild-koji": &OSBuildKojiJobImpl{
+			Store:       store,
+			KojiServers: kojiServers,
+		},
+		"koji-init": &KojiInitJobImpl{
+			KojiServers: kojiServers,
+		},
+		"koji-finalize": &KojiFinalizeJobImpl{
+			KojiServers: kojiServers,
+		},
 	}
 
 	acceptedJobTypes := []string{}
diff --git a/internal/cloudapi/server.go b/internal/cloudapi/server.go
index 3e60438f9..983bbd4e9 100644
--- a/internal/cloudapi/server.go
+++ b/internal/cloudapi/server.go
@@ -188,7 +188,7 @@ func (server *Server) Compose(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	id, err := server.workers.Enqueue(ir.arch, &worker.OSBuildJob{
+	id, err := server.workers.EnqueueOSBuild(ir.arch, &worker.OSBuildJob{
 		Manifest: ir.manifest,
 		Targets:  targets,
 	})
@@ -216,7 +216,7 @@ func (server *Server) ComposeStatus(w http.ResponseWriter, r *http.Request, id s
 	}
 
 	var result worker.OSBuildJobResult
-	status, err := server.workers.JobStatus(jobId, &result)
+	status, _, err := server.workers.JobStatus(jobId, &result)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("Job %s not found: %s", id, err), http.StatusNotFound)
 		return
diff --git a/internal/jobqueue/fsjobqueue/fsjobqueue.go b/internal/jobqueue/fsjobqueue/fsjobqueue.go
index 7949bf748..c041e7b68 100644
--- a/internal/jobqueue/fsjobqueue/fsjobqueue.go
+++ b/internal/jobqueue/fsjobqueue/fsjobqueue.go
@@ -17,7 +17,6 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
-	"sort"
 	"sync"
 	"time"
 
@@ -108,7 +107,7 @@ func (q *fsJobQueue) Enqueue(jobType string, args interface{}, dependencies []uu
 	var j = job{
 		Id:           uuid.New(),
 		Type:         jobType,
-		Dependencies: uniqueUUIDList(dependencies),
+		Dependencies: dependencies,
 		QueuedAt:     time.Now(),
 	}
 
@@ -145,13 +144,13 @@ func (q *fsJobQueue) Enqueue(jobType string, args interface{}, dependencies []uu
 	return j.Id, nil
 }
 
-func (q *fsJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, string, json.RawMessage, error) {
+func (q *fsJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, []uuid.UUID, string, json.RawMessage, error) {
 	q.mu.Lock()
 	defer q.mu.Unlock()
 
 	// Return early if the context is already canceled.
 	if err := ctx.Err(); err != nil {
-		return uuid.Nil, "", nil, err
+		return uuid.Nil, nil, "", nil, err
 	}
 
 	// Filter q.pending by the `jobTypes`. Ignore those job types that this
@@ -173,12 +172,12 @@ func (q *fsJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID,
 		q.mu.Lock()
 
 		if err != nil {
-			return uuid.Nil, "", nil, err
+			return uuid.Nil, nil, "", nil, err
 		}
 
 		j, err = q.readJob(id)
 		if err != nil {
-			return uuid.Nil, "", nil, err
+			return uuid.Nil, nil, "", nil, err
 		}
 
 		if !j.Canceled {
@@ -190,10 +189,10 @@ func (q *fsJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID,
 
 	err := q.db.Write(j.Id.String(), j)
 	if err != nil {
-		return uuid.Nil, "", nil, fmt.Errorf("error writing job %s: %v", j.Id, err)
+		return uuid.Nil, nil, "", nil, fmt.Errorf("error writing job %s: %v", j.Id, err)
 	}
 
-	return j.Id, j.Type, j.Args, nil
+	return j.Id, j.Dependencies, j.Type, j.Args, nil
 }
 
 func (q *fsJobQueue) FinishJob(id uuid.UUID, result interface{}) error {
@@ -264,24 +263,18 @@ func (q *fsJobQueue) CancelJob(id uuid.UUID) error {
 	return nil
 }
 
-func (q *fsJobQueue) JobStatus(id uuid.UUID, result interface{}) (queued, started, finished time.Time, canceled bool, err error) {
+func (q *fsJobQueue) JobStatus(id uuid.UUID) (result json.RawMessage, queued, started, finished time.Time, canceled bool, deps []uuid.UUID, err error) {
 	j, err := q.readJob(id)
 	if err != nil {
 		return
 	}
 
-	if !j.FinishedAt.IsZero() && !j.Canceled {
-		err = json.Unmarshal(j.Result, result)
-		if err != nil {
-			err = fmt.Errorf("error unmarshaling result for job '%s': %v", id, err)
-			return
-		}
-	}
-
+	result = j.Result
 	queued = j.QueuedAt
 	started = j.StartedAt
 	finished = j.FinishedAt
 	canceled = j.Canceled
+	deps = j.Dependencies
 
 	return
 }
@@ -336,30 +329,6 @@ func (q *fsJobQueue) maybeEnqueue(j *job, updateDependants bool) error {
 	return nil
 }
 
-// Sorts and removes duplicates from `ids`.
-func uniqueUUIDList(ids []uuid.UUID) []uuid.UUID {
-	s := map[uuid.UUID]bool{}
-	for _, id := range ids {
-		s[id] = true
-	}
-
-	l := []uuid.UUID{}
-	for id := range s {
-		l = append(l, id)
-	}
-
-	sort.Slice(l, func(i, j int) bool {
-		for b := 0; b < 16; b++ {
-			if l[i][b] < l[j][b] {
-				return true
-			}
-		}
-		return false
-	})
-
-	return l
-}
-
 // Select on a list of `chan uuid.UUID`s. Returns an error if one of the
 // channels is closed.
 //
diff --git a/internal/jobqueue/jobqueue.go b/internal/jobqueue/jobqueue.go
index 751952c35..d95533aa2 100644
--- a/internal/jobqueue/jobqueue.go
+++ b/internal/jobqueue/jobqueue.go
@@ -39,9 +39,9 @@ type JobQueue interface {
 	// Waits until a job with a type of any of `jobTypes` is available, or `ctx` is
 	// canceled.
 	//
-	// Returns the job's id, type, and arguments, or an error. Arguments
+	// Returns the job's id, dependencies, type, and arguments, or an error. Arguments
 	// can be unmarshaled to the type given in Enqueue().
-	Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, string, json.RawMessage, error)
+	Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, []uuid.UUID, string, json.RawMessage, error)
 
 	// Mark the job with `id` as finished. `result` must fit the associated
 	// job type and must be serializable to JSON.
@@ -50,13 +50,15 @@ type JobQueue interface {
 	// Cancel a job. Does nothing if the job has already finished.
 	CancelJob(id uuid.UUID) error
 
+	// If the job has finished, returns the result as raw JSON.
+	//
 	// Returns the current status of the job, in the form of three times:
 	// queued, started, and finished. `started` and `finished` might be the
 	// zero time (check with t.IsZero()), when the job is not running or
 	// finished, respectively.
 	//
-	// If the job is finished, its result will be returned in `result`.
-	JobStatus(id uuid.UUID, result interface{}) (queued, started, finished time.Time, canceled bool, err error)
+	// Lastly, the IDs of the jobs dependencies are returned.
+	JobStatus(id uuid.UUID) (result json.RawMessage, queued, started, finished time.Time, canceled bool, deps []uuid.UUID, err error)
 }
 
 var (
diff --git a/internal/kojiapi/api/api.gen.go b/internal/kojiapi/api/api.gen.go
index 3f9d11972..0cbe1d95c 100644
--- a/internal/kojiapi/api/api.gen.go
+++ b/internal/kojiapi/api/api.gen.go
@@ -12,7 +12,9 @@ import (
 
 // ComposeLogs defines model for ComposeLogs.
 type ComposeLogs struct {
-	ImageLogs []interface{} `json:"image_logs"`
+	ImageLogs      []interface{} `json:"image_logs"`
+	KojiImportLogs interface{}   `json:"koji_import_logs"`
+	KojiInitLogs   interface{}   `json:"koji_init_logs"`
 }
 
 // ComposeRequest defines model for ComposeRequest.
@@ -34,6 +36,7 @@ type ComposeResponse struct {
 // ComposeStatus defines model for ComposeStatus.
 type ComposeStatus struct {
 	ImageStatuses []ImageStatus `json:"image_statuses"`
+	KojiBuildId   *int          `json:"koji_build_id,omitempty"`
 	KojiTaskId    int           `json:"koji_task_id"`
 	Status        string        `json:"status"`
 }
diff --git a/internal/kojiapi/api/openapi.yml b/internal/kojiapi/api/openapi.yml
index 490779edd..923fd7d4d 100644
--- a/internal/kojiapi/api/openapi.yml
+++ b/internal/kojiapi/api/openapi.yml
@@ -147,10 +147,17 @@ components:
         koji_task_id:
           type: integer
           example: 203143
+        koji_build_id:
+          type: integer
+          example: 42
     ComposeLogs:
       required:
+        - koji_init_logs
+        - koji_import_logs
         - image_logs
       properties:
+        koji_init_logs: {}
+        koji_import_logs: {}
         image_logs:
           type: array
     ImageStatus:
diff --git a/internal/kojiapi/server.go b/internal/kojiapi/server.go
index a0b61a68d..09e928bd1 100644
--- a/internal/kojiapi/server.go
+++ b/internal/kojiapi/server.go
@@ -2,13 +2,12 @@
 package kojiapi
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
-	"net/url"
 	"strings"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
@@ -17,8 +16,6 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/kojiapi/api"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
-	"github.com/osbuild/osbuild-composer/internal/target"
-	"github.com/osbuild/osbuild-composer/internal/upload/koji"
 	"github.com/osbuild/osbuild-composer/internal/worker"
 )
 
@@ -28,17 +25,15 @@ type Server struct {
 	workers     *worker.Server
 	rpmMetadata rpmmd.RPMMD
 	distros     *distro.Registry
-	kojiServers map[string]koji.GSSAPICredentials
 }
 
 // NewServer creates a new koji server
-func NewServer(logger *log.Logger, workers *worker.Server, rpmMetadata rpmmd.RPMMD, distros *distro.Registry, kojiServers map[string]koji.GSSAPICredentials) *Server {
+func NewServer(logger *log.Logger, workers *worker.Server, rpmMetadata rpmmd.RPMMD, distros *distro.Registry) *Server {
 	s := &Server{
 		logger:      logger,
 		workers:     workers,
 		rpmMetadata: rpmMetadata,
 		distros:     distros,
-		kojiServers: kojiServers,
 	}
 
 	return s
@@ -76,22 +71,15 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unsupported distribution: %s", request.Distribution))
 	}
 
-	kojiServer, err := url.Parse(request.Koji.Server)
-	if err != nil {
-		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid Koji server: %s", request.Koji.Server))
-	}
-	creds, exists := h.server.kojiServers[kojiServer.Hostname()]
-	if !exists {
-		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Koji server has not been configured: %s", kojiServer.Hostname()))
-	}
-
 	type imageRequest struct {
-		manifest     distro.Manifest
-		arch         string
-		filename     string
-		kojiFilename string
+		manifest distro.Manifest
+		arch     string
+		filename string
 	}
+
 	imageRequests := make([]imageRequest, len(request.ImageRequests))
+	kojiFilenames := make([]string, len(request.ImageRequests))
+	kojiDirectory := "osbuild-composer-koji-" + uuid.New().String()
 
 	for i, ir := range request.ImageRequests {
 		arch, err := d.GetArch(ir.Architecture)
@@ -134,7 +122,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		imageRequests[i].arch = arch.Name()
 		imageRequests[i].filename = imageType.Filename()
 
-		filename := fmt.Sprintf(
+		kojiFilenames[i] = fmt.Sprintf(
 			"%s-%s-%s.%s%s",
 			request.Name,
 			request.Version,
@@ -142,69 +130,73 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			ir.Architecture,
 			splitExtension(imageType.Filename()),
 		)
-
-		imageRequests[i].kojiFilename = filename
 	}
 
-	var ir imageRequest
-	if len(imageRequests) == 1 {
-		// NOTE: the store currently does not support multi-image composes
-		ir = imageRequests[0]
-	} else {
-		return echo.NewHTTPError(http.StatusBadRequest, "Only single-image composes are currently supported")
-	}
-
-	// Koji for some reason needs TLS renegotiation enabled.
-	// Clone the default http transport and enable renegotiation.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	transport.TLSClientConfig = &tls.Config{
-		Renegotiation: tls.RenegotiateOnceAsClient,
-	}
-
-	k, err := koji.NewFromGSSAPI(request.Koji.Server, &creds, transport)
+	initID, err := h.server.workers.EnqueueKojiInit(&worker.KojiInitJob{
+		Server:  request.Koji.Server,
+		Name:    request.Name,
+		Version: request.Version,
+		Release: request.Release,
+	})
 	if err != nil {
-		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Could not log into Koji: %v", err))
+		// This is a programming errror.
+		panic(err)
 	}
 
-	defer func() {
-		err := k.Logout()
+	var buildIDs []uuid.UUID
+	for i, ir := range imageRequests {
+		id, err := h.server.workers.EnqueueOSBuildKoji(ir.arch, &worker.OSBuildKojiJob{
+			Manifest:      ir.manifest,
+			ImageName:     ir.filename,
+			KojiServer:    request.Koji.Server,
+			KojiDirectory: kojiDirectory,
+			KojiFilename:  kojiFilenames[i],
+		}, initID)
 		if err != nil {
-			log.Printf("koji logout failed: %v", err)
+			// This is a programming errror.
+			panic(err)
 		}
-	}()
+		buildIDs = append(buildIDs, id)
+	}
 
-	buildInfo, err := k.CGInitBuild(request.Name, request.Version, request.Release)
-	if err != nil {
-		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Could not initialize build with koji: %v", err))
-	}
-
-	job := worker.OSBuildJob{
-		Manifest: ir.manifest,
-		Targets: []*target.Target{
-			target.NewKojiTarget(&target.KojiTargetOptions{
-				BuildID:         uint64(buildInfo.BuildID),
-				TaskID:          uint64(request.Koji.TaskId),
-				Token:           buildInfo.Token,
-				Name:            request.Name,
-				Version:         request.Version,
-				Release:         request.Release,
-				Filename:        ir.filename,
-				UploadDirectory: "osbuild-composer-koji-" + uuid.New().String(),
-				Server:          request.Koji.Server,
-				KojiFilename:    ir.kojiFilename,
-			}),
-		},
-	}
-
-	id, err := h.server.workers.Enqueue(ir.arch, &job)
+	id, err := h.server.workers.EnqueueKojiFinalize(&worker.KojiFinalizeJob{
+		Server:        request.Koji.Server,
+		Name:          request.Name,
+		Version:       request.Version,
+		Release:       request.Release,
+		KojiFilenames: kojiFilenames,
+		KojiDirectory: kojiDirectory,
+		TaskID:        uint64(request.Koji.TaskId),
+		StartTime:     uint64(time.Now().Unix()),
+	}, initID, buildIDs)
 	if err != nil {
 		// This is a programming errror.
 		panic(err)
 	}
 
+	// TODO: remove
+	// For backwards compatibility we must only return once the
+	// build ID is known. This logic should live in the client,
+	// and `JobStatus()` should have a way to block until it
+	// changes.
+	var initResult worker.KojiInitJobResult
+	for {
+		status, _, err := h.server.workers.JobStatus(initID, &initResult)
+		if err != nil {
+			panic(err)
+		}
+		if !status.Finished.IsZero() || status.Canceled {
+			break
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	if initResult.KojiError != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Could not initialize build with koji: %v", initResult.KojiError))
+	}
+
 	return ctx.JSON(http.StatusCreated, &api.ComposeResponse{
 		Id:          id.String(),
-		KojiBuildId: buildInfo.BuildID,
+		KojiBuildId: int(initResult.BuildID),
 	})
 }
 
@@ -227,31 +219,44 @@ func splitExtension(filename string) string {
 	return "." + strings.Join(filenameParts[1:], ".")
 }
 
-func composeStatusFromJobStatus(js *worker.JobStatus, result *worker.OSBuildJobResult) string {
+func composeStatusFromJobStatus(js *worker.JobStatus, initResult *worker.KojiInitJobResult, buildResults []worker.OSBuildKojiJobResult, result *worker.KojiFinalizeJobResult) string {
 	if js.Canceled {
 		return "failure"
 	}
 
-	if js.Started.IsZero() {
-		return "pending"
+	if initResult.KojiError != nil {
+		return "failure"
+	}
+
+	for _, buildResult := range buildResults {
+		if buildResult.OSBuildOutput != nil && !buildResult.OSBuildOutput.Success {
+			return "failure"
+		}
+		if buildResult.KojiError != nil {
+			return "failure"
+		}
 	}
 
 	if js.Finished.IsZero() {
 		return "pending"
 	}
 
-	if result.Success {
+	if result.KojiError == nil {
 		return "success"
 	}
 
 	return "failure"
 }
 
-func imageStatusFromJobStatus(js *worker.JobStatus, result *worker.OSBuildJobResult) string {
+func imageStatusFromJobStatus(js *worker.JobStatus, initResult *worker.KojiInitJobResult, buildResult *worker.OSBuildKojiJobResult) string {
 	if js.Canceled {
 		return "failure"
 	}
 
+	if initResult.KojiError != nil {
+		return "failure"
+	}
+
 	if js.Started.IsZero() {
 		return "pending"
 	}
@@ -260,7 +265,7 @@ func imageStatusFromJobStatus(js *worker.JobStatus, result *worker.OSBuildJobRes
 		return "building"
 	}
 
-	if result.Success {
+	if buildResult.OSBuildOutput != nil && buildResult.OSBuildOutput.Success && buildResult.KojiError == nil {
 		return "success"
 	}
 
@@ -274,19 +279,41 @@ func (h *apiHandlers) GetComposeId(ctx echo.Context, idstr string) error {
 		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err))
 	}
 
-	var result worker.OSBuildJobResult
-	status, err := h.server.workers.JobStatus(id, &result)
+	var finalizeResult worker.KojiFinalizeJobResult
+	finalizeStatus, deps, err := h.server.workers.JobStatus(id, &finalizeResult)
 	if err != nil {
 		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Job %s not found: %s", idstr, err))
 	}
 
+	var initResult worker.KojiInitJobResult
+	_, _, err = h.server.workers.JobStatus(deps[0], &initResult)
+	if err != nil {
+		// this is a programming error
+		panic(err)
+	}
+
+	var buildResults []worker.OSBuildKojiJobResult
+	var imageStatuses []api.ImageStatus
+	for i := 1; i < len(deps); i++ {
+		var buildResult worker.OSBuildKojiJobResult
+		jobStatus, _, err := h.server.workers.JobStatus(deps[i], &buildResult)
+		if err != nil {
+			// this is a programming error
+			panic(err)
+		}
+		buildResults = append(buildResults, buildResult)
+		imageStatuses = append(imageStatuses, api.ImageStatus{
+			Status: imageStatusFromJobStatus(jobStatus, &initResult, &buildResult),
+		})
+	}
+
 	response := api.ComposeStatus{
-		Status: composeStatusFromJobStatus(status, &result),
-		ImageStatuses: []api.ImageStatus{
-			{
-				Status: imageStatusFromJobStatus(status, &result),
-			},
-		},
+		Status:        composeStatusFromJobStatus(finalizeStatus, &initResult, buildResults, &finalizeResult),
+		ImageStatuses: imageStatuses,
+	}
+	buildID := int(initResult.BuildID)
+	if buildID != 0 {
+		response.KojiBuildId = &buildID
 	}
 	return ctx.JSON(http.StatusOK, response)
 }
@@ -305,19 +332,37 @@ func (h *apiHandlers) GetComposeIdLogs(ctx echo.Context, idstr string) error {
 		return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err))
 	}
 
-	var result worker.OSBuildJobResult
-	_, err = h.server.workers.JobStatus(id, &result)
+	var finalizeResult worker.KojiFinalizeJobResult
+	_, deps, err := h.server.workers.JobStatus(id, &finalizeResult)
 	if err != nil {
 		return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Job %s not found: %s", idstr, err))
 	}
 
-	// Return the OSBuildJobResult as-is for now. The contents of ImageLogs
+	var initResult worker.KojiInitJobResult
+	_, _, err = h.server.workers.JobStatus(deps[0], &initResult)
+	if err != nil {
+		// This is a programming errror.
+		panic(err)
+	}
+
+	var buildResults []interface{}
+	for i := 1; i < len(deps); i++ {
+		var buildResult worker.OSBuildJobResult
+		_, _, err = h.server.workers.JobStatus(deps[i], &buildResult)
+		if err != nil {
+			// This is a programming error.
+			panic(err)
+		}
+		buildResults = append(buildResults, buildResult)
+	}
+
+	// Return the OSBuildJobResults as-is for now. The contents of ImageLogs
 	// is not part of the API. It's meant for a human to be able to access
 	// the logs, which just happen to be in JSON.
 	response := api.ComposeLogs{
-		ImageLogs: []interface{}{
-			result,
-		},
+		KojiInitLogs:   initResult,
+		KojiImportLogs: finalizeResult,
+		ImageLogs:      buildResults,
 	}
 
 	return ctx.JSON(http.StatusOK, response)
diff --git a/internal/weldr/api.go b/internal/weldr/api.go
index 6359ef474..f13614c6a 100644
--- a/internal/weldr/api.go
+++ b/internal/weldr/api.go
@@ -247,8 +247,10 @@ func (api *API) getComposeStatus(compose store.Compose) *composeStatus {
 	// All jobs are "osbuild" jobs.
 	var result worker.OSBuildJobResult
 
-	// is it ok to ignore this error?
-	jobStatus, _ := api.workers.JobStatus(jobId, &result)
+	jobStatus, _, err := api.workers.JobStatus(jobId, &result)
+	if err != nil {
+		panic(err)
+	}
 	return &composeStatus{
 		State:    composeStateFromJobStatus(jobStatus, &result),
 		Queued:   jobStatus.Queued,
@@ -1864,7 +1866,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
 	} else {
 		var jobId uuid.UUID
 
-		jobId, err = api.workers.Enqueue(api.arch.Name(), &worker.OSBuildJob{
+		jobId, err = api.workers.EnqueueOSBuild(api.arch.Name(), &worker.OSBuildJob{
 			Manifest:        manifest,
 			Targets:         targets,
 			ImageName:       imageType.Filename(),
diff --git a/internal/worker/api/openapi.yml b/internal/worker/api/openapi.yml
index c0a9d02ab..478abd603 100644
--- a/internal/worker/api/openapi.yml
+++ b/internal/worker/api/openapi.yml
@@ -63,6 +63,9 @@ paths:
                     enum:
                       - osbuild
                   args: {}
+                  dynamic_args:
+                    type: array
+                    items: {}
                 required:
                   - type
                   - location
diff --git a/internal/worker/client.go b/internal/worker/client.go
index 09dbe6e0e..fcb0697c6 100644
--- a/internal/worker/client.go
+++ b/internal/worker/client.go
@@ -25,6 +25,8 @@ type Job interface {
 	Id() uuid.UUID
 	Type() string
 	Args(args interface{}) error
+	DynamicArgs(i int, args interface{}) error
+	NDynamicArgs() int
 	Update(result interface{}) error
 	Canceled() (bool, error)
 	UploadArtifact(name string, reader io.Reader) error
@@ -37,6 +39,7 @@ type job struct {
 	artifactLocation string
 	jobType          string
 	args             json.RawMessage
+	dynamicArgs      []json.RawMessage
 }
 
 func NewClient(baseURL string, conf *tls.Config) (*Client, error) {
@@ -128,6 +131,7 @@ func (c *Client) RequestJob(types []string) (Job, error) {
 		id:               jr.Id,
 		jobType:          jr.Type,
 		args:             jr.Args,
+		dynamicArgs:      jr.DynamicArgs,
 		location:         location.String(),
 		artifactLocation: artifactLocation.String(),
 	}, nil
@@ -149,6 +153,18 @@ func (j *job) Args(args interface{}) error {
 	return nil
 }
 
+func (j *job) NDynamicArgs() int {
+	return len(j.dynamicArgs)
+}
+
+func (j *job) DynamicArgs(i int, args interface{}) error {
+	err := json.Unmarshal(j.dynamicArgs[i], args)
+	if err != nil {
+		return fmt.Errorf("error parsing job arguments: %v", err)
+	}
+	return nil
+}
+
 func (j *job) Update(result interface{}) error {
 	var buf bytes.Buffer
 	err := json.NewEncoder(&buf).Encode(api.UpdateJobJSONRequestBody{
diff --git a/internal/worker/json.go b/internal/worker/json.go
index b7bfcb348..8828f316b 100644
--- a/internal/worker/json.go
+++ b/internal/worker/json.go
@@ -26,6 +26,51 @@ type OSBuildJobResult struct {
 	TargetErrors  []string        `json:"target_errors,omitempty"`
 }
 
+type KojiInitJob struct {
+	Server  string `json:"server"`
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Release string `json:"release"`
+}
+
+type KojiInitJobResult struct {
+	BuildID   uint64 `json:"build_id"`
+	Token     string `json:"token"`
+	KojiError error  `json:"koji_error"`
+}
+
+type OSBuildKojiJob struct {
+	Manifest      distro.Manifest `json:"manifest"`
+	ImageName     string          `json:"image_name"`
+	KojiServer    string          `json:"koji_server"`
+	KojiDirectory string          `json:"koji_directory"`
+	KojiFilename  string          `json:"koji_filename"`
+}
+
+type OSBuildKojiJobResult struct {
+	HostOS        string          `json:"host_os"`
+	Arch          string          `json:"arch"`
+	OSBuildOutput *osbuild.Result `json:"osbuild_output"`
+	ImageHash     string          `json:"image_hash"`
+	ImageSize     uint64          `json:"image_size"`
+	KojiError     error           `json:"koji_error"`
+}
+
+type KojiFinalizeJob struct {
+	Server        string   `json:"server"`
+	Name          string   `json:"name"`
+	Version       string   `json:"version"`
+	Release       string   `json:"release"`
+	KojiFilenames []string `json:"koji_filenames"`
+	KojiDirectory string   `json:"koji_directory"`
+	TaskID        uint64   `json:"task_id"` /* https://pagure.io/koji/issue/215 */
+	StartTime     uint64   `json:"start_time"`
+}
+
+type KojiFinalizeJobResult struct {
+	KojiError error `json:"koji_error"`
+}
+
 //
 // JSON-serializable types for the HTTP API
 //
@@ -35,11 +80,12 @@ type statusResponse struct {
 }
 
 type requestJobResponse struct {
-	Id               uuid.UUID       `json:"id"`
-	Location         string          `json:"location"`
-	ArtifactLocation string          `json:"artifact_location"`
-	Type             string          `json:"type"`
-	Args             json.RawMessage `json:"args,omitempty"`
+	Id               uuid.UUID         `json:"id"`
+	Location         string            `json:"location"`
+	ArtifactLocation string            `json:"artifact_location"`
+	Type             string            `json:"type"`
+	Args             json.RawMessage   `json:"args,omitempty"`
+	DynamicArgs      []json.RawMessage `json:"dynamic_args,omitempty"`
 }
 
 type getJobResponse struct {
diff --git a/internal/worker/server.go b/internal/worker/server.go
index 7a8fc8432..baa2b7293 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -83,14 +83,33 @@ func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
 	s.server.Handler.ServeHTTP(writer, request)
 }
 
-func (s *Server) Enqueue(arch string, job *OSBuildJob) (uuid.UUID, error) {
+func (s *Server) EnqueueOSBuild(arch string, job *OSBuildJob) (uuid.UUID, error) {
 	return s.jobs.Enqueue("osbuild:"+arch, job, nil)
 }
 
-func (s *Server) JobStatus(id uuid.UUID, result interface{}) (*JobStatus, error) {
-	queued, started, finished, canceled, err := s.jobs.JobStatus(id, result)
+func (s *Server) EnqueueOSBuildKoji(arch string, job *OSBuildKojiJob, initID uuid.UUID) (uuid.UUID, error) {
+	return s.jobs.Enqueue("osbuild-koji:"+arch, job, []uuid.UUID{initID})
+}
+
+func (s *Server) EnqueueKojiInit(job *KojiInitJob) (uuid.UUID, error) {
+	return s.jobs.Enqueue("koji-init", job, nil)
+}
+
+func (s *Server) EnqueueKojiFinalize(job *KojiFinalizeJob, initID uuid.UUID, buildIDs []uuid.UUID) (uuid.UUID, error) {
+	return s.jobs.Enqueue("koji-finalize", job, append([]uuid.UUID{initID}, buildIDs...))
+}
+
+func (s *Server) JobStatus(id uuid.UUID, result interface{}) (*JobStatus, []uuid.UUID, error) {
+	rawResult, queued, started, finished, canceled, deps, err := s.jobs.JobStatus(id)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+
+	if !finished.IsZero() && !canceled {
+		err = json.Unmarshal(rawResult, result)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error unmarshaling result for job '%s': %v", id, err)
+		}
 	}
 
 	// For backwards compatibility: OSBuildJobResult didn't use to have a
@@ -106,7 +125,7 @@ func (s *Server) JobStatus(id uuid.UUID, result interface{}) (*JobStatus, error)
 		Started:  started,
 		Finished: finished,
 		Canceled: canceled,
-	}, nil
+	}, deps, nil
 }
 
 func (s *Server) Cancel(id uuid.UUID) error {
@@ -120,7 +139,7 @@ func (s *Server) JobArtifact(id uuid.UUID, name string) (io.Reader, int64, error
 		return nil, 0, errors.New("Artifacts not enabled")
 	}
 
-	status, err := s.JobStatus(id, &json.RawMessage{})
+	status, _, err := s.JobStatus(id, &json.RawMessage{})
 	if err != nil {
 		return nil, 0, err
 	}
@@ -149,7 +168,7 @@ func (s *Server) DeleteArtifacts(id uuid.UUID) error {
 		return errors.New("Artifacts not enabled")
 	}
 
-	status, err := s.JobStatus(id, &json.RawMessage{})
+	status, _, err := s.JobStatus(id, &json.RawMessage{})
 	if err != nil {
 		return err
 	}
@@ -161,7 +180,7 @@ func (s *Server) DeleteArtifacts(id uuid.UUID) error {
 	return os.RemoveAll(path.Join(s.artifactsDir, id.String()))
 }
 
-func (s *Server) RequestJob(ctx context.Context, arch string, jobTypes []string) (uuid.UUID, uuid.UUID, string, json.RawMessage, error) {
+func (s *Server) RequestJob(ctx context.Context, arch string, jobTypes []string) (uuid.UUID, uuid.UUID, string, json.RawMessage, []json.RawMessage, error) {
 	token := uuid.New()
 
 	// treat osbuild jobs specially until we have found a generic way to
@@ -169,21 +188,27 @@ func (s *Server) RequestJob(ctx context.Context, arch string, jobTypes []string)
 	// restriction: arch for osbuild jobs.
 	jts := []string{}
 	for _, t := range jobTypes {
-		if t == "osbuild" {
-			t = "osbuild:" + arch
+		if t == "osbuild" || t == "osbuild-koji" {
+			t = t + ":" + arch
 		}
 		jts = append(jts, t)
 	}
 
-	jobId, jobType, args, err := s.jobs.Dequeue(ctx, jts)
+	jobId, depIDs, jobType, args, err := s.jobs.Dequeue(ctx, jts)
 	if err != nil {
-		return uuid.Nil, uuid.Nil, "", nil, err
+		return uuid.Nil, uuid.Nil, "", nil, nil, err
+	}
+
+	var dynamicArgs []json.RawMessage
+	for _, depID := range depIDs {
+		result, _, _, _, _, _, _ := s.jobs.JobStatus(depID)
+		dynamicArgs = append(dynamicArgs, result)
 	}
 
 	if s.artifactsDir != "" {
 		err := os.MkdirAll(path.Join(s.artifactsDir, "tmp", token.String()), 0700)
 		if err != nil {
-			return uuid.Nil, uuid.Nil, "", nil, fmt.Errorf("cannot create artifact directory: %v", err)
+			return uuid.Nil, uuid.Nil, "", nil, nil, fmt.Errorf("cannot create artifact directory: %v", err)
 		}
 	}
 
@@ -193,9 +218,11 @@ func (s *Server) RequestJob(ctx context.Context, arch string, jobTypes []string)
 
 	if jobType == "osbuild:"+arch {
 		jobType = "osbuild"
+	} else if jobType == "osbuild-koji:"+arch {
+		jobType = "osbuild-koji"
 	}
 
-	return token, jobId, jobType, args, nil
+	return token, jobId, jobType, args, dynamicArgs, nil
 }
 
 func (s *Server) RunningJob(token uuid.UUID) (uuid.UUID, error) {
@@ -261,7 +288,7 @@ func (h *apiHandlers) RequestJob(ctx echo.Context) error {
 		return err
 	}
 
-	token, jobId, jobType, jobArgs, err := h.server.RequestJob(ctx.Request().Context(), body.Arch, body.Types)
+	token, jobId, jobType, jobArgs, dynamicJobArgs, err := h.server.RequestJob(ctx.Request().Context(), body.Arch, body.Types)
 	if err != nil {
 		return err
 	}
@@ -272,6 +299,7 @@ func (h *apiHandlers) RequestJob(ctx echo.Context) error {
 		ArtifactLocation: fmt.Sprintf("%s/jobs/%v/artifacts/", api.BasePath, token),
 		Type:             jobType,
 		Args:             jobArgs,
+		DynamicArgs:      dynamicJobArgs,
 	})
 }
 
@@ -295,7 +323,7 @@ func (h *apiHandlers) GetJob(ctx echo.Context, tokenstr string) error {
 		return ctx.JSON(http.StatusOK, getJobResponse{})
 	}
 
-	status, err := h.server.JobStatus(jobId, &json.RawMessage{})
+	status, _, err := h.server.JobStatus(jobId, &json.RawMessage{})
 	if err != nil {
 		return err
 	}
diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh
index 701dddc2a..b67722236 100755
--- a/schutzbot/mockbuild.sh
+++ b/schutzbot/mockbuild.sh
@@ -29,17 +29,11 @@ fi
 greenprint "📦 Installing mock requirements"
 sudo dnf -y install createrepo_c make mock rpm-build s3cmd
 
-# Jenkins sets a workspace variable as the root of its working directory.
-WORKSPACE=${WORKSPACE:-$(pwd)}
-
 # Mock configuration file to use for building RPMs.
 MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)"
 
-# Jenkins takes the proposed PR and merges it onto master. Although this
-# creates a new SHA (which is slightly confusing), it ensures that the code
-# merges properly against master and it tests the code against the latest
-# commit in master, which is certainly good.
-POST_MERGE_SHA=$(git rev-parse --short HEAD)
+# The commit this script operates on.
+COMMIT=$(git rev-parse HEAD)
 
 # Bucket in S3 where our artifacts are uploaded
 REPO_BUCKET=osbuild-composer-repos
@@ -47,19 +41,19 @@ REPO_BUCKET=osbuild-composer-repos
 # Public URL for the S3 bucket with our artifacts.
 MOCK_REPO_BASE_URL="http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com"
 
-# Directory to hold the RPMs temporarily before we upload them.
-REPO_DIR=repo/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}_${ARCH}
+# Relative path of the repository – used for constructing both the local and
+# remote paths below, so that they're consistent.
+REPO_PATH=osbuild-composer/${ID}-${VERSION_ID}/${ARCH}/${COMMIT}
 
-# Maintain a directory for the master branch that always contains the latest
-# RPM packages.
-REPO_DIR_LATEST=repo/${JOB_NAME}/latest
+# Directory to hold the RPMs temporarily before we upload them.
+REPO_DIR=repo/${REPO_PATH}
 
 # Full URL to the RPM repository after they are uploaded.
-REPO_URL=${MOCK_REPO_BASE_URL}/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}_${ARCH}
+REPO_URL=${MOCK_REPO_BASE_URL}/${REPO_PATH}
 
 # Print some data.
 greenprint "🧬 Using mock config: ${MOCK_CONFIG}"
-greenprint "📦 Post merge SHA: ${POST_MERGE_SHA}"
+greenprint "📦 SHA: ${COMMIT}"
 greenprint "📤 RPMS will be uploaded to: ${REPO_URL}"
 
 # Build source RPMs.
@@ -75,22 +69,13 @@ sudo mock -r "$MOCK_CONFIG" --resultdir "$REPO_DIR" --with=tests \
 # Change the ownership of all of our repo files from root to our CI user.
 sudo chown -R "$USER" "${REPO_DIR%%/*}"
 
-# Move the logs out of the way.
-greenprint "🧹 Retaining logs from mock build"
-mv "${REPO_DIR}"/*.log "$WORKSPACE"
+greenprint "🧹 Remove logs from mock build"
+rm "${REPO_DIR}"/*.log
 
 # Create a repo of the built RPMs.
 greenprint "⛓️ Creating dnf repository"
 createrepo_c "${REPO_DIR}"
 
-# Copy the current build to the latest directory.
-mkdir -p "$REPO_DIR_LATEST"
-cp -arv "${REPO_DIR}"/ "${REPO_DIR_LATEST}"/
-
-# Remove the previous latest build for this branch.
-# Don't fail if the path is missing.
-s3cmd --recursive rm "s3://${REPO_BUCKET}/${JOB_NAME}/latest/${ID}${VERSION_ID//./}_${ARCH}" || true
-
 # Upload repository to S3.
 greenprint "☁ Uploading RPMs to S3"
 pushd repo
@@ -101,7 +86,7 @@ popd
 greenprint "📜 Generating dnf repository file"
 tee osbuild-mock.repo << EOF
 [osbuild-mock]
-name=osbuild mock ${JOB_NAME}-${POST_MERGE_SHA} ${ID}${VERSION_ID//./}
+name=osbuild mock ${COMMIT}
 baseurl=${REPO_URL}
 enabled=1
 gpgcheck=0
diff --git a/tools/koji-compose.py b/tools/koji-compose.py
index b37b95304..ff6c21901 100755
--- a/tools/koji-compose.py
+++ b/tools/koji-compose.py
@@ -40,6 +40,10 @@ def compose_request(distro, koji, arch):
             "architecture": "x86_64",
             "image_type": "qcow2",
             "repositories": repositories
+        },{
+            "architecture": "x86_64",
+            "image_type": "ami",
+            "repositories": repositories
         }]
     }
 


Test patch:
diff --git a/cmd/osbuild-composer/config_test.go b/cmd/osbuild-composer/config_test.go
index f5341e1bf..25b2e2223 100644
--- a/cmd/osbuild-composer/config_test.go
+++ b/cmd/osbuild-composer/config_test.go
@@ -11,7 +11,6 @@ func TestEmpty(t *testing.T) {
 	config, err := LoadConfig("testdata/empty-config.toml")
 	require.NoError(t, err)
 	require.NotNil(t, config)
-	require.Empty(t, config.Koji.Servers)
 	require.Empty(t, config.Koji.AllowedDomains)
 	require.Empty(t, config.Koji.CA)
 	require.Empty(t, config.Worker.AllowedDomains)
@@ -30,12 +29,6 @@ func TestConfig(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, config)
 
-	server, ok := config.Koji.Servers["example.com"]
-	require.True(t, ok)
-	require.NotNil(t, server.Kerberos)
-	require.Equal(t, server.Kerberos.Principal, "example@osbuild.org")
-	require.Equal(t, server.Kerberos.KeyTab, "/etc/osbuild-composer/osbuild.keytab")
-
 	require.Equal(t, config.Koji.AllowedDomains, []string{"osbuild.org"})
 	require.Equal(t, config.Koji.CA, "/etc/osbuild-composer/ca-crt.pem")
 
diff --git a/cmd/osbuild-composer/testdata/test.toml b/cmd/osbuild-composer/testdata/test.toml
index 2a81ab237..a16843ff2 100644
--- a/cmd/osbuild-composer/testdata/test.toml
+++ b/cmd/osbuild-composer/testdata/test.toml
@@ -2,10 +2,6 @@
 allowed_domains = [ "osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
 
-[koji.servers."example.com".kerberos]
-principal = "example@osbuild.org"
-keytab = "/etc/osbuild-composer/osbuild.keytab"
-
 [worker]
 allowed_domains = [ "osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
diff --git a/internal/jobqueue/fsjobqueue/fsjobqueue_private_test.go b/internal/jobqueue/fsjobqueue/fsjobqueue_private_test.go
deleted file mode 100644
index e122b2239..000000000
--- a/internal/jobqueue/fsjobqueue/fsjobqueue_private_test.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package fsjobqueue
-
-import (
-	"testing"
-
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/require"
-)
-
-func uuidList(t *testing.T, strs ...string) []uuid.UUID {
-	var err error
-	ids := make([]uuid.UUID, len(strs))
-	for i, s := range strs {
-		ids[i], err = uuid.Parse(s)
-		require.NoError(t, err)
-	}
-	return ids
-}
-
-func TestUniqueUUIDList(t *testing.T) {
-	l := uniqueUUIDList([]uuid.UUID{})
-	require.Empty(t, l)
-
-	s := uuidList(t, "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2", "a0ad7428-b813-4efb-a156-da2b524f4868", "36e5817c-f29d-4043-8d7d-95ffaa77ff88")
-	l = uniqueUUIDList(s)
-	require.ElementsMatch(t, s, l)
-
-	s = uuidList(t, "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2", "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2")
-	l = uniqueUUIDList(s)
-	require.ElementsMatch(t, uuidList(t, "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2"), l)
-
-	s = uuidList(t, "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2", "a0ad7428-b813-4efb-a156-da2b524f4868", "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2")
-	l = uniqueUUIDList(s)
-	require.ElementsMatch(t, uuidList(t, "8ad6bbcd-55f9-4cd8-be45-d0370ff079d2", "a0ad7428-b813-4efb-a156-da2b524f4868"), l)
-}
diff --git a/internal/jobqueue/fsjobqueue/fsjobqueue_test.go b/internal/jobqueue/fsjobqueue/fsjobqueue_test.go
index 4c3fec619..1b72fb600 100644
--- a/internal/jobqueue/fsjobqueue/fsjobqueue_test.go
+++ b/internal/jobqueue/fsjobqueue/fsjobqueue_test.go
@@ -42,10 +42,11 @@ func pushTestJob(t *testing.T, q jobqueue.JobQueue, jobType string, args interfa
 	return id
 }
 
-func finishNextTestJob(t *testing.T, q jobqueue.JobQueue, jobType string, result interface{}) uuid.UUID {
-	id, typ, args, err := q.Dequeue(context.Background(), []string{jobType})
+func finishNextTestJob(t *testing.T, q jobqueue.JobQueue, jobType string, result interface{}, deps []uuid.UUID) uuid.UUID {
+	id, d, typ, args, err := q.Dequeue(context.Background(), []string{jobType})
 	require.NoError(t, err)
 	require.NotEmpty(t, id)
+	require.ElementsMatch(t, deps, d)
 	require.Equal(t, jobType, typ)
 	require.NotNil(t, args)
 
@@ -93,17 +94,19 @@ func TestArgs(t *testing.T) {
 
 	var parsedArgs argument
 
-	id, typ, args, err := q.Dequeue(context.Background(), []string{"octopus"})
+	id, deps, typ, args, err := q.Dequeue(context.Background(), []string{"octopus"})
 	require.NoError(t, err)
 	require.Equal(t, two, id)
+	require.Empty(t, deps)
 	require.Equal(t, "octopus", typ)
 	err = json.Unmarshal(args, &parsedArgs)
 	require.NoError(t, err)
 	require.Equal(t, twoargs, parsedArgs)
 
-	id, typ, args, err = q.Dequeue(context.Background(), []string{"fish"})
+	id, deps, typ, args, err = q.Dequeue(context.Background(), []string{"fish"})
 	require.NoError(t, err)
 	require.Equal(t, one, id)
+	require.Empty(t, deps)
 	require.Equal(t, "fish", typ)
 	err = json.Unmarshal(args, &parsedArgs)
 	require.NoError(t, err)
@@ -117,14 +120,15 @@ func TestJobTypes(t *testing.T) {
 	one := pushTestJob(t, q, "octopus", nil, nil)
 	two := pushTestJob(t, q, "clownfish", nil, nil)
 
-	require.Equal(t, two, finishNextTestJob(t, q, "clownfish", testResult{}))
-	require.Equal(t, one, finishNextTestJob(t, q, "octopus", testResult{}))
+	require.Equal(t, two, finishNextTestJob(t, q, "clownfish", testResult{}, nil))
+	require.Equal(t, one, finishNextTestJob(t, q, "octopus", testResult{}, nil))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
-	id, typ, args, err := q.Dequeue(ctx, []string{"zebra"})
+	id, deps, typ, args, err := q.Dequeue(ctx, []string{"zebra"})
 	require.Equal(t, err, context.Canceled)
 	require.Equal(t, uuid.Nil, id)
+	require.Empty(t, deps)
 	require.Equal(t, "", typ)
 	require.Nil(t, args)
 }
@@ -138,26 +142,31 @@ func TestDependencies(t *testing.T) {
 		two := pushTestJob(t, q, "test", nil, nil)
 
 		r := []uuid.UUID{}
-		r = append(r, finishNextTestJob(t, q, "test", testResult{}))
-		r = append(r, finishNextTestJob(t, q, "test", testResult{}))
+		r = append(r, finishNextTestJob(t, q, "test", testResult{}, nil))
+		r = append(r, finishNextTestJob(t, q, "test", testResult{}, nil))
 		require.ElementsMatch(t, []uuid.UUID{one, two}, r)
 
 		j := pushTestJob(t, q, "test", nil, []uuid.UUID{one, two})
-		queued, started, finished, canceled, err := q.JobStatus(j, nil)
+		_, queued, started, finished, canceled, deps, err := q.JobStatus(j)
 		require.NoError(t, err)
 		require.True(t, !queued.IsZero())
 		require.True(t, started.IsZero())
 		require.True(t, finished.IsZero())
 		require.False(t, canceled)
+		require.ElementsMatch(t, deps, []uuid.UUID{one, two})
 
-		require.Equal(t, j, finishNextTestJob(t, q, "test", testResult{}))
+		require.Equal(t, j, finishNextTestJob(t, q, "test", testResult{}, []uuid.UUID{one, two}))
 
-		queued, started, finished, canceled, err = q.JobStatus(j, &testResult{})
+		result, queued, started, finished, canceled, deps, err := q.JobStatus(j)
 		require.NoError(t, err)
 		require.True(t, !queued.IsZero())
 		require.True(t, !started.IsZero())
 		require.True(t, !finished.IsZero())
 		require.False(t, canceled)
+		require.ElementsMatch(t, deps, []uuid.UUID{one, two})
+
+		err = json.Unmarshal(result, &testResult{})
+		require.NoError(t, err)
 	})
 
 	t.Run("done-after-pushing-dependant", func(t *testing.T) {
@@ -165,26 +174,31 @@ func TestDependencies(t *testing.T) {
 		two := pushTestJob(t, q, "test", nil, nil)
 
 		j := pushTestJob(t, q, "test", nil, []uuid.UUID{one, two})
-		queued, started, finished, canceled, err := q.JobStatus(j, nil)
+		_, queued, started, finished, canceled, deps, err := q.JobStatus(j)
 		require.NoError(t, err)
 		require.True(t, !queued.IsZero())
 		require.True(t, started.IsZero())
 		require.True(t, finished.IsZero())
 		require.False(t, canceled)
+		require.ElementsMatch(t, deps, []uuid.UUID{one, two})
 
 		r := []uuid.UUID{}
-		r = append(r, finishNextTestJob(t, q, "test", testResult{}))
-		r = append(r, finishNextTestJob(t, q, "test", testResult{}))
+		r = append(r, finishNextTestJob(t, q, "test", testResult{}, nil))
+		r = append(r, finishNextTestJob(t, q, "test", testResult{}, nil))
 		require.ElementsMatch(t, []uuid.UUID{one, two}, r)
 
-		require.Equal(t, j, finishNextTestJob(t, q, "test", testResult{}))
+		require.Equal(t, j, finishNextTestJob(t, q, "test", testResult{}, []uuid.UUID{one, two}))
 
-		queued, started, finished, canceled, err = q.JobStatus(j, &testResult{})
+		result, queued, started, finished, canceled, deps, err := q.JobStatus(j)
 		require.NoError(t, err)
 		require.True(t, !queued.IsZero())
 		require.True(t, !started.IsZero())
 		require.True(t, !finished.IsZero())
 		require.False(t, canceled)
+		require.ElementsMatch(t, deps, []uuid.UUID{one, two})
+
+		err = json.Unmarshal(result, &testResult{})
+		require.NoError(t, err)
 	})
 }
 
@@ -199,9 +213,10 @@ func TestMultipleWorkers(t *testing.T) {
 		defer close(done)
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
-		id, typ, args, err := q.Dequeue(ctx, []string{"octopus"})
+		id, deps, typ, args, err := q.Dequeue(ctx, []string{"octopus"})
 		require.NoError(t, err)
 		require.NotEmpty(t, id)
+		require.Empty(t, deps)
 		require.Equal(t, "octopus", typ)
 		require.Equal(t, json.RawMessage("null"), args)
 	}()
@@ -212,9 +227,10 @@ func TestMultipleWorkers(t *testing.T) {
 
 	// This call to Dequeue() should not block on the one in the goroutine.
 	id := pushTestJob(t, q, "clownfish", nil, nil)
-	r, typ, args, err := q.Dequeue(context.Background(), []string{"clownfish"})
+	r, deps, typ, args, err := q.Dequeue(context.Background(), []string{"clownfish"})
 	require.NoError(t, err)
 	require.Equal(t, id, r)
+	require.Empty(t, deps)
 	require.Equal(t, "clownfish", typ)
 	require.Equal(t, json.RawMessage("null"), args)
 
@@ -236,41 +252,47 @@ func TestCancel(t *testing.T) {
 	require.NotEmpty(t, id)
 	err = q.CancelJob(id)
 	require.NoError(t, err)
-	_, _, _, canceled, err := q.JobStatus(id, &testResult{})
+	result, _, _, _, canceled, _, err := q.JobStatus(id)
 	require.NoError(t, err)
 	require.True(t, canceled)
+	require.Nil(t, result)
 	err = q.FinishJob(id, &testResult{})
 	require.Error(t, err)
 
 	// Cancel a running job, which should not dequeue the canceled job from above
 	id = pushTestJob(t, q, "clownfish", nil, nil)
 	require.NotEmpty(t, id)
-	r, typ, args, err := q.Dequeue(context.Background(), []string{"clownfish"})
+	r, deps, typ, args, err := q.Dequeue(context.Background(), []string{"clownfish"})
 	require.NoError(t, err)
 	require.Equal(t, id, r)
+	require.Empty(t, deps)
 	require.Equal(t, "clownfish", typ)
 	require.Equal(t, json.RawMessage("null"), args)
 	err = q.CancelJob(id)
 	require.NoError(t, err)
-	_, _, _, canceled, err = q.JobStatus(id, &testResult{})
+	result, _, _, _, canceled, _, err = q.JobStatus(id)
 	require.NoError(t, err)
 	require.True(t, canceled)
+	require.Nil(t, result)
 	err = q.FinishJob(id, &testResult{})
 	require.Error(t, err)
 
 	// Cancel a finished job, which is a no-op
 	id = pushTestJob(t, q, "clownfish", nil, nil)
 	require.NotEmpty(t, id)
-	r, typ, args, err = q.Dequeue(context.Background(), []string{"clownfish"})
+	r, deps, typ, args, err = q.Dequeue(context.Background(), []string{"clownfish"})
 	require.NoError(t, err)
 	require.Equal(t, id, r)
+	require.Empty(t, deps)
 	require.Equal(t, "clownfish", typ)
 	require.Equal(t, json.RawMessage("null"), args)
 	err = q.FinishJob(id, &testResult{})
 	require.NoError(t, err)
 	err = q.CancelJob(id)
 	require.NoError(t, err)
-	_, _, _, canceled, err = q.JobStatus(id, &testResult{})
+	result, _, _, _, canceled, _, err = q.JobStatus(id)
 	require.NoError(t, err)
 	require.False(t, canceled)
+	err = json.Unmarshal(result, &testResult{})
+	require.NoError(t, err)
 }
diff --git a/internal/jobqueue/testjobqueue/testjobqueue.go b/internal/jobqueue/testjobqueue/testjobqueue.go
index dc2635141..25d8ba323 100644
--- a/internal/jobqueue/testjobqueue/testjobqueue.go
+++ b/internal/jobqueue/testjobqueue/testjobqueue.go
@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"sort"
 	"time"
 
 	"github.com/google/uuid"
@@ -38,8 +37,9 @@ type job struct {
 
 func New() *testJobQueue {
 	return &testJobQueue{
-		jobs:    make(map[uuid.UUID]*job),
-		pending: make(map[string][]uuid.UUID),
+		jobs:       make(map[uuid.UUID]*job),
+		pending:    make(map[string][]uuid.UUID),
+		dependants: make(map[uuid.UUID][]uuid.UUID),
 	}
 }
 
@@ -47,7 +47,7 @@ func (q *testJobQueue) Enqueue(jobType string, args interface{}, dependencies []
 	var j = job{
 		Id:           uuid.New(),
 		Type:         jobType,
-		Dependencies: uniqueUUIDList(dependencies),
+		Dependencies: dependencies,
 		QueuedAt:     time.Now(),
 	}
 
@@ -79,7 +79,7 @@ func (q *testJobQueue) Enqueue(jobType string, args interface{}, dependencies []
 	return j.Id, nil
 }
 
-func (q *testJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, string, json.RawMessage, error) {
+func (q *testJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUID, []uuid.UUID, string, json.RawMessage, error) {
 	for _, t := range jobTypes {
 		if len(q.pending[t]) == 0 {
 			continue
@@ -91,10 +91,10 @@ func (q *testJobQueue) Dequeue(ctx context.Context, jobTypes []string) (uuid.UUI
 		j := q.jobs[id]
 
 		j.StartedAt = time.Now()
-		return j.Id, j.Type, j.Args, nil
+		return j.Id, j.Dependencies, j.Type, j.Args, nil
 	}
 
-	return uuid.Nil, "", nil, errors.New("no job available")
+	return uuid.Nil, nil, "", nil, errors.New("no job available")
 }
 
 func (q *testJobQueue) FinishJob(id uuid.UUID, result interface{}) error {
@@ -141,24 +141,19 @@ func (q *testJobQueue) CancelJob(id uuid.UUID) error {
 	return nil
 }
 
-func (q *testJobQueue) JobStatus(id uuid.UUID, result interface{}) (queued, started, finished time.Time, canceled bool, err error) {
+func (q *testJobQueue) JobStatus(id uuid.UUID) (result json.RawMessage, queued, started, finished time.Time, canceled bool, deps []uuid.UUID, err error) {
 	j, exists := q.jobs[id]
 	if !exists {
 		err = jobqueue.ErrNotExist
 		return
 	}
 
-	if !j.FinishedAt.IsZero() {
-		err = json.Unmarshal(j.Result, result)
-		if err != nil {
-			return
-		}
-	}
-
+	result = j.Result
 	queued = j.QueuedAt
 	started = j.StartedAt
 	finished = j.FinishedAt
 	canceled = j.Canceled
+	deps = j.Dependencies
 
 	return
 }
@@ -178,28 +173,3 @@ func (q *testJobQueue) countFinishedJobs(ids []uuid.UUID) (int, error) {
 
 	return n, nil
 }
-
-// Sorts and removes duplicates from `ids`.
-// Copied from fsjobqueue, which also contains a test.
-func uniqueUUIDList(ids []uuid.UUID) []uuid.UUID {
-	s := map[uuid.UUID]bool{}
-	for _, id := range ids {
-		s[id] = true
-	}
-
-	l := []uuid.UUID{}
-	for id := range s {
-		l = append(l, id)
-	}
-
-	sort.Slice(l, func(i, j int) bool {
-		for b := 0; b < 16; b++ {
-			if l[i][b] < l[j][b] {
-				return true
-			}
-		}
-		return false
-	})
-
-	return l
-}
diff --git a/internal/kojiapi/server_test.go b/internal/kojiapi/server_test.go
index f0c18168d..33b5fae9e 100644
--- a/internal/kojiapi/server_test.go
+++ b/internal/kojiapi/server_test.go
@@ -1,22 +1,28 @@
 package kojiapi_test
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"sync"
 	"testing"
 
-	"github.com/osbuild/osbuild-composer/internal/jobqueue/testjobqueue"
+	"github.com/osbuild/osbuild-composer/internal/jobqueue/fsjobqueue"
 	"github.com/osbuild/osbuild-composer/internal/kojiapi"
 	"github.com/osbuild/osbuild-composer/internal/kojiapi/api"
 	distro_mock "github.com/osbuild/osbuild-composer/internal/mocks/distro"
 	rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
-	"github.com/osbuild/osbuild-composer/internal/upload/koji"
+	"github.com/osbuild/osbuild-composer/internal/test"
 	"github.com/osbuild/osbuild-composer/internal/worker"
 	"github.com/stretchr/testify/require"
 )
 
-func newTestKojiServer(t *testing.T) *kojiapi.Server {
+func newTestKojiServer(t *testing.T, dir string) (*kojiapi.Server, *worker.Server) {
 	rpm_fixture := rpmmd_mock.BaseFixture()
 	rpm := rpmmd_mock.NewRPMMDMock(rpm_fixture)
 	require.NotNil(t, rpm)
@@ -25,36 +31,182 @@ func newTestKojiServer(t *testing.T) *kojiapi.Server {
 	require.NoError(t, err)
 	require.NotNil(t, distros)
 
-	workers := worker.NewServer(nil, testjobqueue.New(), "")
-	require.NotNil(t, workers)
+	queue, err := fsjobqueue.New(dir, []string{"osbuild:x86_64", "koji-init", "osbuild-koji:x86_64", "koji-finalize"})
+	require.NoError(t, err)
+
+	workerServer := worker.NewServer(nil, queue, "")
+	require.NotNil(t, workerServer)
 
-	server := kojiapi.NewServer(nil, workers, rpm, distros, map[string]koji.GSSAPICredentials{})
-	require.NotNil(t, server)
+	kojiServer := kojiapi.NewServer(nil, workerServer, rpm, distros)
+	require.NotNil(t, kojiServer)
 
-	return server
+	return kojiServer, workerServer
 }
 
 func TestStatus(t *testing.T) {
-	server := newTestKojiServer(t)
-	handler := server.Handler("/api/composer-koji/v1")
+	dir, err := ioutil.TempDir("", "osbuild-composer-test-kojiapi-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	kojiServer, _ := newTestKojiServer(t, dir)
+	handler := kojiServer.Handler("/api/composer-koji/v1")
+	test.TestRoute(t, handler, false, "GET", "/api/composer-koji/v1/status", ``, http.StatusOK, `{"status":"OK"}`, "message")
+}
 
-	req := httptest.NewRequest("GET", "/api/composer-koji/v1/status", nil)
-	req.Header.Set("Content-Type", "application/json")
+func TestCompose(t *testing.T) {
+	dir, err := ioutil.TempDir("", "osbuild-composer-test-kojiapi-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	kojiServer, workerServer := newTestKojiServer(t, dir)
+	handler := kojiServer.Handler("/api/composer-koji/v1")
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go func() {
+		token, _, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), "x86_64", []string{"koji-init"})
+		require.NoError(t, err)
+		require.Equal(t, "koji-init", jobType)
+
+		var initJob worker.KojiInitJob
+		err = json.Unmarshal(rawJob, &initJob)
+		require.NoError(t, err)
+		require.Equal(t, "koji.example.com", initJob.Server)
+		require.Equal(t, "foo", initJob.Name)
+		require.Equal(t, "1", initJob.Version)
+		require.Equal(t, "2", initJob.Release)
+
+		test.TestRoute(t, workerServer, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), `{
+		"result": {
+			"build_id": 42,
+			"token": "foobar"
+		}
+	}`, http.StatusOK, `{}`)
+		wg.Done()
+	}()
+
+	test.TestRoute(t, handler, false, "POST", "/api/composer-koji/v1/compose", `
+	{
+		"name":"foo",
+		"version":"1",
+		"release":"2",
+		"distribution":"fedora-30",
+		"image_requests": [
+			{
+				"architecture": "x86_64",
+				"image_type": "qcow2",
+				"repositories": [
+					{
+						"baseurl": "https://repo.example.com/"
+					}
+				]
+			},
+			{
+				"architecture": "x86_64",
+				"image_type": "qcow2",
+				"repositories": [
+					{
+						"baseurl": "https://repo.example.com/"
+					}
+				]
+			}
+		],
+		"koji": {
+			"server": "koji.example.com"
+		}
+	}`, http.StatusCreated, `{"koji_build_id":42}`, "id")
+	wg.Wait()
+
+	token, _, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), "x86_64", []string{"osbuild-koji"})
+	require.NoError(t, err)
+	require.Equal(t, "osbuild-koji", jobType)
 
-	rec := httptest.NewRecorder()
-	handler.ServeHTTP(rec, req)
-	resp := rec.Result()
+	var osbuildJob worker.OSBuildKojiJob
+	err = json.Unmarshal(rawJob, &osbuildJob)
+	require.NoError(t, err)
+	require.Equal(t, "koji.example.com", osbuildJob.KojiServer)
+	require.Equal(t, "test.img", osbuildJob.ImageName)
+	require.NotEmpty(t, osbuildJob.KojiDirectory)
+
+	test.TestRoute(t, workerServer, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), `{
+		"result": {
+			"arch": "x86_64",
+			"host_os": "fedora-30",
+			"image_hash": "browns",
+			"image_size": 42,
+			"osbuild_output": {
+				"success": true
+			}
+		}
+	}`, http.StatusOK, `{}`)
+
+	token, _, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), "x86_64", []string{"osbuild-koji"})
+	require.NoError(t, err)
+	require.Equal(t, "osbuild-koji", jobType)
 
-	require.Equal(t, 200, resp.StatusCode)
+	err = json.Unmarshal(rawJob, &osbuildJob)
+	require.NoError(t, err)
+	require.Equal(t, "koji.example.com", osbuildJob.KojiServer)
+	require.Equal(t, "test.img", osbuildJob.ImageName)
+	require.NotEmpty(t, osbuildJob.KojiDirectory)
+
+	test.TestRoute(t, workerServer, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), `{
+		"result": {
+			"arch": "x86_64",
+			"host_os": "fedora-30",
+			"image_hash": "browns",
+			"image_size": 42,
+			"osbuild_output": {
+				"success": true
+			}
+		}
+	}`, http.StatusOK, `{}`)
+
+	token, finalizeID, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), "x86_64", []string{"koji-finalize"})
+	require.NoError(t, err)
+	require.Equal(t, "koji-finalize", jobType)
 
-	var status api.Status
-	err := json.NewDecoder(resp.Body).Decode(&status)
+	var kojiFinalizeJob worker.KojiFinalizeJob
+	err = json.Unmarshal(rawJob, &kojiFinalizeJob)
 	require.NoError(t, err)
-	require.Equal(t, "OK", status.Status)
+	require.Equal(t, "koji.example.com", kojiFinalizeJob.Server)
+	require.Equal(t, "1", kojiFinalizeJob.Version)
+	require.Equal(t, "2", kojiFinalizeJob.Release)
+	require.ElementsMatch(t, []string{"foo-1-2.x86_64.img", "foo-1-2.x86_64.img"}, kojiFinalizeJob.KojiFilenames)
+	require.NotEmpty(t, kojiFinalizeJob.KojiDirectory)
+
+	test.TestRoute(t, workerServer, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), `{
+		"result": {
+		}
+	}`, http.StatusOK, `{}`)
+
+	test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/composer-koji/v1/compose/%v", finalizeID), ``, http.StatusOK, `{
+		"image_statuses": [
+			{
+				"status": "success"
+			},
+			{
+				"status": "success"
+			}
+		],
+		"koji_build_id": 42,
+		"koji_task_id": 0,
+		"status": "success"
+	}`)
 }
 
 func TestRequest(t *testing.T) {
-	server := newTestKojiServer(t)
+	dir, err := ioutil.TempDir("", "osbuild-composer-test-kojiapi-")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	server, _ := newTestKojiServer(t, dir)
 	handler := server.Handler("/api/composer-koji/v1")
 
 	// Make request to an invalid route
@@ -65,7 +217,7 @@ func TestRequest(t *testing.T) {
 	resp := rec.Result()
 
 	var status api.Status
-	err := json.NewDecoder(resp.Body).Decode(&status)
+	err = json.NewDecoder(resp.Body).Decode(&status)
 	require.NoError(t, err)
 	require.Equal(t, http.StatusNotFound, resp.StatusCode)
 
diff --git a/internal/test/helpers.go b/internal/test/helpers.go
index cd83f4927..634476ee6 100644
--- a/internal/test/helpers.go
+++ b/internal/test/helpers.go
@@ -119,6 +119,10 @@ func TestRoute(t *testing.T, api API, external bool, method, path, body string,
 		require.Lenf(t, replyJSON, 0, "%s: expected no response body, but got:\n%s", path, replyJSON)
 	}
 
+	if expectedJSON == "?" {
+		return
+	}
+
 	var reply, expected interface{}
 	err = json.Unmarshal(replyJSON, &reply)
 	require.NoErrorf(t, err, "%s: json.Unmarshal failed for\n%s", path, string(replyJSON))
diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go
index 2057a1b50..a0d7120d0 100644
--- a/internal/weldr/api_test.go
+++ b/internal/weldr/api_test.go
@@ -261,7 +261,9 @@ func TestBlueprintsInfoToml(t *testing.T) {
 		Description: "Test",
 		Version:     "0.0.0",
 		Packages: []blueprint.Package{
-			{"httpd", "2.4.*"},
+			{
+				Name:    "httpd",
+				Version: "2.4.*"},
 		},
 		Groups:  []blueprint.Group{},
 		Modules: []blueprint.Package{},
diff --git a/internal/worker/server_test.go b/internal/worker/server_test.go
index 56aa29668..9f73d6457 100644
--- a/internal/worker/server_test.go
+++ b/internal/worker/server_test.go
@@ -64,7 +64,7 @@ func TestCreate(t *testing.T) {
 	}
 	server := worker.NewServer(nil, testjobqueue.New(), "")
 
-	_, err = server.Enqueue(arch.Name(), &worker.OSBuildJob{Manifest: manifest})
+	_, err = server.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: manifest})
 	require.NoError(t, err)
 
 	test.TestRoute(t, server, false, "POST", "/api/worker/v1/jobs", `{"types":["osbuild"],"arch":"x86_64"}`, http.StatusCreated,
@@ -87,14 +87,18 @@ func TestCancel(t *testing.T) {
 	}
 	server := worker.NewServer(nil, testjobqueue.New(), "")
 
-	jobId, err := server.Enqueue(arch.Name(), &worker.OSBuildJob{Manifest: manifest})
+	jobId, err := server.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: manifest})
 	require.NoError(t, err)
 
-	token, j, typ, args, err := server.RequestJob(context.Background(), arch.Name(), []string{"osbuild"})
+	token, j, typ, args, dynamicArgs, err := server.RequestJob(context.Background(), arch.Name(), []string{"osbuild"})
 	require.NoError(t, err)
 	require.Equal(t, jobId, j)
 	require.Equal(t, "osbuild", typ)
 	require.NotNil(t, args)
+	require.Nil(t, dynamicArgs)
+
+	test.TestRoute(t, server, false, "GET", fmt.Sprintf("/api/worker/v1/jobs/%s", token), `{}`, http.StatusOK,
+		`{"canceled":false}`)
 
 	err = server.Cancel(jobId)
 	require.NoError(t, err)
@@ -102,3 +106,62 @@ func TestCancel(t *testing.T) {
 	test.TestRoute(t, server, false, "GET", fmt.Sprintf("/api/worker/v1/jobs/%s", token), `{}`, http.StatusOK,
 		`{"canceled":true}`)
 }
+
+func TestUpdate(t *testing.T) {
+	distroStruct := fedoratest.New()
+	arch, err := distroStruct.GetArch("x86_64")
+	if err != nil {
+		t.Fatalf("error getting arch from distro")
+	}
+	imageType, err := arch.GetImageType("qcow2")
+	if err != nil {
+		t.Fatalf("error getting image type from arch")
+	}
+	manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("error creating osbuild manifest")
+	}
+	server := worker.NewServer(nil, testjobqueue.New(), "")
+
+	jobId, err := server.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: manifest})
+	require.NoError(t, err)
+
+	token, j, typ, args, dynamicArgs, err := server.RequestJob(context.Background(), arch.Name(), []string{"osbuild"})
+	require.NoError(t, err)
+	require.Equal(t, jobId, j)
+	require.Equal(t, "osbuild", typ)
+	require.NotNil(t, args)
+	require.Nil(t, dynamicArgs)
+
+	test.TestRoute(t, server, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%s", token), `{}`, http.StatusOK, `{}`)
+	test.TestRoute(t, server, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%s", token), `{}`, http.StatusNotFound, `*`)
+}
+
+func TestUpload(t *testing.T) {
+	distroStruct := fedoratest.New()
+	arch, err := distroStruct.GetArch("x86_64")
+	if err != nil {
+		t.Fatalf("error getting arch from distro")
+	}
+	imageType, err := arch.GetImageType("qcow2")
+	if err != nil {
+		t.Fatalf("error getting image type from arch")
+	}
+	manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("error creating osbuild manifest")
+	}
+	server := worker.NewServer(nil, testjobqueue.New(), "")
+
+	jobID, err := server.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: manifest})
+	require.NoError(t, err)
+
+	token, j, typ, args, dynamicArgs, err := server.RequestJob(context.Background(), arch.Name(), []string{"osbuild"})
+	require.NoError(t, err)
+	require.Equal(t, jobID, j)
+	require.Equal(t, "osbuild", typ)
+	require.NotNil(t, args)
+	require.Nil(t, dynamicArgs)
+
+	test.TestRoute(t, server, false, "PUT", fmt.Sprintf("/api/worker/v1/jobs/%s/artifacts/foobar", token), `this is my artifact`, http.StatusOK, `?`)
+}
diff --git a/test/data/composer/osbuild-composer.toml b/test/data/composer/osbuild-composer.toml
index 3d3b573b5..90179d041 100644
--- a/test/data/composer/osbuild-composer.toml
+++ b/test/data/composer/osbuild-composer.toml
@@ -2,10 +2,6 @@
 allowed_domains = [ "localhost", "client.osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
 
-[koji.servers.localhost.kerberos]
-principal = "osbuild-krb@LOCAL"
-keytab = "/etc/osbuild-composer/client.keytab"
-
 [worker]
 allowed_domains = [ "localhost", "worker.osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"