Repository: osbuild/osbuild-composer
Pull request #275 (2020-03-03 20:47:43)

osbuild: introduce `manifest` objects
This allows sources to be passed together with pipelines directly to osbuild on stdin, rather than having to pass sources as a separate file.

For now empty sources are passed, but a follow-up PR will switch from dnf- to rpm-based pipelines, which requires sources to be passed separately.


Patch:
diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go
index 274d4f0f1..0c1d557cc 100644
--- a/cmd/osbuild-pipeline/main.go
+++ b/cmd/osbuild-pipeline/main.go
@@ -106,14 +106,14 @@ func main() {
 	}
 
 	size := d.GetSizeForOutputType(imageType, 0)
-	pipeline, err := d.Pipeline(blueprint, nil, packageSpecs, buildPackageSpecs, checksums, archArg, imageType, size)
+	manifest, err := d.Manifest(blueprint, nil, packageSpecs, buildPackageSpecs, checksums, archArg, imageType, size)
 	if err != nil {
 		panic(err.Error())
 	}
 
-	bytes, err := json.Marshal(pipeline)
+	bytes, err := json.Marshal(manifest)
 	if err != nil {
-		panic("could not marshal pipeline into JSON")
+		panic("could not marshal manifest into JSON")
 	}
 
 	os.Stdout.Write(bytes)
diff --git a/internal/compose/compose.go b/internal/compose/compose.go
index 098f533bf..899800830 100644
--- a/internal/compose/compose.go
+++ b/internal/compose/compose.go
@@ -25,7 +25,7 @@ type ImageBuild struct {
 	Distro      common.Distribution    `json:"distro"`
 	QueueStatus common.ImageBuildState `json:"queue_status"`
 	ImageType   common.ImageType       `json:"image_type"`
-	Pipeline    *osbuild.Pipeline      `json:"pipeline"`
+	Manifest    *osbuild.Manifest      `json:"manifest"`
 	Targets     []*target.Target       `json:"targets"`
 	JobCreated  time.Time              `json:"job_created"`
 	JobStarted  time.Time              `json:"job_started"`
@@ -35,10 +35,10 @@ type ImageBuild struct {
 
 // DeepCopy creates a copy of the ImageBuild structure
 func (ib *ImageBuild) DeepCopy() ImageBuild {
-	var newPipelinePtr *osbuild.Pipeline = nil
-	if ib.Pipeline != nil {
-		pipelineCopy := *ib.Pipeline
-		newPipelinePtr = &pipelineCopy
+	var newManifestPtr *osbuild.Manifest = nil
+	if ib.Manifest != nil {
+		manifestCopy := *ib.Manifest
+		newManifestPtr = &manifestCopy
 	}
 	var newTargets []*target.Target
 	for _, t := range ib.Targets {
@@ -51,7 +51,7 @@ func (ib *ImageBuild) DeepCopy() ImageBuild {
 		Distro:      ib.Distro,
 		QueueStatus: ib.QueueStatus,
 		ImageType:   ib.ImageType,
-		Pipeline:    newPipelinePtr,
+		Manifest:    newManifestPtr,
 		Targets:     newTargets,
 		JobCreated:  ib.JobCreated,
 		JobStarted:  ib.JobStarted,
diff --git a/internal/distro/distro.go b/internal/distro/distro.go
index 2d49d49a7..b9c528b30 100644
--- a/internal/distro/distro.go
+++ b/internal/distro/distro.go
@@ -64,6 +64,11 @@ type Distro interface {
 	// corresponding pipeline containing the given packages.
 	Sources(packages []rpmmd.PackageSpec) *osbuild.Sources
 
+	// Returns an osbuild manifest, containing the sources and pipeline necessary
+	// to generates an image in the given output format with all packages and
+	// customizations specified in the given blueprint.
+	Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error)
+
 	// Returns a osbuild runner that can be used on this distro.
 	Runner() string
 }
diff --git a/internal/distro/fedora30/distro.go b/internal/distro/fedora30/distro.go
index c38130c2c..4bc4fd792 100644
--- a/internal/distro/fedora30/distro.go
+++ b/internal/distro/fedora30/distro.go
@@ -473,6 +473,18 @@ func (r *Fedora30) Sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
 	return &osbuild.Sources{}
 }
 
+func (r *Fedora30) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (r *Fedora30) Runner() string {
 	return "org.osbuild.fedora30"
 }
diff --git a/internal/distro/fedora31/distro.go b/internal/distro/fedora31/distro.go
index 23e7bd36f..0d4f1b4bf 100644
--- a/internal/distro/fedora31/distro.go
+++ b/internal/distro/fedora31/distro.go
@@ -473,6 +473,18 @@ func (r *Fedora31) Sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
 	return &osbuild.Sources{}
 }
 
+func (r *Fedora31) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (r *Fedora31) Runner() string {
 	return "org.osbuild.fedora31"
 }
diff --git a/internal/distro/fedora32/distro.go b/internal/distro/fedora32/distro.go
index 56d237d48..7fd6e8823 100644
--- a/internal/distro/fedora32/distro.go
+++ b/internal/distro/fedora32/distro.go
@@ -473,6 +473,18 @@ func (r *Fedora32) Sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
 	return &osbuild.Sources{}
 }
 
+func (r *Fedora32) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (r *Fedora32) Runner() string {
 	return "org.osbuild.fedora32"
 }
diff --git a/internal/distro/rhel81/distro.go b/internal/distro/rhel81/distro.go
index 397eadb22..12e230873 100644
--- a/internal/distro/rhel81/distro.go
+++ b/internal/distro/rhel81/distro.go
@@ -616,6 +616,18 @@ func (r *RHEL81) Sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
 	return &osbuild.Sources{}
 }
 
+func (r *RHEL81) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (r *RHEL81) Runner() string {
 	return "org.osbuild.rhel81"
 }
diff --git a/internal/distro/rhel82/distro.go b/internal/distro/rhel82/distro.go
index 934a1dd9e..73881065b 100644
--- a/internal/distro/rhel82/distro.go
+++ b/internal/distro/rhel82/distro.go
@@ -616,6 +616,18 @@ func (r *RHEL82) Sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
 	return &osbuild.Sources{}
 }
 
+func (r *RHEL82) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (r *RHEL82) Runner() string {
 	return "org.osbuild.rhel82"
 }
diff --git a/internal/jobqueue/api.go b/internal/jobqueue/api.go
index f588054f8..adbf80642 100644
--- a/internal/jobqueue/api.go
+++ b/internal/jobqueue/api.go
@@ -106,7 +106,7 @@ func (api *API) addJobHandler(writer http.ResponseWriter, request *http.Request,
 		ID:           nextJob.ComposeID,
 		ImageBuildID: nextJob.ImageBuildID,
 		Distro:       nextJob.Distro,
-		Pipeline:     nextJob.Pipeline,
+		Manifest:     nextJob.Manifest,
 		Targets:      nextJob.Targets,
 		OutputType:   nextJob.ImageType,
 	})
diff --git a/internal/jobqueue/job.go b/internal/jobqueue/job.go
index de3f91eaa..74f5aab94 100644
--- a/internal/jobqueue/job.go
+++ b/internal/jobqueue/job.go
@@ -21,7 +21,7 @@ type Job struct {
 	ID           uuid.UUID         `json:"id"`
 	ImageBuildID int               `json:"image_build_id"`
 	Distro       string            `json:"distro"`
-	Pipeline     *osbuild.Pipeline `json:"pipeline"`
+	Manifest     *osbuild.Manifest `json:"manifest"`
 	Targets      []*target.Target  `json:"targets"`
 	OutputType   string            `json:"output_type"`
 }
@@ -102,7 +102,7 @@ func (job *Job) Run(uploader LocalTargetUploader) (*common.ComposeResult, error)
 		return nil, fmt.Errorf("error starting osbuild: %v", err)
 	}
 
-	err = json.NewEncoder(stdin).Encode(job.Pipeline)
+	err = json.NewEncoder(stdin).Encode(job.Manifest)
 	if err != nil {
 		return nil, fmt.Errorf("error encoding osbuild pipeline: %v", err)
 	}
diff --git a/internal/osbuild/osbuild.go b/internal/osbuild/osbuild.go
index aa282c986..760ea7b8c 100644
--- a/internal/osbuild/osbuild.go
+++ b/internal/osbuild/osbuild.go
@@ -2,6 +2,12 @@
 // OSBuild types.
 package osbuild
 
+// A Manifest represents an OSBuild source and pipeline manifest
+type Manifest struct {
+	Sources  Sources  `json:"sources"`
+	Pipeline Pipeline `json:"pipeline"`
+}
+
 // A Pipeline represents an OSBuild pipeline
 type Pipeline struct {
 	// The build environment which can run this pipeline
diff --git a/internal/store/store.go b/internal/store/store.go
index 661a2fa0b..b05888291 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -54,7 +54,7 @@ type Job struct {
 	ComposeID    uuid.UUID
 	ImageBuildID int
 	Distro       string
-	Pipeline     *osbuild.Pipeline
+	Manifest     *osbuild.Manifest
 	Targets      []*target.Target
 	ImageType    string
 }
@@ -179,7 +179,7 @@ func New(stateDir *string, distroArg distro.Distro, distroRegistryArg distro.Reg
 						ComposeID:    composeID,
 						ImageBuildID: imgBuild.Id,
 						Distro:       distroStr,
-						Pipeline:     imgBuild.Pipeline,
+						Manifest:     imgBuild.Manifest,
 						Targets:      imgBuild.Targets,
 						ImageType:    imageTypeCompat,
 					}
@@ -509,7 +509,7 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, packag
 		repos = append(repos, source.RepoConfig())
 	}
 
-	pipelineStruct, err := s.distro.Pipeline(bp, repos, packages, buildPackages, checksums, arch, composeType, size)
+	manifestStruct, err := s.distro.Manifest(bp, repos, packages, buildPackages, checksums, arch, composeType, size)
 	if err != nil {
 		return err
 	}
@@ -520,7 +520,7 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, packag
 			ImageBuilds: []compose.ImageBuild{
 				{
 					QueueStatus: common.IBWaiting,
-					Pipeline:    pipelineStruct,
+					Manifest:    manifestStruct,
 					ImageType:   imageType,
 					Targets:     targets,
 					JobCreated:  time.Now(),
@@ -534,7 +534,7 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, packag
 		ComposeID:    composeID,
 		ImageBuildID: 0,
 		Distro:       s.distro.Name(),
-		Pipeline:     pipelineStruct,
+		Manifest:     manifestStruct,
 		Targets:      targets,
 		ImageType:    composeType,
 	}
@@ -545,7 +545,7 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, packag
 // PushComposeRequest is an alternative to PushCompose which does not assume a pre-defined distro, as such it is better
 // suited for RCM API and possible future API that would respect the fact that we can build any distro and any arch
 func (s *Store) PushComposeRequest(request common.ComposeRequest) error {
-	// This should never happen and once distro.Pipeline is refactored this check will go away
+	// This should never happen and once distro.Manifest is refactored this check will go away
 	arch, exists := request.Arch.ToString()
 	if !exists {
 		panic("fatal error, arch should exist but it does not")
@@ -572,12 +572,12 @@ func (s *Store) PushComposeRequest(request common.ComposeRequest) error {
 	for imageBuildID, imageRequest := range request.RequestedImages {
 		// TODO: handle custom upload targets
 		// TODO: this requires changes in the Compose Request struct
-		// This should never happen and once distro.Pipeline is refactored this check will go away
+		// This should never happen and once distro.Manifest is refactored this check will go away
 		imgTypeCompatStr, exists := imageRequest.ImgType.ToCompatString()
 		if !exists {
 			panic("fatal error, image type should exist but it does not")
 		}
-		pipelineStruct, err := distroStruct.Pipeline(&request.Blueprint, request.Repositories, nil, nil, request.Checksums, arch, imgTypeCompatStr, 0)
+		manifestStruct, err := distroStruct.Manifest(&request.Blueprint, request.Repositories, nil, nil, request.Checksums, arch, imgTypeCompatStr, 0)
 		if err != nil {
 			return err
 		}
@@ -594,7 +594,7 @@ func (s *Store) PushComposeRequest(request common.ComposeRequest) error {
 			ComposeID:    request.ComposeID,
 			ImageBuildID: imageBuildID,
 			Distro:       distroString,
-			Pipeline:     pipelineStruct,
+			Manifest:     manifestStruct,
 			Targets:      []*target.Target{},
 			ImageType:    imgTypeCompatStr,
 		})
@@ -603,7 +603,7 @@ func (s *Store) PushComposeRequest(request common.ComposeRequest) error {
 			Distro:      request.Distro,
 			QueueStatus: common.IBWaiting,
 			ImageType:   imageRequest.ImgType,
-			Pipeline:    pipelineStruct,
+			Manifest:    manifestStruct,
 			Targets:     []*target.Target{},
 			JobCreated:  time.Now(),
 		})


Test patch:
diff --git a/internal/distro/fedoratest/distro.go b/internal/distro/fedoratest/distro.go
index 7ab90d28a..c87e73a4e 100644
--- a/internal/distro/fedoratest/distro.go
+++ b/internal/distro/fedoratest/distro.go
@@ -75,6 +75,18 @@ func (r *FedoraTestDistro) Sources(packages []rpmmd.PackageSpec) *osbuild.Source
 	return &osbuild.Sources{}
 }
 
+func (r *FedoraTestDistro) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (d *FedoraTestDistro) Runner() string {
 	return "org.osbuild.test"
 }
diff --git a/internal/distro/test/distro.go b/internal/distro/test/distro.go
index 92fda8bde..053c5fb07 100644
--- a/internal/distro/test/distro.go
+++ b/internal/distro/test/distro.go
@@ -67,6 +67,18 @@ func (d *TestDistro) Sources(packages []rpmmd.PackageSpec) *osbuild.Sources {
 	return &osbuild.Sources{}
 }
 
+func (r *TestDistro) Manifest(b *blueprint.Blueprint, additionalRepos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, checksums map[string]string, outputArchitecture, outputFormat string, size uint64) (*osbuild.Manifest, error) {
+	pipeline, err := r.Pipeline(b, additionalRepos, packageSpecs, buildPackageSpecs, checksums, outputArchitecture, outputFormat, size)
+	if err != nil {
+		return nil, err
+	}
+
+	return &osbuild.Manifest{
+		Sources:  *r.Sources(append(packageSpecs, buildPackageSpecs...)),
+		Pipeline: *pipeline,
+	}, nil
+}
+
 func (d *TestDistro) Runner() string {
 	return "org.osbuild.test"
 }
diff --git a/internal/jobqueue/api_test.go b/internal/jobqueue/api_test.go
index 0cdf77026..ab7506ff8 100644
--- a/internal/jobqueue/api_test.go
+++ b/internal/jobqueue/api_test.go
@@ -1,10 +1,11 @@
 package jobqueue_test
 
 import (
-	distro_mock "github.com/osbuild/osbuild-composer/internal/mocks/distro"
 	"net/http"
 	"testing"
 
+	distro_mock "github.com/osbuild/osbuild-composer/internal/mocks/distro"
+
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
 	"github.com/osbuild/osbuild-composer/internal/jobqueue"
@@ -60,7 +61,7 @@ func TestCreate(t *testing.T) {
 	}
 
 	test.TestRoute(t, api, false, "POST", "/job-queue/v1/jobs", `{}`, http.StatusCreated,
-		`{"distro":"fedora-30","id":"ffffffff-ffff-ffff-ffff-ffffffffffff","image_build_id":0,"output_type":"qcow2","pipeline":{},"targets":[]}`, "created", "uuid")
+		`{"id":"ffffffff-ffff-ffff-ffff-ffffffffffff","image_build_id":0,"distro":"fedora-30","manifest":{"sources":{},"pipeline":{}},"targets":[],"output_type":"qcow2"}`, "created", "uuid")
 }
 
 func testUpdateTransition(t *testing.T, from, to string, expectedStatus int, expectedResponse string) {
diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go
index a2d0daa94..1691f6adb 100644
--- a/internal/weldr/api_test.go
+++ b/internal/weldr/api_test.go
@@ -3,9 +3,6 @@ package weldr
 import (
 	"archive/tar"
 	"bytes"
-	"github.com/osbuild/osbuild-composer/internal/common"
-	"github.com/osbuild/osbuild-composer/internal/compose"
-	"github.com/osbuild/osbuild-composer/internal/target"
 	"io"
 	"math/rand"
 	"net/http"
@@ -15,6 +12,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/osbuild/osbuild-composer/internal/common"
+	"github.com/osbuild/osbuild-composer/internal/compose"
+	"github.com/osbuild/osbuild-composer/internal/target"
+
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
 	rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
@@ -447,11 +448,11 @@ func TestCompose(t *testing.T) {
 			break
 		}
 
-		if composeStruct.ImageBuilds[0].Pipeline == nil {
+		if composeStruct.ImageBuilds[0].Manifest == nil {
 			t.Fatalf("%s: the compose in the store did not contain a blueprint", c.Path)
 		} else {
 			// TODO: find some (reasonable) way to verify the contents of the pipeline
-			composeStruct.ImageBuilds[0].Pipeline = nil
+			composeStruct.ImageBuilds[0].Manifest = nil
 		}
 
 		if diff := cmp.Diff(composeStruct, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location")); diff != "" {