Repository: osbuild/osbuild-composer
Pull request #155 (2019-12-10 02:15:23)

composer: add provisional multi-arch support
The pipeline generation now takes the architecture as an argument.
Currently only x86_64 is supported. The architecture is detected
at start-up, and passed down to each pipeline translation.

For osbuild-pipeline we now requrie the architecture to be passed
in.

Signed-off-by: Tom Gundersen <teg@jklm.no>


Patch:
diff --git a/.travis.yml b/.travis.yml
index 3ded80474..58a8450b8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,19 @@ addons:
 matrix:
   include:
     - name: unit-tests
+      arch: amd64
+      language: go
+      go: 1.12.x
+      env: GO111MODULE=on # needed even for Go 1.12
+      script: go test -v ./...
+    - name: unit-tests
+      arch: arm64
+      language: go
+      go: 1.12.x
+      env: GO111MODULE=on # needed even for Go 1.12
+      script: go test -v ./...
+    - name: unit-tests
+      arch: s390x
       language: go
       go: 1.12.x
       env: GO111MODULE=on # needed even for Go 1.12
@@ -40,15 +53,6 @@ matrix:
         # ubuntu's rpm package sets dbpath to ~/.rpmdb, which makes rpm fail...
         - sudo sh -c 'mkdir /etc/rpm; echo "%_dbpath /var/lib/rpm" > /etc/rpm/macros'
         - sudo env "PATH=$PATH" "PYTHONUNBUFFERED=1" "OSBUILD_TEST_BUILD_ENV=test/f27-build-from-ubuntu1804.json" test/run --arches aarch64
-    - name: image-test
-      arch: ppc64le
-      language: python
-      python: 3.7
-      before_install: sudo apt-get install -y qemu-kvm genisoimage
-      script:
-        # ubuntu's rpm package sets dbpath to ~/.rpmdb, which makes rpm fail...
-        - sudo sh -c 'mkdir /etc/rpm; echo "%_dbpath /var/lib/rpm" > /etc/rpm/macros'
-        - sudo env "PATH=$PATH" "PYTHONUNBUFFERED=1" "OSBUILD_TEST_BUILD_ENV=test/f27-build-from-ubuntu1804.json" test/run --arches ppc64le
     - name: image-test
       arch: s390x
       language: python
diff --git a/Makefile b/Makefile
index c347b8399..1365b612c 100644
--- a/Makefile
+++ b/Makefile
@@ -22,8 +22,6 @@ install:
 	cp distribution/*.service /etc/systemd/system/
 	cp distribution/*.socket /etc/systemd/system/
 	systemctl daemon-reload
-	systemctl enable osbuild-composer.socket
-	systemctl enable osbuild-worker@1.service
 
 tarball:
 	git archive --prefix=$(PACKAGE_NAME)-$(VERSION)/ --format=tar.gz HEAD > $(PACKAGE_NAME)-$(VERSION).tar.gz
diff --git a/cmd/osbuild-composer/main.go b/cmd/osbuild-composer/main.go
index bf895b10e..d7217adec 100644
--- a/cmd/osbuild-composer/main.go
+++ b/cmd/osbuild-composer/main.go
@@ -4,6 +4,7 @@ import (
 	"flag"
 	"log"
 	"os"
+	"runtime"
 
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/jobqueue"
@@ -14,6 +15,20 @@ import (
 	"github.com/coreos/go-systemd/activation"
 )
 
+func currentArch() string {
+	if runtime.GOARCH == "amd64" {
+		return "x86_64"
+	} else if runtime.GOARCH == "arm64" {
+		return "aarch64"
+	} else if runtime.GOARCH == "ppc64le" {
+		return "ppc64le"
+	} else if runtime.GOARCH == "s390x" {
+		return "s390x"
+	} else {
+		panic("unsupported architecture")
+	}
+}
+
 func main() {
 	var verbose bool
 	flag.BoolVar(&verbose, "v", false, "Print access log")
@@ -23,11 +38,11 @@ func main() {
 
 	listeners, err := activation.Listeners()
 	if err != nil {
-		panic(err)
+		log.Fatalf("Could not get listening sockets: " + err.Error())
 	}
 
 	if len(listeners) != 2 {
-		panic("Unexpected number of sockets. Composer require 2 of them.")
+		log.Fatalf("Unexpected number of listening sockets (%d), expected 2", len(listeners))
 	}
 
 	weldrListener := listeners[0]
@@ -37,7 +52,7 @@ func main() {
 
 	distribution, err := distro.FromHost()
 	if err != nil {
-		panic("cannot detect distro from host: " + err.Error())
+		log.Fatalf("Could not determine distro from host: " + err.Error())
 	}
 
 	var logger *log.Logger
@@ -48,7 +63,7 @@ func main() {
 	store := store.New(&stateDir, distribution)
 
 	jobAPI := jobqueue.New(logger, store)
-	weldrAPI := weldr.New(rpm, distribution, logger, store)
+	weldrAPI := weldr.New(rpm, currentArch(), distribution, logger, store)
 
 	go jobAPI.Serve(jobListener)
 	weldrAPI.Serve(weldrListener)
diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go
index 20b2013a3..afec4c6c4 100644
--- a/cmd/osbuild-pipeline/main.go
+++ b/cmd/osbuild-pipeline/main.go
@@ -14,9 +14,11 @@ import (
 func main() {
 	var format string
 	var blueprintArg string
+	var archArg string
 	var distroArg string
-	flag.StringVar(&format, "output-format", "qcow2", "output format")
+	flag.StringVar(&format, "output-format", "", "output format")
 	flag.StringVar(&blueprintArg, "blueprint", "", "blueprint to translate")
+	flag.StringVar(&archArg, "arch", "", "architecture to create image for")
 	flag.StringVar(&distroArg, "distro", "", "distribution to create")
 	flag.Parse()
 
@@ -55,7 +57,7 @@ func main() {
 		panic(err.Error())
 	}
 
-	pipeline, err := d.Pipeline(blueprint, checksums, format)
+	pipeline, err := d.Pipeline(blueprint, checksums, archArg, format)
 	if err != nil {
 		panic(err.Error())
 	}
diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go
index 459fcd6c8..841496a0d 100644
--- a/cmd/osbuild-worker/main.go
+++ b/cmd/osbuild-worker/main.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"net"
 	"net/http"
 
@@ -76,40 +77,42 @@ func (c *ComposerClient) UpdateJob(job *jobqueue.Job, status string, image *stor
 	return nil
 }
 
-func handleJob(client *ComposerClient, distro distro.Distro) {
+func handleJob(client *ComposerClient, distro distro.Distro) error {
 	fmt.Println("Waiting for a new job...")
 	job, err := client.AddJob()
 	if err != nil {
-		panic(err)
+		return err
 	}
 
-	client.UpdateJob(job, "RUNNING", nil)
+	err = client.UpdateJob(job, "RUNNING", nil)
+	if err != nil {
+		return err
+	}
 
 	fmt.Printf("Running job %s\n", job.ID.String())
 	image, err, errs := job.Run(distro)
 	if err != nil {
-		client.UpdateJob(job, "FAILED", nil)
-		return
+		return client.UpdateJob(job, "FAILED", nil)
 	}
 
 	for _, err := range errs {
 		if err != nil {
-			client.UpdateJob(job, "FAILED", nil)
-			return
+			return client.UpdateJob(job, "FAILED", nil)
 		}
 	}
 
-	client.UpdateJob(job, "FINISHED", image)
+	return client.UpdateJob(job, "FINISHED", image)
 }
 
 func main() {
 	distro, err := distro.FromHost()
 	if err != nil {
-		panic(err)
+		log.Fatalf("Could not determine distro from host: " + err.Error())
 	}
 
 	client := NewClient()
 	for {
-		handleJob(client, distro)
+		err = handleJob(client, distro)
+		log.Fatalf("Failed to handle job: " + err.Error())
 	}
 }
diff --git a/distribution/osbuild-composer.service b/distribution/osbuild-composer.service
index 81fd12724..7ed35f68f 100644
--- a/distribution/osbuild-composer.service
+++ b/distribution/osbuild-composer.service
@@ -2,6 +2,7 @@
 Description=OSBuild Composer
 After=multi-user.target
 Requires=osbuild-composer.socket
+Wants=osbuild-worker@1.service
 
 [Service]
 Type=simple
diff --git a/distribution/osbuild-composer.socket b/distribution/osbuild-composer.socket
index f817fa45f..6a1e471e0 100644
--- a/distribution/osbuild-composer.socket
+++ b/distribution/osbuild-composer.socket
@@ -1,5 +1,5 @@
 [Unit]
-PartOf=osbuild-composer.service
+Description=OSBuild Composer API sockets
 
 [Socket]
 ListenStream=/run/weldr/api.socket
diff --git a/distribution/osbuild-worker@.service b/distribution/osbuild-worker@.service
index 32a3ffb74..a507414bf 100644
--- a/distribution/osbuild-worker@.service
+++ b/distribution/osbuild-worker@.service
@@ -1,5 +1,6 @@
 [Unit]
 Description=OSBuild Composer Worker (%i)
+Requires=osbuild-composer.socket
 After=multi-user.target osbuild-composer.socket
 
 [Service]
@@ -11,5 +12,4 @@ Restart=on-failure
 RestartSec=10s
 
 [Install]
-DefaultInstance=1
-WantedBy=multi-user.target
+WantedBy=osbuild-composer.service
diff --git a/golang-github-osbuild-composer.spec b/golang-github-osbuild-composer.spec
index bf5173424..5c7923885 100644
--- a/golang-github-osbuild-composer.spec
+++ b/golang-github-osbuild-composer.spec
@@ -41,9 +41,8 @@ Provides: osbuild-composer
 %goprep
 
 %build
-for cmd in cmd/* ; do
-  %gobuild -o _bin/$(basename $cmd) %{goipath}/$cmd
-done
+%gobuild -o _bin/osbuild-composer %{goipath}/osbuild-composer
+%gobuild -o _bin/osbuild-worker %{goipath}/osbuild-worker
 
 %install
 install -m 0755 -vd                                         %{buildroot}%{_libexecdir}/osbuild-composer
diff --git a/internal/distro/distro.go b/internal/distro/distro.go
index a0cb3bb1d..8f491d334 100644
--- a/internal/distro/distro.go
+++ b/internal/distro/distro.go
@@ -30,7 +30,7 @@ type Distro interface {
 	// Returns an osbuild pipeline that generates an image in the given
 	// output format with all packages and customizations specified in the
 	// given blueprint.
-	Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputFormat string) (*pipeline.Pipeline, error)
+	Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputArchitecture, outputFormat string) (*pipeline.Pipeline, error)
 
 	// Returns a osbuild runner that can be used on this distro.
 	Runner() string
@@ -41,7 +41,7 @@ var registered map[string]Distro
 func init() {
 	registered = map[string]Distro{
 		"fedora-30": fedora30.New(),
-		"rhel-8.2":    rhel82.New(),
+		"rhel-8.2":  rhel82.New(),
 	}
 }
 
@@ -100,10 +100,10 @@ func readOSRelease(r io.Reader) (map[string]string, error) {
 		key := strings.TrimSpace(parts[0])
 		value := strings.TrimSpace(parts[1])
 		if value[0] == '"' {
-			if len(value) < 2 || value[len(value) - 1] != '"' {
+			if len(value) < 2 || value[len(value)-1] != '"' {
 				return nil, errors.New("readOSRelease: invalid input")
 			}
-			value = value[1:len(value) - 1]
+			value = value[1 : len(value)-1]
 		}
 
 		osrelease[key] = value
diff --git a/internal/distro/fedora30/distro.go b/internal/distro/fedora30/distro.go
index 59b3edea4..0b7d4d2e3 100644
--- a/internal/distro/fedora30/distro.go
+++ b/internal/distro/fedora30/distro.go
@@ -270,12 +270,16 @@ func (r *Fedora30) FilenameFromType(outputFormat string) (string, string, error)
 	return "", "", errors.New("invalid output format: " + outputFormat)
 }
 
-func (r *Fedora30) Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputFormat string) (*pipeline.Pipeline, error) {
+func (r *Fedora30) Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputArchitecture, outputFormat string) (*pipeline.Pipeline, error) {
 	output, exists := r.outputs[outputFormat]
 	if !exists {
 		return nil, errors.New("invalid output format: " + outputFormat)
 	}
 
+	if outputArchitecture != "x86_64" {
+		return nil, errors.New("invalid output architecture: " + outputArchitecture)
+	}
+
 	p := &pipeline.Pipeline{}
 	p.SetBuild(r.buildPipeline(checksums), "org.osbuild.fedora30")
 
diff --git a/internal/distro/rhel82/distro.go b/internal/distro/rhel82/distro.go
index 73497d140..6fde9ac81 100644
--- a/internal/distro/rhel82/distro.go
+++ b/internal/distro/rhel82/distro.go
@@ -107,7 +107,7 @@ func New() *RHEL82 {
 		DefaultTarget: "multi-user.target",
 		IncludeFSTab:  true,
 		KernelOptions: "ro console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295 crashkernel=auto",
-		Assembler:     r.qemuAssembler("raw.xz", "image.raw.xz", 6 * GigaByte),
+		Assembler:     r.qemuAssembler("raw.xz", "image.raw.xz", 6*GigaByte),
 	}
 
 	r.outputs["ext4-filesystem"] = output{
@@ -294,12 +294,16 @@ func (r *RHEL82) FilenameFromType(outputFormat string) (string, string, error) {
 	return "", "", errors.New("invalid output format: " + outputFormat)
 }
 
-func (r *RHEL82) Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputFormat string) (*pipeline.Pipeline, error) {
+func (r *RHEL82) Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputArchitecture, outputFormat string) (*pipeline.Pipeline, error) {
 	output, exists := r.outputs[outputFormat]
 	if !exists {
 		return nil, errors.New("invalid output format: " + outputFormat)
 	}
 
+	if outputArchitecture != "x86_64" {
+		return nil, errors.New("invalid output architecture: " + outputArchitecture)
+	}
+
 	p := &pipeline.Pipeline{}
 	p.SetBuild(r.buildPipeline(checksums), "org.osbuild.rhel82")
 
diff --git a/internal/jobqueue/job.go b/internal/jobqueue/job.go
index db04aa019..8438bfc01 100644
--- a/internal/jobqueue/job.go
+++ b/internal/jobqueue/job.go
@@ -44,9 +44,15 @@ func (job *Job) Run(d distro.Distro) (*store.Image, error, []error) {
 		return nil, err, nil
 	}
 
+	tmpStore, err := ioutil.TempDir("/var/tmp", "osbuild-store")
+	if err != nil {
+		return nil, err, nil
+	}
+	defer os.RemoveAll(tmpStore)
+
 	cmd := exec.Command(
 		"osbuild",
-		"--store", "/var/cache/osbuild-composer/store",
+		"--store", tmpStore,
 		"--build-env", buildFile.Name(),
 		"--json", "-",
 	)
diff --git a/internal/store/store.go b/internal/store/store.go
index 4a978e991..d1723f0cc 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -415,7 +415,7 @@ func (s *Store) DeleteBlueprintFromWorkspace(name string) {
 	})
 }
 
-func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, checksums map[string]string, composeType string, uploadTarget *target.Target) error {
+func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, checksums map[string]string, arch, composeType string, uploadTarget *target.Target) error {
 	targets := []*target.Target{}
 
 	if s.stateDir != nil {
@@ -430,7 +430,7 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, checks
 		targets = append(targets, uploadTarget)
 	}
 
-	pipeline, err := s.distro.Pipeline(bp, checksums, composeType)
+	pipeline, err := s.distro.Pipeline(bp, checksums, arch, composeType)
 	if err != nil {
 		return err
 	}
diff --git a/internal/weldr/api.go b/internal/weldr/api.go
index 79d341279..55dd816bd 100644
--- a/internal/weldr/api.go
+++ b/internal/weldr/api.go
@@ -31,18 +31,20 @@ type API struct {
 	store *store.Store
 
 	rpmmd  rpmmd.RPMMD
+	arch   string
 	distro distro.Distro
 
 	logger *log.Logger
 	router *httprouter.Router
 }
 
-func New(rpmmd rpmmd.RPMMD, distro distro.Distro, logger *log.Logger, store *store.Store) *API {
+func New(rpmmd rpmmd.RPMMD, arch string, distro distro.Distro, logger *log.Logger, store *store.Store) *API {
 	// This needs to be shared with the worker API so that they can communicate with each other
 	// builds := make(chan queue.Build, 200)
 	api := &API{
 		store:  store,
 		rpmmd:  rpmmd,
+		arch:   arch,
 		distro: distro,
 		logger: logger,
 	}
@@ -1221,7 +1223,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
 			return
 		}
 
-		err = api.store.PushCompose(reply.BuildID, bp, checksums, cr.ComposeType, uploadTarget)
+		err = api.store.PushCompose(reply.BuildID, bp, checksums, api.arch, cr.ComposeType, uploadTarget)
 
 		// TODO: we should probably do some kind of blueprint validation in future
 		// for now, let's just 500 and bail out


Test patch:
diff --git a/internal/distro/distro_test.go b/internal/distro/distro_test.go
index 580f43c9c..8b8fc7567 100644
--- a/internal/distro/distro_test.go
+++ b/internal/distro/distro_test.go
@@ -20,6 +20,7 @@ func TestDistro_Pipeline(t *testing.T) {
 	for _, fileInfo := range fileInfos {
 		type compose struct {
 			Distro       string               `json:"distro"`
+			Arch         string               `json:"arch"`
 			OutputFormat string               `json:"output-format"`
 			Checksums    map[string]string    `json:"checksums"`
 			Blueprint    *blueprint.Blueprint `json:"blueprint"`
@@ -46,7 +47,7 @@ func TestDistro_Pipeline(t *testing.T) {
 				t.Errorf("unknown distro: %v", tt.Compose.Distro)
 				return
 			}
-			got, err := d.Pipeline(tt.Compose.Blueprint, tt.Compose.Checksums, tt.Compose.OutputFormat)
+			got, err := d.Pipeline(tt.Compose.Blueprint, tt.Compose.Checksums, tt.Compose.Arch, tt.Compose.OutputFormat)
 			if (err != nil) != (tt.Pipeline == nil) {
 				t.Errorf("distro.Pipeline() error = %v", err)
 				return
diff --git a/internal/distro/test/distro.go b/internal/distro/test/distro.go
index d49a5e0d9..3992f1189 100644
--- a/internal/distro/test/distro.go
+++ b/internal/distro/test/distro.go
@@ -33,8 +33,8 @@ func (d *TestDistro) FilenameFromType(outputFormat string) (string, string, erro
 	return "", "", errors.New("invalid output format: " + outputFormat)
 }
 
-func (d *TestDistro) Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputFormat string) (*pipeline.Pipeline, error) {
-	return nil, errors.New("invalid output format: " + outputFormat)
+func (d *TestDistro) Pipeline(b *blueprint.Blueprint, checksums map[string]string, outputArch, outputFormat string) (*pipeline.Pipeline, error) {
+	return nil, errors.New("invalid output format or arch: " + outputFormat + " @ " + outputArch)
 }
 
 func (d *TestDistro) Runner() string {
diff --git a/internal/jobqueue/api_test.go b/internal/jobqueue/api_test.go
index e1a3b5977..a9f951341 100644
--- a/internal/jobqueue/api_test.go
+++ b/internal/jobqueue/api_test.go
@@ -43,7 +43,7 @@ func TestCreate(t *testing.T) {
 	store := store.New(nil, distro.New("fedora-30"))
 	api := jobqueue.New(nil, store)
 
-	err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"fedora": "test:foo"}, "tar", nil)
+	err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"fedora": "test:foo"}, "x86_64", "tar", nil)
 	if err != nil {
 		t.Fatalf("error pushing compose: %v", err)
 	}
@@ -58,7 +58,7 @@ func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) {
 	api := jobqueue.New(nil, store)
 
 	if from != "VOID" {
-		err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"fedora": "test:foo"}, "tar", nil)
+		err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"fedora": "test:foo"}, "x86_64", "tar", nil)
 		if err != nil {
 			t.Fatalf("error pushing compose: %v", err)
 		}
diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go
index d7dec5b65..8c0e1020b 100644
--- a/internal/weldr/api_test.go
+++ b/internal/weldr/api_test.go
@@ -31,7 +31,7 @@ func createWeldrAPI(fixtureGenerator rpmmd_mock.FixtureGenerator) (*weldr.API, *
 	rpm := rpmmd_mock.NewRPMMDMock(fixture)
 	d := distro.New("test")
 
-	return weldr.New(rpm, d, nil, fixture.Store), fixture.Store
+	return weldr.New(rpm, "x86_64", d, nil, fixture.Store), fixture.Store
 }
 
 func TestBasic(t *testing.T) {