Repository: osbuild/osbuild-composer
Pull request #82 (2019-11-09 11:14:15)

distro: add DetectHost
lorax-composer uses the host's repositories for building images. This is
prone to accidental configuration errors and duplicates functionality
(adding custom repositories via the source API is much more explicit).

However, blueprints don't specify the distribution they're based on.
This is something they should do in the future to enable cross-distro
image builds. Until then, read `/etc/os-release` to detect the host OS
and use that as the base distro for a blueprint.

Detection works by concatenating the ID field with a `-` and the
VERSION_ID field. This mandates how additional distributions should
register themselves to the `distro` package.

Use this detection everywhere that fedora-30 was hard-coded before.

larskarlitski: Note: please don't merge this yet (forgot to click "draft", sorry), because it will break everyone who's on Fedora 31.
larskarlitski: Tests failed on Ubuntu, because we can't build that yet. I've changed the host detection to fall back to `fedora-30` when the host is unknown.

Also added a commit that fails the `jobqueue` test properly in such cases (it was hanging).
larskarlitski: Another update: forgot to add a `_test.go` file.

Patch:
diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go
index 7ac3de033..579ff3461 100644
--- a/cmd/osbuild-pipeline/main.go
+++ b/cmd/osbuild-pipeline/main.go
@@ -31,8 +31,8 @@ func main() {
 		}
 	}
 
-	f30 := distro.New("fedora-30")
-	pipeline, err := f30.Pipeline(blueprint, format)
+	d := distro.New("")
+	pipeline, err := d.Pipeline(blueprint, format)
 	if err != nil {
 		panic(err.Error())
 	}
diff --git a/internal/distro/distro.go b/internal/distro/distro.go
index 6ba2c385c..42bbd6dcf 100644
--- a/internal/distro/distro.go
+++ b/internal/distro/distro.go
@@ -1,6 +1,13 @@
 package distro
 
 import (
+	"bufio"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"strings"
+
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	"github.com/osbuild/osbuild-composer/internal/pipeline"
 )
@@ -33,16 +40,78 @@ func (e *InvalidOutputFormatError) Error() string {
 var registered = map[string]Distro{}
 
 func New(name string) Distro {
+	if name == "" {
+		distro, err := FromHost()
+		if err == nil {
+			return distro
+		} else {
+			log.Println("cannot detect distro from host: " + err.Error())
+			log.Println("falling back to 'fedora-30'")
+			return New("fedora-30")
+		}
+	}
+
 	distro, ok := registered[name]
 	if !ok {
 		panic("unknown distro: " + name)
 	}
+
 	return distro
 }
 
+func FromHost() (Distro, error) {
+	f, err := os.Open("/etc/os-release")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	osrelease, err := readOSRelease(f)
+	if err != nil {
+		return nil, err
+	}
+
+	name := osrelease["ID"] + "-" + osrelease["VERSION_ID"]
+
+	distro, ok := registered[name]
+	if !ok {
+		return nil, errors.New("unknown distro: " + name)
+	}
+	return distro, nil
+}
+
 func Register(name string, distro Distro) {
 	if _, exists := registered[name]; exists {
 		panic("a distro with this name already exists: " + name)
 	}
 	registered[name] = distro
 }
+
+func readOSRelease(r io.Reader) (map[string]string, error) {
+	osrelease := make(map[string]string)
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			return nil, errors.New("readOSRelease: invalid input")
+		}
+
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+		if value[0] == '"' {
+			if len(value) < 2 || value[len(value) - 1] != '"' {
+				return nil, errors.New("readOSRelease: invalid input")
+			}
+			value = value[1:len(value) - 1]
+		}
+
+		osrelease[key] = value
+	}
+
+	return osrelease, nil
+}
diff --git a/internal/store/store.go b/internal/store/store.go
index 102dcd301..6816be987 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -396,8 +396,8 @@ func (s *Store) PushCompose(composeID uuid.UUID, bp *blueprint.Blueprint, compos
 	targets := []*target.Target{
 		target.NewLocalTarget(target.NewLocalTargetOptions("/var/lib/osbuild-composer/outputs/" + composeID.String())),
 	}
-	f30 := distro.New("fedora-30")
-	pipeline, err := f30.Pipeline(bp, composeType)
+	d := distro.New("")
+	pipeline, err := d.Pipeline(bp, composeType)
 	if err != nil {
 		return err
 	}
@@ -475,8 +475,8 @@ func (s *Store) GetImage(composeID uuid.UUID) (*Image, error) {
 		if compose.QueueStatus != "FINISHED" {
 			return nil, &InvalidRequestError{"compose was not finished"}
 		}
-		f30 := distro.New("fedora-30")
-		name, mime, err := f30.FilenameFromType(compose.OutputType)
+		d := distro.New("")
+		name, mime, err := d.FilenameFromType(compose.OutputType)
 		if err != nil {
 			panic("invalid output type")
 		}
diff --git a/internal/weldr/api.go b/internal/weldr/api.go
index fb451547a..818c7c1c6 100644
--- a/internal/weldr/api.go
+++ b/internal/weldr/api.go
@@ -850,8 +850,8 @@ func (api *API) composeTypesHandler(writer http.ResponseWriter, request *http.Re
 		Types []composeType `json:"types"`
 	}
 
-	f30 := distro.New("fedora-30")
-	for _, format := range f30.ListOutputFormats() {
+	d := distro.New("")
+	for _, format := range d.ListOutputFormats() {
 		reply.Types = append(reply.Types, composeType{format, true})
 	}
 


Test patch:
diff --git a/internal/distro/osrelease_test.go b/internal/distro/osrelease_test.go
new file mode 100644
index 000000000..490040bf3
--- /dev/null
+++ b/internal/distro/osrelease_test.go
@@ -0,0 +1,54 @@
+package distro
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestOSRelease(t *testing.T) {
+	var cases = []struct {
+		Input     string
+		OSRelease map[string]string
+	}{
+		{
+			``,
+			map[string]string{},
+		},
+		{
+			`NAME=Fedora
+VERSION="30 (Workstation Edition)"
+ID=fedora
+VERSION_ID=30
+VERSION_CODENAME=""
+PLATFORM_ID="platform:f30"
+PRETTY_NAME="Fedora 30 (Workstation Edition)"
+VARIANT="Workstation Edition"
+VARIANT_ID=workstation`,
+			map[string]string{
+				"NAME":             "Fedora",
+				"VERSION":          "30 (Workstation Edition)",
+				"ID":               "fedora",
+				"VERSION_ID":       "30",
+				"VERSION_CODENAME": "",
+				"PLATFORM_ID":      "platform:f30",
+				"PRETTY_NAME":      "Fedora 30 (Workstation Edition)",
+				"VARIANT":          "Workstation Edition",
+				"VARIANT_ID":       "workstation",
+			},
+		},
+	}
+
+	for i, c := range cases {
+		r := strings.NewReader(c.Input)
+
+		osrelease, err := readOSRelease(r)
+		if err != nil {
+			t.Fatalf("%d: readOSRelease: %v", i, err)
+		}
+
+		if !reflect.DeepEqual(osrelease, c.OSRelease) {
+			t.Fatalf("%d: readOSRelease returned unexpected result: %#v", i, osrelease)
+		}
+	}
+}
diff --git a/internal/jobqueue/api_test.go b/internal/jobqueue/api_test.go
index c50c4d508..3de18f145 100644
--- a/internal/jobqueue/api_test.go
+++ b/internal/jobqueue/api_test.go
@@ -101,7 +101,10 @@ func TestCreate(t *testing.T) {
 	store := store.New(nil)
 	api := jobqueue.New(nil, store)
 
-	store.PushCompose(id, &blueprint.Blueprint{}, "tar")
+	err := store.PushCompose(id, &blueprint.Blueprint{}, "tar")
+	if err != nil {
+		t.Fatalf("error pushing compose: %v", err)
+	}
 
 	testRoute(t, api, "POST", "/job-queue/v1/jobs", `{}`, http.StatusCreated,
 		`{"id":"ffffffff-ffff-ffff-ffff-ffffffffffff","pipeline":{"build":{"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["dnf","e2fsprogs","policycoreutils","qemu-img","systemd","grub2-pc","tar"],"releasever":"30","basearch":"x86_64"}}]},"stages":[{"name":"org.osbuild.dnf","options":{"repos":[{"metalink":"https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch","gpgkey":"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n","checksum":"sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97"}],"packages":["policycoreutils","selinux-policy-targeted","kernel","firewalld","chrony","langpacks-en"],"exclude_packages":["dracut-config-rescue"],"releasever":"30","basearch":"x86_64"}},{"name":"org.osbuild.locale","options":{"language":"en_US"}},{"name":"org.osbuild.grub2","options":{"root_fs_uuid":"76a22bf4-f153-4541-b6c7-0332c0dfaeac","boot_fs_uuid":"00000000-0000-0000-0000-000000000000","kernel_opts":"ro biosdevname=0 net.ifnames=0"}},{"name":"org.osbuild.fix-bls","options":{}},{"name":"org.osbuild.selinux","options":{"file_contexts":"etc/selinux/targeted/contexts/files/file_contexts"}}],"assembler":{"name":"org.osbuild.tar","options":{"filename":"root.tar.xz"}}},"targets":[{"name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"}}]}`)
@@ -113,7 +116,10 @@ func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) {
 	api := jobqueue.New(nil, store)
 
 	if from != "VOID" {
-		store.PushCompose(id, &blueprint.Blueprint{}, "tar")
+		err := store.PushCompose(id, &blueprint.Blueprint{}, "tar")
+		if err != nil {
+			t.Fatalf("error pushing compose: %v", err)
+		}
 		if from != "WAITING" {
 			sendHTTP(api, "POST", "/job-queue/v1/jobs", `{}`)
 			if from != "RUNNING" {