Repository: osbuild/osbuild-composer
Pull request #947 (2020-08-25 23:38:17)

osbuild: rpm metadata
Parse the metadata about the RPM stage as returned from osbuild.


Patch:
diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go
index 1913f068e..3d8610dcb 100644
--- a/cmd/osbuild-worker/main.go
+++ b/cmd/osbuild-worker/main.go
@@ -17,6 +17,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/osbuild/osbuild-composer/internal/common"
+	"github.com/osbuild/osbuild-composer/internal/osbuild"
 	"github.com/osbuild/osbuild-composer/internal/target"
 	"github.com/osbuild/osbuild-composer/internal/upload/awsupload"
 	"github.com/osbuild/osbuild-composer/internal/upload/azure"
@@ -86,7 +87,7 @@ func openAsStreamOptimizedVmdk(imagePath string) (*os.File, error) {
 	return f, err
 }
 
-func RunJob(job *worker.Job, store string, uploadFunc func(uuid.UUID, string, io.Reader) error) (*common.ComposeResult, error) {
+func RunJob(job *worker.Job, store string, uploadFunc func(uuid.UUID, string, io.Reader) error) (*osbuild.Result, error) {
 	outputDirectory, err := ioutil.TempDir("/var/tmp", "osbuild-worker-*")
 	if err != nil {
 		return nil, fmt.Errorf("error creating temporary output directory: %v", err)
@@ -279,7 +280,7 @@ func main() {
 			// This can happen in cases when OSBuild crashes and doesn't produce
 			// a meaningful output. E.g. when the machine runs of disk space.
 			if result == nil {
-				result = &common.ComposeResult{
+				result = &osbuild.Result{
 					Success: false,
 				}
 			}
diff --git a/cmd/osbuild-worker/osbuild.go b/cmd/osbuild-worker/osbuild.go
index bca387dae..d2c19914b 100644
--- a/cmd/osbuild-worker/osbuild.go
+++ b/cmd/osbuild-worker/osbuild.go
@@ -6,20 +6,20 @@ import (
 	"io"
 	"os/exec"
 
-	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/osbuild"
 )
 
 type OSBuildError struct {
 	Message string
-	Result  *common.ComposeResult
+	Result  *osbuild.Result
 }
 
 func (e *OSBuildError) Error() string {
 	return e.Message
 }
 
-func RunOSBuild(manifest distro.Manifest, store, outputDirectory string, errorWriter io.Writer) (*common.ComposeResult, error) {
+func RunOSBuild(manifest distro.Manifest, store, outputDirectory string, errorWriter io.Writer) (*osbuild.Result, error) {
 	cmd := exec.Command(
 		"osbuild",
 		"--store", store,
@@ -50,7 +50,7 @@ func RunOSBuild(manifest distro.Manifest, store, outputDirectory string, errorWr
 	// FIXME: handle or comment this possible error
 	_ = stdin.Close()
 
-	var result common.ComposeResult
+	var result osbuild.Result
 	err = json.NewDecoder(stdout).Decode(&result)
 	if err != nil {
 		return nil, fmt.Errorf("error decoding osbuild output: %#v", err)
diff --git a/internal/common/compose_result.go b/internal/common/compose_result.go
deleted file mode 100644
index fcbff3f65..000000000
--- a/internal/common/compose_result.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package common
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-)
-
-type assembler struct {
-	Name    string          `json:"name"`
-	Options json.RawMessage `json:"options"`
-	Success bool            `json:"success"`
-	Output  string          `json:"output"`
-}
-
-type stage struct {
-	Name    string          `json:"name"`
-	Options json.RawMessage `json:"options"`
-	Success bool            `json:"success"`
-	Output  string          `json:"output"`
-}
-
-type build struct {
-	Stages  []stage `json:"stages"`
-	TreeID  string  `json:"tree_id"`
-	Success bool    `json:"success"`
-}
-
-type ComposeResult struct {
-	TreeID    string     `json:"tree_id"`
-	OutputID  string     `json:"output_id"`
-	Build     *build     `json:"build"`
-	Stages    []stage    `json:"stages"`
-	Assembler *assembler `json:"assembler"`
-	Success   bool       `json:"success"`
-}
-
-func (cr *ComposeResult) Write(writer io.Writer) error {
-	if cr.Build == nil && len(cr.Stages) == 0 && cr.Assembler == nil {
-		fmt.Fprintf(writer, "The compose result is empty.\n")
-	}
-
-	if cr.Build != nil {
-		fmt.Fprintf(writer, "Build pipeline:\n")
-
-		for _, stage := range cr.Build.Stages {
-			fmt.Fprintf(writer, "Stage %s\n", stage.Name)
-			enc := json.NewEncoder(writer)
-			enc.SetIndent("", "  ")
-			err := enc.Encode(stage.Options)
-			if err != nil {
-				return err
-			}
-			fmt.Fprintf(writer, "\nOutput:\n%s\n", stage.Output)
-		}
-	}
-
-	if len(cr.Stages) > 0 {
-		fmt.Fprintf(writer, "Stages:\n")
-		for _, stage := range cr.Stages {
-			fmt.Fprintf(writer, "Stage: %s\n", stage.Name)
-			enc := json.NewEncoder(writer)
-			enc.SetIndent("", "  ")
-			err := enc.Encode(stage.Options)
-			if err != nil {
-				return err
-			}
-			fmt.Fprintf(writer, "\nOutput:\n%s\n", stage.Output)
-		}
-	}
-
-	if cr.Assembler != nil {
-		fmt.Fprintf(writer, "Assembler %s:\n", cr.Assembler.Name)
-		enc := json.NewEncoder(writer)
-		enc.SetIndent("", "  ")
-		err := enc.Encode(cr.Assembler.Options)
-		if err != nil {
-			return err
-		}
-		fmt.Fprintf(writer, "\nOutput:\n%s\n", cr.Assembler.Output)
-	}
-
-	return nil
-}
diff --git a/internal/osbuild/result.go b/internal/osbuild/result.go
new file mode 100644
index 000000000..fe917983c
--- /dev/null
+++ b/internal/osbuild/result.go
@@ -0,0 +1,125 @@
+package osbuild
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+type rawAssemblerResult struct {
+	Name    string          `json:"name"`
+	Options json.RawMessage `json:"options"`
+	Success bool            `json:"success"`
+	Output  string          `json:"output"`
+}
+
+type StageResult struct {
+	Name     string          `json:"name"`
+	Options  json.RawMessage `json:"options"`
+	Success  bool            `json:"success"`
+	Output   string          `json:"output"`
+	Metadata StageMetadata   `json:"metadata"`
+}
+
+// StageMetadata specify the metadata of a given stage-type.
+type StageMetadata interface {
+	isStageMetadata()
+}
+
+type rawStageResult struct {
+	Name     string          `json:"name"`
+	Options  json.RawMessage `json:"options"`
+	Success  bool            `json:"success"`
+	Output   string          `json:"output"`
+	Metadata json.RawMessage `json:"metadata"`
+}
+
+type buildResult struct {
+	Stages  []StageResult `json:"stages"`
+	TreeID  string        `json:"tree_id"`
+	Success bool          `json:"success"`
+}
+
+type Result struct {
+	TreeID    string              `json:"tree_id"`
+	OutputID  string              `json:"output_id"`
+	Build     *buildResult        `json:"build"`
+	Stages    []StageResult       `json:"stages"`
+	Assembler *rawAssemblerResult `json:"assembler"`
+	Success   bool                `json:"success"`
+}
+
+func (result *StageResult) UnmarshalJSON(data []byte) error {
+	var rawStageResult rawStageResult
+	err := json.Unmarshal(data, &rawStageResult)
+	if err != nil {
+		return err
+	}
+	var metadata StageMetadata
+	switch rawStageResult.Name {
+	case "org.osbuild.rpm":
+		metadata = new(RPMStageMetadata)
+		err = json.Unmarshal(rawStageResult.Metadata, metadata)
+		if err != nil {
+			return err
+		}
+	default:
+		metadata = nil
+	}
+
+	result.Name = rawStageResult.Name
+	result.Options = rawStageResult.Options
+	result.Success = rawStageResult.Success
+	result.Output = rawStageResult.Output
+	result.Metadata = metadata
+
+	return nil
+}
+
+func (cr *Result) Write(writer io.Writer) error {
+	if cr.Build == nil && len(cr.Stages) == 0 && cr.Assembler == nil {
+		fmt.Fprintf(writer, "The compose result is empty.\n")
+	}
+
+	if cr.Build != nil {
+		fmt.Fprintf(writer, "Build pipeline:\n")
+
+		for _, stage := range cr.Build.Stages {
+			fmt.Fprintf(writer, "Stage %s\n", stage.Name)
+			enc := json.NewEncoder(writer)
+			enc.SetIndent("", "  ")
+			err := enc.Encode(stage.Options)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(writer, "\nOutput:\n%s\n", stage.Output)
+		}
+	}
+
+	if len(cr.Stages) > 0 {
+		fmt.Fprintf(writer, "Stages:\n")
+		for _, stage := range cr.Stages {
+			fmt.Fprintf(writer, "Stage: %s\n", stage.Name)
+			enc := json.NewEncoder(writer)
+			enc.SetIndent("", "  ")
+			err := enc.Encode(stage.Options)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(writer, "\nOutput:\n%s\n", stage.Output)
+		}
+	}
+
+	if cr.Assembler != nil {
+		fmt.Fprintf(writer, "Assembler %s:\n", cr.Assembler.Name)
+		enc := json.NewEncoder(writer)
+		enc.SetIndent("", "  ")
+		err := enc.Encode(cr.Assembler.Options)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(writer, "\nOutput:\n%s\n", cr.Assembler.Output)
+	}
+
+	return nil
+}
diff --git a/internal/osbuild/rpm_stage.go b/internal/osbuild/rpm_stage.go
index 71cbcd20f..b4979cf4d 100644
--- a/internal/osbuild/rpm_stage.go
+++ b/internal/osbuild/rpm_stage.go
@@ -28,3 +28,20 @@ func NewRPMStage(options *RPMStageOptions) *Stage {
 		Options: options,
 	}
 }
+
+// RPMStageMetadata gives the set of packages installed by the RPM stage
+type RPMStageMetadata struct {
+	Packages []RPMPackageMetadata `json:"packages"`
+}
+
+// RPMPackageMetadata contains the metadata extracted from one RPM header
+type RPMPackageMetadata struct {
+	Name    string  `json:"name"`
+	Version string  `json:"version"`
+	Release string  `json:"release"`
+	Epoch   *string `json:"epoch"`
+	Arch    string  `json:"arch"`
+	SigMD5  string  `json:"sigmd5"`
+}
+
+func (RPMStageMetadata) isStageMetadata() {}
diff --git a/internal/weldr/api.go b/internal/weldr/api.go
index 243242a1b..299f2600e 100644
--- a/internal/weldr/api.go
+++ b/internal/weldr/api.go
@@ -27,6 +27,7 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/jobqueue"
+	"github.com/osbuild/osbuild-composer/internal/osbuild"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/store"
 	"github.com/osbuild/osbuild-composer/internal/target"
@@ -165,7 +166,7 @@ type composeStatus struct {
 	Queued   time.Time
 	Started  time.Time
 	Finished time.Time
-	Result   *common.ComposeResult
+	Result   *osbuild.Result
 }
 
 // Returns the state of the image in `compose` and the times the job was
@@ -194,7 +195,7 @@ func (api *API) getComposeStatus(compose store.Compose) *composeStatus {
 			Queued:   compose.ImageBuild.JobCreated,
 			Started:  compose.ImageBuild.JobStarted,
 			Finished: compose.ImageBuild.JobFinished,
-			Result:   &common.ComposeResult{},
+			Result:   &osbuild.Result{},
 		}
 	}
 
diff --git a/internal/worker/client.go b/internal/worker/client.go
index b369a98ad..042488f77 100644
--- a/internal/worker/client.go
+++ b/internal/worker/client.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/osbuild"
 	"github.com/osbuild/osbuild-composer/internal/target"
 )
 
@@ -111,7 +112,7 @@ func (c *Client) JobCanceled(job *Job) bool {
 	return jr.Canceled
 }
 
-func (c *Client) UpdateJob(job *Job, status common.ImageBuildState, result *common.ComposeResult) error {
+func (c *Client) UpdateJob(job *Job, status common.ImageBuildState, result *osbuild.Result) error {
 	var b bytes.Buffer
 	err := json.NewEncoder(&b).Encode(&updateJobRequest{status, result})
 	if err != nil {
diff --git a/internal/worker/json.go b/internal/worker/json.go
index 774d0c82e..6f969721b 100644
--- a/internal/worker/json.go
+++ b/internal/worker/json.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/osbuild"
 	"github.com/osbuild/osbuild-composer/internal/target"
 )
 
@@ -18,7 +19,7 @@ type OSBuildJob struct {
 }
 
 type OSBuildJobResult struct {
-	OSBuildOutput *common.ComposeResult `json:"osbuild_output,omitempty"`
+	OSBuildOutput *osbuild.Result `json:"osbuild_output,omitempty"`
 }
 
 //
@@ -49,7 +50,7 @@ type jobResponse struct {
 
 type updateJobRequest struct {
 	Status common.ImageBuildState `json:"status"`
-	Result *common.ComposeResult  `json:"result"`
+	Result *osbuild.Result        `json:"result"`
 }
 
 type updateJobResponse struct {


Test patch:
diff --git a/internal/common/compose_result_test.go b/internal/common/compose_result_test.go
deleted file mode 100644
index 501ee171c..000000000
--- a/internal/common/compose_result_test.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package common
-
-import (
-	"bytes"
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestWriteFull(t *testing.T) {
-
-	const testOptions = `{"msg": "test"}`
-
-	testStage := stage{
-		Name:    "testStage",
-		Options: []byte(testOptions),
-		Success: true,
-		Output:  "Finished",
-	}
-
-	testBuild := build{
-		Stages:  []stage{testStage},
-		TreeID:  "treeID",
-		Success: true,
-	}
-
-	testAssembler := assembler{
-		Name:    "testAssembler",
-		Options: []byte(testOptions),
-		Success: true,
-		Output:  "Done",
-	}
-
-	testComposeResult := ComposeResult{
-		TreeID:    "TreeID",
-		OutputID:  "OutputID",
-		Build:     &testBuild,
-		Stages:    []stage{testStage},
-		Assembler: &testAssembler,
-		Success:   true,
-	}
-
-	var b bytes.Buffer
-	assert.NoError(t, testComposeResult.Write(&b))
-	expectedMessage :=
-		`Build pipeline:
-Stage testStage
-{
-  "msg": "test"
-}
-
-Output:
-Finished
-Stages:
-Stage: testStage
-{
-  "msg": "test"
-}
-
-Output:
-Finished
-Assembler testAssembler:
-{
-  "msg": "test"
-}
-
-Output:
-Done
-`
-	assert.Equal(t, expectedMessage, b.String())
-}
-
-func TestWriteEmpty(t *testing.T) {
-
-	testComposeResult := ComposeResult{}
-
-	var b bytes.Buffer
-	assert.NoError(t, testComposeResult.Write(&b))
-	assert.Equal(t, "The compose result is empty.\n", b.String())
-
-}
diff --git a/internal/osbuild/result_test.go b/internal/osbuild/result_test.go
new file mode 100644
index 000000000..98fed411b
--- /dev/null
+++ b/internal/osbuild/result_test.go
@@ -0,0 +1,153 @@
+package osbuild
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnmarshall(t *testing.T) {
+	resultRaw := `{
+		"success": true,
+		"build": {
+		  "success": true,
+		  "stages": [
+			{
+			  "name": "org.osbuild.rpm",
+			  "id": "9eb0a6f6fd6e2995e107f5bcc6aa3b19643b02ec133bdc8a8ac614860b1bbf2d",
+			  "success": true,
+			  "output": "Building...",
+			  "metadata": {
+				"packages": [
+				  {
+					"name": "libgcc",
+					"version": "10.0.1",
+					"release": "0.11.fc32",
+					"epoch": null,
+					"arch": "x86_64",
+					"sigmd5": "84fc907a5047aeebaf8da1642925a417"
+				  },
+				  {
+					"name": "whois-nls",
+					"version": "5.5.6",
+					"release": "1.fc32",
+					"epoch": null,
+					"arch": "noarch",
+					"sigmd5": "f868cd02046630c8ce3a9c48820e2437"
+				  }
+				]
+			  }
+			}
+		  ]
+		}
+	  }`
+
+	var result Result
+	err := json.Unmarshal([]byte(resultRaw), &result)
+	assert.NoError(t, err)
+
+	assert.Equal(t, result.Build.Stages[0].Name, "org.osbuild.rpm")
+	metadata, ok := result.Build.Stages[0].Metadata.(*RPMStageMetadata)
+	assert.True(t, ok)
+	package1 := metadata.Packages[0]
+	assert.Equal(t, package1.Name, "libgcc")
+	assert.Nil(t, package1.Epoch)
+	assert.Equal(t, package1.Version, "10.0.1")
+	assert.Equal(t, package1.Release, "0.11.fc32")
+	assert.Equal(t, package1.Arch, "x86_64")
+	assert.Equal(t, package1.SigMD5, "84fc907a5047aeebaf8da1642925a417")
+}
+
+func TestWriteFull(t *testing.T) {
+
+	const testOptions = `{"msg": "test"}`
+
+	dnfStage := StageResult{
+		Name:    "org.osbuild.rpm",
+		Options: []byte(testOptions),
+		Success: true,
+		Output:  "Finished",
+		Metadata: RPMStageMetadata{
+			Packages: []RPMPackageMetadata{
+				{
+					Name:    "foobar",
+					Epoch:   nil,
+					Version: "1",
+					Release: "1",
+					Arch:    "noarch",
+					SigMD5:  "deadbeef",
+				},
+			},
+		},
+	}
+
+	testStage := StageResult{
+		Name:    "org.osbuild.test",
+		Options: []byte(testOptions),
+		Success: true,
+		Output:  "Finished",
+	}
+
+	testBuild := buildResult{
+		Stages:  []StageResult{testStage},
+		TreeID:  "treeID",
+		Success: true,
+	}
+
+	testAssembler := rawAssemblerResult{
+		Name:    "testAssembler",
+		Options: []byte(testOptions),
+		Success: true,
+		Output:  "Done",
+	}
+
+	testComposeResult := Result{
+		TreeID:    "TreeID",
+		OutputID:  "OutputID",
+		Build:     &testBuild,
+		Stages:    []StageResult{dnfStage},
+		Assembler: &testAssembler,
+		Success:   true,
+	}
+
+	var b bytes.Buffer
+	assert.NoError(t, testComposeResult.Write(&b))
+	expectedMessage :=
+		`Build pipeline:
+Stage org.osbuild.test
+{
+  "msg": "test"
+}
+
+Output:
+Finished
+Stages:
+Stage: org.osbuild.rpm
+{
+  "msg": "test"
+}
+
+Output:
+Finished
+Assembler testAssembler:
+{
+  "msg": "test"
+}
+
+Output:
+Done
+`
+	assert.Equal(t, expectedMessage, b.String())
+}
+
+func TestWriteEmpty(t *testing.T) {
+
+	testComposeResult := Result{}
+
+	var b bytes.Buffer
+	assert.NoError(t, testComposeResult.Write(&b))
+	assert.Equal(t, "The compose result is empty.\n", b.String())
+
+}