Repository: osbuild/osbuild-composer
Pull request #2032 (2021-11-18 09:07:10)

ci: Update Coverity Scan name
curl will URL encode this properly, no need to be encoded first


This pull request includes:

- [ ] adequate testing for the new functionality or fixed issue
- [ ] adequate documentation informing people about the change such as
  - [ ] submit a PR for the [guides](https://github.com/osbuild/guides) repository if this PR changed any behavior described there: https://www.osbuild.org/guides/

<!--
Thanks for proposing a change to osbuild-composer!

Please don't remove the above check list. These are things that each pull
request must have before it is merged. It helps maintainers to not forget
anything.

If the reason for ticking any of the boxes is ambiguous, please add a short
note explaining why.

In addition, if this pull request fixes a downstream issue, please refer to
test/README.md and add these additional items:

- [ ] 1st commit of any `rhbz#` related PR contains bug reproducer; CI reports FAIL or
- [ ] PR contains automated tests for new functionality and
- [ ] QE has approved reproducer/new tests and
- [ ] Subsequent commits provide bug fixes without modifying the reproducer; CI reports PASS and
- [ ] QE approves this PR; RHBZ status is set to `MODIFIED + Verified=Tested`
-->



Patch:
diff --git a/.github/workflows/coverity_scan.yml b/.github/workflows/coverity_scan.yml
index 490696cd7..c7884b79f 100644
--- a/.github/workflows/coverity_scan.yml
+++ b/.github/workflows/coverity_scan.yml
@@ -15,7 +15,7 @@ jobs:
       # https://scan.coverity.com/projects/osbuild-osbuild-composer
       - name: Run coverity scan script
         env:
-          COVERITY_SCAN_PROJECT_NAME: "osbuild%2Fosbuild-composer"
+          COVERITY_SCAN_PROJECT_NAME: "osbuild/osbuild-composer"
           COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }}
           COVERITY_SCAN_EMAIL: ${{ secrets.COVERITY_SCAN_EMAIL }}
         run: |
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ebaec282..7d3d2ab7b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -295,24 +295,9 @@ API:
   parallel:
     matrix:
       - <<: *API_TESTS
-        RUNNER:
-          - aws/fedora-33-x86_64
-          # See COMPOSER-919
-          # - aws/fedora-34-x86_64
-          # See COMPOSER-1118
-          # - aws/centos-stream-8-x86_64
-      - TARGET:
-          - aws
-          - gcp
-          - aws.s3
         RUNNER:
           - aws/centos-stream-8-x86_64
-      - <<: *API_TESTS
-        RUNNER:
           - aws/rhel-8.4-ga-x86_64
-          - aws/rhel-8.6-nightly-x86_64
-          - aws/rhel-9.0-nightly-x86_64
-        INTERNAL_NETWORK: ["true"]
 
 libvirt:
   stage: test
diff --git a/internal/cloudapi/v2/v2.go b/internal/cloudapi/v2/v2.go
index 5ba97170f..c044fd2a3 100644
--- a/internal/cloudapi/v2/v2.go
+++ b/internal/cloudapi/v2/v2.go
@@ -2,9 +2,9 @@
 package v2
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"math/big"
@@ -15,11 +15,13 @@ import (
 	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	"github.com/sirupsen/logrus"
 
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/distroregistry"
+	"github.com/osbuild/osbuild-composer/internal/jobqueue"
 	osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2"
 	"github.com/osbuild/osbuild-composer/internal/ostree"
 	"github.com/osbuild/osbuild-composer/internal/prometheus"
@@ -149,14 +151,6 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		}
 	}
 
-	var imageRequest struct {
-		manifest      distro.Manifest
-		arch          string
-		exports       []string
-		target        *target.Target
-		pipelineNames worker.PipelineNames
-	}
-
 	// use the same seed for all images so we get the same IDs
 	bigSeed, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
 	if err != nil {
@@ -200,30 +194,6 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		return HTTPErrorWithInternal(ErrorEnqueueingJob, err)
 	}
 
-	var depsolveResults worker.DepsolveJobResult
-	for {
-		status, _, err := h.server.workers.JobStatus(depsolveJobID, &depsolveResults)
-		if err != nil {
-			return HTTPErrorWithInternal(ErrorGettingDepsolveJobStatus, err)
-		}
-		if status.Canceled {
-			return HTTPErrorWithInternal(ErrorDepsolveJobCanceled, err)
-		}
-		if !status.Finished.IsZero() {
-			break
-		}
-		time.Sleep(50 * time.Millisecond)
-	}
-
-	if depsolveResults.Error != "" {
-		if depsolveResults.ErrorType == worker.DepsolveErrorType {
-			return HTTPError(ErrorDNFError)
-		}
-		return HTTPErrorWithInternal(ErrorFailedToDepsolve, errors.New(depsolveResults.Error))
-	}
-
-	pkgSpecSets := depsolveResults.PackageSpecs
-
 	imageOptions := distro.ImageOptions{Size: imageType.Size(0)}
 	if request.Customizations != nil && request.Customizations.Subscription != nil {
 		imageOptions.Subscription = &distro.SubscriptionImageOptions{
@@ -279,19 +249,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		}
 	}
 
-	manifest, err := imageType.Manifest(blueprintCustoms, imageOptions, repositories, pkgSpecSets, manifestSeed)
-	if err != nil {
-		return HTTPErrorWithInternal(ErrorFailedToMakeManifest, err)
-	}
-
-	imageRequest.manifest = manifest
-	imageRequest.arch = arch.Name()
-	imageRequest.exports = imageType.Exports()
-	imageRequest.pipelineNames = worker.PipelineNames{
-		Build:   imageType.BuildPipelines(),
-		Payload: imageType.PayloadPipelines(),
-	}
-
+	var irTarget *target.Target
 	/* oneOf is not supported by the openapi generator so marshal and unmarshal the uploadrequest based on the type */
 	switch ir.ImageType {
 	case ImageTypes_aws:
@@ -319,7 +277,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			t.ImageName = key
 		}
 
-		imageRequest.target = t
+		irTarget = t
 	case ImageTypes_edge_installer:
 		fallthrough
 	case ImageTypes_edge_commit:
@@ -342,7 +300,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		})
 		t.ImageName = key
 
-		imageRequest.target = t
+		irTarget = t
 	case ImageTypes_gcp:
 		var gcpUploadOptions GCPUploadOptions
 		jsonUploadOptions, err := json.Marshal(ir.UploadOptions)
@@ -375,7 +333,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			t.ImageName = object
 		}
 
-		imageRequest.target = t
+		irTarget = t
 	case ImageTypes_azure:
 		var azureUploadOptions AzureUploadOptions
 		jsonUploadOptions, err := json.Marshal(ir.UploadOptions)
@@ -401,22 +359,104 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			t.ImageName = fmt.Sprintf("composer-api-%s", uuid.New().String())
 		}
 
-		imageRequest.target = t
+		irTarget = t
 	default:
 		return HTTPError(ErrorUnsupportedImageType)
 	}
 
-	id, err := h.server.workers.EnqueueOSBuild(imageRequest.arch, &worker.OSBuildJob{
-		Manifest:      imageRequest.manifest,
-		Targets:       []*target.Target{imageRequest.target},
-		Exports:       imageRequest.exports,
-		PipelineNames: &imageRequest.pipelineNames,
-	})
+	manifestJobID, err := h.server.workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, depsolveJobID)
+	if err != nil {
+		return HTTPErrorWithInternal(ErrorEnqueueingJob, err)
+	}
+
+	id, err := h.server.workers.EnqueueOSBuildAsDependency(arch.Name(), &worker.OSBuildJob{
+		Targets: []*target.Target{irTarget},
+		Exports: imageType.Exports(),
+	}, manifestJobID)
 	if err != nil {
 		return HTTPErrorWithInternal(ErrorEnqueueingJob, err)
 	}
 
 	ctx.Logger().Infof("Job ID %s enqueued for operationID %s", id, ctx.Get("operationID"))
+	manifestJobContext, manifestCancel := context.WithTimeout(context.Background(), time.Minute*5)
+
+	// start 1 goroutine which requests datajob type
+	go func(workers *worker.Server, manifestJobID uuid.UUID, b *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, seed int64) {
+		defer manifestCancel()
+		// wait until job is in a pending state
+		var token uuid.UUID
+		var dynArgs []json.RawMessage
+		var err error
+		for {
+			_, token, _, _, dynArgs, err = workers.RequestJobById(context.Background(), "", manifestJobID)
+			if err == jobqueue.ErrNotPending {
+				logrus.Debugf("Manifest job %v not pending, waiting for depsolve job to finish", manifestJobID)
+				time.Sleep(time.Millisecond * 50)
+				select {
+				case <-manifestJobContext.Done():
+					logrus.Warnf("Manifest job %v's dependencies took longer than 5 minutes to finish, returning to avoid dangling routines", manifestJobID)
+					return
+				default:
+					continue
+				}
+			}
+			if err != nil {
+				logrus.Errorf("Error requesting manifest job: %v", err)
+				return
+			}
+			break
+		}
+
+		var jobResult *worker.ManifestJobByIDResult = &worker.ManifestJobByIDResult{
+			Manifest: nil,
+			Error:    "",
+		}
+
+		defer func() {
+			if jobResult.Error != "" {
+				logrus.Errorf("Error in manifest job %v: %v", manifestJobID, jobResult.Error)
+			}
+
+			result, err := json.Marshal(jobResult)
+			if err != nil {
+				logrus.Errorf("Error marshalling manifest job %v results: %v", manifestJobID, err)
+			}
+
+			err = workers.FinishJob(token, result)
+			if err != nil {
+				logrus.Errorf("Error finishing manifest job: %v", err)
+			}
+		}()
+
+		if len(dynArgs) == 0 {
+			jobResult.Error = "No dynamic arguments"
+			return
+		}
+
+		var depsolveResults worker.DepsolveJobResult
+		err = json.Unmarshal(dynArgs[0], &depsolveResults)
+		if err != nil {
+			jobResult.Error = "Error parsing dynamic arguments"
+			return
+		}
+
+		if depsolveResults.Error != "" {
+			if depsolveResults.ErrorType == worker.DepsolveErrorType {
+				jobResult.Error = "Error in depsolve job dependency input, bad request"
+				return
+			}
+			jobResult.Error = "Error in depsolve job dependency"
+			return
+		}
+
+		manifest, err := imageType.Manifest(b, options, repos, depsolveResults.PackageSpecs, seed)
+		if err != nil {
+			jobResult.Error = "Error generating manifest"
+			return
+		}
+
+		jobResult.Manifest = manifest
+	}(h.server.workers, manifestJobID, blueprintCustoms, imageOptions, repositories, manifestSeed)
 
 	return ctx.JSON(http.StatusCreated, &ComposeId{
 		ObjectReference: ObjectReference{
diff --git a/internal/worker/json.go b/internal/worker/json.go
index 3cba32372..e194f2e9e 100644
--- a/internal/worker/json.go
+++ b/internal/worker/json.go
@@ -14,7 +14,7 @@ import (
 //
 
 type OSBuildJob struct {
-	Manifest        distro.Manifest  `json:"manifest"`
+	Manifest        distro.Manifest  `json:"manifest,omitempty"`
 	Targets         []*target.Target `json:"targets,omitempty"`
 	ImageName       string           `json:"image_name,omitempty"`
 	StreamOptimized bool             `json:"stream_optimized,omitempty"`
diff --git a/tools/provision.sh b/tools/provision.sh
index 9da0811a3..f5fcb4315 100755
--- a/tools/provision.sh
+++ b/tools/provision.sh
@@ -61,6 +61,24 @@ EOF
     set -x
 fi
 
+# if AWS credentials are defined in the ENV, add them to the worker's configuration
+V2_AWS_ACCESS_KEY_ID="${V2_AWS_ACCESS_KEY_ID:-}"
+V2_AWS_SECRET_ACCESS_KEY="${V2_AWS_SECRET_ACCESS_KEY:-}"
+if [[ -n "$V2_AWS_ACCESS_KEY_ID" && -n "$V2_AWS_SECRET_ACCESS_KEY" ]]; then
+    set +x
+    sudo tee /etc/osbuild-worker/aws-credentials.toml > /dev/null << EOF
+[default]
+aws_access_key_id = "$V2_AWS_ACCESS_KEY_ID"
+aws_secret_access_key = "$V2_AWS_SECRET_ACCESS_KEY"
+EOF
+    sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF
+
+[aws]
+credentials = "/etc/osbuild-worker/aws-credentials.toml"
+EOF
+    set -x
+fi
+
 # Copy rpmrepo snapshots for use in weldr tests
 REPODIR=/etc/osbuild-composer/repositories
 sudo mkdir -p $REPODIR


Test patch:
diff --git a/internal/cloudapi/v2/v2_test.go b/internal/cloudapi/v2/v2_test.go
index 6ec615094..7db56aa6b 100644
--- a/internal/cloudapi/v2/v2_test.go
+++ b/internal/cloudapi/v2/v2_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/distro/test_distro"
 	distro_mock "github.com/osbuild/osbuild-composer/internal/mocks/distro"
 	rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
+	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/test"
 	"github.com/osbuild/osbuild-composer/internal/worker"
 )
@@ -39,9 +40,12 @@ func newV2Server(t *testing.T, dir string) (*v2.Server, *worker.Server, context.
 			if err != nil {
 				continue
 			}
-			rawMsg, err := json.Marshal(&worker.DepsolveJobResult{PackageSpecs: nil, Error: "", ErrorType: worker.ErrorType("")})
+			rawMsg, err := json.Marshal(&worker.DepsolveJobResult{PackageSpecs: map[string][]rpmmd.PackageSpec{"build": []rpmmd.PackageSpec{rpmmd.PackageSpec{Name: "pkg1"}}}, Error: "", ErrorType: worker.ErrorType("")})
 			require.NoError(t, err)
-			require.NoError(t, rpmFixture.Workers.FinishJob(token, rawMsg))
+			err = rpmFixture.Workers.FinishJob(token, rawMsg)
+			if err != nil {
+				return
+			}
 
 			select {
 			case <-depsolveContext.Done():
@@ -272,10 +276,16 @@ func TestComposeStatusSuccess(t *testing.T) {
 		"kind": "ComposeId"
 	}`, "id")
 
-	jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
+	jobId, token, jobType, args, dynArgs, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
 	require.NoError(t, err)
 	require.Equal(t, "osbuild", jobType)
 
+	var osbuildJob worker.OSBuildJob
+	err = json.Unmarshal(args, &osbuildJob)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(osbuildJob.Manifest))
+	require.NotEqual(t, 0, len(dynArgs[0]))
+
 	test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v", jobId), ``, http.StatusOK, fmt.Sprintf(`
 	{
 		"href": "/api/image-builder-composer/v2/composes/%v",
@@ -284,7 +294,6 @@ func TestComposeStatusSuccess(t *testing.T) {
 		"image_status": {"status": "building"}
 	}`, jobId, jobId))
 
-	// todo make it an osbuildjobresult
 	res, err := json.Marshal(&worker.OSBuildJobResult{
 		Success: true,
 	})
@@ -308,7 +317,6 @@ func TestComposeStatusSuccess(t *testing.T) {
 		"code": "IMAGE-BUILDER-COMPOSER-1012",
 		"reason": "OSBuildJobResult does not have expected fields set"
 	}`, "operation_id")
-
 }
 
 func TestComposeStatusFailure(t *testing.T) {
diff --git a/test/cases/api.sh b/test/cases/api.sh
index 2bae8e1ca..6eed1240a 100755
--- a/test/cases/api.sh
+++ b/test/cases/api.sh
@@ -64,10 +64,12 @@ PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PG
 popd
 
 cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
+log_level = "debug"
 [koji]
 allowed_domains = [ "localhost", "client.osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
-
+[koji.aws_config]
+bucket = "${AWS_BUCKET}"
 [worker]
 allowed_domains = [ "localhost", "worker.osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
@@ -353,7 +355,7 @@ case $CLOUD_PROVIDER in
 esac
 
 #
-# Make sure /openapi.json and /version endpoints return success
+# Make sure /openapi and endpoints return success
 #
 
 curl \
@@ -362,15 +364,7 @@ curl \
     --cacert /etc/osbuild-composer/ca-crt.pem \
     --key /etc/osbuild-composer/client-key.pem \
     --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/composer/v1/version | jq .
-
-curl \
-    --silent \
-    --show-error \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/composer/v1/openapi.json | jq .
+    https://localhost/api/image-builder-composer/v2/openapi | jq .
 
 #
 # Prepare a request to be sent to the composer API.
@@ -444,10 +438,10 @@ if [[ "$ID" == "rhel" ]]; then
   SUBSCRIPTION_BLOCK=$(cat <<EndOfMessage
 ,
     "subscription": {
-      "organization": ${API_TEST_SUBSCRIPTION_ORG_ID:-},
-      "activation-key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
-      "base-url": "https://cdn.redhat.com/",
-      "server-url": "subscription.rhsm.redhat.com",
+      "organization": "${API_TEST_SUBSCRIPTION_ORG_ID:-}",
+      "activation_key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
+      "base_url": "https://cdn.redhat.com/",
+      "server_url": "subscription.rhsm.redhat.com",
       "insights": true
     }
 EndOfMessage
@@ -481,30 +475,16 @@ function createReqFileAWS() {
       }
     ]
   },
-  "image_requests": [
-    {
+  "image_request": {
       "architecture": "$ARCH",
-      "image_type": "ami",
+      "image_type": "aws",
       "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_request": {
-          "type": "aws",
-          "options": {
-            "region": "${AWS_REGION}",
-            "s3": {
-              "access_key_id": "${V2_AWS_ACCESS_KEY_ID}",
-              "secret_access_key": "${V2_AWS_SECRET_ACCESS_KEY}",
-              "bucket": "${AWS_BUCKET}"
-            },
-            "ec2": {
-              "access_key_id": "${V2_AWS_ACCESS_KEY_ID}",
-              "secret_access_key": "${V2_AWS_SECRET_ACCESS_KEY}",
-              "snapshot_name": "${AWS_SNAPSHOT_NAME}",
-              "share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"]
-            }
-          }
-      }
+      "upload_options": {
+        "region": "${AWS_REGION}",
+        "snapshot_name": "${AWS_SNAPSHOT_NAME}",
+        "share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"]
     }
-  ]
+  }
 }
 EOF
 }
@@ -523,27 +503,17 @@ function createReqFileAWSS3() {
       "postgresql"
     ]
   },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "rhel-edge-commit",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "ostree": {
-        "ref": "${OSTREE_REF}"
-      },
-      "upload_request": {
-          "type": "aws.s3",
-          "options": {
-            "region": "${AWS_REGION}",
-            "s3": {
-              "access_key_id": "${V2_AWS_ACCESS_KEY_ID}",
-              "secret_access_key": "${V2_AWS_SECRET_ACCESS_KEY}",
-              "bucket": "${AWS_BUCKET}"
-            }
-          }
-      }
+  "image_request": {
+    "architecture": "$ARCH",
+    "image_type": "edge-commit",
+    "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
+    "ostree": {
+      "ref": "${OSTREE_REF}"
+    },
+    "upload_options": {
+      "region": "${AWS_REGION}"
     }
-  ]
+  }
 }
 EOF
 }
@@ -566,22 +536,17 @@ function createReqFileGCP() {
       "postgresql"
     ]${SUBSCRIPTION_BLOCK}
   },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "vhd",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_request": {
-          "type": "gcp",
-          "options": {
-            "bucket": "${GCP_BUCKET}",
-            "region": "${GCP_REGION}",
-            "image_name": "${GCP_IMAGE_NAME}",
-            "share_with_accounts": ["${GCP_API_TEST_SHARE_ACCOUNT}"]
-          }
-      }
+  "image_request": {
+    "architecture": "$ARCH",
+    "image_type": "gcp",
+    "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
+    "upload_options": {
+      "bucket": "${GCP_BUCKET}",
+      "region": "${GCP_REGION}",
+      "image_name": "${GCP_IMAGE_NAME}",
+      "share_with_accounts": ["${GCP_API_TEST_SHARE_ACCOUNT}"]
     }
-  ]
+  }
 }
 EOF
 }
@@ -597,23 +562,18 @@ function createReqFileAzure() {
       "postgresql"
     ]${SUBSCRIPTION_BLOCK}
   },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "vhd",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_request": {
-          "type": "azure",
-          "options": {
-            "tenant_id": "${AZURE_TENANT_ID}",
-            "subscription_id": "${AZURE_SUBSCRIPTION_ID}",
-            "resource_group": "${AZURE_RESOURCE_GROUP}",
-            "location": "${AZURE_LOCATION}",
-            "image_name": "${AZURE_IMAGE_NAME}"
-          }
-      }
+  "image_request": {
+    "architecture": "$ARCH",
+    "image_type": "azure",
+    "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
+    "upload_options": {
+      "tenant_id": "${AZURE_TENANT_ID}",
+      "subscription_id": "${AZURE_SUBSCRIPTION_ID}",
+      "resource_group": "${AZURE_RESOURCE_GROUP}",
+      "location": "${AZURE_LOCATION}",
+      "image_name": "${AZURE_IMAGE_NAME}"
     }
-  ]
+  }
 }
 EOF
 }
@@ -660,10 +620,10 @@ function sendCompose() {
                  --cert /etc/osbuild-composer/client-crt.pem \
                  --header 'Content-Type: application/json' \
                  --request POST \
-                 --data @"$REQUEST_FILE" \
+                 --data @"$1" \
                  --write-out '%{http_code}' \
                  --output "$OUTPUT" \
-                 https://localhost/api/composer/v1/compose)
+                 https://localhost/api/image-builder-composer/v2/compose)
 
     test "$HTTPSTATUS" = "201"
     COMPOSE_ID=$(jq -r '.id' "$OUTPUT")
@@ -671,11 +631,6 @@ function sendCompose() {
 
 function waitForState() {
     local DESIRED_STATE="${1:-success}"
-    local VERSION="${2:-v1}"
-    local URL=https://localhost/api/composer/v1/compose/"$COMPOSE_ID"
-    if [ "$VERSION" = "v2" ]; then
-        URL=https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"
-    fi
 
     while true
     do
@@ -685,7 +640,7 @@ function waitForState() {
                      --cacert /etc/osbuild-composer/ca-crt.pem \
                      --key /etc/osbuild-composer/client-key.pem \
                      --cert /etc/osbuild-composer/client-crt.pem \
-                     "$URL")
+                     "https://localhost/api/image-builder-composer/v2/composes/$COMPOSE_ID")
 
         COMPOSE_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.status')
         UPLOAD_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.status')
@@ -714,9 +669,17 @@ function waitForState() {
     done
 }
 
-sendCompose
+#
+# Make sure that requesting a non existing paquet results in failure
+#
+REQUEST_FILE2="${WORKDIR}/request2.json"
+jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
+
+sendCompose "$REQUEST_FILE2"
+waitForState "failure"
 
 # crashed/stopped/killed worker should result in a failed state
+sendCompose "$REQUEST_FILE"
 waitForState "building"
 sudo systemctl stop "osbuild-worker@*"
 waitForState "failure"
@@ -724,25 +687,23 @@ sudo systemctl start "osbuild-worker@1"
 
 # full integration case
 INIT_COMPOSES="$(collectMetrics)"
-sendCompose
+sendCompose "$REQUEST_FILE"
 waitForState
-# Same state with v2
-waitForState "success" "v2"
 SUBS_COMPOSES="$(collectMetrics)"
 
 test "$UPLOAD_STATUS" = "success"
 test "$UPLOAD_TYPE" = "$CLOUD_PROVIDER"
 test $((INIT_COMPOSES+1)) = "$SUBS_COMPOSES"
 
-# Make sure we get 2 job entries in the db per compose (depsolve + build)
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT * FROM jobs;" | grep "4 rows"
+# Make sure we get 3 job entries in the db per compose (depsolve + manifest + build)
+sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT * FROM jobs;" | grep "9 rows"
 
 #
 # Save the Manifest from the osbuild-composer store
 # NOTE: The rest of the job data can contain sensitive information
 #
 # Suppressing shellcheck.  See https://github.com/koalaman/shellcheck/wiki/SC2024#exceptions
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT args->>'Manifest' FROM jobs" | sudo tee "${ARTIFACTS}/manifest.json"
+sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT result->>'Manifest' FROM jobs" | sudo tee "${ARTIFACTS}/manifest.json"
 
 #
 # Verify the Cloud-provider specific upload_status options
@@ -951,26 +912,8 @@ function verifyInAWSS3() {
       exit 1
   fi
 
-  # verify that the commit hash matches the metadata
-  local API_COMMIT_ID
-  API_COMMIT_ID=$(curl \
-    --silent \
-    --show-error \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/composer/v1/compose/"$COMPOSE_ID"/metadata | jq -r '.ostree_commit')
-
-
   local TAR_COMMIT_ID
   TAR_COMMIT_ID=$(ostree rev-parse --repo "${COMMIT_DIR}/repo" "${OSTREE_REF}")
-
-  if [[ "${API_COMMIT_ID}" != "${TAR_COMMIT_ID}" ]]; then
-      echo "Commit ID returned from API does not match Commit ID in archive 😠"
-      exit 1
-  fi
-
-  # v2 has the same result
   API_COMMIT_ID_V2=$(curl \
     --silent \
     --show-error \
@@ -980,7 +923,7 @@ function verifyInAWSS3() {
     https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata | jq -r '.ostree_commit')
 
   if [[ "${API_COMMIT_ID_V2}" != "${TAR_COMMIT_ID}" ]]; then
-      echo "Commit ID returned from API v2 does not match Commit ID in archive 😠"
+      echo "Commit ID returned from API does not match Commit ID in archive 😠"
       exit 1
   fi
 }
@@ -1122,7 +1065,7 @@ function verifyPackageList() {
       --cacert /etc/osbuild-composer/ca-crt.pem \
       --key /etc/osbuild-composer/client-key.pem \
       --cert /etc/osbuild-composer/client-crt.pem \
-      https://localhost/api/composer/v1/compose/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
+      https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
   local PACKAGENAMES
   PACKAGENAMES=$(jq -rM '.packages[].name' "${ARTIFACTS}/metadata.json")
 
@@ -1134,44 +1077,6 @@ function verifyPackageList() {
 
 verifyPackageList
 
-#
-# Make sure that requesting a non existing paquet returns a 400 error
-#
-REQUEST_FILE2="${WORKDIR}/request2.json"
-jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
-
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/composer/v1/compose)" = "400" ]
-
-#
-# Make sure that a request that makes the dnf-json crash returns a 500 error
-#
-sudo cp -f /usr/libexec/osbuild-composer/dnf-json /usr/libexec/osbuild-composer/dnf-json.bak
-sudo cat << EOF | sudo tee /usr/libexec/osbuild-composer/dnf-json
-#!/usr/bin/python3
-raise Exception()
-EOF
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/composer/v1/compose)" = "500" ]
-
-sudo mv -f /usr/libexec/osbuild-composer/dnf-json.bak /usr/libexec/osbuild-composer/dnf-json
-
 #
 # Verify oauth2
 #
@@ -1226,14 +1131,25 @@ TOKEN="$(curl localhost:8081/token | jq -r .access_token)"
         --output /dev/null \
         --write-out '%{http_code}' \
         --header "Authorization: Bearer $TOKEN" \
-        http://localhost:443/api/composer/v1/version)" = "200" ]
+        http://localhost:443/api/image-builder-composer/v2/openapi)" = "200" ]
 
+# /openapi doesn't need auth
 [ "$(curl \
         --silent \
         --output /dev/null \
         --write-out '%{http_code}' \
         --header "Authorization: Bearer badtoken" \
-        http://localhost:443/api/composer/v1/version)" = "401" ]
+        http://localhost:443/api/image-builder-composer/v2/openapi)" = "200" ]
+
+
+# /composes/$ID does doesn't need auth
+[ "$(curl \
+        --silent \
+        --output /dev/null \
+        --write-out '%{http_code}' \
+        --header "Authorization: Bearer badtoken" \
+        http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "401" ]
+
 
 sudo systemctl start osbuild-remote-worker@localhost:8700.service
 sudo systemctl is-active --quiet osbuild-remote-worker@localhost:8700.service
diff --git a/test/cases/api_v2.sh b/test/cases/api_v2.sh
deleted file mode 100755
index cee6dd58f..000000000
--- a/test/cases/api_v2.sh
+++ /dev/null
@@ -1,739 +0,0 @@
-#!/usr/bin/bash
-
-#
-# Test osbuild-composer's main API endpoint by building a sample image and
-# uploading it to the appropriate cloud provider. The test currently supports
-# AWS and GCP.
-#
-# This script sets `-x` and is meant to always be run like that. This is
-# simpler than adding extensive error reporting, which would make this script
-# considerably more complex. Also, the full trace this produces is very useful
-# for the primary audience: developers of osbuild-composer looking at the log
-# from a run on a remote continuous integration system.
-#
-
-set -euxo pipefail
-
-ARTIFACTS=ci-artifacts
-mkdir -p "${ARTIFACTS}"
-
-source /etc/os-release
-DISTRO_CODE="${DISTRO_CODE:-${ID}_${VERSION_ID//./}}"
-
-# Container image used for cloud provider CLI tools
-CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest"
-
-#
-# Provision the software under test.
-#
-
-/usr/libexec/osbuild-composer-test/provision.sh
-
-#
-# Set up the database queue
-#
-if which podman 2>/dev/null >&2; then
-  CONTAINER_RUNTIME=podman
-elif which docker 2>/dev/null >&2; then
-  CONTAINER_RUNTIME=docker
-else
-  echo No container runtime found, install podman or docker.
-  exit 2
-fi
-
-# Start the db
-sudo ${CONTAINER_RUNTIME} run -d --name osbuild-composer-db \
-    --health-cmd "pg_isready -U postgres -d osbuildcomposer" --health-interval 2s \
-    --health-timeout 2s --health-retries 10 \
-    -e POSTGRES_USER=postgres \
-    -e POSTGRES_PASSWORD=foobar \
-    -e POSTGRES_DB=osbuildcomposer \
-    -p 5432:5432 \
-    quay.io/osbuild/postgres:13-alpine
-
-# Dump the logs once to have a little more output
-sudo ${CONTAINER_RUNTIME} logs osbuild-composer-db
-
-# Initialize a module in a temp dir so we can get tern without introducing
-# vendoring inconsistency
-pushd "$(mktemp -d)"
-sudo dnf install -y go
-go mod init temp
-go get github.com/jackc/tern
-PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PGPORT=5432 \
-      go run github.com/jackc/tern migrate -m /usr/share/tests/osbuild-composer/schemas
-popd
-
-function configure_composer() {
-  cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
-[koji]
-allowed_domains = [ "localhost", "client.osbuild.org" ]
-ca = "/etc/osbuild-composer/ca-crt.pem"
-[koji.aws_config]
-bucket = "${AWS_BUCKET}"
-[worker]
-allowed_domains = [ "localhost", "worker.osbuild.org" ]
-ca = "/etc/osbuild-composer/ca-crt.pem"
-pg_host = "localhost"
-pg_port = "5432"
-pg_database = "osbuildcomposer"
-pg_user = "postgres"
-pg_password = "foobar"
-pg_ssl_mode = "disable"
-EOF
-
-  sudo mkdir -p /etc/osbuild-worker
-  AWS_CREDS_FILE="/etc/osbuild-worker/credentials"
-  cat <<EOF | sudo tee "/etc/osbuild-worker/osbuild-worker.toml"
-[AWS]
-credentials="$AWS_CREDS_FILE"
-EOF
-  cat <<EOF | sudo tee "$AWS_CREDS_FILE"
-[default]
-aws_access_key_id = $V2_AWS_ACCESS_KEY_ID
-aws_secret_access_key = $V2_AWS_SECRET_ACCESS_KEY
-EOF
-  sudo systemctl restart osbuild-composer osbuild-composer-worker@*.service
-}
-
-#
-# Which cloud provider are we testing?
-#
-
-CLOUD_PROVIDER_AWS="aws"
-CLOUD_PROVIDER_GCP="gcp"
-CLOUD_PROVIDER_AZURE="azure"
-CLOUD_PROVIDER_AWS_S3="aws.s3"
-
-CLOUD_PROVIDER=${1:-$CLOUD_PROVIDER_AWS}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    echo "Testing AWS"
-    configure_composer
-    ;;
-  "$CLOUD_PROVIDER_GCP")
-    exit 0
-    ;;
-  "$CLOUD_PROVIDER_AZURE")
-    exit 0
-    ;;
-  "$CLOUD_PROVIDER_AWS_S3")
-    exit 0
-    ;;
-  *)
-    echo "Unknown cloud provider '$CLOUD_PROVIDER'. Supported are '$CLOUD_PROVIDER_AWS', '$CLOUD_PROVIDER_AWS_S3', '$CLOUD_PROVIDER_GCP', '$CLOUD_PROVIDER_AZURE'"
-    exit 1
-    ;;
-esac
-
-#
-# Verify that this script is running in the right environment.
-#
-
-# Check that needed variables are set to access AWS.
-function checkEnvAWS() {
-  printenv AWS_REGION AWS_BUCKET V2_AWS_ACCESS_KEY_ID V2_AWS_SECRET_ACCESS_KEY AWS_API_TEST_SHARE_ACCOUNT > /dev/null
-}
-
-# Check that needed variables are set to register to RHSM (RHEL only)
-function checkEnvSubscription() {
-  printenv API_TEST_SUBSCRIPTION_ORG_ID API_TEST_SUBSCRIPTION_ACTIVATION_KEY > /dev/null
-}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS" | "$CLOUD_PROVIDER_AWS_S3")
-    checkEnvAWS
-    ;;
-esac
-[[ "$ID" == "rhel" ]] && checkEnvSubscription
-
-#
-# Create a temporary directory and ensure it gets deleted when this script
-# terminates in any way.
-#
-
-function cleanupAWS() {
-  # since this function can be called at any time, ensure that we don't expand unbound variables
-  AWS_CMD="${AWS_CMD:-}"
-  AWS_INSTANCE_ID="${AWS_INSTANCE_ID:-}"
-  AMI_IMAGE_ID="${AMI_IMAGE_ID:-}"
-  AWS_SNAPSHOT_ID="${AWS_SNAPSHOT_ID:-}"
-
-  if [ -n "$AWS_CMD" ]; then
-    set +e
-    $AWS_CMD ec2 terminate-instances --instance-ids "$AWS_INSTANCE_ID"
-    $AWS_CMD ec2 deregister-image --image-id "$AMI_IMAGE_ID"
-    $AWS_CMD ec2 delete-snapshot --snapshot-id "$AWS_SNAPSHOT_ID"
-    $AWS_CMD ec2 delete-key-pair --key-name "key-for-$AMI_IMAGE_ID"
-    set -e
-  fi
-}
-
-function cleanupAWSS3() {
-  echo "mock cleanup"
-}
-
-function cleanupGCP() {
-  echo "mock cleanup"
-}
-
-function cleanupAzure() {
-  echo "mock cleanup"
-}
-
-WORKDIR=$(mktemp -d)
-KILL_PIDS=()
-function cleanup() {
-  case $CLOUD_PROVIDER in
-    "$CLOUD_PROVIDER_AWS")
-      cleanupAWS
-      ;;
-    "$CLOUD_PROVIDER_AWS_S3")
-      cleanupAWSS3
-      ;;
-    "$CLOUD_PROVIDER_GCP")
-      cleanupGCP
-      ;;
-    "$CLOUD_PROVIDER_AZURE")
-      cleanupAzure
-      ;;
-  esac
-
-  sudo rm -rf "$WORKDIR"
-
-  for P in "${KILL_PIDS[@]}"; do
-      sudo pkill -P "$P"
-  done
-}
-trap cleanup EXIT
-
-#
-# Install the necessary cloud provider client tools
-#
-
-function installClientAWS() {
-  if ! hash aws; then
-    echo "Using 'awscli' from a container"
-    sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_IMAGE_CLOUD_TOOLS}
-
-    AWS_CMD="sudo ${CONTAINER_RUNTIME} run --rm \
-      -e AWS_ACCESS_KEY_ID=${V2_AWS_ACCESS_KEY_ID} \
-      -e AWS_SECRET_ACCESS_KEY=${V2_AWS_SECRET_ACCESS_KEY} \
-      -v ${WORKDIR}:${WORKDIR}:Z \
-      ${CONTAINER_IMAGE_CLOUD_TOOLS} aws --region $AWS_REGION --output json --color on"
-  else
-    echo "Using pre-installed 'aws' from the system"
-    AWS_CMD="aws --region $AWS_REGION --output json --color on"
-  fi
-  $AWS_CMD --version
-}
-
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS" )
-    installClientAWS
-    ;;
-esac
-
-#
-# Make sure /openapi.json and /version endpoints return success
-#
-
-curl \
-    --silent \
-    --show-error \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/image-builder-composer/v2/openapi | jq .
-
-#
-# Prepare a request to be sent to the composer API.
-#
-
-REQUEST_FILE="${WORKDIR}/request.json"
-ARCH=$(uname -m)
-SSH_USER=
-
-case $(set +x; . /etc/os-release; echo "$ID-$VERSION_ID") in
-  "rhel-9.0")
-    DISTRO="rhel-90"
-    if [[ "$CLOUD_PROVIDER" == "$CLOUD_PROVIDER_AWS" ]]; then
-      SSH_USER="ec2-user"
-    else
-      SSH_USER="cloud-user"
-    fi
-    ;;
-  "rhel-8.6")
-    DISTRO="rhel-85"
-    if [[ "$CLOUD_PROVIDER" == "$CLOUD_PROVIDER_AWS" ]]; then
-      SSH_USER="ec2-user"
-    else
-      SSH_USER="cloud-user"
-    fi
-    ;;
-  "rhel-8.4")
-    DISTRO="rhel-84"
-    SSH_USER="cloud-user"
-    ;;
-  "rhel-8.2" | "rhel-8.3")
-    DISTRO="rhel-8"
-    SSH_USER="cloud-user"
-    ;;
-  "fedora-33")
-    DISTRO="fedora-33"
-    SSH_USER="fedora"
-    ;;
-  "centos-8")
-    DISTRO="centos-8"
-    SSH_USER="cloud-user"
-    ;;
-  "centos-9")
-    DISTRO="centos-9"
-    SSH_USER="cloud-user"
-    ;;
-esac
-
-# Only RHEL need subscription block.
-if [[ "$ID" == "rhel" ]]; then
-  SUBSCRIPTION_BLOCK=$(cat <<EndOfMessage
-,
-    "subscription": {
-      "organization": "${API_TEST_SUBSCRIPTION_ORG_ID:-}",
-      "activation_key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
-      "base_url": "https://cdn.redhat.com/",
-      "server_url": "subscription.rhsm.redhat.com",
-      "insights": true
-    }
-EndOfMessage
-)
-else
-  SUBSCRIPTION_BLOCK=''
-fi
-
-# generate a temp key for user tests
-ssh-keygen -t rsa -f /tmp/usertest -C "usertest" -N ""
-
-function createReqFileAWS() {
-  AWS_SNAPSHOT_NAME=$(uuidgen)
-
-  cat > "$REQUEST_FILE" << EOF
-{
-  "distribution": "$DISTRO",
-  "customizations": {
-    "packages": [
-      "postgresql"
-    ]${SUBSCRIPTION_BLOCK},
-    "users":[
-      {
-        "name": "user1",
-        "groups": ["wheel"],
-        "key": "$(cat /tmp/usertest.pub)"
-      },
-      {
-        "name": "user2",
-        "key": "$(cat /tmp/usertest.pub)"
-      }
-    ]
-  },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "aws",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_options": {
-        "region": "${AWS_REGION}",
-        "share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"],
-        "snapshot_name": "${AWS_SNAPSHOT_NAME}"
-      }
-    }
-  ]
-}
-EOF
-}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    createReqFileAWS
-    ;;
-esac
-
-#
-# Send the request and wait for the job to finish.
-#
-# Separate `curl` and `jq` commands here, because piping them together hides
-# the server's response in case of an error.
-#
-
-function collectMetrics(){
-    METRICS_OUTPUT=$(curl \
-                          --cacert /etc/osbuild-composer/ca-crt.pem \
-                          --key /etc/osbuild-composer/client-key.pem \
-                          --cert /etc/osbuild-composer/client-crt.pem \
-                          https://localhost/metrics)
-
-    echo "$METRICS_OUTPUT" | grep "^total_compose_requests" | cut -f2 -d' '
-}
-
-function sendCompose() {
-
-    if [[ "$ID" == "rhel" ]]; then
-        echo "rhel --------------------------------- "
-        cat "$REQUEST_FILE"
-        echo "rhel --------------------------------- "
-    fi
-    OUTPUT=$(curl \
-                 --silent \
-                 --show-error \
-                 --cacert /etc/osbuild-composer/ca-crt.pem \
-                 --key /etc/osbuild-composer/client-key.pem \
-                 --cert /etc/osbuild-composer/client-crt.pem \
-                 --header 'Content-Type: application/json' \
-                 --request POST \
-                 --data @"$REQUEST_FILE" \
-                 https://localhost/api/image-builder-composer/v2/compose)
-
-    COMPOSE_ID=$(echo "$OUTPUT" | jq -r '.id')
-}
-
-function waitForState() {
-    local DESIRED_STATE="${1:-success}"
-    while true
-    do
-        OUTPUT=$(curl \
-                     --silent \
-                     --show-error \
-                     --cacert /etc/osbuild-composer/ca-crt.pem \
-                     --key /etc/osbuild-composer/client-key.pem \
-                     --cert /etc/osbuild-composer/client-crt.pem \
-                     https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID")
-
-        COMPOSE_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.status')
-        UPLOAD_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.status')
-        UPLOAD_TYPE=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.type')
-        UPLOAD_OPTIONS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.options')
-
-        case "$COMPOSE_STATUS" in
-            "$DESIRED_STATE")
-                break
-                ;;
-            # all valid status values for a compose which hasn't finished yet
-            "pending"|"building"|"uploading"|"registering")
-                ;;
-            # default undesired state
-            "failure")
-                echo "Image compose failed"
-                exit 1
-                ;;
-            *)
-                echo "API returned unexpected image_status.status value: '$COMPOSE_STATUS'"
-                exit 1
-                ;;
-        esac
-
-        sleep 30
-    done
-}
-
-sendCompose
-
-# crashed/stopped/killed worker should result in a failed state
-waitForState "building"
-sudo systemctl stop "osbuild-worker@*"
-waitForState "failure"
-sudo systemctl start "osbuild-worker@1"
-
-# full integration case
-INIT_COMPOSES="$(collectMetrics)"
-sendCompose
-waitForState
-SUBS_COMPOSES="$(collectMetrics)"
-
-test "$UPLOAD_STATUS" = "success"
-test "$UPLOAD_TYPE" = "$CLOUD_PROVIDER"
-test $((INIT_COMPOSES+1)) = "$SUBS_COMPOSES"
-
-
-# Make sure we get 1 job entry in the db per compose
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT COUNT(*) FROM jobs;"
-
-#
-# Save the Manifest from the osbuild-composer store
-# NOTE: The rest of the job data can contain sensitive information
-#
-# Suppressing shellcheck.  See https://github.com/koalaman/shellcheck/wiki/SC2024#exceptions
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT args->>'Manifest' FROM jobs" | sudo tee "${ARTIFACTS}/manifest.json"
-
-#
-# Verify the Cloud-provider specific upload_status options
-#
-
-function checkUploadStatusOptionsAWS() {
-  local AMI
-  AMI=$(echo "$UPLOAD_OPTIONS" | jq -r '.ami')
-  local REGION
-  REGION=$(echo "$UPLOAD_OPTIONS" | jq -r '.region')
-
-  # AWS ID consist of resource identifier followed by a 17-character string
-  echo "$AMI" | grep -e 'ami-[[:alnum:]]\{17\}' -
-  test "$REGION" = "$AWS_REGION"
-}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    checkUploadStatusOptionsAWS
-    ;;
-esac
-
-#
-# Verify the image landed in the appropriate cloud provider, and delete it.
-#
-
-# Reusable function, which waits for a given host to respond to SSH
-function _instanceWaitSSH() {
-  local HOST="$1"
-
-  for LOOP_COUNTER in {0..30}; do
-      if ssh-keyscan "$HOST" > /dev/null 2>&1; then
-          echo "SSH is up!"
-          # ssh-keyscan "$PUBLIC_IP" | sudo tee -a /root/.ssh/known_hosts
-          break
-      fi
-      echo "Retrying in 5 seconds... $LOOP_COUNTER"
-      sleep 5
-  done
-}
-
-function _instanceCheck() {
-  echo "✔️ Instance checking"
-  local _ssh="$1"
-
-  # Check if postgres is installed
-  $_ssh rpm -q postgresql
-
-  # Verify subscribe status. Loop check since the system may not be registered such early(RHEL only)
-  if [[ "$ID" == "rhel" ]]; then
-    set +eu
-    for LOOP_COUNTER in {1..10}; do
-        subscribe_org_id=$($_ssh sudo subscription-manager identity | grep 'org ID')
-        if [[ "$subscribe_org_id" == "org ID: $API_TEST_SUBSCRIPTION_ORG_ID" ]]; then
-            echo "System is subscribed."
-            break
-        else
-            echo "System is not subscribed. Retrying in 30 seconds...($LOOP_COUNTER/10)"
-            sleep 30
-        fi
-    done
-    set -eu
-    [[ "$subscribe_org_id" == "org ID: $API_TEST_SUBSCRIPTION_ORG_ID" ]]
-
-    # Unregister subscription
-    $_ssh sudo subscription-manager unregister
-  else
-    echo "Not RHEL OS. Skip subscription check."
-  fi
-}
-
-# Verify image in EC2 on AWS
-function verifyInAWS() {
-  $AWS_CMD ec2 describe-images \
-    --owners self \
-    --filters Name=name,Values="$AWS_SNAPSHOT_NAME" \
-    > "$WORKDIR/ami.json"
-
-  AMI_IMAGE_ID=$(jq -r '.Images[].ImageId' "$WORKDIR/ami.json")
-  AWS_SNAPSHOT_ID=$(jq -r '.Images[].BlockDeviceMappings[].Ebs.SnapshotId' "$WORKDIR/ami.json")
-  SHARE_OK=1
-
-  # Verify that the ec2 snapshot was shared
-  $AWS_CMD ec2 describe-snapshot-attribute --snapshot-id "$AWS_SNAPSHOT_ID" --attribute createVolumePermission > "$WORKDIR/snapshot-attributes.json"
-
-  SHARED_ID=$(jq -r '.CreateVolumePermissions[0].UserId' "$WORKDIR/snapshot-attributes.json")
-  if [ "$AWS_API_TEST_SHARE_ACCOUNT" != "$SHARED_ID" ]; then
-    SHARE_OK=0
-  fi
-
-  # Verify that the ec2 ami was shared
-  $AWS_CMD ec2 describe-image-attribute --image-id "$AMI_IMAGE_ID" --attribute launchPermission > "$WORKDIR/ami-attributes.json"
-
-  SHARED_ID=$(jq -r '.LaunchPermissions[0].UserId' "$WORKDIR/ami-attributes.json")
-  if [ "$AWS_API_TEST_SHARE_ACCOUNT" != "$SHARED_ID" ]; then
-    SHARE_OK=0
-  fi
-
-  if [ "$SHARE_OK" != 1 ]; then
-    echo "EC2 snapshot wasn't shared with the AWS_API_TEST_SHARE_ACCOUNT. 😢"
-    exit 1
-  fi
-
-  # Create key-pair
-  $AWS_CMD ec2 create-key-pair --key-name "key-for-$AMI_IMAGE_ID" --query 'KeyMaterial' --output text > keypair.pem
-  chmod 400 ./keypair.pem
-
-  # Create an instance based on the ami
-  $AWS_CMD ec2 run-instances --image-id "$AMI_IMAGE_ID" --count 1 --instance-type t2.micro --key-name "key-for-$AMI_IMAGE_ID" > "$WORKDIR/instances.json"
-  AWS_INSTANCE_ID=$(jq -r '.Instances[].InstanceId' "$WORKDIR/instances.json")
-
-  $AWS_CMD ec2 wait instance-running --instance-ids "$AWS_INSTANCE_ID"
-
-  $AWS_CMD ec2 describe-instances --instance-ids "$AWS_INSTANCE_ID" > "$WORKDIR/instances.json"
-  HOST=$(jq -r '.Reservations[].Instances[].PublicIpAddress' "$WORKDIR/instances.json")
-
-  echo "⏱ Waiting for AWS instance to respond to ssh"
-  _instanceWaitSSH "$HOST"
-
-  # Verify image
-  _ssh="ssh -oStrictHostKeyChecking=no -i ./keypair.pem $SSH_USER@$HOST"
-  _instanceCheck "$_ssh"
-
-  # Check access to user1 and user2
-  check_groups=$(ssh -i /tmp/usertest "user1@$HOST" -t 'groups')
-  if [[ $check_groups =~ "wheel" ]]; then
-   echo "✔️  user1 has the group wheel"
-  else
-    echo 'user1 should have the group wheel 😢'
-    exit 1
-  fi
-  check_groups=$(ssh -i /tmp/usertest "user2@$HOST" -t 'groups')
-  if [[ $check_groups =~ "wheel" ]]; then
-    echo 'user2 should not have group wheel 😢'
-    exit 1
-  else
-   echo "✔️  user2 does not have the group wheel"
-  fi
-}
-
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    verifyInAWS
-    ;;
-esac
-
-# Verify selected package (postgresql) is included in package list
-function verifyPackageList() {
-  # Save build metadata to artifacts directory for troubleshooting
-  curl --silent \
-      --show-error \
-      --cacert /etc/osbuild-composer/ca-crt.pem \
-      --key /etc/osbuild-composer/client-key.pem \
-      --cert /etc/osbuild-composer/client-crt.pem \
-      https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
-  local PACKAGENAMES
-  PACKAGENAMES=$(jq -rM '.packages[].name' "${ARTIFACTS}/metadata.json")
-
-  if ! grep -q postgresql <<< "${PACKAGENAMES}"; then
-      echo "'postgresql' not found in compose package list 😠"
-      exit 1
-  fi
-}
-
-verifyPackageList
-
-#
-# Make sure that requesting a non existing paquet returns a 400 error
-#
-REQUEST_FILE2="${WORKDIR}/request2.json"
-jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
-
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/image-builder-composer/v2/compose)" = "400" ]
-
-#
-# Make sure that a request that makes the dnf-json crash returns a 500 error
-#
-sudo cp -f /usr/libexec/osbuild-composer/dnf-json /usr/libexec/osbuild-composer/dnf-json.bak
-sudo cat << EOF | sudo tee /usr/libexec/osbuild-composer/dnf-json
-#!/usr/bin/python3
-raise Exception()
-EOF
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/image-builder-composer/v2/compose)" = "500" ]
-
-sudo mv -f /usr/libexec/osbuild-composer/dnf-json.bak /usr/libexec/osbuild-composer/dnf-json
-
-#
-# Verify oauth2
-#
-cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
-[koji]
-enable_tls = false
-enable_mtls = false
-enable_jwt = true
-jwt_keys_url = "https://localhost:8080/certs"
-jwt_ca_file = "/etc/osbuild-composer/ca-crt.pem"
-jwt_acl_file = ""
-
-[worker]
-allowed_domains = [ "localhost", "worker.osbuild.org" ]
-ca = "/etc/osbuild-composer/ca-crt.pem"
-pg_host = "localhost"
-pg_port = "5432"
-pg_database = "osbuildcomposer"
-pg_user = "postgres"
-pg_password = "foobar"
-pg_ssl_mode = "disable"
-enable_jwt = true
-jwt_keys_url = "https://localhost:8080/certs"
-jwt_ca_file = "/etc/osbuild-composer/ca-crt.pem"
-EOF
-
-cat <<EOF | sudo tee "/etc/osbuild-worker/token"
-offlineToken
-EOF
-
-cat <<EOF | sudo tee "/etc/osbuild-worker/osbuild-worker.toml"
-[authentication]
-oauth_url = http://localhost:8081/token
-offline_token = "/etc/osbuild-worker/token"
-EOF
-
-# Spin up an https instance for the composer-api and worker-api; the auth handler needs to hit an ssl `/certs` endpoint
-sudo /usr/libexec/osbuild-composer-test/osbuild-mock-openid-provider -rsaPubPem /etc/osbuild-composer/client-crt.pem -rsaPem /etc/osbuild-composer/client-key.pem -cert /etc/osbuild-composer/composer-crt.pem -key /etc/osbuild-composer/composer-key.pem &
-KILL_PIDS+=("$!")
-# Spin up an http instance for the worker client to bypass the need to specify an extra CA
-sudo /usr/libexec/osbuild-composer-test/osbuild-mock-openid-provider -a localhost:8081 -rsaPubPem /etc/osbuild-composer/client-crt.pem -rsaPem /etc/osbuild-composer/client-key.pem &
-KILL_PIDS+=("$!")
-
-sudo systemctl restart osbuild-composer
-
-until curl --output /dev/null --silent --fail localhost:8081/token; do
-    sleep 0.5
-done
-TOKEN="$(curl localhost:8081/token | jq -r .access_token)"
-
-[ "$(curl \
-        --silent \
-        --output /dev/null \
-        --write-out '%{http_code}' \
-        --header "Authorization: Bearer $TOKEN" \
-        http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "200" ]
-
-[ "$(curl \
-        --silent \
-        --output /dev/null \
-        --write-out '%{http_code}' \
-        --header "Authorization: Bearer badtoken" \
-        http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "401" ]
-
-sudo systemctl start osbuild-remote-worker@https:--localhost:8700.service
-sudo systemctl is-active --quiet osbuild-remote-worker@https:--localhost:8700.service
-
-exit 0