Repository: osbuild/osbuild-composer
Pull request #2026 (2021-11-16 19:27:56)

Switch api tests to v2 & manifest job in api v2

Gundersanne: @teg I wonder if we want some sort of hard timeout for the manifest job; I can imagine just like 100 manifest goroutines spinning endlessly. Or at least start printing warning messages after a certain amount of time so we're aware.

Ideally we'd capture this in metrics as well but we're only starting with per-job-type metrics.

Gundersanne: ergh had a little error in subscriptions again

Patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2ebaec282..7d3d2ab7b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -295,24 +295,9 @@ API:
   parallel:
     matrix:
       - <<: *API_TESTS
-        RUNNER:
-          - aws/fedora-33-x86_64
-          # See COMPOSER-919
-          # - aws/fedora-34-x86_64
-          # See COMPOSER-1118
-          # - aws/centos-stream-8-x86_64
-      - TARGET:
-          - aws
-          - gcp
-          - aws.s3
         RUNNER:
           - aws/centos-stream-8-x86_64
-      - <<: *API_TESTS
-        RUNNER:
           - aws/rhel-8.4-ga-x86_64
-          - aws/rhel-8.6-nightly-x86_64
-          - aws/rhel-9.0-nightly-x86_64
-        INTERNAL_NETWORK: ["true"]
 
 libvirt:
   stage: test
diff --git a/internal/cloudapi/v2/v2.go b/internal/cloudapi/v2/v2.go
index 5ba97170f..c044fd2a3 100644
--- a/internal/cloudapi/v2/v2.go
+++ b/internal/cloudapi/v2/v2.go
@@ -2,9 +2,9 @@
 package v2
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"math/big"
@@ -15,11 +15,13 @@ import (
 	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	"github.com/sirupsen/logrus"
 
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	"github.com/osbuild/osbuild-composer/internal/common"
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/distroregistry"
+	"github.com/osbuild/osbuild-composer/internal/jobqueue"
 	osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2"
 	"github.com/osbuild/osbuild-composer/internal/ostree"
 	"github.com/osbuild/osbuild-composer/internal/prometheus"
@@ -149,14 +151,6 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		}
 	}
 
-	var imageRequest struct {
-		manifest      distro.Manifest
-		arch          string
-		exports       []string
-		target        *target.Target
-		pipelineNames worker.PipelineNames
-	}
-
 	// use the same seed for all images so we get the same IDs
 	bigSeed, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
 	if err != nil {
@@ -200,30 +194,6 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		return HTTPErrorWithInternal(ErrorEnqueueingJob, err)
 	}
 
-	var depsolveResults worker.DepsolveJobResult
-	for {
-		status, _, err := h.server.workers.JobStatus(depsolveJobID, &depsolveResults)
-		if err != nil {
-			return HTTPErrorWithInternal(ErrorGettingDepsolveJobStatus, err)
-		}
-		if status.Canceled {
-			return HTTPErrorWithInternal(ErrorDepsolveJobCanceled, err)
-		}
-		if !status.Finished.IsZero() {
-			break
-		}
-		time.Sleep(50 * time.Millisecond)
-	}
-
-	if depsolveResults.Error != "" {
-		if depsolveResults.ErrorType == worker.DepsolveErrorType {
-			return HTTPError(ErrorDNFError)
-		}
-		return HTTPErrorWithInternal(ErrorFailedToDepsolve, errors.New(depsolveResults.Error))
-	}
-
-	pkgSpecSets := depsolveResults.PackageSpecs
-
 	imageOptions := distro.ImageOptions{Size: imageType.Size(0)}
 	if request.Customizations != nil && request.Customizations.Subscription != nil {
 		imageOptions.Subscription = &distro.SubscriptionImageOptions{
@@ -279,19 +249,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		}
 	}
 
-	manifest, err := imageType.Manifest(blueprintCustoms, imageOptions, repositories, pkgSpecSets, manifestSeed)
-	if err != nil {
-		return HTTPErrorWithInternal(ErrorFailedToMakeManifest, err)
-	}
-
-	imageRequest.manifest = manifest
-	imageRequest.arch = arch.Name()
-	imageRequest.exports = imageType.Exports()
-	imageRequest.pipelineNames = worker.PipelineNames{
-		Build:   imageType.BuildPipelines(),
-		Payload: imageType.PayloadPipelines(),
-	}
-
+	var irTarget *target.Target
 	/* oneOf is not supported by the openapi generator so marshal and unmarshal the uploadrequest based on the type */
 	switch ir.ImageType {
 	case ImageTypes_aws:
@@ -319,7 +277,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			t.ImageName = key
 		}
 
-		imageRequest.target = t
+		irTarget = t
 	case ImageTypes_edge_installer:
 		fallthrough
 	case ImageTypes_edge_commit:
@@ -342,7 +300,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 		})
 		t.ImageName = key
 
-		imageRequest.target = t
+		irTarget = t
 	case ImageTypes_gcp:
 		var gcpUploadOptions GCPUploadOptions
 		jsonUploadOptions, err := json.Marshal(ir.UploadOptions)
@@ -375,7 +333,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			t.ImageName = object
 		}
 
-		imageRequest.target = t
+		irTarget = t
 	case ImageTypes_azure:
 		var azureUploadOptions AzureUploadOptions
 		jsonUploadOptions, err := json.Marshal(ir.UploadOptions)
@@ -401,22 +359,104 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
 			t.ImageName = fmt.Sprintf("composer-api-%s", uuid.New().String())
 		}
 
-		imageRequest.target = t
+		irTarget = t
 	default:
 		return HTTPError(ErrorUnsupportedImageType)
 	}
 
-	id, err := h.server.workers.EnqueueOSBuild(imageRequest.arch, &worker.OSBuildJob{
-		Manifest:      imageRequest.manifest,
-		Targets:       []*target.Target{imageRequest.target},
-		Exports:       imageRequest.exports,
-		PipelineNames: &imageRequest.pipelineNames,
-	})
+	manifestJobID, err := h.server.workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, depsolveJobID)
+	if err != nil {
+		return HTTPErrorWithInternal(ErrorEnqueueingJob, err)
+	}
+
+	id, err := h.server.workers.EnqueueOSBuildAsDependency(arch.Name(), &worker.OSBuildJob{
+		Targets: []*target.Target{irTarget},
+		Exports: imageType.Exports(),
+	}, manifestJobID)
 	if err != nil {
 		return HTTPErrorWithInternal(ErrorEnqueueingJob, err)
 	}
 
 	ctx.Logger().Infof("Job ID %s enqueued for operationID %s", id, ctx.Get("operationID"))
+	manifestJobContext, manifestCancel := context.WithTimeout(context.Background(), time.Minute*5)
+
+	// start 1 goroutine which requests datajob type
+	go func(workers *worker.Server, manifestJobID uuid.UUID, b *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, seed int64) {
+		defer manifestCancel()
+		// wait until job is in a pending state
+		var token uuid.UUID
+		var dynArgs []json.RawMessage
+		var err error
+		for {
+			_, token, _, _, dynArgs, err = workers.RequestJobById(context.Background(), "", manifestJobID)
+			if err == jobqueue.ErrNotPending {
+				logrus.Debugf("Manifest job %v not pending, waiting for depsolve job to finish", manifestJobID)
+				time.Sleep(time.Millisecond * 50)
+				select {
+				case <-manifestJobContext.Done():
+					logrus.Warnf("Manifest job %v's dependencies took longer than 5 minutes to finish, returning to avoid dangling routines", manifestJobID)
+					return
+				default:
+					continue
+				}
+			}
+			if err != nil {
+				logrus.Errorf("Error requesting manifest job: %v", err)
+				return
+			}
+			break
+		}
+
+		var jobResult *worker.ManifestJobByIDResult = &worker.ManifestJobByIDResult{
+			Manifest: nil,
+			Error:    "",
+		}
+
+		defer func() {
+			if jobResult.Error != "" {
+				logrus.Errorf("Error in manifest job %v: %v", manifestJobID, jobResult.Error)
+			}
+
+			result, err := json.Marshal(jobResult)
+			if err != nil {
+				logrus.Errorf("Error marshalling manifest job %v results: %v", manifestJobID, err)
+			}
+
+			err = workers.FinishJob(token, result)
+			if err != nil {
+				logrus.Errorf("Error finishing manifest job: %v", err)
+			}
+		}()
+
+		if len(dynArgs) == 0 {
+			jobResult.Error = "No dynamic arguments"
+			return
+		}
+
+		var depsolveResults worker.DepsolveJobResult
+		err = json.Unmarshal(dynArgs[0], &depsolveResults)
+		if err != nil {
+			jobResult.Error = "Error parsing dynamic arguments"
+			return
+		}
+
+		if depsolveResults.Error != "" {
+			if depsolveResults.ErrorType == worker.DepsolveErrorType {
+				jobResult.Error = "Error in depsolve job dependency input, bad request"
+				return
+			}
+			jobResult.Error = "Error in depsolve job dependency"
+			return
+		}
+
+		manifest, err := imageType.Manifest(b, options, repos, depsolveResults.PackageSpecs, seed)
+		if err != nil {
+			jobResult.Error = "Error generating manifest"
+			return
+		}
+
+		jobResult.Manifest = manifest
+	}(h.server.workers, manifestJobID, blueprintCustoms, imageOptions, repositories, manifestSeed)
 
 	return ctx.JSON(http.StatusCreated, &ComposeId{
 		ObjectReference: ObjectReference{
diff --git a/internal/worker/json.go b/internal/worker/json.go
index 3cba32372..e194f2e9e 100644
--- a/internal/worker/json.go
+++ b/internal/worker/json.go
@@ -14,7 +14,7 @@ import (
 //
 
 type OSBuildJob struct {
-	Manifest        distro.Manifest  `json:"manifest"`
+	Manifest        distro.Manifest  `json:"manifest,omitempty"`
 	Targets         []*target.Target `json:"targets,omitempty"`
 	ImageName       string           `json:"image_name,omitempty"`
 	StreamOptimized bool             `json:"stream_optimized,omitempty"`
diff --git a/tools/provision.sh b/tools/provision.sh
index 9da0811a3..f5fcb4315 100755
--- a/tools/provision.sh
+++ b/tools/provision.sh
@@ -61,6 +61,24 @@ EOF
     set -x
 fi
 
+# if AWS credentials are defined in the ENV, add them to the worker's configuration
+V2_AWS_ACCESS_KEY_ID="${V2_AWS_ACCESS_KEY_ID:-}"
+V2_AWS_SECRET_ACCESS_KEY="${V2_AWS_SECRET_ACCESS_KEY:-}"
+if [[ -n "$V2_AWS_ACCESS_KEY_ID" && -n "$V2_AWS_SECRET_ACCESS_KEY" ]]; then
+    set +x
+    sudo tee /etc/osbuild-worker/aws-credentials.toml > /dev/null << EOF
+[default]
+aws_access_key_id = "$V2_AWS_ACCESS_KEY_ID"
+aws_secret_access_key = "$V2_AWS_SECRET_ACCESS_KEY"
+EOF
+    sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF
+
+[aws]
+credentials = "/etc/osbuild-worker/aws-credentials.toml"
+EOF
+    set -x
+fi
+
 # Copy rpmrepo snapshots for use in weldr tests
 REPODIR=/etc/osbuild-composer/repositories
 sudo mkdir -p $REPODIR


Test patch:
diff --git a/internal/cloudapi/v2/v2_test.go b/internal/cloudapi/v2/v2_test.go
index 6ec615094..7db56aa6b 100644
--- a/internal/cloudapi/v2/v2_test.go
+++ b/internal/cloudapi/v2/v2_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/distro/test_distro"
 	distro_mock "github.com/osbuild/osbuild-composer/internal/mocks/distro"
 	rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
+	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/test"
 	"github.com/osbuild/osbuild-composer/internal/worker"
 )
@@ -39,9 +40,12 @@ func newV2Server(t *testing.T, dir string) (*v2.Server, *worker.Server, context.
 			if err != nil {
 				continue
 			}
-			rawMsg, err := json.Marshal(&worker.DepsolveJobResult{PackageSpecs: nil, Error: "", ErrorType: worker.ErrorType("")})
+			rawMsg, err := json.Marshal(&worker.DepsolveJobResult{PackageSpecs: map[string][]rpmmd.PackageSpec{"build": []rpmmd.PackageSpec{rpmmd.PackageSpec{Name: "pkg1"}}}, Error: "", ErrorType: worker.ErrorType("")})
 			require.NoError(t, err)
-			require.NoError(t, rpmFixture.Workers.FinishJob(token, rawMsg))
+			err = rpmFixture.Workers.FinishJob(token, rawMsg)
+			if err != nil {
+				return
+			}
 
 			select {
 			case <-depsolveContext.Done():
@@ -272,10 +276,16 @@ func TestComposeStatusSuccess(t *testing.T) {
 		"kind": "ComposeId"
 	}`, "id")
 
-	jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
+	jobId, token, jobType, args, dynArgs, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
 	require.NoError(t, err)
 	require.Equal(t, "osbuild", jobType)
 
+	var osbuildJob worker.OSBuildJob
+	err = json.Unmarshal(args, &osbuildJob)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(osbuildJob.Manifest))
+	require.NotEqual(t, 0, len(dynArgs[0]))
+
 	test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v", jobId), ``, http.StatusOK, fmt.Sprintf(`
 	{
 		"href": "/api/image-builder-composer/v2/composes/%v",
@@ -284,7 +294,6 @@ func TestComposeStatusSuccess(t *testing.T) {
 		"image_status": {"status": "building"}
 	}`, jobId, jobId))
 
-	// todo make it an osbuildjobresult
 	res, err := json.Marshal(&worker.OSBuildJobResult{
 		Success: true,
 	})
@@ -308,7 +317,6 @@ func TestComposeStatusSuccess(t *testing.T) {
 		"code": "IMAGE-BUILDER-COMPOSER-1012",
 		"reason": "OSBuildJobResult does not have expected fields set"
 	}`, "operation_id")
-
 }
 
 func TestComposeStatusFailure(t *testing.T) {
diff --git a/test/cases/api.sh b/test/cases/api.sh
index 2bae8e1ca..6eed1240a 100755
--- a/test/cases/api.sh
+++ b/test/cases/api.sh
@@ -64,10 +64,12 @@ PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PG
 popd
 
 cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
+log_level = "debug"
 [koji]
 allowed_domains = [ "localhost", "client.osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
-
+[koji.aws_config]
+bucket = "${AWS_BUCKET}"
 [worker]
 allowed_domains = [ "localhost", "worker.osbuild.org" ]
 ca = "/etc/osbuild-composer/ca-crt.pem"
@@ -353,7 +355,7 @@ case $CLOUD_PROVIDER in
 esac
 
 #
-# Make sure /openapi.json and /version endpoints return success
+# Make sure /openapi and endpoints return success
 #
 
 curl \
@@ -362,15 +364,7 @@ curl \
     --cacert /etc/osbuild-composer/ca-crt.pem \
     --key /etc/osbuild-composer/client-key.pem \
     --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/composer/v1/version | jq .
-
-curl \
-    --silent \
-    --show-error \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/composer/v1/openapi.json | jq .
+    https://localhost/api/image-builder-composer/v2/openapi | jq .
 
 #
 # Prepare a request to be sent to the composer API.
@@ -444,10 +438,10 @@ if [[ "$ID" == "rhel" ]]; then
   SUBSCRIPTION_BLOCK=$(cat <<EndOfMessage
 ,
     "subscription": {
-      "organization": ${API_TEST_SUBSCRIPTION_ORG_ID:-},
-      "activation-key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
-      "base-url": "https://cdn.redhat.com/",
-      "server-url": "subscription.rhsm.redhat.com",
+      "organization": "${API_TEST_SUBSCRIPTION_ORG_ID:-}",
+      "activation_key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
+      "base_url": "https://cdn.redhat.com/",
+      "server_url": "subscription.rhsm.redhat.com",
       "insights": true
     }
 EndOfMessage
@@ -481,30 +475,16 @@ function createReqFileAWS() {
       }
     ]
   },
-  "image_requests": [
-    {
+  "image_request": {
       "architecture": "$ARCH",
-      "image_type": "ami",
+      "image_type": "aws",
       "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_request": {
-          "type": "aws",
-          "options": {
-            "region": "${AWS_REGION}",
-            "s3": {
-              "access_key_id": "${V2_AWS_ACCESS_KEY_ID}",
-              "secret_access_key": "${V2_AWS_SECRET_ACCESS_KEY}",
-              "bucket": "${AWS_BUCKET}"
-            },
-            "ec2": {
-              "access_key_id": "${V2_AWS_ACCESS_KEY_ID}",
-              "secret_access_key": "${V2_AWS_SECRET_ACCESS_KEY}",
-              "snapshot_name": "${AWS_SNAPSHOT_NAME}",
-              "share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"]
-            }
-          }
-      }
+      "upload_options": {
+        "region": "${AWS_REGION}",
+        "snapshot_name": "${AWS_SNAPSHOT_NAME}",
+        "share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"]
     }
-  ]
+  }
 }
 EOF
 }
@@ -523,27 +503,17 @@ function createReqFileAWSS3() {
       "postgresql"
     ]
   },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "rhel-edge-commit",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "ostree": {
-        "ref": "${OSTREE_REF}"
-      },
-      "upload_request": {
-          "type": "aws.s3",
-          "options": {
-            "region": "${AWS_REGION}",
-            "s3": {
-              "access_key_id": "${V2_AWS_ACCESS_KEY_ID}",
-              "secret_access_key": "${V2_AWS_SECRET_ACCESS_KEY}",
-              "bucket": "${AWS_BUCKET}"
-            }
-          }
-      }
+  "image_request": {
+    "architecture": "$ARCH",
+    "image_type": "edge-commit",
+    "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
+    "ostree": {
+      "ref": "${OSTREE_REF}"
+    },
+    "upload_options": {
+      "region": "${AWS_REGION}"
     }
-  ]
+  }
 }
 EOF
 }
@@ -566,22 +536,17 @@ function createReqFileGCP() {
       "postgresql"
     ]${SUBSCRIPTION_BLOCK}
   },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "vhd",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_request": {
-          "type": "gcp",
-          "options": {
-            "bucket": "${GCP_BUCKET}",
-            "region": "${GCP_REGION}",
-            "image_name": "${GCP_IMAGE_NAME}",
-            "share_with_accounts": ["${GCP_API_TEST_SHARE_ACCOUNT}"]
-          }
-      }
+  "image_request": {
+    "architecture": "$ARCH",
+    "image_type": "gcp",
+    "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
+    "upload_options": {
+      "bucket": "${GCP_BUCKET}",
+      "region": "${GCP_REGION}",
+      "image_name": "${GCP_IMAGE_NAME}",
+      "share_with_accounts": ["${GCP_API_TEST_SHARE_ACCOUNT}"]
     }
-  ]
+  }
 }
 EOF
 }
@@ -597,23 +562,18 @@ function createReqFileAzure() {
       "postgresql"
     ]${SUBSCRIPTION_BLOCK}
   },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "vhd",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_request": {
-          "type": "azure",
-          "options": {
-            "tenant_id": "${AZURE_TENANT_ID}",
-            "subscription_id": "${AZURE_SUBSCRIPTION_ID}",
-            "resource_group": "${AZURE_RESOURCE_GROUP}",
-            "location": "${AZURE_LOCATION}",
-            "image_name": "${AZURE_IMAGE_NAME}"
-          }
-      }
+  "image_request": {
+    "architecture": "$ARCH",
+    "image_type": "azure",
+    "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
+    "upload_options": {
+      "tenant_id": "${AZURE_TENANT_ID}",
+      "subscription_id": "${AZURE_SUBSCRIPTION_ID}",
+      "resource_group": "${AZURE_RESOURCE_GROUP}",
+      "location": "${AZURE_LOCATION}",
+      "image_name": "${AZURE_IMAGE_NAME}"
     }
-  ]
+  }
 }
 EOF
 }
@@ -660,10 +620,10 @@ function sendCompose() {
                  --cert /etc/osbuild-composer/client-crt.pem \
                  --header 'Content-Type: application/json' \
                  --request POST \
-                 --data @"$REQUEST_FILE" \
+                 --data @"$1" \
                  --write-out '%{http_code}' \
                  --output "$OUTPUT" \
-                 https://localhost/api/composer/v1/compose)
+                 https://localhost/api/image-builder-composer/v2/compose)
 
     test "$HTTPSTATUS" = "201"
     COMPOSE_ID=$(jq -r '.id' "$OUTPUT")
@@ -671,11 +631,6 @@ function sendCompose() {
 
 function waitForState() {
     local DESIRED_STATE="${1:-success}"
-    local VERSION="${2:-v1}"
-    local URL=https://localhost/api/composer/v1/compose/"$COMPOSE_ID"
-    if [ "$VERSION" = "v2" ]; then
-        URL=https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"
-    fi
 
     while true
     do
@@ -685,7 +640,7 @@ function waitForState() {
                      --cacert /etc/osbuild-composer/ca-crt.pem \
                      --key /etc/osbuild-composer/client-key.pem \
                      --cert /etc/osbuild-composer/client-crt.pem \
-                     "$URL")
+                     "https://localhost/api/image-builder-composer/v2/composes/$COMPOSE_ID")
 
         COMPOSE_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.status')
         UPLOAD_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.status')
@@ -714,9 +669,17 @@ function waitForState() {
     done
 }
 
-sendCompose
+#
+# Make sure that requesting a non existing paquet results in failure
+#
+REQUEST_FILE2="${WORKDIR}/request2.json"
+jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
+
+sendCompose "$REQUEST_FILE2"
+waitForState "failure"
 
 # crashed/stopped/killed worker should result in a failed state
+sendCompose "$REQUEST_FILE"
 waitForState "building"
 sudo systemctl stop "osbuild-worker@*"
 waitForState "failure"
@@ -724,25 +687,23 @@ sudo systemctl start "osbuild-worker@1"
 
 # full integration case
 INIT_COMPOSES="$(collectMetrics)"
-sendCompose
+sendCompose "$REQUEST_FILE"
 waitForState
-# Same state with v2
-waitForState "success" "v2"
 SUBS_COMPOSES="$(collectMetrics)"
 
 test "$UPLOAD_STATUS" = "success"
 test "$UPLOAD_TYPE" = "$CLOUD_PROVIDER"
 test $((INIT_COMPOSES+1)) = "$SUBS_COMPOSES"
 
-# Make sure we get 2 job entries in the db per compose (depsolve + build)
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT * FROM jobs;" | grep "4 rows"
+# Make sure we get 3 job entries in the db per compose (depsolve + manifest + build)
+sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT * FROM jobs;" | grep "9 rows"
 
 #
 # Save the Manifest from the osbuild-composer store
 # NOTE: The rest of the job data can contain sensitive information
 #
 # Suppressing shellcheck.  See https://github.com/koalaman/shellcheck/wiki/SC2024#exceptions
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT args->>'Manifest' FROM jobs" | sudo tee "${ARTIFACTS}/manifest.json"
+sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT result->>'Manifest' FROM jobs" | sudo tee "${ARTIFACTS}/manifest.json"
 
 #
 # Verify the Cloud-provider specific upload_status options
@@ -951,26 +912,8 @@ function verifyInAWSS3() {
       exit 1
   fi
 
-  # verify that the commit hash matches the metadata
-  local API_COMMIT_ID
-  API_COMMIT_ID=$(curl \
-    --silent \
-    --show-error \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/composer/v1/compose/"$COMPOSE_ID"/metadata | jq -r '.ostree_commit')
-
-
   local TAR_COMMIT_ID
   TAR_COMMIT_ID=$(ostree rev-parse --repo "${COMMIT_DIR}/repo" "${OSTREE_REF}")
-
-  if [[ "${API_COMMIT_ID}" != "${TAR_COMMIT_ID}" ]]; then
-      echo "Commit ID returned from API does not match Commit ID in archive 😠"
-      exit 1
-  fi
-
-  # v2 has the same result
   API_COMMIT_ID_V2=$(curl \
     --silent \
     --show-error \
@@ -980,7 +923,7 @@ function verifyInAWSS3() {
     https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata | jq -r '.ostree_commit')
 
   if [[ "${API_COMMIT_ID_V2}" != "${TAR_COMMIT_ID}" ]]; then
-      echo "Commit ID returned from API v2 does not match Commit ID in archive 😠"
+      echo "Commit ID returned from API does not match Commit ID in archive 😠"
       exit 1
   fi
 }
@@ -1122,7 +1065,7 @@ function verifyPackageList() {
       --cacert /etc/osbuild-composer/ca-crt.pem \
       --key /etc/osbuild-composer/client-key.pem \
       --cert /etc/osbuild-composer/client-crt.pem \
-      https://localhost/api/composer/v1/compose/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
+      https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
   local PACKAGENAMES
   PACKAGENAMES=$(jq -rM '.packages[].name' "${ARTIFACTS}/metadata.json")
 
@@ -1134,44 +1077,6 @@ function verifyPackageList() {
 
 verifyPackageList
 
-#
-# Make sure that requesting a non existing paquet returns a 400 error
-#
-REQUEST_FILE2="${WORKDIR}/request2.json"
-jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
-
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/composer/v1/compose)" = "400" ]
-
-#
-# Make sure that a request that makes the dnf-json crash returns a 500 error
-#
-sudo cp -f /usr/libexec/osbuild-composer/dnf-json /usr/libexec/osbuild-composer/dnf-json.bak
-sudo cat << EOF | sudo tee /usr/libexec/osbuild-composer/dnf-json
-#!/usr/bin/python3
-raise Exception()
-EOF
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/composer/v1/compose)" = "500" ]
-
-sudo mv -f /usr/libexec/osbuild-composer/dnf-json.bak /usr/libexec/osbuild-composer/dnf-json
-
 #
 # Verify oauth2
 #
@@ -1226,14 +1131,25 @@ TOKEN="$(curl localhost:8081/token | jq -r .access_token)"
         --output /dev/null \
         --write-out '%{http_code}' \
         --header "Authorization: Bearer $TOKEN" \
-        http://localhost:443/api/composer/v1/version)" = "200" ]
+        http://localhost:443/api/image-builder-composer/v2/openapi)" = "200" ]
 
+# /openapi doesn't need auth
 [ "$(curl \
         --silent \
         --output /dev/null \
         --write-out '%{http_code}' \
         --header "Authorization: Bearer badtoken" \
-        http://localhost:443/api/composer/v1/version)" = "401" ]
+        http://localhost:443/api/image-builder-composer/v2/openapi)" = "200" ]
+
+
+# /composes/$ID does doesn't need auth
+[ "$(curl \
+        --silent \
+        --output /dev/null \
+        --write-out '%{http_code}' \
+        --header "Authorization: Bearer badtoken" \
+        http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "401" ]
+
 
 sudo systemctl start osbuild-remote-worker@localhost:8700.service
 sudo systemctl is-active --quiet osbuild-remote-worker@localhost:8700.service
diff --git a/test/cases/api_v2.sh b/test/cases/api_v2.sh
deleted file mode 100755
index cee6dd58f..000000000
--- a/test/cases/api_v2.sh
+++ /dev/null
@@ -1,739 +0,0 @@
-#!/usr/bin/bash
-
-#
-# Test osbuild-composer's main API endpoint by building a sample image and
-# uploading it to the appropriate cloud provider. The test currently supports
-# AWS and GCP.
-#
-# This script sets `-x` and is meant to always be run like that. This is
-# simpler than adding extensive error reporting, which would make this script
-# considerably more complex. Also, the full trace this produces is very useful
-# for the primary audience: developers of osbuild-composer looking at the log
-# from a run on a remote continuous integration system.
-#
-
-set -euxo pipefail
-
-ARTIFACTS=ci-artifacts
-mkdir -p "${ARTIFACTS}"
-
-source /etc/os-release
-DISTRO_CODE="${DISTRO_CODE:-${ID}_${VERSION_ID//./}}"
-
-# Container image used for cloud provider CLI tools
-CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest"
-
-#
-# Provision the software under test.
-#
-
-/usr/libexec/osbuild-composer-test/provision.sh
-
-#
-# Set up the database queue
-#
-if which podman 2>/dev/null >&2; then
-  CONTAINER_RUNTIME=podman
-elif which docker 2>/dev/null >&2; then
-  CONTAINER_RUNTIME=docker
-else
-  echo No container runtime found, install podman or docker.
-  exit 2
-fi
-
-# Start the db
-sudo ${CONTAINER_RUNTIME} run -d --name osbuild-composer-db \
-    --health-cmd "pg_isready -U postgres -d osbuildcomposer" --health-interval 2s \
-    --health-timeout 2s --health-retries 10 \
-    -e POSTGRES_USER=postgres \
-    -e POSTGRES_PASSWORD=foobar \
-    -e POSTGRES_DB=osbuildcomposer \
-    -p 5432:5432 \
-    quay.io/osbuild/postgres:13-alpine
-
-# Dump the logs once to have a little more output
-sudo ${CONTAINER_RUNTIME} logs osbuild-composer-db
-
-# Initialize a module in a temp dir so we can get tern without introducing
-# vendoring inconsistency
-pushd "$(mktemp -d)"
-sudo dnf install -y go
-go mod init temp
-go get github.com/jackc/tern
-PGUSER=postgres PGPASSWORD=foobar PGDATABASE=osbuildcomposer PGHOST=localhost PGPORT=5432 \
-      go run github.com/jackc/tern migrate -m /usr/share/tests/osbuild-composer/schemas
-popd
-
-function configure_composer() {
-  cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
-[koji]
-allowed_domains = [ "localhost", "client.osbuild.org" ]
-ca = "/etc/osbuild-composer/ca-crt.pem"
-[koji.aws_config]
-bucket = "${AWS_BUCKET}"
-[worker]
-allowed_domains = [ "localhost", "worker.osbuild.org" ]
-ca = "/etc/osbuild-composer/ca-crt.pem"
-pg_host = "localhost"
-pg_port = "5432"
-pg_database = "osbuildcomposer"
-pg_user = "postgres"
-pg_password = "foobar"
-pg_ssl_mode = "disable"
-EOF
-
-  sudo mkdir -p /etc/osbuild-worker
-  AWS_CREDS_FILE="/etc/osbuild-worker/credentials"
-  cat <<EOF | sudo tee "/etc/osbuild-worker/osbuild-worker.toml"
-[AWS]
-credentials="$AWS_CREDS_FILE"
-EOF
-  cat <<EOF | sudo tee "$AWS_CREDS_FILE"
-[default]
-aws_access_key_id = $V2_AWS_ACCESS_KEY_ID
-aws_secret_access_key = $V2_AWS_SECRET_ACCESS_KEY
-EOF
-  sudo systemctl restart osbuild-composer osbuild-composer-worker@*.service
-}
-
-#
-# Which cloud provider are we testing?
-#
-
-CLOUD_PROVIDER_AWS="aws"
-CLOUD_PROVIDER_GCP="gcp"
-CLOUD_PROVIDER_AZURE="azure"
-CLOUD_PROVIDER_AWS_S3="aws.s3"
-
-CLOUD_PROVIDER=${1:-$CLOUD_PROVIDER_AWS}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    echo "Testing AWS"
-    configure_composer
-    ;;
-  "$CLOUD_PROVIDER_GCP")
-    exit 0
-    ;;
-  "$CLOUD_PROVIDER_AZURE")
-    exit 0
-    ;;
-  "$CLOUD_PROVIDER_AWS_S3")
-    exit 0
-    ;;
-  *)
-    echo "Unknown cloud provider '$CLOUD_PROVIDER'. Supported are '$CLOUD_PROVIDER_AWS', '$CLOUD_PROVIDER_AWS_S3', '$CLOUD_PROVIDER_GCP', '$CLOUD_PROVIDER_AZURE'"
-    exit 1
-    ;;
-esac
-
-#
-# Verify that this script is running in the right environment.
-#
-
-# Check that needed variables are set to access AWS.
-function checkEnvAWS() {
-  printenv AWS_REGION AWS_BUCKET V2_AWS_ACCESS_KEY_ID V2_AWS_SECRET_ACCESS_KEY AWS_API_TEST_SHARE_ACCOUNT > /dev/null
-}
-
-# Check that needed variables are set to register to RHSM (RHEL only)
-function checkEnvSubscription() {
-  printenv API_TEST_SUBSCRIPTION_ORG_ID API_TEST_SUBSCRIPTION_ACTIVATION_KEY > /dev/null
-}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS" | "$CLOUD_PROVIDER_AWS_S3")
-    checkEnvAWS
-    ;;
-esac
-[[ "$ID" == "rhel" ]] && checkEnvSubscription
-
-#
-# Create a temporary directory and ensure it gets deleted when this script
-# terminates in any way.
-#
-
-function cleanupAWS() {
-  # since this function can be called at any time, ensure that we don't expand unbound variables
-  AWS_CMD="${AWS_CMD:-}"
-  AWS_INSTANCE_ID="${AWS_INSTANCE_ID:-}"
-  AMI_IMAGE_ID="${AMI_IMAGE_ID:-}"
-  AWS_SNAPSHOT_ID="${AWS_SNAPSHOT_ID:-}"
-
-  if [ -n "$AWS_CMD" ]; then
-    set +e
-    $AWS_CMD ec2 terminate-instances --instance-ids "$AWS_INSTANCE_ID"
-    $AWS_CMD ec2 deregister-image --image-id "$AMI_IMAGE_ID"
-    $AWS_CMD ec2 delete-snapshot --snapshot-id "$AWS_SNAPSHOT_ID"
-    $AWS_CMD ec2 delete-key-pair --key-name "key-for-$AMI_IMAGE_ID"
-    set -e
-  fi
-}
-
-function cleanupAWSS3() {
-  echo "mock cleanup"
-}
-
-function cleanupGCP() {
-  echo "mock cleanup"
-}
-
-function cleanupAzure() {
-  echo "mock cleanup"
-}
-
-WORKDIR=$(mktemp -d)
-KILL_PIDS=()
-function cleanup() {
-  case $CLOUD_PROVIDER in
-    "$CLOUD_PROVIDER_AWS")
-      cleanupAWS
-      ;;
-    "$CLOUD_PROVIDER_AWS_S3")
-      cleanupAWSS3
-      ;;
-    "$CLOUD_PROVIDER_GCP")
-      cleanupGCP
-      ;;
-    "$CLOUD_PROVIDER_AZURE")
-      cleanupAzure
-      ;;
-  esac
-
-  sudo rm -rf "$WORKDIR"
-
-  for P in "${KILL_PIDS[@]}"; do
-      sudo pkill -P "$P"
-  done
-}
-trap cleanup EXIT
-
-#
-# Install the necessary cloud provider client tools
-#
-
-function installClientAWS() {
-  if ! hash aws; then
-    echo "Using 'awscli' from a container"
-    sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_IMAGE_CLOUD_TOOLS}
-
-    AWS_CMD="sudo ${CONTAINER_RUNTIME} run --rm \
-      -e AWS_ACCESS_KEY_ID=${V2_AWS_ACCESS_KEY_ID} \
-      -e AWS_SECRET_ACCESS_KEY=${V2_AWS_SECRET_ACCESS_KEY} \
-      -v ${WORKDIR}:${WORKDIR}:Z \
-      ${CONTAINER_IMAGE_CLOUD_TOOLS} aws --region $AWS_REGION --output json --color on"
-  else
-    echo "Using pre-installed 'aws' from the system"
-    AWS_CMD="aws --region $AWS_REGION --output json --color on"
-  fi
-  $AWS_CMD --version
-}
-
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS" )
-    installClientAWS
-    ;;
-esac
-
-#
-# Make sure /openapi.json and /version endpoints return success
-#
-
-curl \
-    --silent \
-    --show-error \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    https://localhost/api/image-builder-composer/v2/openapi | jq .
-
-#
-# Prepare a request to be sent to the composer API.
-#
-
-REQUEST_FILE="${WORKDIR}/request.json"
-ARCH=$(uname -m)
-SSH_USER=
-
-case $(set +x; . /etc/os-release; echo "$ID-$VERSION_ID") in
-  "rhel-9.0")
-    DISTRO="rhel-90"
-    if [[ "$CLOUD_PROVIDER" == "$CLOUD_PROVIDER_AWS" ]]; then
-      SSH_USER="ec2-user"
-    else
-      SSH_USER="cloud-user"
-    fi
-    ;;
-  "rhel-8.6")
-    DISTRO="rhel-85"
-    if [[ "$CLOUD_PROVIDER" == "$CLOUD_PROVIDER_AWS" ]]; then
-      SSH_USER="ec2-user"
-    else
-      SSH_USER="cloud-user"
-    fi
-    ;;
-  "rhel-8.4")
-    DISTRO="rhel-84"
-    SSH_USER="cloud-user"
-    ;;
-  "rhel-8.2" | "rhel-8.3")
-    DISTRO="rhel-8"
-    SSH_USER="cloud-user"
-    ;;
-  "fedora-33")
-    DISTRO="fedora-33"
-    SSH_USER="fedora"
-    ;;
-  "centos-8")
-    DISTRO="centos-8"
-    SSH_USER="cloud-user"
-    ;;
-  "centos-9")
-    DISTRO="centos-9"
-    SSH_USER="cloud-user"
-    ;;
-esac
-
-# Only RHEL need subscription block.
-if [[ "$ID" == "rhel" ]]; then
-  SUBSCRIPTION_BLOCK=$(cat <<EndOfMessage
-,
-    "subscription": {
-      "organization": "${API_TEST_SUBSCRIPTION_ORG_ID:-}",
-      "activation_key": "${API_TEST_SUBSCRIPTION_ACTIVATION_KEY:-}",
-      "base_url": "https://cdn.redhat.com/",
-      "server_url": "subscription.rhsm.redhat.com",
-      "insights": true
-    }
-EndOfMessage
-)
-else
-  SUBSCRIPTION_BLOCK=''
-fi
-
-# generate a temp key for user tests
-ssh-keygen -t rsa -f /tmp/usertest -C "usertest" -N ""
-
-function createReqFileAWS() {
-  AWS_SNAPSHOT_NAME=$(uuidgen)
-
-  cat > "$REQUEST_FILE" << EOF
-{
-  "distribution": "$DISTRO",
-  "customizations": {
-    "packages": [
-      "postgresql"
-    ]${SUBSCRIPTION_BLOCK},
-    "users":[
-      {
-        "name": "user1",
-        "groups": ["wheel"],
-        "key": "$(cat /tmp/usertest.pub)"
-      },
-      {
-        "name": "user2",
-        "key": "$(cat /tmp/usertest.pub)"
-      }
-    ]
-  },
-  "image_requests": [
-    {
-      "architecture": "$ARCH",
-      "image_type": "aws",
-      "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json),
-      "upload_options": {
-        "region": "${AWS_REGION}",
-        "share_with_accounts": ["${AWS_API_TEST_SHARE_ACCOUNT}"],
-        "snapshot_name": "${AWS_SNAPSHOT_NAME}"
-      }
-    }
-  ]
-}
-EOF
-}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    createReqFileAWS
-    ;;
-esac
-
-#
-# Send the request and wait for the job to finish.
-#
-# Separate `curl` and `jq` commands here, because piping them together hides
-# the server's response in case of an error.
-#
-
-function collectMetrics(){
-    METRICS_OUTPUT=$(curl \
-                          --cacert /etc/osbuild-composer/ca-crt.pem \
-                          --key /etc/osbuild-composer/client-key.pem \
-                          --cert /etc/osbuild-composer/client-crt.pem \
-                          https://localhost/metrics)
-
-    echo "$METRICS_OUTPUT" | grep "^total_compose_requests" | cut -f2 -d' '
-}
-
-function sendCompose() {
-
-    if [[ "$ID" == "rhel" ]]; then
-        echo "rhel --------------------------------- "
-        cat "$REQUEST_FILE"
-        echo "rhel --------------------------------- "
-    fi
-    OUTPUT=$(curl \
-                 --silent \
-                 --show-error \
-                 --cacert /etc/osbuild-composer/ca-crt.pem \
-                 --key /etc/osbuild-composer/client-key.pem \
-                 --cert /etc/osbuild-composer/client-crt.pem \
-                 --header 'Content-Type: application/json' \
-                 --request POST \
-                 --data @"$REQUEST_FILE" \
-                 https://localhost/api/image-builder-composer/v2/compose)
-
-    COMPOSE_ID=$(echo "$OUTPUT" | jq -r '.id')
-}
-
-function waitForState() {
-    local DESIRED_STATE="${1:-success}"
-    while true
-    do
-        OUTPUT=$(curl \
-                     --silent \
-                     --show-error \
-                     --cacert /etc/osbuild-composer/ca-crt.pem \
-                     --key /etc/osbuild-composer/client-key.pem \
-                     --cert /etc/osbuild-composer/client-crt.pem \
-                     https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID")
-
-        COMPOSE_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.status')
-        UPLOAD_STATUS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.status')
-        UPLOAD_TYPE=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.type')
-        UPLOAD_OPTIONS=$(echo "$OUTPUT" | jq -r '.image_status.upload_status.options')
-
-        case "$COMPOSE_STATUS" in
-            "$DESIRED_STATE")
-                break
-                ;;
-            # all valid status values for a compose which hasn't finished yet
-            "pending"|"building"|"uploading"|"registering")
-                ;;
-            # default undesired state
-            "failure")
-                echo "Image compose failed"
-                exit 1
-                ;;
-            *)
-                echo "API returned unexpected image_status.status value: '$COMPOSE_STATUS'"
-                exit 1
-                ;;
-        esac
-
-        sleep 30
-    done
-}
-
-sendCompose
-
-# crashed/stopped/killed worker should result in a failed state
-waitForState "building"
-sudo systemctl stop "osbuild-worker@*"
-waitForState "failure"
-sudo systemctl start "osbuild-worker@1"
-
-# full integration case
-INIT_COMPOSES="$(collectMetrics)"
-sendCompose
-waitForState
-SUBS_COMPOSES="$(collectMetrics)"
-
-test "$UPLOAD_STATUS" = "success"
-test "$UPLOAD_TYPE" = "$CLOUD_PROVIDER"
-test $((INIT_COMPOSES+1)) = "$SUBS_COMPOSES"
-
-
-# Make sure we get 1 job entry in the db per compose
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT COUNT(*) FROM jobs;"
-
-#
-# Save the Manifest from the osbuild-composer store
-# NOTE: The rest of the job data can contain sensitive information
-#
-# Suppressing shellcheck.  See https://github.com/koalaman/shellcheck/wiki/SC2024#exceptions
-sudo podman exec osbuild-composer-db psql -U postgres -d osbuildcomposer -c "SELECT args->>'Manifest' FROM jobs" | sudo tee "${ARTIFACTS}/manifest.json"
-
-#
-# Verify the Cloud-provider specific upload_status options
-#
-
-function checkUploadStatusOptionsAWS() {
-  local AMI
-  AMI=$(echo "$UPLOAD_OPTIONS" | jq -r '.ami')
-  local REGION
-  REGION=$(echo "$UPLOAD_OPTIONS" | jq -r '.region')
-
-  # AWS ID consist of resource identifier followed by a 17-character string
-  echo "$AMI" | grep -e 'ami-[[:alnum:]]\{17\}' -
-  test "$REGION" = "$AWS_REGION"
-}
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    checkUploadStatusOptionsAWS
-    ;;
-esac
-
-#
-# Verify the image landed in the appropriate cloud provider, and delete it.
-#
-
-# Reusable function, which waits for a given host to respond to SSH
-function _instanceWaitSSH() {
-  local HOST="$1"
-
-  for LOOP_COUNTER in {0..30}; do
-      if ssh-keyscan "$HOST" > /dev/null 2>&1; then
-          echo "SSH is up!"
-          # ssh-keyscan "$PUBLIC_IP" | sudo tee -a /root/.ssh/known_hosts
-          break
-      fi
-      echo "Retrying in 5 seconds... $LOOP_COUNTER"
-      sleep 5
-  done
-}
-
-function _instanceCheck() {
-  echo "✔️ Instance checking"
-  local _ssh="$1"
-
-  # Check if postgres is installed
-  $_ssh rpm -q postgresql
-
-  # Verify subscribe status. Loop check since the system may not be registered such early(RHEL only)
-  if [[ "$ID" == "rhel" ]]; then
-    set +eu
-    for LOOP_COUNTER in {1..10}; do
-        subscribe_org_id=$($_ssh sudo subscription-manager identity | grep 'org ID')
-        if [[ "$subscribe_org_id" == "org ID: $API_TEST_SUBSCRIPTION_ORG_ID" ]]; then
-            echo "System is subscribed."
-            break
-        else
-            echo "System is not subscribed. Retrying in 30 seconds...($LOOP_COUNTER/10)"
-            sleep 30
-        fi
-    done
-    set -eu
-    [[ "$subscribe_org_id" == "org ID: $API_TEST_SUBSCRIPTION_ORG_ID" ]]
-
-    # Unregister subscription
-    $_ssh sudo subscription-manager unregister
-  else
-    echo "Not RHEL OS. Skip subscription check."
-  fi
-}
-
-# Verify image in EC2 on AWS
-function verifyInAWS() {
-  $AWS_CMD ec2 describe-images \
-    --owners self \
-    --filters Name=name,Values="$AWS_SNAPSHOT_NAME" \
-    > "$WORKDIR/ami.json"
-
-  AMI_IMAGE_ID=$(jq -r '.Images[].ImageId' "$WORKDIR/ami.json")
-  AWS_SNAPSHOT_ID=$(jq -r '.Images[].BlockDeviceMappings[].Ebs.SnapshotId' "$WORKDIR/ami.json")
-  SHARE_OK=1
-
-  # Verify that the ec2 snapshot was shared
-  $AWS_CMD ec2 describe-snapshot-attribute --snapshot-id "$AWS_SNAPSHOT_ID" --attribute createVolumePermission > "$WORKDIR/snapshot-attributes.json"
-
-  SHARED_ID=$(jq -r '.CreateVolumePermissions[0].UserId' "$WORKDIR/snapshot-attributes.json")
-  if [ "$AWS_API_TEST_SHARE_ACCOUNT" != "$SHARED_ID" ]; then
-    SHARE_OK=0
-  fi
-
-  # Verify that the ec2 ami was shared
-  $AWS_CMD ec2 describe-image-attribute --image-id "$AMI_IMAGE_ID" --attribute launchPermission > "$WORKDIR/ami-attributes.json"
-
-  SHARED_ID=$(jq -r '.LaunchPermissions[0].UserId' "$WORKDIR/ami-attributes.json")
-  if [ "$AWS_API_TEST_SHARE_ACCOUNT" != "$SHARED_ID" ]; then
-    SHARE_OK=0
-  fi
-
-  if [ "$SHARE_OK" != 1 ]; then
-    echo "EC2 snapshot wasn't shared with the AWS_API_TEST_SHARE_ACCOUNT. 😢"
-    exit 1
-  fi
-
-  # Create key-pair
-  $AWS_CMD ec2 create-key-pair --key-name "key-for-$AMI_IMAGE_ID" --query 'KeyMaterial' --output text > keypair.pem
-  chmod 400 ./keypair.pem
-
-  # Create an instance based on the ami
-  $AWS_CMD ec2 run-instances --image-id "$AMI_IMAGE_ID" --count 1 --instance-type t2.micro --key-name "key-for-$AMI_IMAGE_ID" > "$WORKDIR/instances.json"
-  AWS_INSTANCE_ID=$(jq -r '.Instances[].InstanceId' "$WORKDIR/instances.json")
-
-  $AWS_CMD ec2 wait instance-running --instance-ids "$AWS_INSTANCE_ID"
-
-  $AWS_CMD ec2 describe-instances --instance-ids "$AWS_INSTANCE_ID" > "$WORKDIR/instances.json"
-  HOST=$(jq -r '.Reservations[].Instances[].PublicIpAddress' "$WORKDIR/instances.json")
-
-  echo "⏱ Waiting for AWS instance to respond to ssh"
-  _instanceWaitSSH "$HOST"
-
-  # Verify image
-  _ssh="ssh -oStrictHostKeyChecking=no -i ./keypair.pem $SSH_USER@$HOST"
-  _instanceCheck "$_ssh"
-
-  # Check access to user1 and user2
-  check_groups=$(ssh -i /tmp/usertest "user1@$HOST" -t 'groups')
-  if [[ $check_groups =~ "wheel" ]]; then
-   echo "✔️  user1 has the group wheel"
-  else
-    echo 'user1 should have the group wheel 😢'
-    exit 1
-  fi
-  check_groups=$(ssh -i /tmp/usertest "user2@$HOST" -t 'groups')
-  if [[ $check_groups =~ "wheel" ]]; then
-    echo 'user2 should not have group wheel 😢'
-    exit 1
-  else
-   echo "✔️  user2 does not have the group wheel"
-  fi
-}
-
-
-case $CLOUD_PROVIDER in
-  "$CLOUD_PROVIDER_AWS")
-    verifyInAWS
-    ;;
-esac
-
-# Verify selected package (postgresql) is included in package list
-function verifyPackageList() {
-  # Save build metadata to artifacts directory for troubleshooting
-  curl --silent \
-      --show-error \
-      --cacert /etc/osbuild-composer/ca-crt.pem \
-      --key /etc/osbuild-composer/client-key.pem \
-      --cert /etc/osbuild-composer/client-crt.pem \
-      https://localhost/api/image-builder-composer/v2/composes/"$COMPOSE_ID"/metadata --output "${ARTIFACTS}/metadata.json"
-  local PACKAGENAMES
-  PACKAGENAMES=$(jq -rM '.packages[].name' "${ARTIFACTS}/metadata.json")
-
-  if ! grep -q postgresql <<< "${PACKAGENAMES}"; then
-      echo "'postgresql' not found in compose package list 😠"
-      exit 1
-  fi
-}
-
-verifyPackageList
-
-#
-# Make sure that requesting a non existing paquet returns a 400 error
-#
-REQUEST_FILE2="${WORKDIR}/request2.json"
-jq '.customizations.packages = [ "jesuisunpaquetquinexistepas" ]' "$REQUEST_FILE" > "$REQUEST_FILE2"
-
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/image-builder-composer/v2/compose)" = "400" ]
-
-#
-# Make sure that a request that makes the dnf-json crash returns a 500 error
-#
-sudo cp -f /usr/libexec/osbuild-composer/dnf-json /usr/libexec/osbuild-composer/dnf-json.bak
-sudo cat << EOF | sudo tee /usr/libexec/osbuild-composer/dnf-json
-#!/usr/bin/python3
-raise Exception()
-EOF
-[ "$(curl \
-    --silent \
-    --cacert /etc/osbuild-composer/ca-crt.pem \
-    --key /etc/osbuild-composer/client-key.pem \
-    --cert /etc/osbuild-composer/client-crt.pem \
-    --output /dev/null \
-    --write-out '%{http_code}' \
-    -H "Content-Type: application/json" \
-    --data @"$REQUEST_FILE2" \
-    https://localhost/api/image-builder-composer/v2/compose)" = "500" ]
-
-sudo mv -f /usr/libexec/osbuild-composer/dnf-json.bak /usr/libexec/osbuild-composer/dnf-json
-
-#
-# Verify oauth2
-#
-cat <<EOF | sudo tee "/etc/osbuild-composer/osbuild-composer.toml"
-[koji]
-enable_tls = false
-enable_mtls = false
-enable_jwt = true
-jwt_keys_url = "https://localhost:8080/certs"
-jwt_ca_file = "/etc/osbuild-composer/ca-crt.pem"
-jwt_acl_file = ""
-
-[worker]
-allowed_domains = [ "localhost", "worker.osbuild.org" ]
-ca = "/etc/osbuild-composer/ca-crt.pem"
-pg_host = "localhost"
-pg_port = "5432"
-pg_database = "osbuildcomposer"
-pg_user = "postgres"
-pg_password = "foobar"
-pg_ssl_mode = "disable"
-enable_jwt = true
-jwt_keys_url = "https://localhost:8080/certs"
-jwt_ca_file = "/etc/osbuild-composer/ca-crt.pem"
-EOF
-
-cat <<EOF | sudo tee "/etc/osbuild-worker/token"
-offlineToken
-EOF
-
-cat <<EOF | sudo tee "/etc/osbuild-worker/osbuild-worker.toml"
-[authentication]
-oauth_url = http://localhost:8081/token
-offline_token = "/etc/osbuild-worker/token"
-EOF
-
-# Spin up an https instance for the composer-api and worker-api; the auth handler needs to hit an ssl `/certs` endpoint
-sudo /usr/libexec/osbuild-composer-test/osbuild-mock-openid-provider -rsaPubPem /etc/osbuild-composer/client-crt.pem -rsaPem /etc/osbuild-composer/client-key.pem -cert /etc/osbuild-composer/composer-crt.pem -key /etc/osbuild-composer/composer-key.pem &
-KILL_PIDS+=("$!")
-# Spin up an http instance for the worker client to bypass the need to specify an extra CA
-sudo /usr/libexec/osbuild-composer-test/osbuild-mock-openid-provider -a localhost:8081 -rsaPubPem /etc/osbuild-composer/client-crt.pem -rsaPem /etc/osbuild-composer/client-key.pem &
-KILL_PIDS+=("$!")
-
-sudo systemctl restart osbuild-composer
-
-until curl --output /dev/null --silent --fail localhost:8081/token; do
-    sleep 0.5
-done
-TOKEN="$(curl localhost:8081/token | jq -r .access_token)"
-
-[ "$(curl \
-        --silent \
-        --output /dev/null \
-        --write-out '%{http_code}' \
-        --header "Authorization: Bearer $TOKEN" \
-        http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "200" ]
-
-[ "$(curl \
-        --silent \
-        --output /dev/null \
-        --write-out '%{http_code}' \
-        --header "Authorization: Bearer badtoken" \
-        http://localhost:443/api/image-builder-composer/v2/composes/"$COMPOSE_ID")" = "401" ]
-
-sudo systemctl start osbuild-remote-worker@https:--localhost:8700.service
-sudo systemctl is-active --quiet osbuild-remote-worker@https:--localhost:8700.service
-
-exit 0