Repository: stackrox/stackrox
Pull request #14801: ROX-28693: Direct Helm deployment method in scanner-v4-tests
Author: login: mclasmeier
Labels: name: backport release-4.6
description: Create a PR to backport this PR to release-4.6
Created: 2025-03-28T11:49:48Z | Merged: 2025-04-25T14:06:33Z

### Description

See [ROX-28693](https://issues.redhat.com/browse/ROX-28693), which contains an extensive description of this change.

tl,dr: This PR provides a new and much more direct way to trigger Helm-based deployments of our charts. One test has been rewritten to use these new deployment functions.

#### Automated testing

Note that this PR only touches existing testing code.

- [x] modified existing tests

#### How I validated my change

Running the test suite manually against remote clusters.
CI.



Diff:
diff --git a/tests/e2e/run-scanner-v4-install.bats b/tests/e2e/run-scanner-v4-install.bats
index 513528d1f65b8..1c271400ef487 100755
--- a/tests/e2e/run-scanner-v4-install.bats
+++ b/tests/e2e/run-scanner-v4-install.bats
@@ -142,9 +142,9 @@ EOT
     info "Using MAIN_IMAGE_TAG=$MAIN_IMAGE_TAG"
 
     export CURRENT_MAIN_IMAGE_TAG=${CURRENT_MAIN_IMAGE_TAG:-} # Setting a tag can be useful for local testing.
-    export EARLIER_CHART_VERSION="4.3.0"
+    export EARLIER_CHART_VERSION="4.6.0"
     export EARLIER_MAIN_IMAGE_TAG=$EARLIER_CHART_VERSION
-    export USE_LOCAL_ROXCTL=true
+    export USE_LOCAL_ROXCTL="${USE_LOCAL_ROXCTL:-true}"
     export ROX_PRODUCT_BRANDING=RHACS_BRANDING
     export CI=${CI:-false}
     OS="$(uname | tr '[:upper:]' '[:lower:]')"
@@ -156,6 +156,10 @@ EOT
     if [[ "${ORCHESTRATOR_FLAVOR:-}" == "openshift" ]]; then
       export ROX_OPENSHIFT_VERSION=4
     fi
+    if [[ -z "${ROX_ADMIN_PASSWORD:-}" ]]; then
+        ROX_ADMIN_PASSWORD="$(openssl rand -base64 20 | tr -d '/=+')"
+    fi
+    export ROX_ADMIN_PASSWORD
 
     if ! command -v roxctl >/dev/null; then
         die "roxctl not found, please make sure it can be resolved via PATH."
@@ -210,9 +214,44 @@ EOT
     # have any logs for investigation the situation.
     export BATS_TEST_TIMEOUT=1800 # Seconds
 
+   if [[ -z "${HEAD_HELM_CHART_CENTRAL_SERVICES_DIR:-}" ]]; then
+        HEAD_HELM_CHART_CENTRAL_SERVICES_DIR=$(mktemp -d)
+        echo "Rendering fresh central-services Helm chart and writing to ${HEAD_HELM_CHART_CENTRAL_SERVICES_DIR}..."
+        roxctl helm output central-services \
+            --debug --debug-path="${ROOT}/image" \
+            --output-dir="${HEAD_HELM_CHART_CENTRAL_SERVICES_DIR}" --remove
+        export HEAD_HELM_CHART_CENTRAL_SERVICES_DIR
+    fi
+
+   if [[ -z "${HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR:-}" ]]; then
+        HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR=$(mktemp -d)
+        echo "Rendering fresh secured-cluster-services Helm chart and writing to ${HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR}..."
+        roxctl helm output secured-cluster-services \
+            --debug --debug-path="${ROOT}/image" \
+            --output-dir="${HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR}" --remove
+        export HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR
+    fi
+
+    # For installation testing we don't need to deploy collector per-node, it is sufficient to deploy
+    # collector on a single node.
+    local _worker; _worker=$(select_worker_node)
+
+    echo "Patching node ${_worker} to include label run-collector=true"
+    "${ORCH_CMD}" </dev/null label node "$_worker" run-collector=true
+    echo "collector will only be scheduled on node ${_worker}"
+
     _end
 }
 
+select_worker_node() {
+    local select_filter="true"
+    if [[ "$ORCHESTRATOR_FLAVOR" == "openshift" ]]; then
+        select_filter=".metadata.labels[\"node-role.kubernetes.io/worker\"] != null"
+    fi
+
+    "${ORCH_CMD}" </dev/null get nodes -o json | jq -r ".items | map(select(${select_filter})) | map(.metadata.name) | sort | first"
+}
+
 teardown_file() {
     local test_suite_end_timestamp=$(date +%s)
     _begin "teardown-file"
@@ -259,7 +298,8 @@ setup() {
 
 # CRD needs to be owned by Helm if upgrading to 4.8+ from 4.7.x via Helm
 apply_crd_ownership_for_upgrade() {
-    local namespace={$1:-stackrox}
+    local namespace=$1
+    echo "Making sure that SecurityPolicies CRD has the correct metadata..."
     "${ORCH_CMD}" </dev/null annotate crd/securitypolicies.config.stackrox.io meta.helm.sh/release-name=stackrox-central-services || true
     "${ORCH_CMD}" </dev/null annotate crd/securitypolicies.config.stackrox.io meta.helm.sh/release-namespace="$namespace" || true
     "${ORCH_CMD}" </dev/null label crd/securitypolicies.config.stackrox.io app.kubernetes.io/managed-by=Helm || true
@@ -360,51 +400,73 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploy-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "upgrade-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        --set scannerV4.disable=false
 
-    # shellcheck disable=SC2030,SC2031
-    export SENSOR_SCANNER_V4_SUPPORT=true
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    apply_crd_ownership_for_upgrade "stackrox"
-    _deploy_stackrox
+    _begin "deploy-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "verify"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Verify that Scanner V2 and V4 are up.
-    verify_scannerV2_deployed "stackrox"
-    verify_scannerV4_deployed "stackrox"
-    verify_deployment_scannerV4_env_var_set "stackrox" "central"
-    verify_deployment_scannerV4_env_var_set "stackrox" "sensor"
+    _step "upgrade-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
+
+    _step "verify-scanner-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-secured-cluster"
+    # Without creating the scanner-db-password secret manually Scanner V2 doesn't come up.
+    # Let's just reuse an existing password for this for simplicity.
+    "$ORCH_CMD" </dev/null -n "$CUSTOM_SENSOR_NAMESPACE" create secret generic scanner-db-password \
+        --from-file=password=<(echo "$ROX_ADMIN_PASSWORD")
+
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" "" \
+        --set scannerV4.disable=false \
+        --set scanner.disable=false
+
+    _step "verify-scanner-V4-deployed"
+    verify_scannerV4_indexer_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
     _end
 }
@@ -664,7 +726,7 @@ EOF
 
     wait_until_central_validation_webhook_is_ready "${CUSTOM_CENTRAL_NAMESPACE}"
 
-    _step "patching-central"
+    _step "patch-central"
 
     # Enable Scanner V4 on central side.
     info "Patching Central"
@@ -710,7 +772,7 @@ EOT
     sleep 60
     "${ORCH_CMD}" </dev/null -n "${CUSTOM_CENTRAL_NAMESPACE}" wait --for=condition=Ready pods -l app=central || true
 
-    _step "patching-secured-cluster"
+    _step "patch-secured-cluster"
 
     info "Patching SecuredCluster"
     # Enable Scanner V4 on secured-cluster side
@@ -809,7 +871,7 @@ EOT
     _end
 }
 
-@test "Upgrade from old version without Scanner V4 support to the version which supports Scanner V4" {
+@test "Upgrade from old version without Scanner V4 to HEAD with Scanner V4 enabled" {
     _begin "deploy-stackrox"
 
     if [[ "$CI" = "true" ]]; then
@@ -820,7 +882,7 @@ EOT
     # shellcheck disable=SC2030,SC2031
     export OUTPUT_FORMAT=""
     info "Using roxctl executable ${EARLIER_ROXCTL_PATH}/roxctl for generating pre-Scanner V4 deployment bundles"
-    PATH="${EARLIER_ROXCTL_PATH}:${PATH}" MAIN_IMAGE_TAG="${EARLIER_MAIN_IMAGE_TAG}" _deploy_stackrox
+    PATH="${EARLIER_ROXCTL_PATH}:${PATH}" MAIN_IMAGE_TAG="${EARLIER_MAIN_IMAGE_TAG}" ROX_SCANNER_V4=false _deploy_stackrox
 
     _step "verify"
 
@@ -832,7 +894,7 @@ EOT
     _step "upgrade-stackrox"
 
     info "Upgrading StackRox using HEAD deployment bundles"
-    _deploy_stackrox
+    ROX_SCANNER_V4=true _deploy_stackrox
 
     _step "verify"
 
@@ -844,6 +906,19 @@ EOT
     _end
 }
 
+get_central_endpoint() {
+    local namespace="$1"
+    local central_ip="$("${ORCH_CMD}" -n "$CUSTOM_CENTRAL_NAMESPACE" </dev/null get service central-loadbalancer \
+        -o json | service_get_endpoint)"
+    echo "${central_ip}:443"
+}
+
+get_cluster_name() {
+    local prefix="${1:-sc}"
+    echo "${prefix}-${RANDOM}"
+}
+
+
 verify_deployment_deletion() {
     local deployment_names_file="$1"; shift
     local namespace="$1"; shift
@@ -855,7 +930,7 @@ verify_deployment_deletion() {
 
     local deleted
     while [[ -s "$deployment_names_file" ]]; do
-        active_deployments=$("${ORCH_CMD}" -n "$namespace" get deployments -o json)
+        active_deployments=$("${ORCH_CMD}" </dev/null -n "$namespace" get deployments -o json)
         deployment_names=$(cat "$deployment_names_file")
 
         for deployment_name in $deployment_names; do
@@ -1025,6 +1100,294 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        echo "{ \"apiVersion\": \"v1\", \"kind\": \"Namespace\", \"metadata\": { \"name\": \"$central_namespace\" } }" | "${ORCH_CMD}" apply -f -
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+        apply_crd_ownership_for_upgrade "$central_namespace"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # TODO(ROX-28903): For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+        fi
+        base_helm_values=$(cat <<EOT
+clusterName: "$cluster_name"
+centralEndpoint: "$central_endpoint"
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+admissionControl:
+  replicas: 1
+
+collector:
+  nodeSelector:
+    run-collector: "true"
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-secured-cluster-services Helm chart \"${helm_chart_dir}\" into namespace ${sensor_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${sensor_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        -f <(echo "$init_bundle") \
+        "$@" \
+        stackrox-secured-cluster-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # TODO(ROX-28903): For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${sensor_namespace}"
+    fi
+
+    echo "Sensor deployed. Waiting for sensor to be up"
+    sensor_wait "${sensor_namespace}"
+    wait_for_collectors_to_be_operational "${sensor_namespace}"
+}
+
+bounce_pods() {
+    local namespace="$1"
+    echo "Bouncing all workload pods..."
+    "${ORCH_CMD}" </dev/null -n "$namespace" delete pod --all --force --grace-period=0
+}
+
 patch_down_central() {
     local central_namespace="$1"
     # shellcheck disable=SC2031
diff --git a/tools/allowed-large-files b/tools/allowed-large-files
index 3a6c316bcc45b..975465bf82567 100644
--- a/tools/allowed-large-files
+++ b/tools/allowed-large-files
@@ -52,6 +52,7 @@ scripts/ci/lib.sh
 sensor/tests/data/policies.json
 sensor/upgrader/preflight/testdata/k8s-1.22.json.gz
 tests/e2e/lib.sh
+tests/e2e/run-scanner-v4-install.bats
 tests/performance/load/package-lock.json
 tests/performance/scale/config/metrics-acs.yml
 tests/performance/scale/config/metrics-full.yml


Reviews:
user: login: porridge
body: Some nitpicks and a bunch of questions.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T10:38:57Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:03:14Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:06:52Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:14:23Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:17:48Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:36:16Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:40:29Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:48:32Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:49:03Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:51:56Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:53:06Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:54:26Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T21:59:52Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T22:02:00Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T22:07:06Z
user: login: porridge
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T08:41:07Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T09:54:16Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T09:57:02Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T10:03:00Z
user: login: porridge
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T10:12:28Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T10:12:32Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T10:13:53Z
user: login: porridge
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T10:24:12Z
user: login: porridge
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T10:27:27Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T06:57:25Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T06:58:06Z
user: login: mclasmeier
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T07:45:50Z

Review comments:
diff_hunk: @@ -138,9 +138,9 @@ EOT
     info "Using MAIN_IMAGE_TAG=$MAIN_IMAGE_TAG"
 
     export CURRENT_MAIN_IMAGE_TAG=${CURRENT_MAIN_IMAGE_TAG:-} # Setting a tag can be useful for local testing.
-    export EARLIER_CHART_VERSION="4.3.0"
+    export EARLIER_CHART_VERSION="4.6.0"
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Probably not something to fix in scope of this PR, but this hardcoded version looks concerning.
What's the semantics of "earlier" here? The earlier roxctl below is mentioned as something not having scanner v4 support. Is this in similar spirit?
Or are we expected to keep this up to date? Can you please file a ticket to address this?
created_at: 2025-04-09T09:37:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Some labels use gerund, others not. Let's be consistent.
```suggestion
    _begin "deploy-old-central"
```
Similar elsewhere.
created_at: 2025-04-09T09:57:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Do we need the variable? How about:
```suggestion
    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
        -f <(cat <<EOT
central:
  adminPassword:
    value: "$ROX_ADMIN_PASSWORD"
EOT
    )
```
created_at: 2025-04-09T09:59:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: I guess you're doing it this way to avoid showing the password on the command line? Otherwise `--set central.adminPassword.value=$ROX_ADMIN_PASSWORD` would also work, right?
created_at: 2025-04-09T10:02:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Would the following also work but be simpler?
```suggestion
        --set scannerV4.disable=false
```
created_at: 2025-04-09T10:03:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: How about making these optional and:
```suggestion
    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE"
```

Similar below.
created_at: 2025-04-09T10:05:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
 
-    # shellcheck disable=SC2030,SC2031
-    export SENSOR_SCANNER_V4_SUPPORT=true
+    _step "verify-scanner-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    _deploy_stackrox
+    _begin "enabling-scanner-V4-in-secured-cluster"
+    # Without creating the scanner-db-password secret manually Scanner V2 doesn't come up.
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: 😱 Is this intended?
created_at: 2025-04-09T10:08:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Should we also make assertions about scanner v2 state here and below?
created_at: 2025-04-09T10:09:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -830,6 +897,19 @@ EOT
     _end
 }
 
+get_central_endpoint() {
+    local namespace="$1"
+    local central_ip="$("${ORCH_CMD}" -n "$CUSTOM_CENTRAL_NAMESPACE" </dev/null get service central-loadbalancer \
+        -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: For debuggability and better platform support:
```suggestion
    local central_ip="$("${ORCH_CMD}" -n "$CUSTOM_CENTRAL_NAMESPACE" </dev/null get service central-loadbalancer \
        -o json | service_get_endpoint"
```
created_at: 2025-04-09T10:16:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: If you wanted to take care of the corner case when the `create` fails for a transient reason:
```suggestion
        echo '{ "apiVersion": "v1", "kind": "Namespace", "metadata": { "name": "$central_namespace" } }' | "${ORCH_CMD}" apply -f -

```
created_at: 2025-04-09T10:20:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Why do we need this? It's not something that users typically do, right?
created_at: 2025-04-09T10:22:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: This seems like some elaborate hack. What is the purpose of this? How do actual users deal with this?
created_at: 2025-04-09T10:24:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Isn't this obsolete?
created_at: 2025-04-09T10:25:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Why disable scanner-v4-db persistence?
created_at: 2025-04-09T10:26:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Please add a TODO to track this down and remove this workaround.
Or is it the documented practice? 😨 
created_at: 2025-04-09T10:28:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Just wondering: is this one really needed? Isn't the `stackrox` one sufficient?
created_at: 2025-04-09T10:31:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+        fi
+        base_helm_values=$(cat <<EOT
+clusterName: "$cluster_name"
+centralEndpoint: "$central_endpoint"
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+admissionControl:
+  replicas: 1
+
+collector:
+  nodeSelector:
+    run-collector: "true"
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-secured-cluster-services Helm chart \"${helm_chart_dir}\" into namespace ${sensor_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${sensor_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        -f <(echo "$init_bundle") \
+        "$@" \
+        stackrox-secured-cluster-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${sensor_namespace}"
+    fi
+
+    echo "Sensor deployed. Waiting for sensor to be up"
+    sensor_wait "${sensor_namespace}"
+    wait_for_collectors_to_be_operational "${sensor_namespace}"
+}
+
+bounce_pods() {
+    local namespace="$1"
+    echo "Bouncing all workload pods..."
+    {
+        "${ORCH_CMD}" </dev/null -n "$namespace" get deployments -o name
+        "${ORCH_CMD}" </dev/null -n "$namespace" get daemonsets -o name
+    } | cut -d / -f 2 | while read -r workload; do
+        "${ORCH_CMD}" </dev/null -n "$namespace" delete pod -l "app=$workload" --force --grace-period=0
+    done
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Would
```suggestion
    "${ORCH_CMD}" </dev/null -n "$namespace" delete pod --all --force --grace-period=0
```
also work?
created_at: 2025-04-09T10:35:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -52,6 +52,7 @@ scripts/ci/lib.sh
 sensor/tests/data/policies.json
 sensor/upgrader/preflight/testdata/k8s-1.22.json.gz
 tests/e2e/lib.sh
+tests/e2e/run-scanner-v4-install.bats
path: tools/allowed-large-files
user: login: porridge
body: Ouch, perhaps it's time to move some helper functions to `tests/e2e/lib.sh` instead?
created_at: 2025-04-09T10:38:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -138,9 +138,9 @@ EOT
     info "Using MAIN_IMAGE_TAG=$MAIN_IMAGE_TAG"
 
     export CURRENT_MAIN_IMAGE_TAG=${CURRENT_MAIN_IMAGE_TAG:-} # Setting a tag can be useful for local testing.
-    export EARLIER_CHART_VERSION="4.3.0"
+    export EARLIER_CHART_VERSION="4.6.0"
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Initially, for the Scanner V4 rollout, it was the previous version, which was also the latest version without any Scanner V4 support. I believe nowadays it should simply be the *previous* version, because that is what users usually upgrade from.
created_at: 2025-04-09T21:03:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -138,9 +138,9 @@ EOT
     info "Using MAIN_IMAGE_TAG=$MAIN_IMAGE_TAG"
 
     export CURRENT_MAIN_IMAGE_TAG=${CURRENT_MAIN_IMAGE_TAG:-} # Setting a tag can be useful for local testing.
-    export EARLIER_CHART_VERSION="4.3.0"
+    export EARLIER_CHART_VERSION="4.6.0"
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Created ROX-28901 for this.
created_at: 2025-04-09T21:06:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: correct.
created_at: 2025-04-09T21:14:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Not sure how to combine this with the approach of passing remaining arguments verbatim to the Helm CLI.

created_at: 2025-04-09T21:17:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
 
-    # shellcheck disable=SC2030,SC2031
-    export SENSOR_SCANNER_V4_SUPPORT=true
+    _step "verify-scanner-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    _deploy_stackrox
+    _begin "enabling-scanner-V4-in-secured-cluster"
+    # Without creating the scanner-db-password secret manually Scanner V2 doesn't come up.
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Probably not.
Just checked the Helm charts... it is probably due to [this special-handling here](https://github.com/stackrox/stackrox/blob/59f180537d5f5873b4916f1031fa925ef5d31b57/image/templates/helm/shared/templates/_scanner-v4_init.tpl.htpl#L116) not being present for scanner V2. Maybe it is not necessary anymore to fix this...?
created_at: 2025-04-09T21:36:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: To keep things as light-weight as possible.
created_at: 2025-04-09T21:40:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: done: ROX-28903.
created_at: 2025-04-09T21:48:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: According to my tests both are needed. :-( 
created_at: 2025-04-09T21:49:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+        fi
+        base_helm_values=$(cat <<EOT
+clusterName: "$cluster_name"
+centralEndpoint: "$central_endpoint"
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+admissionControl:
+  replicas: 1
+
+collector:
+  nodeSelector:
+    run-collector: "true"
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-secured-cluster-services Helm chart \"${helm_chart_dir}\" into namespace ${sensor_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${sensor_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        -f <(echo "$init_bundle") \
+        "$@" \
+        stackrox-secured-cluster-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${sensor_namespace}"
+    fi
+
+    echo "Sensor deployed. Waiting for sensor to be up"
+    sensor_wait "${sensor_namespace}"
+    wait_for_collectors_to_be_operational "${sensor_namespace}"
+}
+
+bounce_pods() {
+    local namespace="$1"
+    echo "Bouncing all workload pods..."
+    {
+        "${ORCH_CMD}" </dev/null -n "$namespace" get deployments -o name
+        "${ORCH_CMD}" </dev/null -n "$namespace" get daemonsets -o name
+    } | cut -d / -f 2 | while read -r workload; do
+        "${ORCH_CMD}" </dev/null -n "$namespace" delete pod -l "app=$workload" --force --grace-period=0
+    done
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Probably yes. Don't remember why I did it this way.
created_at: 2025-04-09T21:51:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -52,6 +52,7 @@ scripts/ci/lib.sh
 sensor/tests/data/policies.json
 sensor/upgrader/preflight/testdata/k8s-1.22.json.gz
 tests/e2e/lib.sh
+tests/e2e/run-scanner-v4-install.bats
path: tools/allowed-large-files
user: login: mclasmeier
body: Some small helpers functions, probably. But if that is so useful, I don't know.

created_at: 2025-04-09T21:53:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Aren't these eqivalent? What are the advantages of your suggestion?
created_at: 2025-04-09T21:54:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: We don't have these helper functions in this file as of now. And it felt like scope creep to me to extend the coverage further as part of this PR, as it's primary intent is primarily to pave the way to better testing building blocks.
created_at: 2025-04-09T21:59:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Oh, this allows for convenient testing locally by working on a dev branch and running the test suite with MAIN_IMAGE_TAG set in the environment pointing to some fixed tag (in case one doesn't need new images, but only wants to run the test cases).
created_at: 2025-04-09T22:02:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Actually this should be a documented procedure.

When you install the central-services Helm chart a command like this is shown to the user as part of the post-install NOTES. It fetches the data generated by the Helm chart in the form of a Helm values file.

There are situation in which you need to provide this during Helm upgrades, for example when enabling a new component like scanner V4, because this requires the CA to be provided so that a new cert can be issued for scanner V4.
created_at: 2025-04-09T22:07:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: `apply` is idempotent.

So if the namespace does not exist, and the operation fails for some intermittent issue:
1. in `create || true` case it will be ignored by the `|| true` and then the `pull-secret` script will fail complaining about the missing namespace.
2. in the `apply` case, it will fail right there mentioning the cause.

Not a big difference here, since in the former case the error from create will be right above anyway, and the latter case is somewhat uglier, so up to you.
created_at: 2025-04-16T08:29:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: I see, maybe mention that this is for local dev testing on [this line](https://github.com/stackrox/stackrox/pull/14801/files#diff-96ce4b6130b5dedd6d30c7916417e2b6595655045af9d1bf64d16c8c75520c83R1098)?
created_at: 2025-04-16T08:32:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: OMG this is the first time I'm learning about this. I wonder how many users are blissfully unaware.
This does not seem to be documented in [the upgrade instructions](https://docs.redhat.com/en/documentation/red_hat_advanced_cluster_security_for_kubernetes/4.7/html/upgrading/upgrade-helm#upgrade-helm-chart_upgrade-helm). Can you please file a ticket about this?
created_at: 2025-04-16T08:36:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Ping?
created_at: 2025-04-16T08:36:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: OK, thought with each such modification we're decreasing coverage... 😟 
created_at: 2025-04-16T08:37:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Interesting, I think I installed SC components from `quay.io/rhacs-eng` a few times in the past and never created anything else than the `stackrox` secret, init bundle, and the CRs 🤔 
Let me try now.
created_at: 2025-04-16T08:39:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+        fi
+        base_helm_values=$(cat <<EOT
+clusterName: "$cluster_name"
+centralEndpoint: "$central_endpoint"
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+admissionControl:
+  replicas: 1
+
+collector:
+  nodeSelector:
+    run-collector: "true"
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-secured-cluster-services Helm chart \"${helm_chart_dir}\" into namespace ${sensor_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${sensor_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        -f <(echo "$init_bundle") \
+        "$@" \
+        stackrox-secured-cluster-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${sensor_namespace}"
+    fi
+
+    echo "Sensor deployed. Waiting for sensor to be up"
+    sensor_wait "${sensor_namespace}"
+    wait_for_collectors_to_be_operational "${sensor_namespace}"
+}
+
+bounce_pods() {
+    local namespace="$1"
+    echo "Bouncing all workload pods..."
+    {
+        "${ORCH_CMD}" </dev/null -n "$namespace" get deployments -o name
+        "${ORCH_CMD}" </dev/null -n "$namespace" get daemonsets -o name
+    } | cut -d / -f 2 | while read -r workload; do
+        "${ORCH_CMD}" </dev/null -n "$namespace" delete pod -l "app=$workload" --force --grace-period=0
+    done
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Can you please change it to the one-liner then?
created_at: 2025-04-16T08:40:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Ok, well, I have enabled it again.
created_at: 2025-04-16T09:54:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: thanks, removed.
created_at: 2025-04-16T09:57:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Actually this is not directly related. The mechanism behind your link is used, for example, for installing old (non-HEAD) charts for testing upgrades.
created_at: 2025-04-16T10:02:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
+            > "$helm_generated_values_file"
+        command=("upgrade" "--install" "--reuse-values" "-f" "$helm_generated_values_file")
+        upgrade="true"
+    else
+        base_helm_values=$(cat <<EOT
+central:
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+  telemetry:
+    enabled: false
+  exposure:
+    loadBalancer:
+      enabled: true
+  persistence:
+    none: true
+  db:
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+
+scanner:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "500Mi"
+    limits:
+      cpu: "2000m"
+      memory: "2500Mi"
+  dbResources:
+    requests:
+      cpu: "400m"
+      memory: "512Mi"
+    limits:
+      cpu: "2000m"
+      memory: "4Gi"
+  replicas: 1
+  autoscaling:
+    disable: true
+
+scannerV4:
+  indexer:
+    replicas: 1
+    autoscaling:
+      disable: true
+  matcher:
+    replicas: 1
+    autoscaling:
+      disable: true
+  db:
+    persistence:
+      none: true
+
+allowNonstandardNamespace: true
+EOT
+        )
+    fi
+
+    echo "Deploying stackrox-central-services Helm chart \"${helm_chart_dir}\" into namespace ${central_namespace} with the following settings:"
+    if [[ -n "$base_helm_values" ]]; then
+        echo "base Helm values:"
+        echo "$base_helm_values" | sed -e 's/^/  |/;'
+    fi
+    if [[ -n "$image_overwrites" ]]; then
+        echo "image overwrites:"
+        echo "$image_overwrites" | sed -e 's/^/  |/;'
+    fi
+    echo "additional arguments:"
+    echo "  | $*"
+
+    helm -n "${central_namespace}" "${command[@]}" \
+        -f <(echo "$image_overwrites") \
+        -f <(echo "$base_helm_values") \
+        "$@" \
+        stackrox-central-services "${helm_chart_dir}"
+
+    if [[ "$upgrade" == "true" ]]; then
+        # For some reason pods don't always terminate smoothly after an upgrade.
+        bounce_pods "${central_namespace}"
+    fi
+
+    echo "Waiting for API..."
+    wait_for_api "${central_namespace}"
+
+    if [[ "$upgrade" == "false" ]]; then
+        echo "Setting up client TLS certs..."
+        setup_client_TLS_certs ""
+        echo "Recording build info..."
+        record_build_info "${central_namespace}"
+    fi
+}
+
+# shellcheck disable=SC2120
+# deploy_sensor_with_helm "<central namespace>" "<sensor namespace>"
+#   "<main image tag>" "<helm chart dir overwrite>"
+#   "<cluster name>" "<central admin password>" "<central endpoint>" [ additional args for helm ... ]
+deploy_sensor_with_helm() {
+    echo "Deploying secured-cluster-services via Helm..."
+    local central_namespace="$1"; shift
+    local sensor_namespace="$1"; shift
+    local main_image_tag="${1:-$MAIN_IMAGE_TAG}"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_SECURED_CLUSTER_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+    local cluster_name="$1"; shift
+    local central_password="$1"; shift
+    local central_endpoint="$1"; shift
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+  main:
+    tag: "$main_image_tag"
+  scannerV4:
+    tag: "$main_image_tag"
+  scannerV4DB:
+    tag: "$main_image_tag"
+
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$sensor_namespace" -o json | jq -e '.[] | select(.name == "stackrox-secured-cluster-services")' > /dev/null 2>&1; then
+        command=("upgrade" "--install" "--reuse-values")
+        upgrade="true"
+    else
+        # Later this will be replaced by CRS.
+        echo "Retrieving init-bundle from Central..."
+        init_bundle="$("${ORCH_CMD}" </dev/null -n "$central_namespace" exec deploy/central -- roxctl \
+            --insecure-skip-tls-verify \
+            -p "$central_password" \
+            -e "central.${central_namespace}.svc:443" \
+            central init-bundles generate "$cluster_name" --output=-)"
+
+        if [[ "${CI:-}" != "true" ]]; then
+            info "Creating image pull secrets..."
+            "${ORCH_CMD}" </dev/null create namespace "$sensor_namespace" || true
+            "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
+            "${ROOT}/deploy/common/pull-secret.sh" collector-stackrox quay.io | "${ORCH_CMD}" -n "$sensor_namespace" apply -f -
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Ah, no I remember now I explicitly configured the pull secret in the CR.
created_at: 2025-04-16T10:12:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
+image:
+  registry: "$DEFAULT_IMAGE_REGISTRY"
+central:
+  db:
+    image:
+      tag: "$main_image_tag"
+  image:
+    tag: "$main_image_tag"
+
+scannerV4:
+  image:
+    tag: "$main_image_tag"
+  db:
+    image:
+      tag: "$main_image_tag"
+EOT
+        )
+    fi
+
+    command=("install" "--create-namespace")
+    if helm list -n "$central_namespace" -o json | jq -e '.[] | select(.name == "stackrox-central-services")' > /dev/null 2>&1; then
+        helm_generated_values_file=$(mktemp)
+        "${ORCH_CMD}" -n "$central_namespace" get secrets -o json \
+            </dev/null \
+            | jq -r '.items[] | select(.metadata.name | startswith("stackrox-generated-")) | .data["generated-values.yaml"] | @base64d' \
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Done: https://issues.redhat.com/browse/ROX-29000.
created_at: 2025-04-16T10:12:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Ok to resolve for now? Definitely not ruling out that we can come up with something nicer during a future iteration on this.
created_at: 2025-04-16T10:13:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: Right, I now noticed that that line **disables** this feature. So why are we specifying these image overrides by default? I'm concerned this causes us to drift away by default from what a typical user sets.
created_at: 2025-04-23T10:23:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
 
-    # shellcheck disable=SC2030,SC2031
-    export SENSOR_SCANNER_V4_SUPPORT=true
+    _step "verify-scanner-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    _deploy_stackrox
+    _begin "enabling-scanner-V4-in-secured-cluster"
+    # Without creating the scanner-db-password secret manually Scanner V2 doesn't come up.
path: tests/e2e/run-scanner-v4-install.bats
user: login: porridge
body: I don't know, I don't fully grasp the scope of this issue. My understanding is: using helm to install secured cluster with local scanner v2 enabled is broken by default. One needs to go an fiddle with secrets manually afterwards.

If this is accurate, then I think we should fix it.
created_at: 2025-04-23T10:27:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: We are specifying the overrides for two reasons:

1. We allow for injecting a fixed tag, which is practically a necessity when working on the *tests* without the need to build now images. If this is not overwritten, then what is inserted essentially comes from `make tag`, which is also the tag that is injected in the HEAD charts.

2. We inject a custom registry. Currently, by default the charts will be rendered out having the registy set to `quay.io/stackrox-io`. In a [separate PR](https://github.com/stackrox/stackrox/pull/15048) I am setting this registry to `quay.io/rhacs-eng` to match what the effect of the image prefetcher.

created_at: 2025-04-25T06:57:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1011,6 +1091,303 @@ _deploy_central() {
     patch_down_central "${central_namespace}"
 }
 
+# shellcheck disable=SC2120
+# deploy_central_with_helm "<central namespace>" "<main image tag>" "<helm chart dir overwrite>" [ additional args for helm ... ]
+deploy_central_with_helm() {
+    echo "Deploying central-services via Helm..."
+    local central_namespace="$1"; shift
+    local main_image_tag="$1"; shift
+    local helm_chart_dir="$HEAD_HELM_CHART_CENTRAL_SERVICES_DIR"
+    local use_default_chart="true"
+    if [[ -n "$1" ]]; then
+        # overwrite Helm chart path.
+        helm_chart_dir="$1"
+        use_default_chart="false"
+    fi; shift
+
+    if [[ "${CI:-}" != "true" ]]; then
+        info "Creating namespace and image pull secrets..."
+        "${ORCH_CMD}" </dev/null create namespace "$central_namespace" || true
+        "${ROOT}/deploy/common/pull-secret.sh" stackrox quay.io | "${ORCH_CMD}" -n "$central_namespace" apply -f -
+    fi
+
+    local image_overwrites=""
+    local base_helm_values=""
+    local upgrade="false"
+
+    if [[ "$use_default_chart" == "true" ]]; then
+        image_overwrites=$(cat <<EOT
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: Will resolve this for now for merging, if you have further concerns we can of course discuss this later.
created_at: 2025-04-25T06:58:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -347,50 +386,78 @@ teardown() {
 @test "Upgrade from old Helm chart to HEAD Helm chart with Scanner v4 enabled" {
     init
 
-    # shellcheck disable=SC2030,SC2031
-    export OUTPUT_FORMAT=helm
-    export ROX_DEPLOY_SENSOR_WITH_CRS=false
-
     local main_image_tag="${MAIN_IMAGE_TAG}"
 
     # Deploy earlier version without Scanner V4.
-    local _CENTRAL_CHART_DIR_OVERRIDE="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_central_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/central-services"
+    local old_sensor_chart="${CHART_REPOSITORY}${CHART_BASE}/${EARLIER_CHART_VERSION}/secured-cluster-services"
 
-    _begin "deploying-stackrox"
+    _begin "deploying-old-central"
+    info "Deploying StackRox central-services using chart ${old_central_chart}"
+    password_setting=$(cat <<EOT
+central:
+  adminPassword:
+    value: "$ROX_ADMIN_PASSWORD"
+EOT
+    )
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$EARLIER_MAIN_IMAGE_TAG" "$old_central_chart" \
+        -f <(echo "$password_setting")
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "upgrading-to-HEAD-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values
+
+    _step "verify-scanner-V4-not-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_no_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+
+    _begin "enable-scanner-V4-in-central"
+    deploy_central_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$MAIN_IMAGE_TAG" "" \
+        --reuse-values \
+        -f <(cat <<EOT
+scannerV4:
+  disable: false
+EOT
+    )
 
-    info "Deploying StackRox services using chart ${_CENTRAL_CHART_DIR_OVERRIDE}"
+    _step "verify-scanners-deployed"
+    verify_scannerV2_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_scannerV4_deployed "$CUSTOM_CENTRAL_NAMESPACE"
+    verify_deployment_scannerV4_env_var_set "$CUSTOM_CENTRAL_NAMESPACE" "central"
 
-    if [[ -n "${EARLIER_MAIN_IMAGE_TAG:-}" ]]; then
-        MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG
-        info "Overriding MAIN_IMAGE_TAG=$EARLIER_MAIN_IMAGE_TAG"
-    fi
-    local _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE
-    _ROX_CENTRAL_EXTRA_HELM_VALUES_FILE=$(mktemp)
-    cat <<EOF >"$_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE"
-central:
-  persistence:
-    none: true
-EOF
-    ROX_CENTRAL_EXTRA_HELM_VALUES_FILE="${_ROX_CENTRAL_EXTRA_HELM_VALUES_FILE}" CENTRAL_CHART_DIR_OVERRIDE="${_CENTRAL_CHART_DIR_OVERRIDE}" _deploy_stackrox
+    _begin "deploying-old-sensor"
+    info "Deploying StackRox secured-cluster-services using chart ${old_sensor_chart}"
+    local central_endpoint="$(get_central_endpoint "$CUSTOM_CENTRAL_NAMESPACE")"
+    local secured_cluster_name="$(get_cluster_name)"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" \
+        "$EARLIER_MAIN_IMAGE_TAG" "$old_sensor_chart" \
+        "$secured_cluster_name" "$ROX_ADMIN_PASSWORD" "$central_endpoint"
 
-    _step "upgrading-stackrox"
+    _step "verify-scanner-V4-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    # Upgrade to HEAD chart without explicit disabling of Scanner V4.
-    info "Upgrading StackRox using HEAD Helm chart"
-    MAIN_IMAGE_TAG="${main_image_tag}"
+    _step "upgrading-to-HEAD-sensor"
+    deploy_sensor_with_helm "$CUSTOM_CENTRAL_NAMESPACE" "$CUSTOM_SENSOR_NAMESPACE" "" "" "" "" ""
 
-    # shellcheck disable=SC2030,SC2031
-    export SENSOR_SCANNER_V4_SUPPORT=true
+    _step "verify-scanner-not-deployed"
+    verify_no_scannerV4_deployed "$CUSTOM_SENSOR_NAMESPACE"
 
-    _deploy_stackrox
+    _begin "enabling-scanner-V4-in-secured-cluster"
+    # Without creating the scanner-db-password secret manually Scanner V2 doesn't come up.
path: tests/e2e/run-scanner-v4-install.bats
user: login: mclasmeier
body: I have created ROX-29076 for this. Hence I will resolve this.
created_at: 2025-04-25T07:45:50Z
author_association: CONTRIBUTOR

PR comments:
user: login: rhacs-bot
created_at: 2025-03-28T12:22:37Z
body: [//]: # (meta:type=build)

Images are ready for the commit at 5a19e33.

To use with deploy scripts, first `export MAIN_IMAGE_TAG=4.8.x-569-g5a19e33ad9`.
author_association: CONTRIBUTOR
user: login: codecov[bot]
created_at: 2025-03-28T23:57:55Z
body: ## [Codecov](https://app.codecov.io/gh/stackrox/stackrox/pull/14801?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 48.99%. Comparing base [(`23b1014`)](https://app.codecov.io/gh/stackrox/stackrox/commit/23b101484b9ef4666bc20ba24ddf7ea1629eea83?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox) to head [(`5a19e33`)](https://app.codecov.io/gh/stackrox/stackrox/commit/5a19e33ad9bf53a2561bc48c32fe484490f4b75c?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox).
> Report is 139 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master   #14801      +/-   ##
==========================================
- Coverage   48.99%   48.99%   -0.01%     
==========================================
  Files        2550     2550              
  Lines      187230   187230              
==========================================
- Hits        91733    91731       -2     
  Misses      88249    88249              
- Partials     7248     7250       +2     
```

| [Flag](https://app.codecov.io/gh/stackrox/stackrox/pull/14801/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox) | Coverage Δ | |
|---|---|---|
| [go-unit-tests](https://app.codecov.io/gh/stackrox/stackrox/pull/14801/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox) | `48.99% <ø> (-0.01%)` | :arrow_down: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox#carryforward-flags-in-the-pull-request-comment) to find out more.

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/stackrox/stackrox/pull/14801?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stackrox).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE
user: login: mclasmeier
created_at: 2025-04-01T05:53:47Z
body: /retest
author_association: CONTRIBUTOR
user: login: mclasmeier
created_at: 2025-04-03T08:05:54Z
body: /retest
author_association: CONTRIBUTOR
user: login: mclasmeier
created_at: 2025-04-03T17:03:52Z
body: /retest
author_association: CONTRIBUTOR
user: login: mclasmeier
created_at: 2025-04-14T11:52:16Z
body: /retest
author_association: CONTRIBUTOR
user: login: mclasmeier
created_at: 2025-04-16T13:34:41Z
body: /retest
author_association: CONTRIBUTOR
user: login: rhacs-bot
created_at: 2025-05-12T23:29:58Z
body: The backport to `release-4.6` failed:
```
The process '/usr/bin/git' failed with exit code 1
```
To backport manually, run these commands in your terminal:
```bash
# Fetch latest updates from GitHub
git fetch
# Create a new working tree
git worktree add .worktrees/backport-release-4.6 release-4.6
# Navigate to the new working tree
cd .worktrees/backport-release-4.6
# Create a new branch
git switch --create backport-14801-to-release-4.6
# Cherry-pick the merged commit of this pull request and resolve the conflicts
git cherry-pick -x --mainline 1 428f3beafd629cef62dbdc726280ca26b720ec20
# Push it to GitHub
git push --set-upstream origin backport-14801-to-release-4.6
# Go back to the original working tree
cd ../..
# Delete the working tree
git worktree remove .worktrees/backport-release-4.6
```
Then, create a pull request where the `base` branch is `release-4.6` and the `compare`/`head` branch is `backport-14801-to-release-4.6`.
author_association: CONTRIBUTOR