Repository: konflux-ci/konflux-ci
Pull request #1626: test: validate docker-build-oci-ta pipelines
Author: login: psturc
Created: 2025-03-20T15:49:24Z | Merged: 2025-03-31T14:53:38Z

closes https://github.com/konflux-ci/konflux-ci/issues/812

depends on https://github.com/konflux-ci/e2e-tests/pull/1550 (should be merged after that one is in)


Diff:
diff --git a/test/e2e/customize-docker-pipeline.sh b/test/e2e/customize-docker-pipeline.sh
index e85bc84656..397c0b70ed 100755
--- a/test/e2e/customize-docker-pipeline.sh
+++ b/test/e2e/customize-docker-pipeline.sh
@@ -1,8 +1,44 @@
-#!/bin/bash -e
-set -x
+#!/bin/bash
+set -euo pipefail
 
 script_path="$(dirname -- "${BASH_SOURCE[0]}")"
 
+update_pipeline() {
+    local pipeline_name=$1
+    local tmp_pipeline_path="/tmp/customized-${pipeline_name}.yaml"
+    local original_pipeline_bundle_ref
+
+    original_pipeline_bundle_ref=$(yq ".data[\"config.yaml\"]" konflux-ci/build-service/core/build-pipeline-config.yaml | yq ".pipelines[] | select(.name == \"${pipeline_name}\").bundle")
+    tkn bundle list --remote-skip-tls "$original_pipeline_bundle_ref" -o yaml > "${tmp_pipeline_path}"
+
+    for task_name in $(yq "(.spec.tasks[] | .name)" "${tmp_pipeline_path}");
+    do
+        local original_task_bundle_ref
+        local tmp_task_path="/tmp/customized-task.yaml"
+        local customized_task_image_ref_localhost="localhost:30001/test/test:customized-task-${task_name}-${pipeline_name}"
+        local customized_task_image_ref_cluster="registry-service.kind-registry/test/test:customized-task-${task_name}-${pipeline_name}"
+
+        original_task_bundle_ref=$(yq "(.spec.tasks[] | select(.name == \"${task_name}\").taskRef.params[] | select(.name == \"bundle\").value)" "${tmp_pipeline_path}")
+        tkn bundle list --remote-skip-tls "$original_task_bundle_ref" -o yaml > $tmp_task_path || continue
+
+        yq e -i '(.spec.steps[] | select(has("computeResources")) | .computeResources.requests.cpu) |= "100m"' $tmp_task_path
+        yq e -i '(.spec.steps[] | select(has("computeResources")) | .computeResources.requests.memory) |= "256Mi"' $tmp_task_path
+        yq e -i '(.spec.steps[] | select(has("resources")) | .resources.requests.cpu) |= "100m"' $tmp_task_path
+        yq e -i '(.spec.steps[] | select(has("resources")) | .resources.requests.memory) |= "256Mi"' $tmp_task_path
+
+        yq e -i "(.spec.tasks[] | select(.name == \"${task_name}\").taskRef.params[] | select(.name == \"bundle\").value) |= \"${customized_task_image_ref_cluster}\"" "${tmp_pipeline_path}"
+
+        tkn bundle push --remote-skip-tls "$customized_task_image_ref_localhost" -f $tmp_task_path
+    done
+
+    local customized_pipeline_image_ref_localhost="localhost:30001/test/test:customized-${pipeline_name}-pipeline"
+    tkn bundle push --remote-skip-tls "$customized_pipeline_image_ref_localhost" -f "${tmp_pipeline_path}"
+
+    # Update the bundle ref in build-service pipeline configmap
+    local customized_pipeline_image_ref_cluster="registry-service.kind-registry/test/test:customized-${pipeline_name}-pipeline"
+    sed -i "s|bundle:.*${pipeline_name}@.*|bundle: ${customized_pipeline_image_ref_cluster}|g" konflux-ci/build-service/core/build-pipeline-config.yaml
+}
+
 main() {
     echo "Updating docker-build pipeline to workaround the issues with and cpu/mem requests for build task" >&2
     # shellcheck disable=SC1091
@@ -11,28 +47,7 @@ main() {
     # This is required in order to push the modified pipeline to local registry
     kubectl port-forward -n kind-registry svc/registry-service 30001:443 &
 
-    local original_docker_build_bundle_ref
-    local tmp_pipeline_path="/tmp/customized-docker-pipeline.yaml"
-    local tmp_build_task_path="/tmp/customized-build-container-task.yaml"
-    # Pull the required Pipeline and Task bundles
-    original_docker_build_bundle_ref=$(yq ".data[\"config.yaml\"]" konflux-ci/build-service/core/build-pipeline-config.yaml | yq ".pipelines[] | select(.name == \"docker-build\").bundle")
-    tkn bundle list --remote-skip-tls "$original_docker_build_bundle_ref" -o yaml > $tmp_pipeline_path
-    original_build_task_bundle_ref=$(yq "(.spec.tasks[] | select(.name == \"build-container\").taskRef.params[] | select(.name == \"bundle\").value)" $tmp_pipeline_path)
-    tkn bundle list --remote-skip-tls "$original_build_task_bundle_ref" -o yaml > $tmp_build_task_path
-    # Workaround
-    # CPU and memory requests are too high for the build-container Task's "build" step to run in GitHub actions
-    # Thus this workaround lowers ".computeResources.requests" from the "build" step and updates the build Pipeline that is used in CI to reference the updated Task
-    yq e -i "(.spec.steps[].computeResources.requests.cpu) |= \"100m\"" $tmp_build_task_path
-    yq e -i "(.spec.steps[].computeResources.requests.memory) |= \"256Mi\"" $tmp_build_task_path
-    yq e -i "(.spec.tasks[] | select(.name == \"build-container\").taskRef.params[] | select(.name == \"bundle\").value) |= \"${CUSTOMIZED_BUILD_TASK_IMAGE_REF_CLUSTER}\"" $tmp_pipeline_path
-
-    # Push the modified Pipeline and Task bundles to the local container registry
-    tkn bundle push --remote-skip-tls "$CUSTOMIZED_DOCKER_PIPELINE_IMAGE_REF_LOCALHOST" \
-        -f $tmp_pipeline_path
-    tkn bundle push --remote-skip-tls "$CUSTOMIZED_BUILD_TASK_IMAGE_REF_LOCALHOST" \
-        -f $tmp_build_task_path
-    # Update the bundle ref in build-service pipeline configmap
-    sed -i "s|bundle:.*docker-build.*|bundle: ${CUSTOMIZED_DOCKER_PIPELINE_IMAGE_REF_CLUSTER}|g" konflux-ci/build-service/core/build-pipeline-config.yaml
+    update_pipeline docker-build-oci-ta
 }
 
 
diff --git a/test/e2e/run-e2e.sh b/test/e2e/run-e2e.sh
index 3f72fabb01..9ba6f9f3df 100755
--- a/test/e2e/run-e2e.sh
+++ b/test/e2e/run-e2e.sh
@@ -9,7 +9,7 @@ main() {
     local github_org="${GH_ORG:?A GitHub org where the https://github.com/redhat-appstudio-qe/konflux-ci-sample repo is present/forked should be provided}"
     local github_token="${GH_TOKEN:?A GitHub token should be provided}"
     local quay_dockerconfig="${QUAY_DOCKERCONFIGJSON:?quay.io credentials in .dockerconfig format should be provided}"
-    docker run --network=host -v ~/.kube/config:/kube/config --env KUBECONFIG=/kube/config -e GITHUB_TOKEN="$github_token" -e QUAY_TOKEN="$(base64 <<< "$quay_dockerconfig")" -e MY_GITHUB_ORG="$github_org" -e E2E_APPLICATIONS_NAMESPACE=user-ns2 "$E2E_TEST_IMAGE" /konflux-e2e/konflux-e2e.test "--ginkgo.label-filter=upstream-konflux" "--ginkgo.focus=Test local" "--ginkgo.v"
+    docker run --network=host -v ~/.kube/config:/kube/config --env KUBECONFIG=/kube/config -e GITHUB_TOKEN="$github_token" -e QUAY_TOKEN="$(base64 <<< "$quay_dockerconfig")" -e MY_GITHUB_ORG="$github_org" -e E2E_APPLICATIONS_NAMESPACE=user-ns2 "$E2E_TEST_IMAGE"  /bin/bash -c "ginkgo -v --label-filter=upstream-konflux --focus=\"Test local\" /konflux-e2e/konflux-e2e.test"
 }
 
 
diff --git a/test/e2e/vars.sh b/test/e2e/vars.sh
index c8c0afd6e5..17c5323433 100755
--- a/test/e2e/vars.sh
+++ b/test/e2e/vars.sh
@@ -1,8 +1,4 @@
 #!/bin/bash -e
-export E2E_TEST_IMAGE CUSTOMIZED_DOCKER_PIPELINE_IMAGE_REF_LOCALHOST CUSTOMIZED_DOCKER_PIPELINE_IMAGE_REF_CLUSTER CUSTOMIZED_BUILD_TASK_IMAGE_REF_LOCALHOST CUSTOMIZED_BUILD_TASK_IMAGE_REF_CLUSTER
+export E2E_TEST_IMAGE
 # https://github.com/konflux-ci/e2e-tests/commit/<COMMIT>
-E2E_TEST_IMAGE=quay.io/redhat-user-workloads/konflux-qe-team-tenant/konflux-e2e/konflux-e2e-tests@sha256:667faab108b533a46e8ff308132dd26b4dd94dcd1bdd75d2b537beccda1c0d4f
-CUSTOMIZED_DOCKER_PIPELINE_IMAGE_REF_LOCALHOST="localhost:30001/test/test:customized-docker-pipeline"
-CUSTOMIZED_BUILD_TASK_IMAGE_REF_LOCALHOST="localhost:30001/test/test:customized-build-task"
-CUSTOMIZED_DOCKER_PIPELINE_IMAGE_REF_CLUSTER="registry-service.kind-registry/test/test:customized-docker-pipeline"
-CUSTOMIZED_BUILD_TASK_IMAGE_REF_CLUSTER="registry-service.kind-registry/test/test:customized-build-task"
+E2E_TEST_IMAGE=quay.io/redhat-user-workloads/konflux-qe-team-tenant/konflux-e2e/konflux-e2e-tests@sha256:e9a739d65c5f7809c29103217d8bc36a419cf3ff41be04c89f10324c4be9fd21


Reviews:
user: login: github-advanced-security[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-21T17:39:36Z
user: login: yftacherzog
body: lgtm. Thank you!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-26T07:54:36Z

Review comments:
diff_hunk: @@ -3,19 +3,18 @@
 
 script_path="$(dirname -- "${BASH_SOURCE[0]}")"
 
-main() {
-    echo "Updating docker-build pipeline to workaround the issues with and cpu/mem requests for build task" >&2
-    # shellcheck disable=SC1091
-    source "${script_path}/vars.sh"
-
-    # This is required in order to push the modified pipeline to local registry
-    kubectl port-forward -n kind-registry svc/registry-service 30001:443 &
+update_pipelines() {
+    local pipeline_name=$1
+    local customized_build_task_image_ref_localhost="localhost:30001/test/test:customized-build-task-${pipeline_name}"
path: test/e2e/customize-docker-pipeline.sh
user: login: github-advanced-security[bot]
body: ## SC2034

customized_build_task_image_ref_localhost appears unused. Verify use (or export if used externally).

[Show more details](https://github.com/konflux-ci/konflux-ci/security/code-scanning/35)
created_at: 2025-03-21T17:39:36Z
author_association: NONE
diff_hunk: @@ -3,19 +3,18 @@
 
 script_path="$(dirname -- "${BASH_SOURCE[0]}")"
 
-main() {
-    echo "Updating docker-build pipeline to workaround the issues with and cpu/mem requests for build task" >&2
-    # shellcheck disable=SC1091
-    source "${script_path}/vars.sh"
-
-    # This is required in order to push the modified pipeline to local registry
-    kubectl port-forward -n kind-registry svc/registry-service 30001:443 &
+update_pipelines() {
+    local pipeline_name=$1
+    local customized_build_task_image_ref_localhost="localhost:30001/test/test:customized-build-task-${pipeline_name}"
+    local customized_build_task_image_ref_cluster="registry-service.kind-registry/test/test:customized-build-task-${pipeline_name}"
path: test/e2e/customize-docker-pipeline.sh
user: login: github-advanced-security[bot]
body: ## SC2034

customized_build_task_image_ref_cluster appears unused. Verify use (or export if used externally).

[Show more details](https://github.com/konflux-ci/konflux-ci/security/code-scanning/36)
created_at: 2025-03-21T17:39:36Z
author_association: NONE

PR comments:
user: login: psturc
created_at: 2025-03-25T05:52:54Z
body: Hi @ralphbean @yftacherzog 

following up on [this comment](https://github.com/konflux-ci/konflux-ci/pull/803#issuecomment-2503560205) - in this PR I've been trying to update the test to validate 2 pipelines: `docker-build` and `docker-build-oci-ta` (instead of only the `docker-build`, which is currently being tested in PR checks of this repo).

All the time I'm hitting an issue with insufficient memory in github actions, even after [customizing all build pipelines' Tasks to request a minimum cpu/memory](https://github.com/konflux-ci/konflux-ci/pull/1626/files#diff-fd17c12f6a9db21ecea8c3d688e76cad475c81ac6b6ba0ef6eaa958c5bee288eR24-R27) and [increasing the system-reserved memory to 12Gi](https://github.com/konflux-ci/konflux-ci/pull/1626/files#diff-b085642d962662414dd873573d4e087a3c45e50a036271863583244f93fa04a6R12).

I'm at the stage where I think that it's not worth to test both of those pipelines (even if we'd overcome the memory limitation, the test would take ~10 minutes more).

So if we're not going to increase the compute resources on github actions' node size (it'd be no longer free), I'd suggest to update our test to validate only `docker-build-oci-ta` pipeline (the default one) for now in PR checks of this repo (I think it's not a bad situation, because other build pipelines are still being tested in build-definitions PR checks).

Wdyt?
author_association: MEMBER
user: login: yftacherzog
created_at: 2025-03-25T07:59:02Z
body: > Hi @ralphbean @yftacherzog
> 
> following up on [this comment](https://github.com/konflux-ci/konflux-ci/pull/803#issuecomment-2503560205) - in this PR I've been trying to update the test to validate 2 pipelines: `docker-build` and `docker-build-oci-ta` (instead of only the `docker-build`, which is currently being tested in PR checks of this repo).
> 
> All the time I'm hitting an issue with insufficient memory in github actions, even after [customizing all build pipelines' Tasks to request a minimum cpu/memory](https://github.com/konflux-ci/konflux-ci/pull/1626/files#diff-fd17c12f6a9db21ecea8c3d688e76cad475c81ac6b6ba0ef6eaa958c5bee288eR24-R27) and [increasing the system-reserved memory to 12Gi](https://github.com/konflux-ci/konflux-ci/pull/1626/files#diff-b085642d962662414dd873573d4e087a3c45e50a036271863583244f93fa04a6R12).
> 
> I'm at the stage where I think that it's not worth to test both of those pipelines (even if we'd overcome the memory limitation, the test would take ~10 minutes more).
> 
> So if we're not going to increase the compute resources on github actions' node size (it'd be no longer free), I'd suggest to update our test to validate only `docker-build-oci-ta` pipeline (the default one) for now in PR checks of this repo (I think it's not a bad situation, because other build pipelines are still being tested in build-definitions PR checks).
> 
> Wdyt?

@psturc thank you for picking this up!

I think it would be fine to only have the oci-ta variant for now (if we decide we need both eventually, then we can create another workflow that will use additional runner. No?).

In parallel, we're (@amisstea is) also working on making this oci-ta pipeline compatible with the in-cluster registry, so it should be what we will use for all our documented use cases.

All in all, having the oci-ta variant tested here instead of the non-oci-ta variant will be a great improvement!

cc @gbenhaim @avi-biton 

author_association: MEMBER
user: login: amisstea
created_at: 2025-03-25T13:09:21Z
body: @psturc the latest bundle of the `docker-build-oci-ta` pipeline now (within the last ~24hours) fully supports the internal registry in case you want to start testing with that.
author_association: MEMBER
user: login: yftacherzog
created_at: 2025-03-31T11:31:59Z
body: @psturc is it ready to be merged?
author_association: MEMBER
user: login: psturc
created_at: 2025-03-31T11:49:40Z
body: @yftacherzog yes I had to update the e2e image after https://github.com/konflux-ci/e2e-tests/pull/1550 was merged, so let's wait for the e2e to finish and then merge 🚀 
author_association: MEMBER
user: login: yftacherzog
created_at: 2025-03-31T11:52:20Z
body: > @yftacherzog yes I had to update the e2e image after [konflux-ci/e2e-tests#1550](https://github.com/konflux-ci/e2e-tests/pull/1550) was merged, so let's wait for the e2e to finish and then merge 🚀

Alright. Feel free to merge it once you're ready.
author_association: MEMBER