Repository: kubernetes-sigs/kueue

Issue:
[Flaky E2E Test]: Deployment should admit workloads after change queue-name if AvailableReplicas = 0
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!

If the matter is security related, please disclose it privately via https://kubernetes.io/security/
-->

**What happened**:
End To End Suite: kindest/node:v1.30.10: [It] Deployment should admit workloads after change queue-name if AvailableReplicas = 0

```
{Expected
    <[]v1.Condition | len:0, cap:0>: nil
to have condition type QuotaReserved and status False failed [FAILED] Expected
    <[]v1.Condition | len:0, cap:0>: nil
to have condition type QuotaReserved and status False
In [It] at: /home/prow/go/src/sigs.k8s.io/kueue/test/e2e/singlecluster/deployment_test.go:168 @ 05/05/25 11:22:26.192
}
```

**What you expected to happen**:

No errors

**How to reproduce it (as minimally and precisely as possible)**:

https://prow.k8s.io/view/gs/kubernetes-ci-logs/pr-logs/pull/kubernetes-sigs_kueue/5160/pull-kueue-test-e2e-main-1-30/1919348839219728384

**Anything else we need to know?**:

/kind flake

**Environment**:
- Kubernetes version (use `kubectl version`):
- Kueue version (use `git describe --tags --dirty --always`):
- Cloud provider or hardware configuration:
- OS (e.g: `cat /etc/os-release`):
- Kernel (e.g. `uname -a`):
- Install tools:
- Others:

Discussion:
@mbobrovskyi: /assign

Solution patch:
diff --git a/.gitignore b/.gitignore
index b8fa17518..c9f4a438b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,8 @@ testbin/*
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# build_logs directory, where by default are downloaded artifacts from the CI job
+build_logs
 
 # editor and IDE paraphernalia
 .idea
diff --git a/hack/get_build_logs.sh b/hack/get_build_logs.sh
new file mode 100755
index 000000000..7247580e9
--- /dev/null
+++ b/hack/get_build_logs.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script is designed to facilitate the build logs from a Kueue CI/CD (GCS) bucket using the gcloud CLI.
+# It transforms a given URL copied from the CI into a format compatible with the gcloud storage cp command.
+# It ensures that the gcloud CLI is installed and that given URL is in fact pointing to Kueue related GCS bucket.
+# Second command line argument is optional and allows to define output directory, which by default is located in build_logs at project direcotry level.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+transform_url() {
+    local url="$1"
+    local prefix="https://prow.k8s.io/view/gs/"  # Prefix to remove
+    local replacement="gs://"  # Replacement prefix
+
+    # Check if the URL starts with the prefix
+    if [[ "$url" == "$prefix"* ]]; then
+        # Replace the prefix with 'gs://'
+        local transformed_url="${url/$prefix/$replacement}"
+        echo "$transformed_url"
+    else
+        echo "Error: URL does not start with the expected prefix: $prefix"
+        return 1
+    fi
+}
+
+if ! command -v gcloud &> /dev/null; then
+    echo "Can't find Google Cloud SDK (gcloud CLI) in PATH."
+    echo "Please install it by following the instructions at: https://cloud.google.com/sdk/docs/install"
+    exit 1
+fi
+
+if [[ "$1" != *"kubernetes-sigs_kueue"* ]]; then
+    echo "The URL does not point to Kueue build logs bucket."
+fi
+
+transformed_url=$(transform_url "$1")
+root_dir="$(cd "$(dirname "$0")/.." && pwd)"
+output_dir=${2:-"$root_dir/build_logs"}
+mkdir -p "$output_dir"
+gsutil -m cp -r "$transformed_url" "$output_dir"
+



Test patch:
diff --git a/test/e2e/singlecluster/deployment_test.go b/test/e2e/singlecluster/deployment_test.go
index f89481bc7..f017ab63c 100644
--- a/test/e2e/singlecluster/deployment_test.go
+++ b/test/e2e/singlecluster/deployment_test.go
@@ -27,8 +27,6 @@ import (
 	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 	"sigs.k8s.io/kueue/pkg/controller/constants"
 	"sigs.k8s.io/kueue/pkg/controller/jobs/pod"
-	podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants"
-	utilpod "sigs.k8s.io/kueue/pkg/util/pod"
 	"sigs.k8s.io/kueue/pkg/util/testing"
 	deploymenttesting "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment"
 	"sigs.k8s.io/kueue/test/util"
@@ -154,20 +152,17 @@ var _ = ginkgo.Describe("Deployment", func() {
 		pods := &corev1.PodList{}
 		gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
 			client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(gomega.Succeed())
+		gomega.Expect(pods.Items).To(gomega.HaveLen(3))
 
-		createdWorkloads := make([]*kueue.Workload, 0, len(pods.Items))
+		createdWorkloads := &kueue.WorkloadList{}
 		ginkgo.By("Check that workloads are created but not admitted", func() {
-			for _, p := range pods.Items {
-				gomega.Expect(utilpod.HasGate(&p, podconstants.SchedulingGateName)).Should(gomega.BeTrue())
-				createdWorkload := &kueue.Workload{}
-				wlLookupKey := types.NamespacedName{
-					Name:      pod.GetWorkloadNameForPod(p.Name, p.UID),
-					Namespace: p.Namespace,
+			gomega.Eventually(func(g gomega.Gomega) {
+				g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(ns.Name))).To(gomega.Succeed())
+				g.Expect(createdWorkloads.Items).To(gomega.HaveLen(3))
+				for _, wl := range createdWorkloads.Items {
+					g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved))
 				}
-				gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed())
-				gomega.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved))
-				createdWorkloads = append(createdWorkloads, createdWorkload)
-			}
+			}, util.Timeout, util.Interval).Should(gomega.Succeed())
 		})
 
 		ginkgo.By("Update queue-name on the deployment", func() {
@@ -195,19 +190,20 @@ var _ = ginkgo.Describe("Deployment", func() {
 			}, util.LongTimeout, util.Interval).Should(gomega.Succeed())
 		})
 
+		ginkgo.By("Check previous workloads are deleted", func() {
+			for _, wl := range createdWorkloads.Items {
+				util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sClient, &wl, false, util.LongTimeout)
+			}
+		})
+
 		ginkgo.By("Check that workloads are created and admitted", func() {
-			gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name),
-				client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(gomega.Succeed())
-			for _, p := range pods.Items {
-				createdWorkload := &kueue.Workload{}
-				wlLookupKey := types.NamespacedName{
-					Name:      pod.GetWorkloadNameForPod(p.Name, p.UID),
-					Namespace: p.Namespace,
+			gomega.Eventually(func(g gomega.Gomega) {
+				g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(ns.Name))).To(gomega.Succeed())
+				g.Expect(createdWorkloads.Items).To(gomega.HaveLen(3))
+				for _, wl := range createdWorkloads.Items {
+					g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted))
 				}
-				gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed())
-				gomega.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted))
-				createdWorkloads = append(createdWorkloads, createdWorkload)
-			}
+			}, util.Timeout, util.Interval).Should(gomega.Succeed())
 		})
 
 		ginkgo.By("Delete the deployment", func() {
@@ -215,8 +211,8 @@ var _ = ginkgo.Describe("Deployment", func() {
 		})
 
 		ginkgo.By("Check that workloads are deleted", func() {
-			for _, wl := range createdWorkloads {
-				util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sClient, wl, false, util.LongTimeout)
+			for _, wl := range createdWorkloads.Items {
+				util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sClient, &wl, false, util.LongTimeout)
 			}
 		})
 	})
diff --git a/test/util/e2e.go b/test/util/e2e.go
index 00ce4b8d7..76384b9e6 100644
--- a/test/util/e2e.go
+++ b/test/util/e2e.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"time"
 
 	cmv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
@@ -137,6 +138,9 @@ func CreateVisibilityClient(user string) visibilityv1beta1.VisibilityV1beta1Inte
 }
 
 func rolloutOperatorDeployment(ctx context.Context, k8sClient client.Client, key types.NamespacedName) {
+	// Export logs before the rollout to preserve logs from the previous version.
+	exportKindLogs(ctx)
+
 	deployment := &appsv1.Deployment{}
 	var deploymentCondition *appsv1.DeploymentCondition
 	expectedDeploymentCondition := &appsv1.DeploymentCondition{
@@ -166,6 +170,20 @@ func rolloutOperatorDeployment(ctx context.Context, k8sClient client.Client, key
 	}, StartUpTimeout, Interval).Should(gomega.Succeed())
 }
 
+func exportKindLogs(ctx context.Context) {
+	// Path to the kind binary
+	kind := os.Getenv("KIND")
+	// Path to the artifacts
+	artifacts := os.Getenv("ARTIFACTS")
+
+	if kind != "" && artifacts != "" {
+		cmd := exec.CommandContext(ctx, kind, "export", "logs", artifacts)
+		cmd.Stdout = ginkgo.GinkgoWriter
+		cmd.Stderr = ginkgo.GinkgoWriter
+		gomega.Expect(cmd.Run()).To(gomega.Succeed())
+	}
+}
+
 func waitForOperatorAvailability(ctx context.Context, k8sClient client.Client, key types.NamespacedName) {
 	deployment := &appsv1.Deployment{}
 	pods := &corev1.PodList{}