Repository: openshift-kni/numaresources-operator
Pull request #1248: e2e:serial: add wait on deployment deletion
Author: login: shajmakh
Labels: name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: Indicates that a PR is ready to be merged.
Created: 2025-04-08T11:31:24Z | Merged: 2025-04-28T12:48:59Z

The test can be flaky sometimes when the new deployment that is created post the scheduler deletion succeeds to be scheduled instead of pending. Add a wait period after the scheduler deletion to ensure it got deleted before creating new pods and moving forward with the test.


Diff:
diff --git a/test/e2e/serial/tests/scheduler_removal.go b/test/e2e/serial/tests/scheduler_removal.go
index 802df20b54..91f4056c90 100644
--- a/test/e2e/serial/tests/scheduler_removal.go
+++ b/test/e2e/serial/tests/scheduler_removal.go
@@ -45,27 +45,38 @@ import (
 
 var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources scheduler removal on a live cluster", Serial, Label("disruptive", "scheduler", "schedrst"), Label("feature:schedrst"), func() {
 	var fxt *e2efixture.Fixture
+	var nroSchedObj *nropv1.NUMAResourcesScheduler
+	var config *textlogger.Config
+	var dpNName nropv1.NamespacedName
 
 	BeforeEach(func() {
 		Expect(serialconfig.Config).ToNot(BeNil())
 		Expect(serialconfig.Config.Ready()).To(BeTrue(), "NUMA fixture initialization failed")
 
+		config = textlogger.NewConfig(textlogger.Verbosity(1))
+
 		var err error
 		fxt, err = e2efixture.Setup("e2e-test-sched-remove", serialconfig.Config.NRTList)
 		Expect(err).ToNot(HaveOccurred(), "unable to setup test fixture")
 
-		nrosched.CheckNROSchedulerAvailable(fxt.Client, serialconfig.Config.NROSchedObj.Name)
+		nroSchedObj = nrosched.CheckNROSchedulerAvailable(fxt.Client, serialconfig.Config.NROSchedObj.Name)
+		Expect(nroSchedObj).ToNot(BeNil())
+		dpNName = nroSchedObj.Status.Deployment
+
 	})
 
 	AfterEach(func() {
-		restoreScheduler(fxt, serialconfig.Config.NROSchedObj)
-		nrosched.CheckNROSchedulerAvailable(fxt.Client, serialconfig.Config.NROSchedObj.Name)
-
 		err := e2efixture.Teardown(fxt)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
 	When("removing the topology aware scheduler from a live cluster", func() {
+		AfterEach(func() {
+			restoreScheduler(fxt, serialconfig.Config.NROSchedObj)
+			nroSchedObj = nrosched.CheckNROSchedulerAvailable(fxt.Client, serialconfig.Config.NROSchedObj.Name)
+			Expect(nroSchedObj).ToNot(BeNil())
+		})
+
 		It("[case:1][test_id:47593] should keep existing workloads running", Label(label.Tier1), func() {
 			var err error
 
@@ -75,6 +86,10 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 			err = fxt.Client.Delete(context.TODO(), serialconfig.Config.NROSchedObj)
 			Expect(err).ToNot(HaveOccurred())
 
+			// make sure scheduler deployment is gone
+			err = depwait.With(fxt.Client, textlogger.NewLogger(config)).Interval(10*time.Second).Timeout(time.Minute).ForDeploymentDeleted(context.TODO(), dpNName.Namespace, dpNName.Name)
+			Expect(err).ToNot(HaveOccurred())
+
 			maxStep := 3
 			for step := 0; step < maxStep; step++ {
 				time.Sleep(10 * time.Second)
@@ -96,6 +111,10 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 			err = fxt.Client.Delete(context.TODO(), serialconfig.Config.NROSchedObj)
 			Expect(err).ToNot(HaveOccurred())
 
+			// make sure scheduler deployment is gone
+			err = depwait.With(fxt.Client, textlogger.NewLogger(config)).Interval(10*time.Second).Timeout(time.Minute).ForDeploymentDeleted(context.TODO(), dpNName.Namespace, dpNName.Name)
+			Expect(err).ToNot(HaveOccurred())
+
 			dp := createDeployment(fxt, "testdp", serialconfig.Config.SchedulerName)
 
 			maxStep := 3
@@ -112,53 +131,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 			}
 		})
 	})
-})
-
-var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources scheduler restart on a live cluster", Serial, Label("disruptive", "scheduler", "schedrst"), Label("feature:schedrst"), func() {
-	var fxt *e2efixture.Fixture
-	var nroSchedObj *nropv1.NUMAResourcesScheduler
-	var schedulerName string
-
-	BeforeEach(func() {
-		var err error
-		fxt, err = e2efixture.Setup("e2e-test-sched-remove", serialconfig.Config.NRTList)
-		Expect(err).ToNot(HaveOccurred(), "unable to setup test fixture")
-
-		nroSchedObj = &nropv1.NUMAResourcesScheduler{}
-		nroSchedKey := objects.NROSchedObjectKey()
-		err = fxt.Client.Get(context.TODO(), nroSchedKey, nroSchedObj)
-		Expect(err).ToNot(HaveOccurred(), "cannot get %q in the cluster", nroSchedKey.String())
-
-		schedulerName = nroSchedObj.Status.SchedulerName
-		Expect(schedulerName).ToNot(BeEmpty(), "cannot autodetect the TAS scheduler name from the cluster")
-
-		nrosched.CheckNROSchedulerAvailable(fxt.Client, nroSchedObj.Name)
-	})
-
-	AfterEach(func() {
-		err := e2efixture.Teardown(fxt)
-		Expect(err).NotTo(HaveOccurred())
-	})
 
 	When("restarting the topology aware scheduler in a live cluster", func() {
 		It("[case:1][test_id:48069] should schedule any pending workloads submitted while the scheduler was unavailable", Label(label.Tier2), func() {
 			var err error
 
-			dpNName := nroSchedObj.Status.Deployment // shortcut
-			if dpNName.Namespace == "" || dpNName.Name == "" {
-				Fail(fmt.Sprintf("scheduler deployment missing: %q", dpNName.String()))
-			}
-
 			By(fmt.Sprintf("deleting the NRO Scheduler object: %s", nroSchedObj.Name))
 			err = fxt.Client.Delete(context.TODO(), nroSchedObj)
 			Expect(err).ToNot(HaveOccurred())
 
 			// make sure scheduler deployment is gone
-			config := textlogger.NewConfig(textlogger.Verbosity(1))
 			err = depwait.With(fxt.Client, textlogger.NewLogger(config)).Interval(10*time.Second).Timeout(time.Minute).ForDeploymentDeleted(context.TODO(), dpNName.Namespace, dpNName.Name)
 			Expect(err).ToNot(HaveOccurred())
 
-			dp := createDeployment(fxt, "testdp", schedulerName)
+			dp := createDeployment(fxt, "testdp", nroSchedObj.Status.SchedulerName)
 
 			updatedDp := &appsv1.Deployment{}
 			maxStep := 3
@@ -174,13 +160,15 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 			}
 
 			restoreScheduler(fxt, nroSchedObj)
-			nrosched.CheckNROSchedulerAvailable(fxt.Client, nroSchedObj.Name)
+			nroSchedObj = nrosched.CheckNROSchedulerAvailable(fxt.Client, nroSchedObj.Name)
+			Expect(nroSchedObj).ToNot(BeNil())
 
 			By(fmt.Sprintf("waiting for the test deployment %q to become complete and ready", updatedDp.Name))
 			_, err = wait.With(fxt.Client).Interval(2*time.Second).Interval(30*time.Second).ForDeploymentComplete(context.TODO(), updatedDp)
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})
+
 })
 
 func restoreScheduler(fxt *e2efixture.Fixture, nroSchedObj *nropv1.NUMAResourcesScheduler) {
diff --git a/test/internal/nrosched/nrosched.go b/test/internal/nrosched/nrosched.go
index 58094db017..cb4085420f 100644
--- a/test/internal/nrosched/nrosched.go
+++ b/test/internal/nrosched/nrosched.go
@@ -176,6 +176,13 @@ func CheckNROSchedulerAvailable(cli client.Client, schedObjName string) *nropv1.
 
 		return cond.Status == metav1.ConditionTrue
 	}, 5*time.Minute, 10*time.Second).Should(BeTrue(), "Scheduler condition did not become available")
+
+	dpNName := nroSchedObj.Status.Deployment
+	if dpNName.Namespace == "" || dpNName.Name == "" {
+		klog.Errorf("scheduler deployment missing: %q", dpNName.String())
+		return nil
+	}
+
 	return nroSchedObj
 }
 


Reviews:
user: login: Tal-or
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T12:24:19Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T20:46:04Z
user: login: shajmakh
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-14T12:58:52Z
user: login: shajmakh
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T11:08:51Z
user: login: shajmakh
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T11:09:34Z
user: login: Tal-or
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-28T09:56:23Z

Review comments:
diff_hunk: @@ -71,10 +72,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 
 			dp := createDeploymentSync(fxt, "testdp", serialconfig.Config.SchedulerName)
 
+			dpNName := nroSchedObj.Status.Deployment
path: test/e2e/serial/tests/scheduler_removal.go
user: login: Tal-or
body: A more direct approach would be `Get` the deployment
created_at: 2025-04-10T12:23:21Z
author_association: COLLABORATOR
diff_hunk: @@ -92,10 +103,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 		It("[case:2][test_id:49093]should keep new scheduled workloads pending", Label(label.Tier1, "unsched", "feature:unsched"), func() {
 			var err error
 
+			dpNName := nroSchedObj.Status.Deployment
path: test/e2e/serial/tests/scheduler_removal.go
user: login: Tal-or
body: ditto
created_at: 2025-04-10T12:23:52Z
author_association: COLLABORATOR
diff_hunk: @@ -71,10 +72,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 
 			dp := createDeploymentSync(fxt, "testdp", serialconfig.Config.SchedulerName)
 
+			dpNName := nroSchedObj.Status.Deployment
+			if dpNName.Namespace == "" || dpNName.Name == "" {
+				Fail(fmt.Sprintf("scheduler deployment missing: %q", dpNName.String()))
+			}
+
 			By(fmt.Sprintf("deleting the NRO Scheduler object: %s", serialconfig.Config.NROSchedObj.Name))
 			err = fxt.Client.Delete(context.TODO(), serialconfig.Config.NROSchedObj)
 			Expect(err).ToNot(HaveOccurred())
 
+			// make sure scheduler deployment is gone
+			config := textlogger.NewConfig(textlogger.Verbosity(1))
path: test/e2e/serial/tests/scheduler_removal.go
user: login: ffromani
body: we can and should reuse a single logger for all the use in this group of test, just move it in `Describe`, initialize in the top level `BeforeEach` and use when needed
created_at: 2025-04-10T20:40:16Z
author_association: MEMBER
diff_hunk: @@ -71,10 +72,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 
 			dp := createDeploymentSync(fxt, "testdp", serialconfig.Config.SchedulerName)
 
+			dpNName := nroSchedObj.Status.Deployment
path: test/e2e/serial/tests/scheduler_removal.go
user: login: ffromani
body: well, this is what we do usually, but arguably this current approach implemented here is the one more direct, because the operator is reporting in its status the namespace/name of the Deployment to enable direct lookup (and easy matching)

Continuing with this line of thinking, we actually report a narrowed down ObjectReference exploiting the fact the object reference is always a Depoloyment, so it may be unusual but doesn't seem too wrong. One way or the other, it's part of our API.

All that said, we should have a test to ensure the reported daemonset namespaced name matches the reality (in a separate PR of course) but I slightly prefer the current approach because it also work as live example of our API should be used (dogfooding)
created_at: 2025-04-10T20:45:06Z
author_association: MEMBER
diff_hunk: @@ -71,10 +72,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 
 			dp := createDeploymentSync(fxt, "testdp", serialconfig.Config.SchedulerName)
 
+			dpNName := nroSchedObj.Status.Deployment
+			if dpNName.Namespace == "" || dpNName.Name == "" {
path: test/e2e/serial/tests/scheduler_removal.go
user: login: ffromani
body: how can this happen? should we check in `BeforeEach`?
created_at: 2025-04-10T20:45:49Z
author_association: MEMBER
diff_hunk: @@ -92,10 +103,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 		It("[case:2][test_id:49093]should keep new scheduled workloads pending", Label(label.Tier1, "unsched", "feature:unsched"), func() {
 			var err error
 
+			dpNName := nroSchedObj.Status.Deployment
+			if dpNName.Namespace == "" || dpNName.Name == "" {
+				Fail(fmt.Sprintf("scheduler deployment missing: %q", dpNName.String()))
+			}
+
 			By(fmt.Sprintf("deleting the NRO Scheduler object: %s", serialconfig.Config.NROSchedObj.Name))
 			err = fxt.Client.Delete(context.TODO(), serialconfig.Config.NROSchedObj)
 			Expect(err).ToNot(HaveOccurred())
 
+			// make sure scheduler deployment is gone
+			config := textlogger.NewConfig(textlogger.Verbosity(1))
path: test/e2e/serial/tests/scheduler_removal.go
user: login: ffromani
body: ditto
created_at: 2025-04-10T20:46:01Z
author_association: MEMBER
diff_hunk: @@ -71,10 +72,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 
 			dp := createDeploymentSync(fxt, "testdp", serialconfig.Config.SchedulerName)
 
+			dpNName := nroSchedObj.Status.Deployment
+			if dpNName.Namespace == "" || dpNName.Name == "" {
path: test/e2e/serial/tests/scheduler_removal.go
user: login: shajmakh
body: we don't but we should ensure that, I'll fix this in the called function 

created_at: 2025-04-14T12:58:52Z
author_association: MEMBER
diff_hunk: @@ -92,10 +103,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 		It("[case:2][test_id:49093]should keep new scheduled workloads pending", Label(label.Tier1, "unsched", "feature:unsched"), func() {
 			var err error
 
+			dpNName := nroSchedObj.Status.Deployment
path: test/e2e/serial/tests/scheduler_removal.go
user: login: shajmakh
body: Thanks, as discussed in other comments, I have added this check as part of CheckNROSchedulerAvailable
created_at: 2025-04-24T11:08:51Z
author_association: MEMBER
diff_hunk: @@ -71,10 +72,20 @@ var _ = Describe("[serial][disruptive][scheduler][schedrst] numaresources schedu
 
 			dp := createDeploymentSync(fxt, "testdp", serialconfig.Config.SchedulerName)
 
+			dpNName := nroSchedObj.Status.Deployment
path: test/e2e/serial/tests/scheduler_removal.go
user: login: shajmakh
body: as discussed in other comments, I have added this check as part of CheckNROSchedulerAvailable
created_at: 2025-04-24T11:09:34Z
author_association: MEMBER

PR comments:
user: login: shajmakh
created_at: 2025-04-10T10:59:31Z
body: /cc @SargunNarula 
author_association: MEMBER
user: login: shajmakh
created_at: 2025-04-24T14:46:44Z
body: /retest
author_association: MEMBER
user: login: openshift-ci[bot]
created_at: 2025-04-28T09:56:33Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/openshift-kni/numaresources-operator/pull/1248#" title="Author self-approved">shajmakh</a>*, *<a href="https://github.com/openshift-kni/numaresources-operator/pull/1248#pullrequestreview-2798752347" title="Approved">Tal-or</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=openshift-kni%2Fnumaresources-operator).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[test/OWNERS](https://github.com/openshift-kni/numaresources-operator/blob/main/test/OWNERS)~~ [Tal-or,shajmakh]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR