Repository: submariner-io/submariner
Pull request #3346: Handle stale remote Endpoints removed in the route-agent health checker
Author: login: tpantelis
Labels: name: backport
description: This change requires a backport to eligible release branches
name: backport-handled
name: release-note-needed
description: Should be mentioned in the release notes
name: ready-to-test
description: When a PR is ready for full E2E testing
name: release-note-handled
Created: 2025-03-11T18:32:20Z | Merged: 2025-03-13T16:21:35Z

See commits for details.

Fixes https://github.com/submariner-io/submariner/issues/3335


Diff:
diff --git a/pkg/event/controller/controller_test.go b/pkg/event/controller/controller_test.go
index e7a9b253b..5398cd603 100644
--- a/pkg/event/controller/controller_test.go
+++ b/pkg/event/controller/controller_test.go
@@ -137,9 +137,12 @@ var _ = Describe("Event controller", func() {
 				Spec:       submV1.EndpointSpec{ClusterID: "east"},
 			})
 			t.ensureNoEvents()
+			Expect(t.handler.remoteEndpoints.Load()).To(Equal([]submV1.Endpoint{*latestEndpoint}))
 
 			t.DeleteEndpoint(staleEndpoint.GetName())
-			t.ensureNoEvents()
+			Consistently(func() any {
+				return t.handler.remoteEndpoints.Load()
+			}).Should(Equal([]submV1.Endpoint{*latestEndpoint}))
 
 			t.DeleteEndpoint(latestEndpoint.GetName())
 			t.awaitEvent(testing.EvRemoteEndpointRemoved, latestEndpoint)
@@ -148,6 +151,29 @@ var _ = Describe("Event controller", func() {
 			t.awaitEvent(testing.EvRemoteEndpointCreated, staleEndpoint)
 		})
 	})
+
+	When("a remote Endpoint is deleted after a newer Endpoint from the same cluster", func() {
+		It("should notify the handler of the stale removed Endpoint", func() {
+			now := time.Now()
+			staleEndpoint := t.CreateEndpoint(&submV1.Endpoint{
+				ObjectMeta: v1meta.ObjectMeta{Name: "stale", CreationTimestamp: v1meta.NewTime(now)},
+				Spec:       submV1.EndpointSpec{ClusterID: "east"},
+			})
+			t.awaitEvent(testing.EvRemoteEndpointCreated, staleEndpoint)
+
+			latestEndpoint := t.CreateEndpoint(&submV1.Endpoint{
+				ObjectMeta: v1meta.ObjectMeta{Name: "latest", CreationTimestamp: v1meta.NewTime(now.Add(2 * time.Second))},
+				Spec:       submV1.EndpointSpec{ClusterID: "east"},
+			})
+			t.awaitEvent(testing.EvRemoteEndpointCreated, latestEndpoint)
+			Expect(t.handler.remoteEndpoints.Load()).To(HaveLen(2))
+
+			t.DeleteEndpoint(staleEndpoint.GetName())
+			t.awaitEvent(testing.EvStaleRemoteEndpointRemoved, staleEndpoint)
+			t.ensureNoEvents()
+			Expect(t.handler.remoteEndpoints.Load()).To(Equal([]submV1.Endpoint{*latestEndpoint}))
+		})
+	})
 })
 
 type testDriver struct {
@@ -285,6 +311,11 @@ func (t *TestHandler) RemoteEndpointRemoved(endpoint *submV1.Endpoint) error {
 	return t.TestHandler.RemoteEndpointRemoved(endpoint)
 }
 
+func (t *TestHandler) StaleRemoteEndpointRemoved(endpoint *submV1.Endpoint) error {
+	t.storeRemoteEndpoints()
+	return t.TestHandler.StaleRemoteEndpointRemoved(endpoint)
+}
+
 func (t *TestHandler) storeRemoteEndpoints() {
 	eps := t.State().GetRemoteEndpoints()
 	if eps == nil {
diff --git a/pkg/event/controller/endpoint_removed.go b/pkg/event/controller/endpoint_removed.go
index f3fbdaeb2..a7349ea74 100644
--- a/pkg/event/controller/endpoint_removed.go
+++ b/pkg/event/controller/endpoint_removed.go
@@ -67,16 +67,15 @@ func (c *handlerController) handleRemovedLocalEndpoint(endpoint *smv1.Endpoint)
 }
 
 func (c *handlerController) handleRemovedRemoteEndpoint(endpoint *smv1.Endpoint) error {
+	c.handlerState.remoteEndpoints.Delete(endpoint.Name)
+
 	lastProcessedTime, ok := c.remoteEndpointTimeStamp[endpoint.Spec.ClusterID]
 
 	if ok && lastProcessedTime.After(endpoint.CreationTimestamp.Time) {
-		logger.Infof("Ignoring deleted remote %#v since a later endpoint was already"+
-			"processed", endpoint)
-		return nil
+		return c.handler.StaleRemoteEndpointRemoved(endpoint) //nolint:wrapcheck  // Let the caller wrap it
 	}
 
 	delete(c.remoteEndpointTimeStamp, endpoint.Spec.ClusterID)
-	c.handlerState.remoteEndpoints.Delete(endpoint.Name)
 
 	return c.handler.RemoteEndpointRemoved(endpoint) //nolint:wrapcheck  // Let the caller wrap it
 }
diff --git a/pkg/event/handler.go b/pkg/event/handler.go
index f794523af..54bb58da8 100644
--- a/pkg/event/handler.go
+++ b/pkg/event/handler.go
@@ -67,6 +67,10 @@ type EndpointHandler interface {
 
 	// RemoteEndpointRemoved is called when an endpoint associated with a remote cluster is removed
 	RemoteEndpointRemoved(endpoint *submV1.Endpoint) error
+
+	// StaleRemoteEndpointRemoved is called in lieu of RemoteEndpointRemoved when a notification for an endpoint
+	// associated with a remote cluster is received after a newer endpoint is received from the cluster.
+	StaleRemoteEndpointRemoved(endpoint *submV1.Endpoint) error
 }
 
 type NodeHandler interface {
@@ -164,6 +168,10 @@ func (ev *HandlerBase) RemoteEndpointRemoved(_ *submV1.Endpoint) error {
 	return nil
 }
 
+func (ev *HandlerBase) StaleRemoteEndpointRemoved(_ *submV1.Endpoint) error {
+	return nil
+}
+
 type NodeHandlerBase struct{}
 
 func (h *NodeHandlerBase) NodeCreated(_ *k8sV1.Node) error {
diff --git a/pkg/event/logger/handler.go b/pkg/event/logger/handler.go
index f4c33ee34..ea3ab10ab 100644
--- a/pkg/event/logger/handler.go
+++ b/pkg/event/logger/handler.go
@@ -76,13 +76,13 @@ func (l *Handler) RemoteEndpointCreated(endpoint *submV1.Endpoint) error {
 }
 
 func (l *Handler) RemoteEndpointUpdated(endpoint *submV1.Endpoint) error {
-	logger.V(log.DEBUG).Infof("A new Endpoint for remote cluster %q has been updated: %#v",
+	logger.V(log.DEBUG).Infof("An Endpoint for remote cluster %q has been updated: %#v",
 		endpoint.Spec.ClusterID, endpoint.Spec)
 	return nil
 }
 
 func (l *Handler) RemoteEndpointRemoved(endpoint *submV1.Endpoint) error {
-	logger.V(log.DEBUG).Infof("A new Endpoint for remote cluster %q has been removed: %#v",
+	logger.V(log.DEBUG).Infof("An Endpoint for remote cluster %q has been removed: %#v",
 		endpoint.Spec.ClusterID, endpoint.Spec)
 	return nil
 }
diff --git a/pkg/event/testing/testing.go b/pkg/event/testing/testing.go
index 4927880cc..66f729a3d 100644
--- a/pkg/event/testing/testing.go
+++ b/pkg/event/testing/testing.go
@@ -125,20 +125,21 @@ func (t *TestHandlerBase) GetNetworkPlugins() []string {
 }
 
 const (
-	EvTransitionToNonGateway = "TransitionToNonGateway"
-	EvTransitionToGateway    = "TransitionToGateway"
-	EvLocalEndpointCreated   = "LocalEndpointCreated"
-	EvLocalEndpointUpdated   = "LocalEndpointUpdated"
-	EvLocalEndpointRemoved   = "LocalEndpointRemoved"
-	EvRemoteEndpointCreated  = "RemoteEndpointCreated"
-	EvRemoteEndpointUpdated  = "RemoteEndpointUpdated"
-	EvRemoteEndpointRemoved  = "RemoteEndpointRemoved"
-	EvNodeCreated            = "NodeCreated"
-	EvNodeUpdated            = "NodeUpdated"
-	EvNodeRemoved            = "NodeRemoved"
-	EvStop                   = "Stop"
-	EvUninstall              = "Uninstall"
-	EvInit                   = "Init"
+	EvTransitionToNonGateway     = "TransitionToNonGateway"
+	EvTransitionToGateway        = "TransitionToGateway"
+	EvLocalEndpointCreated       = "LocalEndpointCreated"
+	EvLocalEndpointUpdated       = "LocalEndpointUpdated"
+	EvLocalEndpointRemoved       = "LocalEndpointRemoved"
+	EvRemoteEndpointCreated      = "RemoteEndpointCreated"
+	EvRemoteEndpointUpdated      = "RemoteEndpointUpdated"
+	EvRemoteEndpointRemoved      = "RemoteEndpointRemoved"
+	EvStaleRemoteEndpointRemoved = "StaleRemoteEndpointRemoved"
+	EvNodeCreated                = "NodeCreated"
+	EvNodeUpdated                = "NodeUpdated"
+	EvNodeRemoved                = "NodeRemoved"
+	EvStop                       = "Stop"
+	EvUninstall                  = "Uninstall"
+	EvInit                       = "Init"
 )
 
 func (t *TestHandlerBase) Stop() error {
@@ -181,6 +182,10 @@ func (t *TestHandlerBase) RemoteEndpointRemoved(endpoint *v1.Endpoint) error {
 	return t.addEvent(EvRemoteEndpointRemoved, endpoint)
 }
 
+func (t *TestHandlerBase) StaleRemoteEndpointRemoved(endpoint *v1.Endpoint) error {
+	return t.addEvent(EvStaleRemoteEndpointRemoved, endpoint)
+}
+
 func (t *TestHandler) NodeCreated(node *v12.Node) error {
 	return t.addEvent(EvNodeCreated, node)
 }
diff --git a/pkg/routeagent_driver/handlers/healthchecker/healthchecker.go b/pkg/routeagent_driver/handlers/healthchecker/healthchecker.go
index 0bd0eda61..fe7fb86df 100644
--- a/pkg/routeagent_driver/handlers/healthchecker/healthchecker.go
+++ b/pkg/routeagent_driver/handlers/healthchecker/healthchecker.go
@@ -160,6 +160,8 @@ func (h *controller) processEndpointCreatedOrUpdated(endpoint *submarinerv1.Endp
 }
 
 func (h *controller) RemoteEndpointRemoved(endpoint *submarinerv1.Endpoint) error {
+	logger.Infof("Processing removed Endpoint %q", endpoint.Spec.CableName)
+
 	h.Lock()
 	defer h.Unlock()
 
@@ -171,6 +173,10 @@ func (h *controller) RemoteEndpointRemoved(endpoint *submarinerv1.Endpoint) erro
 	return nil
 }
 
+func (h *controller) StaleRemoteEndpointRemoved(endpoint *submarinerv1.Endpoint) error {
+	return h.RemoteEndpointRemoved(endpoint)
+}
+
 func (h *controller) Init(_ context.Context) error {
 	//nolint:contextcheck // Ignore "should pass the context parameter"
 	go func() {
diff --git a/pkg/routeagent_driver/handlers/healthchecker/healthchecker_test.go b/pkg/routeagent_driver/handlers/healthchecker/healthchecker_test.go
index b3321a6f9..4fb28f309 100644
--- a/pkg/routeagent_driver/handlers/healthchecker/healthchecker_test.go
+++ b/pkg/routeagent_driver/handlers/healthchecker/healthchecker_test.go
@@ -20,7 +20,6 @@ package healthchecker_test
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -34,9 +33,7 @@ import (
 	"github.com/submariner-io/submariner/pkg/pinger"
 	"github.com/submariner-io/submariner/pkg/pinger/fake"
 	"github.com/submariner-io/submariner/pkg/routeagent_driver/handlers/healthchecker"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/dynamic"
 	dynamicfake "k8s.io/client-go/dynamic/fake"
 	kubeScheme "k8s.io/client-go/kubernetes/scheme"
@@ -60,27 +57,57 @@ var _ = Describe("RouteAgent syncing", func() {
 
 	When("a remote Endpoint is created/updated/deleted", func() {
 		It("should add/update/delete its RemoteEndpoint information in the RouteAgent resource", func() {
-			endpoint1 := t.CreateEndpoint(t.newSubmEndpoint(healthCheckIP1))
+			endpoint := t.CreateEndpoint(t.newSubmEndpoint(healthCheckIP1))
 
-			remoteEndpoint := t.awaitRemoteEndpoint(nil)
-			Expect(remoteEndpoint.Spec).To(Equal(endpoint1.Spec))
+			t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+				g.Expect(ep.Spec).To(Equal(endpoint.Spec))
+			})
 
 			By("Updating remote endpoint")
 
-			endpoint1.Spec.Hostname = "newHostName"
-			t.UpdateEndpoint(endpoint1)
+			endpoint.Spec.Hostname = "newHostName"
+			t.UpdateEndpoint(endpoint)
 
-			remoteEndpoint = t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint) bool {
-				return ep.Spec.Hostname == endpoint1.Spec.Hostname
+			t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+				g.Expect(ep.Spec.Hostname).To(Equal(endpoint.Spec.Hostname))
+				g.Expect(ep.Spec).To(Equal(endpoint.Spec))
 			})
-			Expect(remoteEndpoint.Spec).To(Equal(endpoint1.Spec))
 
 			By("Deleting remote endpoint")
 
+			t.DeleteEndpoint(endpoint.Name)
+
+			t.awaitRouteAgent(func(ra *submarinerv1.RouteAgent, g Gomega) {
+				g.Expect(ra.Status.RemoteEndpoints).To(BeEmpty())
+			})
+		})
+	})
+
+	When("a stale remote Endpoint is deleted", func() {
+		It("should remove its RemoteEndpoint information in the RouteAgent resource", func() {
+			endpoint1 := t.CreateEndpoint(t.newSubmEndpoint(healthCheckIP1))
+
+			t.awaitRemoteEndpoint(nil)
+
+			By("Creating new remote endpoint")
+
+			endpoint2 := t.newSubmEndpoint(healthCheckIP2)
+			endpoint2.Spec.CableName = "new-cable"
+			endpoint2.Name = "new-endpoint"
+			endpoint2.CreationTimestamp = metav1.Time{Time: metav1.Now().Add(time.Second)}
+			t.CreateEndpoint(endpoint2)
+
+			t.awaitRouteAgent(func(ra *submarinerv1.RouteAgent, g Gomega) {
+				g.Expect(ra.Status.RemoteEndpoints).To(HaveLen(2))
+			})
+
+			By("Deleting stale remote endpoint")
+
 			t.DeleteEndpoint(endpoint1.Name)
 
-			t.awaitRouteAgent(func(ra *submarinerv1.RouteAgent) bool {
-				return len(ra.Status.RemoteEndpoints) == 0
+			t.awaitRouteAgent(func(ra *submarinerv1.RouteAgent, g Gomega) {
+				g.Expect(ra.Status.RemoteEndpoints).To(HaveLen(1))
+				g.Expect(ra.Status.RemoteEndpoints[0].Spec.GetHealthCheckIP(k8snet.IPv4)).To(Equal(healthCheckIP2))
 			})
 		})
 	})
@@ -97,12 +124,10 @@ var _ = Describe("RemoteEndpoint latency info", func() {
 			latencyInfo := t.newLatencyInfo()
 			t.setLatencyInfo(healthCheckIP1, latencyInfo)
 
-			remoteEndpoint := t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint) bool {
-				return ep.Status == submarinerv1.Connected
+			t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+				g.Expect(ep.Status).To(Equal(submarinerv1.Connected))
+				g.Expect(ep.LatencyRTT).To(Equal(latencyInfo.Spec))
 			})
-
-			Expect(remoteEndpoint.Status).To(Equal(submarinerv1.Connected))
-			Expect(remoteEndpoint.LatencyRTT).To(Equal(latencyInfo.Spec))
 		})
 
 		Context("with no HealthCheckIP", func() {
@@ -110,12 +135,10 @@ var _ = Describe("RemoteEndpoint latency info", func() {
 				endpoint1 := t.CreateEndpoint(t.newSubmEndpoint(""))
 				t.pingerMap[healthCheckIP1].AwaitNoStart()
 
-				remoteEndpoint := t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint) bool {
-					return ep.Status == submarinerv1.ConnectionNone
+				t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+					g.Expect(ep.Status).To(Equal(submarinerv1.ConnectionNone))
+					g.Expect(ep.Spec).To(Equal(endpoint1.Spec))
 				})
-
-				Expect(remoteEndpoint.Status).To(Equal(submarinerv1.ConnectionNone))
-				Expect(remoteEndpoint.Spec).To(Equal(endpoint1.Spec))
 			})
 		})
 
@@ -125,12 +148,10 @@ var _ = Describe("RemoteEndpoint latency info", func() {
 				endpoint1 := t.CreateEndpoint(t.newSubmEndpoint(healthCheckIP1))
 				t.pingerMap[healthCheckIP1].AwaitNoStart()
 
-				remoteEndpoint := t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint) bool {
-					return ep.Status == submarinerv1.ConnectionNone
+				t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+					g.Expect(ep.Status).To(Equal(submarinerv1.ConnectionNone))
+					g.Expect(ep.Spec).To(Equal(endpoint1.Spec))
 				})
-
-				Expect(remoteEndpoint.Status).To(Equal(submarinerv1.ConnectionNone))
-				Expect(remoteEndpoint.Spec).To(Equal(endpoint1.Spec))
 			})
 		})
 
@@ -143,12 +164,10 @@ var _ = Describe("RemoteEndpoint latency info", func() {
 				endpoint1 := t.CreateEndpoint(t.newSubmEndpoint(healthCheckIP1))
 				t.pingerMap[healthCheckIP1].AwaitNoStart()
 
-				remoteEndpoint := t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint) bool {
-					return ep.Status == submarinerv1.ConnectionNone
+				t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+					g.Expect(ep.Status).To(Equal(submarinerv1.ConnectionNone))
+					g.Expect(ep.Spec).To(Equal(endpoint1.Spec))
 				})
-
-				Expect(remoteEndpoint.Status).To(Equal(submarinerv1.ConnectionNone))
-				Expect(remoteEndpoint.Spec).To(Equal(endpoint1.Spec))
 			})
 		})
 	})
@@ -159,7 +178,6 @@ var _ = Describe("RemoteEndpoint latency info", func() {
 				endpoint1 := t.CreateEndpoint(t.newSubmEndpoint(healthCheckIP1))
 
 				t.pingerMap[healthCheckIP1].AwaitStart()
-				t.pingerMap[healthCheckIP2] = fake.NewPinger(healthCheckIP2)
 
 				endpoint1.Spec.HealthCheckIPs = []string{healthCheckIP2}
 
@@ -207,12 +225,10 @@ var _ = Describe("RemoteEndpoint latency info", func() {
 
 			t.setLatencyInfo(healthCheckIP1, latencyInfo)
 
-			remoteEndpoint := t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint) bool {
-				return ep.Status == submarinerv1.ConnectionError
+			t.awaitRemoteEndpoint(func(ep *submarinerv1.RemoteEndpoint, g Gomega) {
+				g.Expect(ep.Status).To(Equal(submarinerv1.ConnectionError))
+				g.Expect(ep.StatusMessage).To(Equal(latencyInfo.ConnectionError))
 			})
-
-			Expect(remoteEndpoint.Status).To(Equal(submarinerv1.ConnectionError))
-			Expect(remoteEndpoint.StatusMessage).To(Equal(latencyInfo.ConnectionError))
 		})
 	})
 })
@@ -269,6 +285,7 @@ func newTestDriver() *testDriver {
 		t.client = clientset.SubmarinerV1().RouteAgents(namespace)
 		t.pingerMap = map[string]*fake.Pinger{
 			healthCheckIP1: fake.NewPinger(healthCheckIP1),
+			healthCheckIP2: fake.NewPinger(healthCheckIP2),
 		}
 	})
 
@@ -314,7 +331,8 @@ func (t *testDriver) newSubmEndpoint(healthCheckIP string) *submarinerv1.Endpoin
 
 	endpoint := &submarinerv1.Endpoint{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: endpointName,
+			Name:              endpointName,
+			CreationTimestamp: metav1.Now(),
 		},
 		Spec: *endpointSpec,
 	}
@@ -344,30 +362,23 @@ func (t *testDriver) Start(handler event.Handler) {
 	t.ControllerSupport.Start(handler)
 }
 
-func (t *testDriver) awaitRouteAgent(verify func(*submarinerv1.RouteAgent) bool) *submarinerv1.RouteAgent {
-	var routeAgent *submarinerv1.RouteAgent
-
-	_ = wait.PollUntilContextTimeout(context.TODO(), 20*time.Millisecond,
-		5*time.Second, true, func(ctx context.Context) (bool, error) {
-			ra, err := t.client.Get(ctx, localNodeName, metav1.GetOptions{})
-			if apierrors.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) {
-				return false, nil
-			}
-
-			Expect(err).ToNot(HaveOccurred(), "Error retrieving RouteAgent")
+func (t *testDriver) awaitRouteAgent(verify func(*submarinerv1.RouteAgent, Gomega)) {
+	Eventually(func(g Gomega) {
+		ra, err := t.client.Get(context.TODO(), localNodeName, metav1.GetOptions{})
+		g.Expect(err).ToNot(HaveOccurred(), "Error retrieving RouteAgent")
 
-			routeAgent = ra
-
-			return verify == nil || verify(routeAgent), nil
-		})
-
-	return routeAgent
+		if verify != nil {
+			verify(ra, g)
+		}
+	}).Within(5 * time.Second).Should(Succeed())
 }
 
-func (t *testDriver) awaitRemoteEndpoint(verify func(*submarinerv1.RemoteEndpoint) bool) *submarinerv1.RemoteEndpoint {
-	routeAgent := t.awaitRouteAgent(func(ra *submarinerv1.RouteAgent) bool {
-		return len(ra.Status.RemoteEndpoints) != 0 && (verify == nil || verify(&ra.Status.RemoteEndpoints[0]))
-	})
+func (t *testDriver) awaitRemoteEndpoint(verify func(*submarinerv1.RemoteEndpoint, Gomega)) {
+	t.awaitRouteAgent(func(ra *submarinerv1.RouteAgent, g Gomega) {
+		g.Expect(ra.Status.RemoteEndpoints).ToNot(BeEmpty())
 
-	return &routeAgent.Status.RemoteEndpoints[0]
+		if verify != nil {
+			verify(&ra.Status.RemoteEndpoints[0], g)
+		}
+	})
 }


Reviews:
user: login: yboaron
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T15:13:07Z
user: login: tpantelis
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T15:23:15Z
user: login: tpantelis
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T15:27:09Z
user: login: yboaron
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:59:45Z
user: login: yboaron
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T11:59:52Z
user: login: dfarrell07
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-13T15:44:09Z

Review comments:
diff_hunk: @@ -67,16 +67,15 @@ func (c *handlerController) handleRemovedLocalEndpoint(endpoint *smv1.Endpoint)
 }
 
 func (c *handlerController) handleRemovedRemoteEndpoint(endpoint *smv1.Endpoint) error {
+	c.handlerState.remoteEndpoints.Delete(endpoint.Name)
+
 	lastProcessedTime, ok := c.remoteEndpointTimeStamp[endpoint.Spec.ClusterID]
 
 	if ok && lastProcessedTime.After(endpoint.CreationTimestamp.Time) {
path: pkg/event/controller/endpoint_removed.go
user: login: yboaron
body: I'm not very familiar with the endpoints creation/deletion order in case of multiple GW failover use case, could this change affect ep events order for multi GW fail over use case ?
created_at: 2025-03-12T15:13:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,16 +67,15 @@ func (c *handlerController) handleRemovedLocalEndpoint(endpoint *smv1.Endpoint)
 }
 
 func (c *handlerController) handleRemovedRemoteEndpoint(endpoint *smv1.Endpoint) error {
+	c.handlerState.remoteEndpoints.Delete(endpoint.Name)
+
 	lastProcessedTime, ok := c.remoteEndpointTimeStamp[endpoint.Spec.ClusterID]
 
 	if ok && lastProcessedTime.After(endpoint.CreationTimestamp.Time) {
path: pkg/event/controller/endpoint_removed.go
user: login: tpantelis
body: Sorry not sure I follow. Can you elaborate more, perhaps with an example?
created_at: 2025-03-12T15:23:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,16 +67,15 @@ func (c *handlerController) handleRemovedLocalEndpoint(endpoint *smv1.Endpoint)
 }
 
 func (c *handlerController) handleRemovedRemoteEndpoint(endpoint *smv1.Endpoint) error {
+	c.handlerState.remoteEndpoints.Delete(endpoint.Name)
+
 	lastProcessedTime, ok := c.remoteEndpointTimeStamp[endpoint.Spec.ClusterID]
 
 	if ok && lastProcessedTime.After(endpoint.CreationTimestamp.Time) {
path: pkg/event/controller/endpoint_removed.go
user: login: tpantelis
body: Are you referring to moving line `c.handlerState.remoteEndpoints.Delete(endpoint.Name)` up to the top? If so, we have to always remove the deleted endpoint from `remoteEndpoints`, regardless if stale or not. This was a latent bug uncovered by the RA health checker unit tests.
created_at: 2025-03-12T15:27:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,16 +67,15 @@ func (c *handlerController) handleRemovedLocalEndpoint(endpoint *smv1.Endpoint)
 }
 
 func (c *handlerController) handleRemovedRemoteEndpoint(endpoint *smv1.Endpoint) error {
+	c.handlerState.remoteEndpoints.Delete(endpoint.Name)
+
 	lastProcessedTime, ok := c.remoteEndpointTimeStamp[endpoint.Spec.ClusterID]
 
 	if ok && lastProcessedTime.After(endpoint.CreationTimestamp.Time) {
path: pkg/event/controller/endpoint_removed.go
user: login: yboaron
body: > Are you referring to moving line c.handlerState.remoteEndpoints.Delete(endpoint.Name) up to the top?

Yep. got it.
created_at: 2025-03-13T11:59:45Z
author_association: CONTRIBUTOR

PR comments:
user: login: submariner-bot
created_at: 2025-03-11T18:32:24Z
body: 🤖 Created branch: z_pr3346/tpantelis/stale_ra_endpoints 
🚀 Full E2E won't run until the "ready-to-test" label is applied. I will add it automatically once the PR has 2 approvals, or you can add it manually.
author_association: CONTRIBUTOR
user: login: submariner-bot
created_at: 2025-03-13T16:21:38Z
body: 🤖 Closed branches: [z_pr3346/tpantelis/stale_ra_endpoints]
author_association: CONTRIBUTOR