Repository: gravitational/teleport (Go)

Issue:
Dynamic Windows desktops should detect when agents are removed
When agents that are serving a dynamic_windows_desktop disappear, the corresponding windows_desktop resource should be removed so that Teleport doesn't try to route connections through an agent that no longer exists.

I suspect that we are currently watching the dynamic resource, but not watching the agent resources correctly.
Dynamic Windows apps are not refreshed
### Expected behavior:

If a `windows_desktop` object is manually deleted/edited outside the agent, the agent should recreate/announce it on its next heartbeat and/or announce interval. This is consistent with other dynamically loaded resources.

### Current behavior:

A deleted `windows_desktop` resource won't repopulate until the next time an agent cold-starts.

### Bug details:
#### Teleport version: 17.4.9
#### Recreation steps

* Create a `dynamic_windows_desktop` resource.
* Configure a Teleport Windows Desktop agent to load it.
* Observe a corresponding `windows_desktop` object is created for each agent that loads the resource.
* Delete one or more `windows_desktop` objects that correspond with a running agent.
* Observe that the `windows_desktop` object never repopulates.

Repeating the test with the `app` object instead of `dynamic_windows_desktop`, along with `app_server` instead of `windows_desktop`, and the app service _does_ repopulate the `app_server` resource.



Discussion:

Patch:
diff --git a/lib/services/watcher.go b/lib/services/watcher.go
index d4eeb46a6cf7d..cd0a5f27800d2 100644
--- a/lib/services/watcher.go
+++ b/lib/services/watcher.go
@@ -556,7 +556,7 @@ type DynamicWindowsDesktopWatcherConfig struct {
 // NewDynamicWindowsDesktopWatcher returns a new instance of DynamicWindowsDesktopWatcher.
 func NewDynamicWindowsDesktopWatcher(ctx context.Context, cfg DynamicWindowsDesktopWatcherConfig) (*GenericWatcher[types.DynamicWindowsDesktop, readonly.DynamicWindowsDesktop], error) {
 	if cfg.DynamicWindowsDesktopGetter == nil {
-		return nil, trace.BadParameter("KubernetesClusterGetter must be provided")
+		return nil, trace.BadParameter("DynamicWindowsDesktopGetter must be provided")
 	}
 
 	w, err := NewGenericResourceWatcher(ctx, GenericWatcherConfig[types.DynamicWindowsDesktop, readonly.DynamicWindowsDesktop]{
diff --git a/lib/srv/desktop/discovery.go b/lib/srv/desktop/discovery.go
index 3ae2011128bff..cab51b65f6a32 100644
--- a/lib/srv/desktop/discovery.go
+++ b/lib/srv/desktop/discovery.go
@@ -387,6 +387,7 @@ func (s *WindowsService) startDynamicReconciler(ctx context.Context) (*services.
 			Client:    s.cfg.AccessPoint,
 		},
 	})
+
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -399,6 +400,13 @@ func (s *WindowsService) startDynamicReconciler(ctx context.Context) (*services.
 			return services.MatchResourceLabels(s.cfg.ResourceMatchers, desktop.GetAllLabels())
 		},
 		GetCurrentResources: func() map[string]types.WindowsDesktop {
+			maps.DeleteFunc(currentResources, func(_ string, v types.WindowsDesktop) bool {
+				d, err := s.cfg.AuthClient.GetWindowsDesktops(ctx, types.WindowsDesktopFilter{
+					HostID: v.GetHostID(),
+					Name:   v.GetName(),
+				})
+				return err != nil || len(d) == 0
+			})
 			return currentResources
 		},
 		GetNewResources: func() map[string]types.WindowsDesktop {
@@ -414,12 +422,17 @@ func (s *WindowsService) startDynamicReconciler(ctx context.Context) (*services.
 	go func() {
 		defer s.cfg.Logger.DebugContext(ctx, "DynamicWindowsDesktop resource watcher done.")
 		defer watcher.Close()
+		tickDuration := 5 * time.Minute
+		expiryDuration := tickDuration + 2*time.Minute
+		tick := s.cfg.Clock.NewTicker(tickDuration)
+		defer tick.Stop()
 		for {
 			select {
 			case desktops := <-watcher.ResourcesC:
 				newResources = make(map[string]types.WindowsDesktop)
 				for _, dynamicDesktop := range desktops {
 					desktop, err := s.toWindowsDesktop(dynamicDesktop)
+					desktop.SetExpiry(s.cfg.Clock.Now().Add(expiryDuration))
 					if err != nil {
 						s.cfg.Logger.WarnContext(ctx, "Can't create desktop resource", "error", err)
 						continue
@@ -431,6 +444,17 @@ func (s *WindowsService) startDynamicReconciler(ctx context.Context) (*services.
 					continue
 				}
 				currentResources = newResources
+			case <-tick.Chan():
+				newResources = make(map[string]types.WindowsDesktop)
+				for k, v := range currentResources {
+					newResources[k] = v.Copy()
+					newResources[k].SetExpiry(s.cfg.Clock.Now().Add(expiryDuration))
+				}
+				if err := reconciler.Reconcile(ctx); err != nil {
+					s.cfg.Logger.WarnContext(ctx, "Reconciliation failed, will retry", "error", err)
+					continue
+				}
+				currentResources = newResources
 			case <-watcher.Done():
 				return
 			case <-ctx.Done():


Test patch:
diff --git a/lib/srv/desktop/discovery_test.go b/lib/srv/desktop/discovery_test.go
index c96e3324babb7..28eaf8297fd34 100644
--- a/lib/srv/desktop/discovery_test.go
+++ b/lib/srv/desktop/discovery_test.go
@@ -317,3 +317,89 @@ func TestDynamicWindowsDiscovery(t *testing.T) {
 		})
 	}
 }
+
+func TestDynamicWindowsDiscoveryExpiry(t *testing.T) {
+	authServer, err := auth.NewTestAuthServer(auth.TestAuthServerConfig{
+		ClusterName: "test",
+		Dir:         t.TempDir(),
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, authServer.Close())
+	})
+
+	tlsServer, err := authServer.NewTestTLSServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, tlsServer.Close())
+	})
+
+	client, err := tlsServer.NewClient(auth.TestServerID(types.RoleWindowsDesktop, "test-host-id"))
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, client.Close())
+	})
+
+	dynamicWindowsClient := client.DynamicDesktopClient()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	clock := clockwork.NewFakeClock()
+	s := &WindowsService{
+		cfg: WindowsServiceConfig{
+			Heartbeat: HeartbeatConfig{
+				HostUUID: "1234",
+			},
+			Logger:      slog.New(logutils.NewSlogTextHandler(io.Discard, logutils.SlogTextHandlerConfig{})),
+			Clock:       clock,
+			AuthClient:  client,
+			AccessPoint: client,
+			ResourceMatchers: []services.ResourceMatcher{{
+				Labels: types.Labels{
+					"foo": {"bar"},
+				},
+			}},
+		},
+		dnsResolver: &net.Resolver{
+			PreferGo: true,
+			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+				return nil, errors.New("this resolver always fails")
+			},
+		},
+	}
+	_, err = s.startDynamicReconciler(ctx)
+	require.NoError(t, err)
+
+	desktop, err := types.NewDynamicWindowsDesktopV1("test", map[string]string{
+		"foo": "bar",
+	}, types.DynamicWindowsDesktopSpecV1{
+		Addr: "addr",
+	})
+	require.NoError(t, err)
+
+	_, err = dynamicWindowsClient.CreateDynamicWindowsDesktop(ctx, desktop)
+	require.NoError(t, err)
+
+	require.EventuallyWithT(t, func(t *assert.CollectT) {
+		desktops, err := client.GetWindowsDesktops(ctx, types.WindowsDesktopFilter{})
+		require.NoError(t, err)
+		require.Len(t, desktops, 1)
+		require.Equal(t, "test", desktops[0].GetName())
+	}, 5*time.Second, 50*time.Millisecond)
+
+	err = client.DeleteWindowsDesktop(ctx, s.cfg.Heartbeat.HostUUID, "test")
+	require.NoError(t, err)
+	desktops, err := client.GetWindowsDesktops(ctx, types.WindowsDesktopFilter{})
+	require.NoError(t, err)
+	require.Empty(t, desktops)
+
+	clock.Advance(5 * time.Minute)
+
+	require.EventuallyWithT(t, func(t *assert.CollectT) {
+		desktops, err := client.GetWindowsDesktops(ctx, types.WindowsDesktopFilter{})
+		require.NoError(t, err)
+		require.Len(t, desktops, 1)
+		require.Equal(t, "test", desktops[0].GetName())
+	}, 5*time.Second, 50*time.Millisecond)
+}