Repository: hetznercloud/hcloud-cloud-controller-manager (go)

Issue:
server Y: no network with id X
Hi, I set up HA cluster with K3s. Everything works so far except that the cloud-controller was not able to find the network by id. This makes no sense because I already joined two control plane nodes. After restarting the cloud controller manually the error was gone. Looks like a caching issue. Even after minutes, the CCM was not able to recover. The issue isn't reproducible every time.

https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/7a1e3c76996f9f8351555a4c5d4a935318008104/hcloud/routes.go#L86

Script: https://github.com/StarpTech/k-andy

Pull request:
Fix cache issue in CreateRoute
Signed-off-by: Lukas Kämmerling <lukas.kaemmerling@hetzner-cloud.de>

Fixes #166

Patch:
diff --git a/hcloud/routes.go b/hcloud/routes.go
index 6cfb24bb..0312ee3c 100644
--- a/hcloud/routes.go
+++ b/hcloud/routes.go
@@ -83,7 +83,16 @@ func (r *routes) CreateRoute(ctx context.Context, clusterName string, nameHint s
 
 	privNet, ok := findServerPrivateNetByID(srv, r.network.ID)
 	if !ok {
-		return fmt.Errorf("%s: server %v: no network with id: %d", op, route.TargetNode, r.network.ID)
+		r.serverCache.InvalidateCache()
+		srv, err = r.serverCache.ByName(string(route.TargetNode))
+		if err != nil {
+			return fmt.Errorf("%s: %v", op, err)
+		}
+
+		privNet, ok = findServerPrivateNetByID(srv, r.network.ID)
+		if !ok {
+			return fmt.Errorf("%s: server %v: network with id %d not attached to this server ", op, route.TargetNode, r.network.ID)
+		}
 	}
 	ip := privNet.IP
 
diff --git a/internal/hcops/server.go b/internal/hcops/server.go
index 3aec08c7..fa02fab5 100644
--- a/internal/hcops/server.go
+++ b/internal/hcops/server.go
@@ -117,6 +117,14 @@ func (c *AllServersCache) getCache(getSrv func() (*hcloud.Server, bool)) (*hclou
 	return nil, fmt.Errorf("%s: %w", op, ErrNotFound)
 }
 
+// InvalidateCache invalidates the cache so that on the next cache call the cache gets refreshed
+func (c *AllServersCache) InvalidateCache() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.lastRefresh = time.Time{}
+}
+
 func (c *AllServersCache) isExpired() bool {
 	maxAge := c.MaxAge
 	if maxAge == 0 {


Test patch:
diff --git a/internal/hcops/server_test.go b/internal/hcops/server_test.go
index 22de059d..f727aee9 100644
--- a/internal/hcops/server_test.go
+++ b/internal/hcops/server_test.go
@@ -71,6 +71,48 @@ func TestAllServersCache_CacheHit(t *testing.T) {
 	runAllServersCacheTests(t, "Cache hit", tmpl, cacheOps)
 }
 
+func TestAllServersCache_InvalidateCache(t *testing.T) {
+	srv := &hcloud.Server{
+		ID:   54321,
+		Name: "cache-hit",
+		PrivateNet: []hcloud.ServerPrivateNet{
+			{
+				IP: net.ParseIP("10.0.0.3"),
+			},
+		},
+	}
+	cacheOps := newAllServersCacheOps(t, srv)
+	tmpl := allServersCacheTestCase{
+		SetUp: func(t *testing.T, tt *allServersCacheTestCase) {
+			tt.ServerClient.
+				On("All", mock.Anything).
+				Return([]*hcloud.Server{srv}, nil).
+				Times(2)
+
+			// Perform any cache op to initialize caches
+			if _, err := tt.Cache.ByName(srv.Name); err != nil {
+				t.Fatalf("SetUp: %v", err)
+			}
+
+			// Invalidate Cache
+			tt.Cache.InvalidateCache()
+
+			// Perform a second cache lookup
+			if _, err := tt.Cache.ByName(srv.Name); err != nil {
+				t.Fatalf("SetUp: %v", err)
+			}
+		},
+		Assert: func(t *testing.T, tt *allServersCacheTestCase) {
+			// All must be called twice. This call has happened during the
+			// test SetUp method.
+			tt.ServerClient.AssertNumberOfCalls(t, "All", 2)
+		},
+		Expected: srv,
+	}
+
+	runAllServersCacheTests(t, "Cache hit", tmpl, cacheOps)
+}
+
 func TestAllServersCache_CacheRefresh(t *testing.T) {
 	srv := &hcloud.Server{
 		ID:   56789,