Repository: k8sgpt-ai/k8sgpt-operator

Issue:
[Question]: mutation-controller, K8sGPT client not ready, requeuing
### Checklist

- [x] I've searched for similar issues and couldn't find anything matching
- [x] I've included steps to reproduce the behavior

### Affected Components

- [ ] K8sGPT (CLI)
- [x] K8sGPT Operator

### K8sGPT Version

v.0.4.1

### Kubernetes Version

v1.32.5+k3s1 

### Host OS and its Version

Ubuntu 22.04.2 LTS

### Steps to reproduce

 I have used this feature
`apiVersion: core.k8sgpt.ai/v1alpha1
kind: K8sGPT
metadata:
  name: k8sgpt-auto-remediation-sample
spec:
  ai:
    autoRemediation:
      enabled: true
      similarityRequirement: "90"
      resources:
        - Pod
        - Service
        - Deployment
    enabled: true
    model: gpt-4o-mini
    backend: openai
    secret:
      name: k8sgpt-sample-secret
      key: openai-api-key
    # anonymized: false
    # language: english
  noCache: false
  repository: ghcr.io/k8sgpt-ai/k8sgpt
  version: v0.4.1`
and in the operator logs I am seeing the following messages:
`2025-08-15T11:23:02Z	INFO	mutation-controller	Reconciling mutation	{"mutation": "defaultbrokenpull"}
2025-08-15T11:23:02Z	INFO	mutation-controller	K8sGPT client not ready, requeuing`



### Expected behaviour

I have checked the secret and API key, and they are valid. However, I am still constantly receiving this message.

 

### Actual behaviour

Perhaps someone could help me with this issue?

### Additional Information

_No response_

Discussion:
@AlexsJones: Which operator version is this using?

Solution patch:
diff --git a/cmd/main.go b/cmd/main.go
index 80d7d4c..84c8cdc 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/k8sgpt-ai/k8sgpt-operator/internal/controller/k8sgpt"
 	"github.com/k8sgpt-ai/k8sgpt-operator/internal/controller/mutation"
+	"github.com/k8sgpt-ai/k8sgpt-operator/internal/controller/shared"
 	"github.com/k8sgpt-ai/k8sgpt-operator/internal/controller/types"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -135,6 +136,20 @@ func main() {
 	// This is a necessity for the mutation system to work
 	ready := make(chan types.InterControllerSignal, 10)
 
+	// Start a goroutine to listen for signals and set the shared ServerQueryClient
+	go func() {
+		for signal := range ready {
+			setupLog.Info("Received signal from K8sGPT controller, setting shared client")
+			if signal.K8sGPTClient != nil {
+				client := signal.K8sGPTClient.GetServerQueryServiceClient()
+				shared.SetServerQueryClient(&client)
+				shared.SetRemoteBackend(signal.Backend)
+				shared.SetK8sGPTConfig(signal.K8sGPT)
+				setupLog.Info("Shared ServerQueryClient set successfully", "backend", signal.Backend)
+			}
+		}
+	}()
+
 	if err = (&mutation.MutationReconciler{
 		Client:         mgr.GetClient(),
 		Scheme:         mgr.GetScheme(),
diff --git a/internal/controller/mutation/mutation_controller.go b/internal/controller/mutation/mutation_controller.go
index 6e3f1fb..8f63113 100644
--- a/internal/controller/mutation/mutation_controller.go
+++ b/internal/controller/mutation/mutation_controller.go
@@ -87,6 +87,11 @@ func (r *MutationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		mutationControllerLog.Info("K8sGPT client not ready, requeuing")
 		return ctrl.Result{RequeueAfter: util.ErrorRequeueTime}, nil
 	}
+
+	// Get the shared backend and K8sGPT config
+	backend := shared.GetRemoteBackend()
+	k8sgptConfig := shared.GetK8sGPTConfig()
+
 	// check if the object is being deleted
 
 	// check if object has a finalizer
@@ -120,7 +125,7 @@ func (r *MutationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		}
 
 		queryResponse, err := (*r.ServerQueryClient).Query(context.Background(), &schemav1.QueryRequest{
-			Backend: r.RemoteBackend,
+			Backend: backend,
 			Query: fmt.Sprintf(prompts.Mutation_prompt, result.Spec.Details,
 				mutation.Spec.OriginConfiguration),
 		})
@@ -163,14 +168,14 @@ func (r *MutationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 			mutationControllerLog.Info("Target configuration is not set, this shouldn't occur at this phase", "mutation", mutation.Name)
 			return ctrl.Result{RequeueAfter: util.ErrorRequeueTime}, nil
 		}
-		if r.K8sGPT != nil {
-			if r.K8sGPT.Spec.AI.AutoRemediation.SimilarityRequirement != "" {
+		if k8sgptConfig != nil {
+			if k8sgptConfig.Spec.AI.AutoRemediation.SimilarityRequirement != "" {
 				// If the current Similarity score is less than the riskThreshold, we should not apply the mutation
 				ss, err := strconv.ParseFloat(strings.TrimSpace(mutation.Spec.SimilarityScore), 64)
 				if err != nil {
 					mutationControllerLog.Error(err, "unable to parse similarity score", "mutation", mutation.Name)
 				} else {
-					rt, err := strconv.ParseFloat(r.K8sGPT.Spec.AI.AutoRemediation.SimilarityRequirement, 64)
+					rt, err := strconv.ParseFloat(k8sgptConfig.Spec.AI.AutoRemediation.SimilarityRequirement, 64)
 					if err != nil {
 						mutationControllerLog.Error(err, "unable to parse risk threshold", "mutation", mutation.Name)
 					} else {
@@ -214,7 +219,7 @@ func (r *MutationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 			Rc:          r.Client,
 			Log:         mutationControllerLog,
 			Obj:         obj,
-			Backend:     r.RemoteBackend,
+			Backend:     backend,
 			Mutation:    mutation,
 			QueryClient: *r.ServerQueryClient,
 		})
diff --git a/internal/controller/shared/client.go b/internal/controller/shared/client.go
index 0017b50..345d042 100644
--- a/internal/controller/shared/client.go
+++ b/internal/controller/shared/client.go
@@ -4,11 +4,16 @@ import (
 	"sync"
 
 	rpc "buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go/schema/v1/schemav1grpc"
+	corev1alpha1 "github.com/k8sgpt-ai/k8sgpt-operator/api/v1alpha1"
 )
 
 var (
 	serverQueryClient     *rpc.ServerQueryServiceClient
 	serverQueryClientLock sync.RWMutex
+	remoteBackend         string
+	remoteBackendLock     sync.RWMutex
+	k8sgptConfig          *corev1alpha1.K8sGPT
+	k8sgptConfigLock      sync.RWMutex
 )
 
 // SetServerQueryClient sets the shared ServerQueryClient.
@@ -24,3 +29,31 @@ func GetServerQueryClient() *rpc.ServerQueryServiceClient {
 	defer serverQueryClientLock.RUnlock()
 	return serverQueryClient
 }
+
+// SetRemoteBackend sets the shared backend.
+func SetRemoteBackend(backend string) {
+	remoteBackendLock.Lock()
+	defer remoteBackendLock.Unlock()
+	remoteBackend = backend
+}
+
+// GetRemoteBackend gets the shared backend.
+func GetRemoteBackend() string {
+	remoteBackendLock.RLock()
+	defer remoteBackendLock.RUnlock()
+	return remoteBackend
+}
+
+// SetK8sGPTConfig sets the shared K8sGPT config.
+func SetK8sGPTConfig(config *corev1alpha1.K8sGPT) {
+	k8sgptConfigLock.Lock()
+	defer k8sgptConfigLock.Unlock()
+	k8sgptConfig = config
+}
+
+// GetK8sGPTConfig gets the shared K8sGPT config.
+func GetK8sGPTConfig() *corev1alpha1.K8sGPT {
+	k8sgptConfigLock.RLock()
+	defer k8sgptConfigLock.RUnlock()
+	return k8sgptConfig
+}
diff --git a/pkg/client/client.go b/pkg/client/client.go
index e864366..609f7ae 100644
--- a/pkg/client/client.go
+++ b/pkg/client/client.go
@@ -20,6 +20,7 @@ import (
 	"os"
 	"time"
 
+	rpc "buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go/schema/v1/schemav1grpc"
 	"github.com/k8sgpt-ai/k8sgpt-operator/api/v1alpha1"
 	"google.golang.org/grpc"
 	corev1 "k8s.io/api/core/v1"
@@ -36,6 +37,11 @@ func (c *Client) Close() error {
 	return c.Conn.Close()
 }
 
+// GetServerQueryServiceClient returns a new ServerQueryServiceClient
+func (c *Client) GetServerQueryServiceClient() rpc.ServerQueryServiceClient {
+	return rpc.NewServerQueryServiceClient(c.Conn)
+}
+
 func NewClient(address string) (*Client, error) {
 	// Connect to the K8sGPT server and create a new client
 	conn, err := grpc.Dial(address, grpc.WithInsecure())



Test patch:
diff --git a/internal/controller/shared/client_test.go b/internal/controller/shared/client_test.go
new file mode 100644
index 0000000..b29c8ad
--- /dev/null
+++ b/internal/controller/shared/client_test.go
@@ -0,0 +1,64 @@
+package shared
+
+import (
+	"testing"
+
+	rpc "buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go/schema/v1/schemav1grpc"
+	corev1alpha1 "github.com/k8sgpt-ai/k8sgpt-operator/api/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestServerQueryClient(t *testing.T) {
+	// Test that initially the client is nil
+	if client := GetServerQueryClient(); client != nil {
+		t.Error("Expected GetServerQueryClient to return nil initially")
+	}
+
+	// Create a mock client (using nil as we can't easily create a real one in unit tests)
+	var mockClient rpc.ServerQueryServiceClient
+	SetServerQueryClient(&mockClient)
+
+	// Test that we can retrieve the client
+	if client := GetServerQueryClient(); client == nil {
+		t.Error("Expected GetServerQueryClient to return non-nil after setting")
+	}
+}
+
+func TestRemoteBackend(t *testing.T) {
+	// Test that initially the backend is empty
+	if backend := GetRemoteBackend(); backend != "" {
+		t.Errorf("Expected GetRemoteBackend to return empty string initially, got %s", backend)
+	}
+
+	// Set a backend
+	testBackend := "openai"
+	SetRemoteBackend(testBackend)
+
+	// Test that we can retrieve the backend
+	if backend := GetRemoteBackend(); backend != testBackend {
+		t.Errorf("Expected GetRemoteBackend to return %s, got %s", testBackend, backend)
+	}
+}
+
+func TestK8sGPTConfig(t *testing.T) {
+	// Test that initially the config is nil
+	if config := GetK8sGPTConfig(); config != nil {
+		t.Error("Expected GetK8sGPTConfig to return nil initially")
+	}
+
+	// Create a mock config
+	testConfig := &corev1alpha1.K8sGPT{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-k8sgpt",
+			Namespace: "default",
+		},
+	}
+	SetK8sGPTConfig(testConfig)
+
+	// Test that we can retrieve the config
+	if config := GetK8sGPTConfig(); config == nil {
+		t.Error("Expected GetK8sGPTConfig to return non-nil after setting")
+	} else if config.Name != "test-k8sgpt" {
+		t.Errorf("Expected config name to be test-k8sgpt, got %s", config.Name)
+	}
+}
diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go
new file mode 100644
index 0000000..3aacb21
--- /dev/null
+++ b/pkg/client/client_test.go
@@ -0,0 +1,38 @@
+/*
+Copyright 2023 The K8sGPT Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"testing"
+
+	"google.golang.org/grpc"
+)
+
+func TestGetServerQueryServiceClient(t *testing.T) {
+	// Create a client with a mock connection
+	// Note: We can't fully test this without a real gRPC server, but we can verify it doesn't panic
+	conn, err := grpc.Dial("localhost:9999", grpc.WithInsecure())
+	if err != nil {
+		t.Fatalf("Failed to create mock connection: %v", err)
+	}
+	defer conn.Close()
+
+	client := &Client{Conn: conn}
+
+	// This should not panic
+	queryClient := client.GetServerQueryServiceClient()
+	if queryClient == nil {
+		t.Error("Expected GetServerQueryServiceClient to return non-nil client")
+	}
+}