Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1461 (2022-12-16 15:27:06)

Add first status check for the analyze command
# Description

This is the first step to analyze different conditions from a cluster status json.

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

-

## Testing

I going to add dedicated test cases for this.

## Documentation

-

## Follow-up

-

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 56fab3ed5fcf408682dce20265ca59b592b045a5
* Duration 2:21:51
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/59c7652f-91cb-4e10-b38c-bf45fee92519.gz) (available for 30 days)
* [Build Artifact](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/59c7652f-91cb-4e10-b38c-bf45fee92519/workspace.zip) (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 7bc83756..4524b732 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -139,6 +139,19 @@ type FoundationDBStatusProcessInfo struct {
 
 	// Roles contains a slice of all roles of the process
 	Roles []FoundationDBStatusProcessRoleInfo `json:"roles,omitempty"`
+
+	// Messages contains error messages from that fdbserver process instance
+	Messages []FoundationDBStatusProcessMessage `json:"messages,omitempty"`
+}
+
+// FoundationDBStatusProcessMessage represents an error message in the status json
+type FoundationDBStatusProcessMessage struct {
+	// Time when the error was observed
+	Time float64 `json:"time,omitempty"`
+	// The name of the error
+	Name string `json:"name,omitempty"`
+	// The type of the error
+	Type string `json:"type,omitempty"`
 }
 
 // FoundationDBStatusProcessRoleInfo contains the minimal information from the process status
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index c70de7c4..449eda63 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -1137,6 +1137,11 @@ func (in *FoundationDBStatusProcessInfo) DeepCopyInto(out *FoundationDBStatusPro
 		*out = make([]FoundationDBStatusProcessRoleInfo, len(*in))
 		copy(*out, *in)
 	}
+	if in.Messages != nil {
+		in, out := &in.Messages, &out.Messages
+		*out = make([]FoundationDBStatusProcessMessage, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusProcessInfo.
@@ -1149,6 +1154,21 @@ func (in *FoundationDBStatusProcessInfo) DeepCopy() *FoundationDBStatusProcessIn
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBStatusProcessMessage) DeepCopyInto(out *FoundationDBStatusProcessMessage) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusProcessMessage.
+func (in *FoundationDBStatusProcessMessage) DeepCopy() *FoundationDBStatusProcessMessage {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBStatusProcessMessage)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FoundationDBStatusProcessRoleInfo) DeepCopyInto(out *FoundationDBStatusProcessRoleInfo) {
 	*out = *in
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index 53412951..2e6bf061 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -21,7 +21,11 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"strings"
 	"time"
 
@@ -100,6 +104,17 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
+			config, err := o.configFlags.ToRESTConfig()
+			if err != nil {
+				return err
+			}
+
+			clientSet, err := kubernetes.NewForConfig(config)
+			if err != nil {
+				return err
+			}
+
+			// TODO (jscheuermann): Don't load clusters twice if we check all clusters
 			var clusters []string
 			if allClusters {
 				var clusterList fdbv1beta2.FoundationDBClusterList
@@ -116,8 +131,21 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			}
 
 			var errs []error
-			for _, cluster := range clusters {
-				err := analyzeCluster(cmd, kubeClient, cluster, namespace, autoFix, wait, ignoreConditions, ignoreRemovals, sleep)
+			for _, clusterName := range clusters {
+				cluster, err := loadCluster(kubeClient, namespace, clusterName)
+				if err != nil {
+					if k8serrors.IsNotFound(err) {
+						errs = append(errs, fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName))
+					}
+					errs = append(errs, err)
+				}
+
+				err = analyzeCluster(cmd, kubeClient, cluster, autoFix, wait, ignoreConditions, ignoreRemovals, sleep)
+				if err != nil {
+					errs = append(errs, err)
+				}
+
+				err = analyzeStatus(cmd, config, clientSet, kubeClient, cluster, autoFix)
 				if err != nil {
 					errs = append(errs, err)
 				}
@@ -194,12 +222,12 @@ func allConditionsValid(conditions []string) error {
 	return fmt.Errorf(errString.String())
 }
 
-type messageType string
+type messageType int
 
 const (
-	errorMessage messageType = "error"
-	warnMessage  messageType = "warn"
-	goodMessage  messageType = "good"
+	errorMessage messageType = iota
+	warnMessage
+	goodMessage
 )
 
 func printStatement(cmd *cobra.Command, line string, mesType messageType) {
@@ -222,18 +250,9 @@ func printStatement(cmd *cobra.Command, line string, mesType messageType) {
 	color.Unset()
 }
 
-func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName string, namespace string, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool, sleep uint16) error {
-	foundIssues := false
-	cluster, err := loadCluster(kubeClient, namespace, clusterName)
-
-	if err != nil {
-		if k8serrors.IsNotFound(err) {
-			return fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName)
-		}
-		return err
-	}
-
-	cmd.Printf("Checking cluster: %s/%s\n", namespace, clusterName)
+func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool, sleep uint16) error {
+	var foundIssues bool
+	cmd.Printf("Checking cluster: %s/%s\n", cluster.Namespace, cluster.Name)
 
 	// 1. Check if the cluster is available
 	if cluster.Status.Health.Available {
@@ -325,7 +344,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 	}
 
 	// 4. Check if all Pods are up and running
-	pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+	pods, err := getPodsForCluster(kubeClient, cluster)
 	if err != nil {
 		return err
 	}
@@ -346,7 +365,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 
 		if pod.DeletionTimestamp != nil && pod.DeletionTimestamp.Add(5*time.Minute).Before(time.Now()) {
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s has been stuck in terminating since %s", namespace, pod.Name, pod.DeletionTimestamp)
+			statement := fmt.Sprintf("Pod %s/%s has been stuck in terminating since %s", pod.Namespace, pod.Name, pod.DeletionTimestamp)
 			printStatement(cmd, statement, errorMessage)
 
 			// The process groups that should be deleted, so we can safely replace it
@@ -359,7 +378,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 
 		if pod.Status.Phase != corev1.PodRunning {
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s has unexpected Phase %s with Reason: %s", namespace, pod.Name, pod.Status.Phase, pod.Status.Reason)
+			statement := fmt.Sprintf("Pod %s/%s has unexpected Phase %s with Reason: %s", pod.Namespace, pod.Name, pod.Status.Phase, pod.Status.Reason)
 			printStatement(cmd, statement, errorMessage)
 		}
 
@@ -370,7 +389,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 			}
 
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s has an unready container: %s", namespace, pod.Name, container.Name)
+			statement := fmt.Sprintf("Pod %s/%s has an unready container: %s", pod.Namespace, pod.Name, container.Name)
 			printStatement(cmd, statement, errorMessage)
 
 			// Replace the Pod if the container is unready for more then 30 minutes
@@ -386,7 +405,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		id := pod.Labels[processGroupIDLabel]
 		if _, ok := processGroupMap[id]; !ok {
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s with the ID %s is not part of the cluster spec status", namespace, pod.Name, id)
+			statement := fmt.Sprintf("Pod %s/%s with the ID %s is not part of the cluster spec status", pod.Namespace, pod.Name, id)
 			printStatement(cmd, statement, errorMessage)
 		}
 	}
@@ -397,13 +416,12 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		foundIssues = true
 	}
 
-	// TODO: add more checks using the FDB status directly e.g. error message or something else or overloaded or matching versions
 	// We could add more auto fixes in the future.
 	if autoFix {
 		confirmed := false
 
 		if len(failedProcessGroups) > 0 {
-			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, namespace, true, wait, false, true, sleep)
+			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, cluster.Namespace, true, wait, false, true, sleep)
 			if err != nil {
 				return err
 			}
@@ -416,7 +434,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 				podNames = append(podNames, pod.Name)
 			}
 
-			confirmed = confirmAction(fmt.Sprintf("Delete Pods %v in cluster %s/%s", strings.Join(podNames, ","), namespace, clusterName))
+			confirmed = confirmAction(fmt.Sprintf("Delete Pods %v in cluster %s/%s", strings.Join(podNames, ","), cluster.Namespace, cluster.Name))
 		}
 
 		if !wait || confirmed {
@@ -455,3 +473,101 @@ func filterDeletePods(replacements []string, killPods []corev1.Pod) []corev1.Pod
 
 	return res
 }
+
+func getStatus(restConfig *rest.Config, clientSet *kubernetes.Clientset, pod *corev1.Pod) (*fdbv1beta2.FoundationDBStatus, error) {
+	stdout, stderr, err := executeCmd(restConfig, clientSet, pod.Name, pod.Namespace, "fdbcli --exec 'status json'")
+	if err != nil {
+		return nil, fmt.Errorf("error getting status: %s, %w", stderr, err)
+	}
+
+	content, err := internal.RemoveWarningsInJSON(stdout.String())
+	if err != nil {
+		fmt.Println(stdout.String())
+		return nil, err
+	}
+
+	status := &fdbv1beta2.FoundationDBStatus{}
+	err = json.Unmarshal(content, status)
+	if err != nil {
+		return nil, err
+	}
+
+	return status, nil
+}
+
+func analyzeStatus(cmd *cobra.Command, restConfig *rest.Config, clientSet *kubernetes.Clientset, kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, autoFix bool) error {
+	cmd.Printf("Checking cluster: %s/%s with auto-fix: %t\n", cluster.Namespace, cluster.Name, autoFix)
+
+	pods, err := getPodsForCluster(kubeClient, cluster)
+	if err != nil {
+		return err
+	}
+
+	pod, err := chooseRandomPod(pods)
+	if err != nil {
+		return err
+	}
+
+	var status *fdbv1beta2.FoundationDBStatus
+	var tries int
+
+	// Try to get the status for 5 times
+	for tries < 5 {
+		status, err = getStatus(restConfig, clientSet, pod)
+		if err == nil {
+			break
+		}
+
+		tries++
+		printStatement(cmd, err.Error(), errorMessage)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return analyzeStatusInternal(cmd, restConfig, clientSet, status, pod, autoFix, cluster.Name)
+}
+
+func analyzeStatusInternal(cmd *cobra.Command, restConfig *rest.Config, clientSet *kubernetes.Clientset, status *fdbv1beta2.FoundationDBStatus, pod *corev1.Pod, autoFix bool, clusterName string) error {
+	var foundIssues bool
+
+	processesWithError := make([]string, 0)
+	for _, process := range status.Cluster.Processes {
+		if len(process.Messages) == 0 {
+			continue
+		}
+
+		foundIssues = true
+		addr := process.Address.StringWithoutFlags()
+		processesWithError = append(processesWithError, addr)
+		for _, message := range process.Messages {
+			printStatement(cmd, fmt.Sprintf("Process: %s with address: %s error: %s type: %s, time: %s",
+				process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey],
+				addr,
+				message.Name,
+				message.Type,
+				time.Unix(int64(int(message.Time)), 0).String()), errorMessage)
+		}
+	}
+
+	if len(processesWithError) > 0 && autoFix {
+		// Restart one process by another. The restart will remove the condition, assuming the condition was only
+		// intermediate.
+		for _, process := range processesWithError {
+			cmd.Println("Start killing", process)
+			killCmd := fmt.Sprintf("kill; kill %s; sleep 5; status", process)
+			_, stderr, err := executeCmd(restConfig, clientSet, pod.Name, pod.Namespace, fmt.Sprintf("fdbcli --exec '%s'", killCmd))
+			if err != nil {
+				return fmt.Errorf("error killing process %s status: %s, %w", process, stderr.String(), err)
+			}
+			time.Sleep(5 * time.Second)
+		}
+	}
+
+	if foundIssues {
+		return fmt.Errorf("found issues in status json for cluster %s. Please check them", clusterName)
+	}
+
+	return nil
+}
diff --git a/kubectl-fdb/cmd/exclusion_status.go b/kubectl-fdb/cmd/exclusion_status.go
index aeacf466..4a1f93fe 100644
--- a/kubectl-fdb/cmd/exclusion_status.go
+++ b/kubectl-fdb/cmd/exclusion_status.go
@@ -79,7 +79,7 @@ func newExclusionStatusCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+			pods, err := getPodsForCluster(kubeClient, cluster)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index c7b9c906..1d951c82 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -24,6 +24,7 @@ import (
 	"bytes"
 	ctx "context"
 	"fmt"
+	"math/rand"
 	"strings"
 
 	fdbv1beta1 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta1"
@@ -111,13 +112,13 @@ func getNodes(kubeClient client.Client, nodeSelector map[string]string) ([]strin
 	return nodes, nil
 }
 
-func getPodsForCluster(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, namespace string) (*corev1.PodList, error) {
+func getPodsForCluster(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster) (*corev1.PodList, error) {
 	var podList corev1.PodList
 	err := kubeClient.List(
 		ctx.Background(),
 		&podList,
 		client.MatchingLabels(cluster.GetMatchLabels()),
-		client.InNamespace(namespace))
+		client.InNamespace(cluster.GetNamespace()))
 
 	return &podList, err
 }
@@ -178,7 +179,7 @@ func getAllPodsFromClusterWithCondition(kubeClient client.Client, clusterName st
 	}
 
 	processes := make([]string, 0, len(processesSet))
-	pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+	pods, err := getPodsForCluster(kubeClient, cluster)
 	if err != nil {
 		return processes, err
 	}
@@ -219,3 +220,20 @@ func getProcessGroupIDsFromPodName(cluster *fdbv1beta2.FoundationDBCluster, podN
 
 	return processGroupIDs, nil
 }
+
+func chooseRandomPod(pods *corev1.PodList) (*corev1.Pod, error) {
+	items := pods.Items
+	if len(items) == 0 {
+		return nil, fmt.Errorf("no pods available")
+	}
+
+	var candidate *corev1.Pod
+
+	var tries int
+	for candidate == nil || !candidate.GetDeletionTimestamp().IsZero() || tries > 10 {
+		candidate = &items[rand.Intn(len(items))]
+		tries++
+	}
+
+	return candidate, nil
+}
diff --git a/kubectl-fdb/cmd/restart.go b/kubectl-fdb/cmd/restart.go
index bac88f8d..3dd5e5f8 100644
--- a/kubectl-fdb/cmd/restart.go
+++ b/kubectl-fdb/cmd/restart.go
@@ -96,7 +96,7 @@ func newRestartCmd(streams genericclioptions.IOStreams) *cobra.Command {
 
 			var processes []string
 			if allProcesses {
-				pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+				pods, err := getPodsForCluster(kubeClient, cluster)
 				if err != nil {
 					return err
 				}
diff --git a/kubectl-fdb/cmd/root.go b/kubectl-fdb/cmd/root.go
index 49e654fd..c0701ca6 100644
--- a/kubectl-fdb/cmd/root.go
+++ b/kubectl-fdb/cmd/root.go
@@ -24,7 +24,9 @@ import (
 	"bufio"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
+	"time"
 
 	"strings"
 
@@ -51,6 +53,8 @@ func newFDBOptions(streams genericclioptions.IOStreams) *fdbBOptions {
 
 // NewRootCmd provides a cobra command wrapping FDB actions
 func NewRootCmd(streams genericclioptions.IOStreams) *cobra.Command {
+	rand.Seed(time.Now().Unix())
+
 	o := newFDBOptions(streams)
 
 	cmd := &cobra.Command{


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index 8e566319..2842f283 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -107,6 +107,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "log",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"c813e585043a7ab55a4905f465c4aa52": {
 							Address: ProcessAddress{
@@ -132,6 +133,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "storage",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"f9efa90fc104f4e277b140baf89aab66": {
 							Address: ProcessAddress{
@@ -157,6 +159,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "storage",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"5a633d7f4e98a6c938c84b97ec4aedbf": {
 							Address: ProcessAddress{
@@ -182,6 +185,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "log",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"5c1b68147a0ef34ce005a38245851270": {
 							Address: ProcessAddress{
@@ -207,6 +211,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "resolver",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"653defde43cf1fdef131e2fb82bd192d": {
 							Address: ProcessAddress{
@@ -241,6 +246,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "log",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"9c93d3b70118f16c72f7cb3f53e49f4c": {
 							Address: ProcessAddress{
@@ -269,6 +275,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "storage",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 					},
 					Data: FoundationDBStatusDataStatistics{
@@ -497,6 +504,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: "grv_proxy"},
 						{Role: "storage"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"eab0db1aa7aae81a50ca97e9814a1b7d": {
 					Address: ProcessAddress{
@@ -519,6 +527,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: string(ProcessClassStorage)},
 						{Role: "resolver"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"f483247d4d5f279ef02c549680cbde64": {
 					Address: ProcessAddress{
@@ -541,6 +550,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: "commit_proxy"},
 						{Role: "storage"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"f6e0f7fd80da429d20329ad95d793ca3": {
 					Address: ProcessAddress{
@@ -563,6 +573,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: "data_distributor"},
 						{Role: "ratekeeper"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"f75644abdf1b06c803b5c3c124fdd0a0": {
 					Address: ProcessAddress{
@@ -583,6 +594,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: string(ProcessClassClusterController)},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"105bf6c041f8ec315d03e889c2746ecf": {
 					Address: ProcessAddress{
@@ -603,6 +615,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: "log"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"78c1c84af4481f0df628d40358f0930a": {
 					Address: ProcessAddress{
@@ -623,6 +636,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: "log"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"83084479b50c9c3a09b0286297be3796": {
 					Address: ProcessAddress{
@@ -643,6 +657,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: "log"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 			},
 			Data: FoundationDBStatusDataStatistics{
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index 6883b7b8..b4522449 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -40,7 +40,7 @@ func getCluster(clusterName string, namespace string, available bool, healthy bo
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       clusterName,
 			Namespace:  namespace,
-			Generation: 0,
+			Generation: 1,
 		},
 		Spec: fdbv1beta2.FoundationDBClusterSpec{
 			ProcessCounts: fdbv1beta2.ProcessCounts{
@@ -97,8 +97,6 @@ var _ = Describe("[plugin] analyze cluster", func() {
 
 		DescribeTable("return all successful and failed checks",
 			func(tc testCase) {
-				Expect(k8sClient.Delete(context.TODO(), tc.cluster)).NotTo(HaveOccurred())
-				Expect(k8sClient.Create(context.TODO(), tc.cluster)).NotTo(HaveOccurred())
 				for _, pod := range tc.podList.Items {
 					Expect(k8sClient.Create(context.TODO(), &pod)).NotTo(HaveOccurred())
 				}
@@ -109,7 +107,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				inBuffer := bytes.Buffer{}
 
 				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := analyzeCluster(cmd, k8sClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
+				err := analyzeCluster(cmd, k8sClient, tc.cluster, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
 
 				if err != nil && !tc.HasErrors {
 					Expect(err).To(HaveOccurred())