Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1417 (2022-10-27 09:57:02)

Upgrade used go version
# Description

Upgrade to the latest go 1.19 version.

## Type of change

*Please select one of the options below.*

- Other

## Discussion

Updating to match the go version from Kubernetes. I also adjusted our linters to reflect 1.19.

## Testing

Local testing and unit tests.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 151aa4f30f7fa3657e7e0d81adc04e1858853097
* Duration 2:00:40
* Result: :x: **FAILED**
* Error: `Error while executing command: if $(grep -q -- "--- FAIL:" logs/*.log); then echo "TESTS FAILED SEE THESE LOGS:"; echo ; grep -l -- "--- FAIL:" logs/*.log; exit 1; fi. Reason: exit status 1`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/ff7c14b2-161a-4c68-b9d4-0c084fd2c617.gz) (available for 30 days)
* [Build Artifact](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/ff7c14b2-161a-4c68-b9d4-0c084fd2c617/workspace.zip) (available for 30 days)


Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index d709e1da..b394fd0f 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18.6
+          go-version: 1.19.2
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
@@ -44,7 +44,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18.6
+        go-version: 1.19.2
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -104,7 +104,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18.6
+        go-version: 1.19.2
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -126,10 +126,7 @@ jobs:
         # Install dependencies either from cache or freshly downloaded
         sudo dpkg -i fdb.deb
     - name: Run tests
-      # Currently the default runner has 2 vCPU:
-      # https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources
       env:
-        GOMAXPROCS: "2"
         TEST_RACE_CONDITIONS: "1"
       run: |
         make test
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 0c04c389..b1825d59 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18.6
+          go-version: 1.19.2
       #  https://github.com/goreleaser/goreleaser/issues/1311
       - name: Get current semver tag
         run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
diff --git a/.golangci.yml b/.golangci.yml
index b2b452d7..854ab805 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -2,43 +2,63 @@ issues:
   # don't skip warning about doc comments
   # don't exclude the default set of lint
   exclude-use-default: false
+  exclude:
+    - "ST1000: at least one file in a package should have a package comment"
   exclude-rules:
   # exclude deprecation warning otherwise we get an error for our own deprecations
-  - linters: [staticcheck]
+  - linters:
+    - staticcheck
     text: "SA1019:"
-  - linters: [ govet ]
+  - linters:
+    - govet
     text: 'declaration of "(err|ctx)" shadows declaration at'
+  - linters:
+      - revive
+    text: "package-comments: should have a package comment"
+  # TODO (johscheuer): Ignore those linting issues for now and change it once we cut a new major release.
+  - linters:
+      - stylecheck
+    text: "ST1003: const StorageEngineRocksDbExperimental"
+  - linters:
+      - stylecheck
+    text: "ST1003: const StorageEngineRocksDbV1"
 
 linters:
   disable-all: true
   # TODO(johscheuer): activate linters again and fix issues
   enable:
-    - deadcode
-    #- dupl
+    - asciicheck
+    - bodyclose
+    - depguard
+    - dogsled
     - errcheck
-    #- goconst
-    #- gocyclo
+#    - errorlint
+    - exportloopref
+#    - goconst
+#    - gocritic
+#    - gocyclo
     - gofmt
-    - goimports
-    - revive
-    #- gosec
+    #- goimports
+    - goprintffuncname
+#    - gosec
     - gosimple
     - govet
+    - importas
     - ineffassign
-    #- interfacer
-    #- lll
-    #- maligned
     - misspell
     - nakedret
+    - nilerr
+#    - nolintlint
     - prealloc
-    - exportloopref
+    - revive
+    - rowserrcheck
     - staticcheck
-    - structcheck
+    - stylecheck
     - typecheck
     - unconvert
     - unparam
     - unused
-    - varcheck
+    - whitespace
 
 linters-settings:
   errcheck:
@@ -48,7 +68,13 @@ linters-settings:
   govet:
     enable:
       - shadow
+  staticcheck:
+    go: "1.19"
+  stylecheck:
+    go: "1.19"
+
 run:
-  deadline: 5m
+  timeout: 10m
   skip-files:
-    - ".*_generated.*.go$"
+    - "zz_generated.*\\.go$"
+  allow-parallel-runners: true
diff --git a/Dockerfile b/Dockerfile
index ca6e9d03..1ccf2acd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 ARG BASE_IMAGE=docker.io/debian:bullseye
 
 # Build the manager binary
-FROM docker.io/library/golang:1.18.6 as builder
+FROM docker.io/library/golang:1.19.2 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.2.29
diff --git a/Makefile b/Makefile
index 48296e5a..fe4d475e 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,7 @@ CONTROLLER_GEN_PKG?=sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 KUSTOMIZE_PKG?=sigs.k8s.io/kustomize/kustomize/v4@v4.5.2
 KUSTOMIZE=$(GOBIN)/kustomize
-GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.2
+GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1
 GOLANGCI_LINT=$(GOBIN)/golangci-lint
 GORELEASER_PKG=github.com/goreleaser/goreleaser@v1.6.3
 GORELEASER=$(GOBIN)/goreleaser
diff --git a/api/v1beta1/foundationdb_database_configuration.go b/api/v1beta1/foundationdb_database_configuration.go
index acea2c8e..c973bd17 100644
--- a/api/v1beta1/foundationdb_database_configuration.go
+++ b/api/v1beta1/foundationdb_database_configuration.go
@@ -335,7 +335,6 @@ func (configuration DatabaseConfiguration) GetNextConfigurationChange(finalConfi
 								} else if dataCenter.Priority > 0 {
 									hasAlternativePrimary = true
 								}
-
 							}
 						}
 					}
diff --git a/api/v1beta1/foundationdbcluster_types.go b/api/v1beta1/foundationdbcluster_types.go
index 73220e4b..921e63ae 100644
--- a/api/v1beta1/foundationdbcluster_types.go
+++ b/api/v1beta1/foundationdbcluster_types.go
@@ -789,7 +789,6 @@ func (clusterStatus FoundationDBClusterStatus) ProcessGroupsByProcessClass(proce
 		if groupStatus.ProcessClass == processClass {
 			result = append(result, groupStatus)
 		}
-
 	}
 
 	return result
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index 84f7d482..27605976 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -444,7 +444,6 @@ func (configuration DatabaseConfiguration) GetNextConfigurationChange(finalConfi
 								} else if dataCenter.Priority > 0 {
 									hasAlternativePrimary = true
 								}
-
 							}
 						}
 					}
@@ -579,7 +578,6 @@ func (configuration DatabaseConfiguration) AreSeparatedProxiesConfigured() bool
 // AreSeparatedProxiesConfigured(), then this function will return the
 // string "commit_proxies=%d grv_proxies=%d", otherwise just
 // "proxies=%d" using the correct counts of the configuration object.
-//
 func (configuration DatabaseConfiguration) GetProxiesString(version Version) string {
 	counts := configuration.GetRoleCountsWithDefaults(version, DesiredFaultTolerance(configuration.RedundancyMode))
 	if version.HasSeparatedProxies() && configuration.AreSeparatedProxiesConfigured() {
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index e2463915..8edf3d46 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -620,7 +620,6 @@ func (clusterStatus FoundationDBClusterStatus) ProcessGroupsByProcessClass(proce
 		if groupStatus.ProcessClass == processClass {
 			result = append(result, groupStatus)
 		}
-
 	}
 
 	return result
@@ -1207,6 +1206,10 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 		if len(processGroup.ProcessGroupConditions) > 0 {
 			conditions := make([]ProcessGroupConditionType, 0, len(processGroup.ProcessGroupConditions))
 			for _, condition := range processGroup.ProcessGroupConditions {
+				if condition.ProcessGroupConditionType == IncorrectCommandLine && cluster.Status.Generations.NeedsBounce == 0 {
+					logger.Info("Pending restart of fdbserver processes", "state", "NeedsBounce")
+					cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
+				}
 				conditions = append(conditions, condition.ProcessGroupConditionType)
 			}
 
diff --git a/api/v1beta2/groupversion_info.go b/api/v1beta2/groupversion_info.go
index 0d7c79e5..96fd86a4 100644
--- a/api/v1beta2/groupversion_info.go
+++ b/api/v1beta2/groupversion_info.go
@@ -15,8 +15,8 @@ limitations under the License.
 */
 
 // Package v1beta2 contains API Schema definitions for the apps v1beta2 API group
-//+kubebuilder:object:generate=true
-//+groupName=apps.foundationdb.org
+// +kubebuilder:object:generate=true
+// +groupName=apps.foundationdb.org
 package v1beta2
 
 import (
diff --git a/cmd/po-docgen/api.go b/cmd/po-docgen/api.go
index cf95f2e3..50b090cb 100644
--- a/cmd/po-docgen/api.go
+++ b/cmd/po-docgen/api.go
@@ -124,7 +124,6 @@ func ParseDocumentationFrom(srcs []string) []KubeTypes {
 		pkg := astFrom(src)
 
 		for _, kubType := range pkg.Types {
-
 			if structType, ok := kubType.Decl.Specs[0].(*ast.TypeSpec).Type.(*ast.StructType); ok {
 				var ks KubeTypes
 				ks = append(ks, Pair{kubType.Name, fmtRawDoc(kubType.Doc), "", false})
diff --git a/controllers/admin_client_mock.go b/controllers/admin_client_mock.go
index dc44ad40..f647a7d4 100644
--- a/controllers/admin_client_mock.go
+++ b/controllers/admin_client_mock.go
@@ -256,7 +256,6 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				Version:       client.Cluster.Status.RunningVersion,
 				UptimeSeconds: uptimeSeconds,
 			}
-
 		}
 	}
 
diff --git a/controllers/bounce_processes.go b/controllers/bounce_processes.go
index 0aec4f9e..e261197a 100644
--- a/controllers/bounce_processes.go
+++ b/controllers/bounce_processes.go
@@ -23,18 +23,15 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"math"
 	"time"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient"
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
-
-	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
-
+	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/pointer"
-
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 )
 
 // bounceProcesses provides a reconciliation step for bouncing fdbserver
@@ -43,6 +40,10 @@ type bounceProcesses struct{}
 
 // reconcile runs the reconciler's work.
 func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
+	if !pointer.BoolDeref(cluster.Spec.AutomationOptions.KillProcesses, true) {
+		return nil
+	}
+
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "bounceProcesses")
 	adminClient, err := r.getDatabaseClientProvider().GetAdminClient(cluster, r)
 	if err != nil {
@@ -55,15 +56,103 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		return &requeue{curError: err}
 	}
 
-	minimumUptime, addressMap, err := getMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	minimumUptime, addressMap, err := internal.GetMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+
+	addresses, req := getProcessesReadyForRestart(ctx, logger, r, cluster, addressMap)
+	if req != nil {
+		return req
+	}
+
+	if len(addresses) == 0 {
+		return nil
+	}
+
+	if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
+		r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
+			fmt.Sprintf("Spec require a bounce of some processes, but the cluster has only been up for %f seconds", minimumUptime))
+		cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
+		err = r.updateOrApply(ctx, cluster)
+		if err != nil {
+			logger.Error(err, "Error updating cluster status")
+		}
+
+		// Retry after we waited the minimum uptime
+		return &requeue{
+			message: "Cluster needs to stabilize before bouncing",
+			delay:   time.Second * time.Duration(cluster.GetMinimumUptimeSecondsForBounce()-int(minimumUptime)),
+		}
+	}
+
+	var lockClient fdbadminclient.LockClient
+	useLocks := cluster.ShouldUseLocks()
+	if useLocks {
+		lockClient, err = r.getLockClient(cluster)
+		if err != nil {
+			return &requeue{curError: err}
+		}
+	}
+	version, err := fdbv1beta2.ParseFdbVersion(cluster.Spec.Version)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+
+	upgrading := cluster.Status.RunningVersion != cluster.Spec.Version
+
+	if useLocks && upgrading {
+		processGroupIDs := make([]string, 0, len(cluster.Status.ProcessGroups))
+		for _, processGroup := range cluster.Status.ProcessGroups {
+			processGroupIDs = append(processGroupIDs, processGroup.ProcessGroupID)
+		}
+		err = lockClient.AddPendingUpgrades(version, processGroupIDs)
+		if err != nil {
+			return &requeue{curError: err}
+		}
+	}
+
+	hasLock, err := r.takeLock(cluster, fmt.Sprintf("bouncing processes: %v", addresses))
+	if !hasLock {
+		return &requeue{curError: err}
+	}
+
+	if useLocks && upgrading {
+		var req *requeue
+		addresses, req = getAddressesForUpgrade(logger, r, status, lockClient, cluster, version)
+		if req != nil {
+			return req
+		}
+		if addresses == nil {
+			return &requeue{curError: fmt.Errorf("unknown error when getting addresses that are ready for upgrade")}
+		}
+	}
+
+	logger.Info("Bouncing processes", "addresses", addresses, "upgrading", upgrading)
+	r.Recorder.Event(cluster, corev1.EventTypeNormal, "BouncingProcesses", fmt.Sprintf("Bouncing processes: %v", addresses))
+	err = adminClient.KillProcesses(addresses)
 	if err != nil {
 		return &requeue{curError: err}
 	}
 
+	// If the cluster was upgraded we will requeue and let the update_status command set the correct version.
+	// Updating the version in this method has the drawback that we upgrade the version independent of the success
+	// of the kill command. The kill command is not reliable, which means that some kill request might not be
+	// delivered and the return value will still not contain any error.
+	if upgrading {
+		return &requeue{message: "fetch latest status after upgrade"}
+	}
+
+	return nil
+}
+
+// getProcessesReadyForRestart returns a slice of process addresses that can be restarted. If addresses are missing or not all processes
+// have the latest configuration this method will return a requeue struct with more details.
+func getProcessesReadyForRestart(ctx context.Context, logger logr.Logger, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, addressMap map[string][]fdbv1beta2.ProcessAddress) ([]fdbv1beta2.ProcessAddress, *requeue) {
 	processesToBounce := fdbv1beta2.FilterByConditions(cluster.Status.ProcessGroups, map[fdbv1beta2.ProcessGroupConditionType]bool{
 		fdbv1beta2.IncorrectCommandLine: true,
 		fdbv1beta2.IncorrectPodSpec:     false,
-		fdbv1beta2.SidecarUnreachable:   false, // ignore all Process groups that are not reachable and therefore will not get any config map updates.
+		fdbv1beta2.SidecarUnreachable:   false, // ignore all Process groups that are not reachable and therefore will not get any ConfigMap updates.
 	}, true)
 
 	addresses := make([]fdbv1beta2.ProcessAddress, 0, len(processesToBounce))
@@ -95,10 +184,10 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		processGroupID := podmanager.GetProcessGroupIDFromProcessID(process)
 		pod, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetSinglePodListOptions(cluster, processGroupID)...)
 		if err != nil {
-			return &requeue{curError: err}
+			return nil, &requeue{curError: err}
 		}
 		if len(pod) == 0 {
-			return &requeue{message: fmt.Sprintf("No pod defined for process group ID: \"%s\"", processGroupID), delay: podSchedulingDelayDuration}
+			return nil, &requeue{message: fmt.Sprintf("No pod defined for process group ID: \"%s\"", processGroupID), delay: podSchedulingDelayDuration}
 		}
 
 		synced, err := r.updatePodDynamicConf(cluster, pod[0])
@@ -109,107 +198,19 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 	}
 
 	if len(missingAddress) > 0 {
-		return &requeue{curError: fmt.Errorf("could not find address for processes: %s", missingAddress)}
+		return nil, &requeue{curError: fmt.Errorf("could not find address for processes: %s", missingAddress), delayedRequeue: true}
 	}
 
 	if !allSynced {
-		return &requeue{message: "Waiting for config map to sync to all pods"}
-	}
-
-	if len(addresses) > 0 {
-		if !pointer.BoolDeref(cluster.Spec.AutomationOptions.KillProcesses, true) {
-			r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
-				"Spec require a bounce of some processes, but killing processes is disabled")
-			cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
-			err = r.updateOrApply(ctx, cluster)
-			if err != nil {
-				logger.Error(err, "Error updating cluster status")
-			}
-
-			return &requeue{message: "Kills are disabled"}
-		}
-
-		if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
-			r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
-				fmt.Sprintf("Spec require a bounce of some processes, but the cluster has only been up for %f seconds", minimumUptime))
-			cluster.Status.Generations.NeedsBounce = cluster.ObjectMeta.Generation
-			err = r.updateOrApply(ctx, cluster)
-			if err != nil {
-				logger.Error(err, "Error updating cluster status")
-			}
-
-			// Retry after we waited the minimum uptime
-			return &requeue{
-				message: "Cluster needs to stabilize before bouncing",
-				delay:   time.Second * time.Duration(cluster.GetMinimumUptimeSecondsForBounce()-int(minimumUptime)),
-			}
-		}
-
-		var lockClient fdbadminclient.LockClient
-		useLocks := cluster.ShouldUseLocks()
-		if useLocks {
-			lockClient, err = r.getLockClient(cluster)
-			if err != nil {
-				return &requeue{curError: err}
-			}
-		}
-		version, err := fdbv1beta2.ParseFdbVersion(cluster.Spec.Version)
-		if err != nil {
-			return &requeue{curError: err}
-		}
-
-		upgrading := cluster.Status.RunningVersion != cluster.Spec.Version
-
-		if useLocks && upgrading {
-			processGroupIDs := make([]string, 0, len(cluster.Status.ProcessGroups))
-			for _, processGroup := range cluster.Status.ProcessGroups {
-				processGroupIDs = append(processGroupIDs, processGroup.ProcessGroupID)
-			}
-			err = lockClient.AddPendingUpgrades(version, processGroupIDs)
-			if err != nil {
-				return &requeue{curError: err}
-			}
-		}
-
-		hasLock, err := r.takeLock(cluster, fmt.Sprintf("bouncing processes: %v", addresses))
-		if !hasLock {
-			return &requeue{curError: err}
-		}
-
-		if useLocks && upgrading {
-			var req *requeue
-			addresses, req = getAddressesForUpgrade(r, status, lockClient, cluster, version)
-			if req != nil {
-				return req
-			}
-			if addresses == nil {
-				return &requeue{curError: fmt.Errorf("unknown error when getting addresses that are ready for upgrade")}
-			}
-		}
-
-		logger.Info("Bouncing processes", "addresses", addresses, "upgrading", upgrading)
-		r.Recorder.Event(cluster, corev1.EventTypeNormal, "BouncingProcesses", fmt.Sprintf("Bouncing processes: %v", addresses))
-		err = adminClient.KillProcesses(addresses)
-		if err != nil {
-			return &requeue{curError: err}
-		}
-
-		// If the cluster was upgraded we will requeue and let the update_status command set the correct version.
-		// Updating the version in this method has the drawback that we upgrade the version independent of the success
-		// of the kill command. The kill command is not reliable, which means that some kill request might not be
-		// delivered and the return value will still not contain any error.
-		if upgrading {
-			return &requeue{message: "fetch latest status after upgrade"}
-		}
+		return nil, &requeue{message: "Waiting for config map to sync to all pods", delayedRequeue: true}
 	}
 
-	return nil
+	return addresses, nil
 }
 
 // getAddressesForUpgrade checks that all processes in a cluster are ready to be
 // upgraded and returns the full list of addresses.
-func getAddressesForUpgrade(r *FoundationDBClusterReconciler, databaseStatus *fdbv1beta2.FoundationDBStatus, lockClient fdbadminclient.LockClient, cluster *fdbv1beta2.FoundationDBCluster, version fdbv1beta2.Version) ([]fdbv1beta2.ProcessAddress, *requeue) {
-	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "bounceProcesses")
+func getAddressesForUpgrade(logger logr.Logger, r *FoundationDBClusterReconciler, databaseStatus *fdbv1beta2.FoundationDBStatus, lockClient fdbadminclient.LockClient, cluster *fdbv1beta2.FoundationDBCluster, version fdbv1beta2.Version) ([]fdbv1beta2.ProcessAddress, *requeue) {
 	pendingUpgrades, err := lockClient.GetPendingUpgrades(version)
 	if err != nil {
 		return nil, &requeue{curError: err}
@@ -247,36 +248,3 @@ func getAddressesForUpgrade(r *FoundationDBClusterReconciler, databaseStatus *fd
 
 	return addresses, nil
 }
-
-// getMinimumUptimeAndAddressMap returns address map of the processes included the the foundationdb status. The minimum
-// uptime will be either secondsSinceLastRecovered if the recovery state is supported and enabled otherwise we will
-// take the minimum uptime of all processes.
-func getMinimumUptimeAndAddressMap(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, recoveryStateEnabled bool) (float64, map[string][]fdbv1beta2.ProcessAddress, error) {
-	runningVersion, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion())
-	if err != nil {
-		return 0, nil, err
-	}
-
-	useRecoveryState := runningVersion.SupportsRecoveryState() && recoveryStateEnabled
-
-	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
-
-	minimumUptime := math.Inf(1)
-	if useRecoveryState {
-		minimumUptime = status.Cluster.RecoveryState.SecondsSinceLastRecovered
-	}
-
-	for _, process := range status.Cluster.Processes {
-		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
-
-		if useRecoveryState || process.Excluded {
-			continue
-		}
-
-		if process.UptimeSeconds < minimumUptime {
-			minimumUptime = process.UptimeSeconds
-		}
-	}
-
-	return minimumUptime, addressMap, nil
-}
diff --git a/controllers/change_coordinators.go b/controllers/change_coordinators.go
index 3b0de6e5..e3ca5f15 100644
--- a/controllers/change_coordinators.go
+++ b/controllers/change_coordinators.go
@@ -154,7 +154,7 @@ func selectCoordinators(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1be
 
 	candidates, err := selectCandidates(cluster, status)
 	if err != nil {
-		return []localityInfo{}, nil
+		return []localityInfo{}, err
 	}
 
 	coordinators, err := chooseDistributedProcesses(cluster, candidates, coordinatorCount, processSelectionConstraint{
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 0e5ff5d9..9a8b7159 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -89,7 +89,6 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
 			return ctrl.Result{}, nil
-
 		}
 		// Error reading the object - requeue the request.
 		return ctrl.Result{}, err
diff --git a/controllers/remove_incompatible_processes.go b/controllers/remove_incompatible_processes.go
index 410e344d..d9762171 100644
--- a/controllers/remove_incompatible_processes.go
+++ b/controllers/remove_incompatible_processes.go
@@ -90,18 +90,32 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 		return nil
 	}
 
-	logger.Info("incompatible connections", "incompatibleConnections", status.Cluster.IncompatibleConnections)
-	incompatibleConnections := map[string]fdbv1beta2.None{}
-	for _, incompatibleAddress := range status.Cluster.IncompatibleConnections {
-		address, err := fdbv1beta2.ParseProcessAddress(incompatibleAddress)
-		if err != nil {
-			logger.Error(err, "could not parse address in incompatible connections", "address", incompatibleAddress)
-			continue
-		}
+	// Wait until the cluster is running for the minimum uptime before looking for incompatible processes.
+	minimumUptime, _, err := internal.GetMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	if err != nil {
+		return err
+	}
 
-		incompatibleConnections[address.IPAddress.String()] = fdbv1beta2.None{}
+	if minimumUptime < float64(cluster.GetMinimumUptimeSecondsForBounce()) {
+		logger.V(1).Info("Skipping reconciler and waiting until cluster is for up minimum uptime")
+		return nil
+	}
+
+	// Ensure the cluster is running at fault tolerance before recreating Pods.
+	hasDesiredFaultTolerance, err := internal.HasDesiredFaultToleranceFromStatus(logger, status, cluster)
+	if err != nil {
+		logger.V(1).Info("Cluster doesn't have required fault tolerance won't delete Pods")
+		return err
 	}
 
+	if !hasDesiredFaultTolerance {
+		logger.V(1).Info("Skipping reconciler and waiting until cluster has desired fault tolerance")
+		return nil
+	}
+
+	logger.Info("incompatible connections", "incompatibleConnections", status.Cluster.IncompatibleConnections)
+	incompatibleConnections := parseIncompatibleConnections(logger, status.Cluster.IncompatibleConnections)
+
 	incompatiblePods := make([]*corev1.Pod, 0, len(incompatibleConnections))
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		pod, ok := podMap[processGroup.ProcessGroupID]
@@ -111,8 +125,14 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 			continue
 		}
 
+		if pod.DeletionTimestamp != nil {
+			logger.V(1).Info("Skipping Pod that is already marked for deletion",
+				"processGroupID", processGroup.ProcessGroupID)
+			continue
+		}
+
 		if isIncompatible(incompatibleConnections, processGroup) {
-			logger.Info("recreate Pod for process group with incompatible version", "processGroupID", processGroup.ProcessGroupID)
+			logger.Info("recreate Pod for process group with incompatible version", "processGroupID", processGroup.ProcessGroupID, "address", processGroup.Addresses)
 			incompatiblePods = append(incompatiblePods, pod)
 		}
 	}
@@ -121,6 +141,27 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 	return r.PodLifecycleManager.UpdatePods(ctx, r, cluster, incompatiblePods, true)
 }
 
+// parseIncompatibleConnections parses the incompatible connections string slice to a map
+func parseIncompatibleConnections(logger logr.Logger, incompatibleConnections []string) map[string]fdbv1beta2.None {
+	result := make(map[string]fdbv1beta2.None)
+	for _, incompatibleAddress := range incompatibleConnections {
+		address, err := fdbv1beta2.ParseProcessAddress(incompatibleAddress)
+		if err != nil {
+			logger.Error(err, "could not parse address in incompatible connections", "address", incompatibleAddress)
+			continue
+		}
+
+		if address.Port == 0 {
+			logger.V(1).Info("Ignore incompatible connection with port 0", "address", address)
+			continue
+		}
+
+		result[address.IPAddress.String()] = fdbv1beta2.None{}
+	}
+
+	return result
+}
+
 // isIncompatible checks if the process group is in the list of incompatible connections.
 func isIncompatible(incompatibleConnections map[string]fdbv1beta2.None, processGroup *fdbv1beta2.ProcessGroupStatus) bool {
 	for _, address := range processGroup.Addresses {
diff --git a/controllers/remove_process_groups.go b/controllers/remove_process_groups.go
index 696b99a4..522d1034 100644
--- a/controllers/remove_process_groups.go
+++ b/controllers/remove_process_groups.go
@@ -71,12 +71,17 @@ func (u removeProcessGroups) reconcile(ctx context.Context, r *FoundationDBClust
 		}
 	}
 
+	status, err := adminClient.GetStatus()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+
 	// We don't use the "cached" of the cluster status from the CRD to minimize the window between data loss (e.g. a node
 	// or a set of Pods is not reachable anymore). We still end up with the risk to actually query the FDB cluster and after that
 	// query the cluster gets into a degraded state.
 	// We could be smarter here and only block removals that target stateful processes by e.g. filtering those out of the
 	// processGroupsToRemove slice.
-	hasDesiredFaultTolerance, err := internal.HasDesiredFaultTolerance(logger, adminClient, cluster)
+	hasDesiredFaultTolerance, err := internal.HasDesiredFaultToleranceFromStatus(logger, status, cluster)
 	if err != nil {
 		return &requeue{curError: err}
 	}
@@ -88,11 +93,6 @@ func (u removeProcessGroups) reconcile(ctx context.Context, r *FoundationDBClust
 		}
 	}
 
-	status, err := adminClient.GetStatus()
-	if err != nil {
-		return &requeue{curError: err}
-	}
-
 	// In addition to that we should add the same logic as in the exclude step
 	// to ensure we never exclude/remove more process groups than desired.
 	zonedRemovals, lastDeletion, err := removals.GetZonedRemovals(status, processGroupsToRemove)
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 70ff4dff..26c2fb82 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -341,7 +341,7 @@ func tryConnectionOptions(cluster *fdbv1beta2.FoundationDBCluster, r *Foundation
 			if err == nil {
 				logger.Info("Chose connection option",
 					"version", version, "connectionString", activeConnectionString)
-				return version, activeConnectionString, err
+				return version, activeConnectionString, nil
 			}
 			logger.Error(err, "Error getting connection string from cluster",
 				"version", version, "connectionString", connectionString)
diff --git a/go.mod b/go.mod
index 318090b8..c20d7bf1 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/FoundationDB/fdb-kubernetes-operator
 
-go 1.18
+go 1.19
 
 require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20201222225940-f3aef311ccfb
diff --git a/internal/deprecations.go b/internal/deprecations.go
index 18372226..7ba8bf93 100644
--- a/internal/deprecations.go
+++ b/internal/deprecations.go
@@ -25,7 +25,6 @@ import (
 
 	fdbtypes "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 )
 
@@ -61,12 +60,12 @@ func NormalizeClusterSpec(cluster *fdbtypes.FoundationDBCluster, options Depreca
 
 	if !options.OnlyShowChanges {
 		// Set up resource requirements for the main container.
-		updatePodTemplates(&cluster.Spec, func(template *v1.PodTemplateSpec) {
+		updatePodTemplates(&cluster.Spec, func(template *corev1.PodTemplateSpec) {
 			template.Spec.Containers, _ = ensureContainerPresent(template.Spec.Containers, "foundationdb", 0)
 
-			template.Spec.Containers = customizeContainerFromList(template.Spec.Containers, "foundationdb", func(container *v1.Container) {
+			template.Spec.Containers = customizeContainerFromList(template.Spec.Containers, "foundationdb", func(container *corev1.Container) {
 				if container.Resources.Requests == nil {
-					container.Resources.Requests = v1.ResourceList{
+					container.Resources.Requests = corev1.ResourceList{
 						"cpu":    resource.MustParse("1"),
 						"memory": resource.MustParse("1Gi"),
 					}
@@ -77,9 +76,9 @@ func NormalizeClusterSpec(cluster *fdbtypes.FoundationDBCluster, options Depreca
 				}
 			})
 
-			sidecarUpdater := func(container *v1.Container) {
+			sidecarUpdater := func(container *corev1.Container) {
 				if container.Resources.Requests == nil {
-					container.Resources.Requests = v1.ResourceList{
+					container.Resources.Requests = corev1.ResourceList{
 						"cpu":    resource.MustParse("100m"),
 						"memory": resource.MustParse("256Mi"),
 					}
diff --git a/internal/fault_tolerance.go b/internal/fault_tolerance.go
index 6d3d629e..b1c76fcb 100644
--- a/internal/fault_tolerance.go
+++ b/internal/fault_tolerance.go
@@ -39,10 +39,15 @@ func HasDesiredFaultTolerance(log logr.Logger, adminClient fdbadminclient.AdminC
 		return false, err
 	}
 
+	return HasDesiredFaultToleranceFromStatus(log, status, cluster)
+}
+
+// HasDesiredFaultToleranceFromStatus checks if the cluster has the desired fault tolerance based on the provided status.
+func HasDesiredFaultToleranceFromStatus(log logr.Logger, status *fdbv1beta2.FoundationDBStatus, cluster *fdbv1beta2.FoundationDBCluster) (bool, error) {
 	if !status.Client.DatabaseStatus.Available {
-		//log.V(0).Info("Cluster is not available",
-		//	"namespace", cluster.Namespace,
-		//	"cluster", cluster.Name)
+		log.Info("Cluster is not available",
+			"namespace", cluster.Namespace,
+			"cluster", cluster.Name)
 
 		return false, nil
 	}
diff --git a/internal/foundationdb_status.go b/internal/foundationdb_status.go
index 1c4cf254..bb6ece78 100644
--- a/internal/foundationdb_status.go
+++ b/internal/foundationdb_status.go
@@ -21,26 +21,61 @@
 package internal
 
 import (
-	"github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"math"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 )
 
 // GetCoordinatorsFromStatus gets the current coordinators from the status.
 // The returning set will contain all processes by their process group ID.
-func GetCoordinatorsFromStatus(status *v1beta2.FoundationDBStatus) map[string]struct{} {
+func GetCoordinatorsFromStatus(status *fdbv1beta2.FoundationDBStatus) map[string]struct{} {
 	coordinators := make(map[string]struct{})
 
 	for _, pInfo := range status.Cluster.Processes {
 		for _, roleInfo := range pInfo.Roles {
-			if roleInfo.Role != string(v1beta2.ProcessRoleCoordinator) {
+			if roleInfo.Role != string(fdbv1beta2.ProcessRoleCoordinator) {
 				continue
 			}
 
 			// We don't have to check for duplicates here, if the process group ID is already
 			// set we just overwrite it.
-			coordinators[pInfo.Locality[v1beta2.FDBLocalityInstanceIDKey]] = struct{}{}
+			coordinators[pInfo.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = struct{}{}
 			break
 		}
 	}
 
 	return coordinators
 }
+
+// GetMinimumUptimeAndAddressMap returns address map of the processes included the the foundationdb status. The minimum
+// uptime will be either secondsSinceLastRecovered if the recovery state is supported and enabled otherwise we will
+// take the minimum uptime of all processes.
+func GetMinimumUptimeAndAddressMap(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, recoveryStateEnabled bool) (float64, map[string][]fdbv1beta2.ProcessAddress, error) {
+	runningVersion, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion())
+	if err != nil {
+		return 0, nil, err
+	}
+
+	useRecoveryState := runningVersion.SupportsRecoveryState() && recoveryStateEnabled
+
+	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
+
+	minimumUptime := math.Inf(1)
+	if useRecoveryState {
+		minimumUptime = status.Cluster.RecoveryState.SecondsSinceLastRecovered
+	}
+
+	for _, process := range status.Cluster.Processes {
+		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
+
+		if useRecoveryState || process.Excluded {
+			continue
+		}
+
+		if process.UptimeSeconds < minimumUptime {
+			minimumUptime = process.UptimeSeconds
+		}
+	}
+
+	return minimumUptime, addressMap, nil
+}
diff --git a/internal/pod_client.go b/internal/pod_client.go
index 6d60e3bb..e76ddcc4 100644
--- a/internal/pod_client.go
+++ b/internal/pod_client.go
@@ -173,7 +173,6 @@ func (client *realFdbPodSidecarClient) getListenIP() string {
 
 // makeRequest submits a request to the sidecar.
 func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string, int, error) {
-	var resp *http.Response
 	var err error
 
 	protocol := "http"
@@ -190,13 +189,16 @@ func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string,
 	}
 
 	url := fmt.Sprintf("%s://%s:8080/%s", protocol, client.getListenIP(), path)
+	var resp *http.Response
 	switch method {
 	case http.MethodGet:
 		retryClient.HTTPClient.Timeout = client.getTimeout
 		resp, err = retryClient.Get(url)
+		defer resp.Body.Close()
 	case http.MethodPost:
 		retryClient.HTTPClient.Timeout = client.postTimeout
 		resp, err = retryClient.Post(url, "application/json", strings.NewReader(""))
+		defer resp.Body.Close()
 	default:
 		return "", 0, fmt.Errorf("unknown HTTP method %s", method)
 	}
@@ -205,8 +207,6 @@ func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string,
 		return "", 0, err
 	}
 
-	defer resp.Body.Close()
-
 	body, err := io.ReadAll(resp.Body)
 	bodyText := string(body)
 
@@ -221,7 +221,7 @@ func (client *realFdbPodSidecarClient) makeRequest(method, path string) (string,
 func (client *realFdbPodSidecarClient) IsPresent(filename string) (bool, error) {
 	version, err := fdbv1beta2.ParseFdbVersion(client.Cluster.Spec.Version)
 	if err != nil {
-		return false, nil
+		return false, err
 	}
 
 	path := "check_hash"
diff --git a/internal/pod_models.go b/internal/pod_models.go
index 5e736aa5..a442ba3b 100644
--- a/internal/pod_models.go
+++ b/internal/pod_models.go
@@ -852,7 +852,6 @@ func GetBackupDeployment(backup *fdbv1beta2.FoundationDBBackup) (*appsv1.Deploym
 		for _, customParameter := range backup.Spec.CustomParameters {
 			args = append(args, fmt.Sprintf("--%s", customParameter))
 		}
-
 	}
 
 	mainContainer.Args = args
diff --git a/kubectl-fdb/cmd/exec.go b/kubectl-fdb/cmd/exec.go
index f245ddb3..dfde0c40 100644
--- a/kubectl-fdb/cmd/exec.go
+++ b/kubectl-fdb/cmd/exec.go
@@ -111,14 +111,14 @@ func buildCommand(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBClus
 	for key, value := range cluster.GetMatchLabels() {
 		requirement, err := labels.NewRequirement(key, selection.Equals, []string{value})
 		if err != nil {
-			return exec.Cmd{}, nil
+			return exec.Cmd{}, err
 		}
 		selector = selector.Add(*requirement)
 	}
 
 	processClassRequirement, err := labels.NewRequirement(cluster.GetProcessClassLabel(), selection.Exists, nil)
 	if err != nil {
-		return exec.Cmd{}, nil
+		return exec.Cmd{}, err
 	}
 	selector = selector.Add(*processClassRequirement)
 
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index 35b874b9..340da95b 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -37,7 +37,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
@@ -143,7 +142,7 @@ func executeCmd(restConfig *rest.Config, kubeClient *kubernetes.Clientset, podNa
 	}
 	req.VersionedParams(
 		option,
-		scheme.ParameterCodec,
+		clientgoscheme.ParameterCodec,
 	)
 	exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL())
 	if err != nil {
diff --git a/kubectl-fdb/cmd/version.go b/kubectl-fdb/cmd/version.go
index 735ebf53..2f83c916 100644
--- a/kubectl-fdb/cmd/version.go
+++ b/kubectl-fdb/cmd/version.go
@@ -47,11 +47,11 @@ func newVersionCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			}
 			clientOnly, err := cmd.Flags().GetBool("client-only")
 			if err != nil {
-				return nil
+				return err
 			}
 			containerName, err := cmd.Flags().GetString("container-name")
 			if err != nil {
-				return nil
+				return err
 			}
 
 			if !clientOnly {


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index b851f87f..1a55aee5 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -3141,6 +3141,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(cluster.Status.Generations).To(Equal(ClusterGenerationStatus{
 					Reconciled:          1,
 					HasUnhealthyProcess: 2,
+					NeedsBounce:         2,
 				}))
 
 				cluster = createCluster()
@@ -3392,6 +3393,7 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				Expect(cluster.Status.Generations).To(Equal(ClusterGenerationStatus{
 					Reconciled:          1,
 					HasUnhealthyProcess: 2,
+					NeedsBounce:         2,
 				}))
 
 				cluster = createCluster()
diff --git a/controllers/bounce_processes_test.go b/controllers/bounce_processes_test.go
index fbaccc28..9b50f89a 100644
--- a/controllers/bounce_processes_test.go
+++ b/controllers/bounce_processes_test.go
@@ -23,7 +23,6 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"net"
 	"time"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -382,111 +381,4 @@ var _ = Describe("bounceProcesses", func() {
 			})
 		})
 	})
-
-	DescribeTable("when getting the minimum uptime and the address map", func(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, useRecoveryState bool, expectedMinimumUptime float64, expectedAddressMap map[string][]fdbv1beta2.ProcessAddress) {
-		minimumUptime, addressMap, err := getMinimumUptimeAndAddressMap(cluster, status, useRecoveryState)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(minimumUptime).To(BeNumerically("==", expectedMinimumUptime))
-		Expect(len(addressMap)).To(BeNumerically("==", len(expectedAddressMap)))
-		for key, value := range expectedAddressMap {
-			Expect(addressMap).To(HaveKeyWithValue(key, value))
-		}
-	},
-		Entry("when recovered since is not available",
-			&fdbv1beta2.FoundationDBCluster{
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					Version: fdbv1beta2.Versions.Default.String(),
-				},
-			}, &fdbv1beta2.FoundationDBStatus{
-				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-						"test": {
-							Address: fdbv1beta2.ProcessAddress{
-								IPAddress: net.ParseIP("127.0.0.1"),
-							},
-							Locality: map[string]string{
-								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
-							},
-							UptimeSeconds: 30.0,
-						},
-					},
-					RecoveryState: fdbv1beta2.RecoveryState{
-						SecondsSinceLastRecovered: 90.0,
-					},
-				},
-			},
-			true,
-			30.0,
-			map[string][]fdbv1beta2.ProcessAddress{
-				"test": {
-					{
-						IPAddress: net.ParseIP("127.0.0.1"),
-					},
-				},
-			}),
-		Entry("when recovered since is enabled and version supports it",
-			&fdbv1beta2.FoundationDBCluster{
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
-				},
-			}, &fdbv1beta2.FoundationDBStatus{
-				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-						"test": {
-							Address: fdbv1beta2.ProcessAddress{
-								IPAddress: net.ParseIP("127.0.0.1"),
-							},
-							Locality: map[string]string{
-								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
-							},
-							UptimeSeconds: 30.0,
-						},
-					},
-					RecoveryState: fdbv1beta2.RecoveryState{
-						SecondsSinceLastRecovered: 90.0,
-					},
-				},
-			},
-			true,
-			90.0,
-			map[string][]fdbv1beta2.ProcessAddress{
-				"test": {
-					{
-						IPAddress: net.ParseIP("127.0.0.1"),
-					},
-				},
-			}),
-		Entry("when recovered since is disabled and version supports it",
-			&fdbv1beta2.FoundationDBCluster{
-				Spec: fdbv1beta2.FoundationDBClusterSpec{
-					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
-				},
-			}, &fdbv1beta2.FoundationDBStatus{
-				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
-						"test": {
-							Address: fdbv1beta2.ProcessAddress{
-								IPAddress: net.ParseIP("127.0.0.1"),
-							},
-							Locality: map[string]string{
-								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
-							},
-							UptimeSeconds: 30.0,
-						},
-					},
-					RecoveryState: fdbv1beta2.RecoveryState{
-						SecondsSinceLastRecovered: 90.0,
-					},
-				},
-			},
-			false,
-			30.0,
-			map[string][]fdbv1beta2.ProcessAddress{
-				"test": {
-					{
-						IPAddress: net.ParseIP("127.0.0.1"),
-					},
-				},
-			}),
-	)
 })
diff --git a/controllers/remove_incompatible_processes_test.go b/controllers/remove_incompatible_processes_test.go
index 5c479848..f6054137 100644
--- a/controllers/remove_incompatible_processes_test.go
+++ b/controllers/remove_incompatible_processes_test.go
@@ -66,6 +66,23 @@ var _ = Describe("restart_incompatible_pods", func() {
 			true),
 	)
 
+	DescribeTable("when parsing incompatible connections", func(incompatibleConnections []string, expected map[string]fdbv1beta2.None) {
+		Expect(parseIncompatibleConnections(logr.Discard(), incompatibleConnections)).To(Equal(expected))
+	},
+		Entry("empty incompatible map",
+			[]string{},
+			map[string]fdbv1beta2.None{}),
+		Entry("nil incompatible map",
+			nil,
+			map[string]fdbv1beta2.None{}),
+		Entry("incompatible map contains one address",
+			[]string{"1.1.1.1:4500:tls"},
+			map[string]fdbv1beta2.None{"1.1.1.1": {}}),
+		Entry("incompatible map contains multiple addresses",
+			[]string{"1.1.1.1:4500:tls", "1.1.1.2:0:tls"},
+			map[string]fdbv1beta2.None{"1.1.1.1": {}}),
+	)
+
 	When("running a reconcile for the restart incompatible process reconciler", func() {
 		var cluster *fdbv1beta2.FoundationDBCluster
 		var initialCount int
@@ -94,7 +111,7 @@ var _ = Describe("restart_incompatible_pods", func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		When("no incompatible processes are reported", func() {
+		When("the subreconciler is enabled", func() {
 			BeforeEach(func() {
 				clusterReconciler.EnableRestartIncompatibleProcesses = true
 				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
@@ -106,55 +123,119 @@ var _ = Describe("restart_incompatible_pods", func() {
 				}
 			})
 
-			It("should have no deletions", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+			When("no incompatible processes are reported", func() {
+				BeforeEach(func() {
+					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+					Expect(err).NotTo(HaveOccurred())
+					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							IncompatibleConnections: []string{},
+						},
+					}
+				})
+
+				It("should have no deletions", func() {
+					pods := &corev1.PodList{}
+					err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+				})
 			})
-		})
 
-		When("no matching incompatible processes are reported", func() {
-			BeforeEach(func() {
-				clusterReconciler.EnableRestartIncompatibleProcesses = true
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						IncompatibleConnections: []string{
-							"192.192.192.192",
+			When("no matching incompatible processes are reported", func() {
+				BeforeEach(func() {
+					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+					Expect(err).NotTo(HaveOccurred())
+					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							IncompatibleConnections: []string{
+								"192.192.192.192:4500:tls",
+							},
 						},
-					},
-				}
-			})
+					}
+				})
 
-			It("should have no deletions", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+				It("should have no deletions", func() {
+					pods := &corev1.PodList{}
+					err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+				})
 			})
-		})
 
-		When("matching incompatible processes are reported and the subreconciler is enabled", func() {
-			BeforeEach(func() {
-				clusterReconciler.EnableRestartIncompatibleProcesses = true
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						IncompatibleConnections: []string{
-							cluster.Status.ProcessGroups[0].Addresses[0],
+			When("matching incompatible processes are reported", func() {
+				BeforeEach(func() {
+					adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+					Expect(err).NotTo(HaveOccurred())
+					adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
+						Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+							DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+								Available: true,
+							},
 						},
-					},
-				}
-			})
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							FaultTolerance: fdbv1beta2.FaultTolerance{
+								MaxZoneFailuresWithoutLosingAvailability: 2,
+								MaxZoneFailuresWithoutLosingData:         2,
+							},
+							IncompatibleConnections: []string{
+								cluster.Status.ProcessGroups[0].Addresses[0] + ":4500:tls",
+							},
+						},
+					}
+				})
 
-			It("should have one deletion", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount-1))
+				It("should have one deletion", func() {
+					pods := &corev1.PodList{}
+					err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(len(pods.Items)).To(BeNumerically("==", initialCount-1))
+				})
+
+				When("the database is unavailable", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.frozenStatus.Client.DatabaseStatus.Available = false
+					})
+
+					It("should have no deletions", func() {
+						pods := &corev1.PodList{}
+						err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+					})
+				})
+
+				When("matching incompatible processes are reported but port is zero", func() {
+					BeforeEach(func() {
+						adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
+						Expect(err).NotTo(HaveOccurred())
+						adminClient.frozenStatus.Cluster.IncompatibleConnections[0] = cluster.Status.ProcessGroups[0].Addresses[0] + ":0:tls"
+					})
+
+					It("should have no deletions", func() {
+						pods := &corev1.PodList{}
+						err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+					})
+				})
+
+				When("the cluster is currently upgraded", func() {
+					BeforeEach(func() {
+						cluster.Spec.Version = fdbv1beta2.Versions.NextMajorVersion.String()
+						err := k8sClient.Update(context.Background(), cluster)
+						Expect(err).NotTo(HaveOccurred())
+					})
+
+					It("should have no deletions", func() {
+						pods := &corev1.PodList{}
+						err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
+					})
+				})
 			})
 		})
 
@@ -164,34 +245,18 @@ var _ = Describe("restart_incompatible_pods", func() {
 				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
 				Expect(err).NotTo(HaveOccurred())
 				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
-					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
-						IncompatibleConnections: []string{
-							cluster.Status.ProcessGroups[0].Addresses[0],
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
 						},
 					},
-				}
-			})
-
-			It("should have no deletions", func() {
-				pods := &corev1.PodList{}
-				err := k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(pods.Items)).To(BeNumerically("==", initialCount))
-			})
-		})
-
-		When("matching incompatible processes are reported and the cluster is currently upgraded", func() {
-			BeforeEach(func() {
-				cluster.Spec.Version = fdbv1beta2.Versions.NextMajorVersion.String()
-				err := k8sClient.Update(context.Background(), cluster)
-				Expect(err).NotTo(HaveOccurred())
-				clusterReconciler.EnableRestartIncompatibleProcesses = false
-				adminClient, err := newMockAdminClientUncast(cluster, k8sClient)
-				Expect(err).NotTo(HaveOccurred())
-				adminClient.frozenStatus = &fdbv1beta2.FoundationDBStatus{
 					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+							MaxZoneFailuresWithoutLosingData:         2,
+						},
 						IncompatibleConnections: []string{
-							cluster.Status.ProcessGroups[0].Addresses[0],
+							cluster.Status.ProcessGroups[0].Addresses[0] + ":4500:tls",
 						},
 					},
 				}
diff --git a/internal/fault_tolerance_test.go b/internal/fault_tolerance_test.go
index 3a5421fe..22574bae 100644
--- a/internal/fault_tolerance_test.go
+++ b/internal/fault_tolerance_test.go
@@ -21,8 +21,11 @@
 package internal
 
 import (
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"github.com/go-logr/logr"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 var _ = Describe("fault_tolerance", func() {
@@ -64,4 +67,104 @@ var _ = Describe("fault_tolerance", func() {
 				}),
 		)
 	})
+
+	When("checking if the cluster has the desired fault tolerance from the status", func() {
+		log := logr.New(logf.NewDelegatingLogSink(logf.NullLogSink{}))
+
+		DescribeTable("should return if the cluster has the desired fault tolerance",
+			func(status *fdbv1beta2.FoundationDBStatus, cluster *fdbv1beta2.FoundationDBCluster, expected bool) {
+				Expect(HasDesiredFaultToleranceFromStatus(log, status, cluster)).To(Equal(expected))
+			},
+			Entry("cluster is fully replicated",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         2,
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				true),
+			Entry("database is unavailable",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: false,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         2,
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				false),
+			Entry("data is degraded",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         1,
+							MaxZoneFailuresWithoutLosingAvailability: 2,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				false),
+			Entry("availability is degraded",
+				&fdbv1beta2.FoundationDBStatus{
+					Client: fdbv1beta2.FoundationDBStatusLocalClientInfo{
+						DatabaseStatus: fdbv1beta2.FoundationDBStatusClientDBStatus{
+							Available: true,
+						},
+					},
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						FaultTolerance: fdbv1beta2.FaultTolerance{
+							MaxZoneFailuresWithoutLosingData:         2,
+							MaxZoneFailuresWithoutLosingAvailability: 1,
+						},
+					},
+				},
+				&fdbv1beta2.FoundationDBCluster{
+					Spec: fdbv1beta2.FoundationDBClusterSpec{
+						DatabaseConfiguration: fdbv1beta2.DatabaseConfiguration{
+							RedundancyMode: fdbv1beta2.RedundancyModeTriple,
+						},
+					},
+				},
+				false),
+		)
+	})
+
+	// HasDesiredFaultToleranceFromStatus
 })
diff --git a/internal/foundationdb_status_test.go b/internal/foundationdb_status_test.go
index 41b15fb7..eb6aced2 100644
--- a/internal/foundationdb_status_test.go
+++ b/internal/foundationdb_status_test.go
@@ -21,7 +21,9 @@
 package internal
 
 import (
-	"github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	"net"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -29,7 +31,7 @@ import (
 var _ = Describe("Internal FoundationDBStatus", func() {
 	When("parsing the status for coordinators", func() {
 		type testCase struct {
-			status   *v1beta2.FoundationDBStatus
+			status   *fdbv1beta2.FoundationDBStatus
 			expected map[string]struct{}
 		}
 
@@ -40,19 +42,19 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 			},
 			Entry("no coordinators",
 				testCase{
-					status:   &v1beta2.FoundationDBStatus{},
+					status:   &fdbv1beta2.FoundationDBStatus{},
 					expected: map[string]struct{}{},
 				}),
 			Entry("single coordinators",
 				testCase{
-					status: &v1beta2.FoundationDBStatus{
-						Cluster: v1beta2.FoundationDBStatusClusterInfo{
-							Processes: map[string]v1beta2.FoundationDBStatusProcessInfo{
+					status: &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
 								"foo": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "foo",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "foo",
 									},
-									Roles: []v1beta2.FoundationDBStatusProcessRoleInfo{
+									Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
 										{
 											Role: "coordinator",
 										},
@@ -60,7 +62,7 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 								},
 								"bar": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "bar",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "bar",
 									},
 								},
 							},
@@ -72,14 +74,14 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 				}),
 			Entry("multiple coordinators",
 				testCase{
-					status: &v1beta2.FoundationDBStatus{
-						Cluster: v1beta2.FoundationDBStatusClusterInfo{
-							Processes: map[string]v1beta2.FoundationDBStatusProcessInfo{
+					status: &fdbv1beta2.FoundationDBStatus{
+						Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+							Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
 								"foo": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "foo",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "foo",
 									},
-									Roles: []v1beta2.FoundationDBStatusProcessRoleInfo{
+									Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
 										{
 											Role: "coordinator",
 										},
@@ -87,9 +89,9 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 								},
 								"bar": {
 									Locality: map[string]string{
-										v1beta2.FDBLocalityInstanceIDKey: "bar",
+										fdbv1beta2.FDBLocalityInstanceIDKey: "bar",
 									},
-									Roles: []v1beta2.FoundationDBStatusProcessRoleInfo{
+									Roles: []fdbv1beta2.FoundationDBStatusProcessRoleInfo{
 										{
 											Role: "coordinator",
 										},
@@ -105,4 +107,111 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 				}),
 		)
 	})
+
+	DescribeTable("when getting the minimum uptime and the address map", func(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, useRecoveryState bool, expectedMinimumUptime float64, expectedAddressMap map[string][]fdbv1beta2.ProcessAddress) {
+		minimumUptime, addressMap, err := GetMinimumUptimeAndAddressMap(cluster, status, useRecoveryState)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(minimumUptime).To(BeNumerically("==", expectedMinimumUptime))
+		Expect(len(addressMap)).To(BeNumerically("==", len(expectedAddressMap)))
+		for key, value := range expectedAddressMap {
+			Expect(addressMap).To(HaveKeyWithValue(key, value))
+		}
+	},
+		Entry("when recovered since is not available",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.Default.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			true,
+			30.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+		Entry("when recovered since is enabled and version supports it",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			true,
+			90.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+		Entry("when recovered since is disabled and version supports it",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			false,
+			30.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+	)
 })
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index 20da895c..1f2cbaba 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -79,7 +79,6 @@ func getPodList(clusterName string, namespace string, status corev1.PodStatus, d
 			},
 		},
 	}
-
 }
 
 var _ = Describe("[plugin] analyze cluster", func() {