Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1343 (2022-09-01 14:06:36)

Add support for recovery state recovered since
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/804

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

I decided to create a flag to enable/disable this feature (enabled per default) to make it easier to remove this in the future (and always use it).

## Testing

Local test + unit tests.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 5978f8374e0407cb61f2173929e5310c0a999aaa
* Duration 4:06:43
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/1575e8b7-1434-4d98-b897-1c48157aa096.gz) (available for 30 days)
* [Build Artifact](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/1575e8b7-1434-4d98-b897-1c48157aa096/workspace.zip) (available for 30 days)


Patch:
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index f9775a4c..d709e1da 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18.5
+          go-version: 1.18.6
       - uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
@@ -44,7 +44,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18.5
+        go-version: 1.18.6
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -104,7 +104,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18.5
+        go-version: 1.18.6
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
@@ -134,7 +134,6 @@ jobs:
       run: |
         make test
   build_images:
-    if: github.ref != 'refs/heads/main'
     name: Build Docker images
     runs-on: ubuntu-latest
     strategy:
@@ -149,65 +148,26 @@ jobs:
             name: foundationdb/fdb-kubernetes-operator
             tagSuffix: ""
             file: ./Dockerfile
+            baseImage: "docker.io/debian:bullseye"
           - image: fdb-kubernetes-operator-distroless
             context: ./
             name: foundationdb/fdb-kubernetes-operator
             tagSuffix: -distrolesss
-            file: ./distrolesss.dockerfile
-          - image: fdb-data-loader
-            context: ./sample-apps/data-loader
-            name: foundationdb/fdb-data-loader
-            tagSuffix: ""
-            file: ./sample-apps/data-loader/Dockerfile
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Get the sha
-        id: get_sha
-        run: echo ::set-output name=TAG::${GITHUB_SHA}
-      - name: Build image
-        uses: docker/build-push-action@v2
-        with:
-          build-args: TAG=${{ steps.get_sha.outputs.TAG }}
-          push: false
-          context: ${{ matrix.context }}
-          tags: ${{ matrix.name }}:latest${{ matrix.tagSuffix }}
-          file: ${{ matrix.file }}
-  push_images:
-    if: github.ref == 'refs/heads/main'
-    needs: build
-    name: Push Docker images
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        image:
-          - fdb-kubernetes-operator
-          - fdb-data-loader
-          - fdb-kubernetes-operator-distroless
-        include:
-          - image: fdb-kubernetes-operator
-            context: ./
-            name: foundationdb/fdb-kubernetes-operator
-            tagSuffix: ""
             file: ./Dockerfile
-          - image: fdb-kubernetes-operator-distroless
-            context: ./
-            name: foundationdb/fdb-kubernetes-operator
-            tagSuffix: -distrolesss
-            file: ./distrolesss.dockerfile
+            baseImage: "gcr.io/distroless/base"
           - image: fdb-data-loader
             context: ./sample-apps/data-loader
             name: foundationdb/fdb-data-loader
             tagSuffix: ""
             file: ./sample-apps/data-loader/Dockerfile
+            baseImage: ""
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
+        if: github.ref == 'refs/heads/main'
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -215,11 +175,13 @@ jobs:
       - name: Get the sha
         id: get_sha
         run: echo ::set-output name=TAG::${GITHUB_SHA}
-      - name: Build and push to registry
+      - name: Build image
         uses: docker/build-push-action@v2
         with:
-          build-args: TAG=${{ steps.get_sha.outputs.TAG }}
-          push: true
+          build-args: |
+            TAG=${{ steps.get_sha.outputs.TAG }}
+            BASE_IMAGE=${{ matrix.baseImage }}
+          push: ${{ github.ref == 'refs/heads/main' }}
           context: ${{ matrix.context }}
           tags: ${{ matrix.name }}:latest${{ matrix.tagSuffix }}
           file: ${{ matrix.file }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 89845ecd..0c04c389 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18.5
+          go-version: 1.18.6
       #  https://github.com/goreleaser/goreleaser/issues/1311
       - name: Get current semver tag
         run: echo "::set-output name=CURRENT_TAG::$(git describe --tags --match "v*" --abbrev=0)"
@@ -69,16 +69,19 @@ jobs:
             name: foundationdb/fdb-kubernetes-operator
             tagSuffix: ""
             file: ./Dockerfile
+            baseImage: "docker.io/debian:bullseye"
           - image: fdb-kubernetes-operator-distroless
             context: ./
             name: foundationdb/fdb-kubernetes-operator
             tagSuffix: -distrolesss
-            file: ./distrolesss.dockerfile
+            file: ./Dockerfile
+            baseImage: "gcr.io/distroless/base"
           - image: fdb-data-loader
             context: ./sample-apps/data-loader
             name: foundationdb/fdb-data-loader
             tagSuffix: ""
             file: ./sample-apps/data-loader/Dockerfile
+            baseImage: ""
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
@@ -95,7 +98,9 @@ jobs:
       - name: Build and push to registry
         uses: docker/build-push-action@v2
         with:
-          build-args: TAG=${{ steps.get_tag.outputs.TAG }}
+          build-args: |
+            TAG=${{ steps.get_sha.outputs.TAG }}
+            BASE_IMAGE=${{ matrix.baseImage }}
           push: true
           context: ${{ matrix.context }}
           tags: ${{ matrix.name }}:${{ steps.get_tag.outputs.TAG }}${{ matrix.tagSuffix }}
diff --git a/Dockerfile b/Dockerfile
index 16bd0a35..ca6e9d03 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,7 @@
+ARG BASE_IMAGE=docker.io/debian:bullseye
+
 # Build the manager binary
-FROM docker.io/library/golang:1.18.5 as builder
+FROM docker.io/library/golang:1.18.6 as builder
 
 # Install FDB this version is only required to compile the fdb operator
 ARG FDB_VERSION=6.2.29
@@ -40,7 +42,7 @@ RUN groupadd --gid 4059 fdb && \
 	mkdir -p /var/log/fdb && \
 	touch /var/log/fdb/.keep
 
-FROM docker.io/debian:bullseye
+FROM $BASE_IMAGE
 
 VOLUME /usr/lib/fdb
 
diff --git a/README.md b/README.md
index 9aba4726..b6697b2a 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ If you already have Docker installed you can use the following steps to build an
 
 ```bash
 # Create a container with amd64 which contains go
-docker run --rm --entrypoint=/bin/bash -ti --platform="linux/amd64" -v $(pwd):/work -w /work docker.io/library/golang:1.18.5 ./scripts/setup_container.sh
+docker run --rm --entrypoint=/bin/bash -ti --platform="linux/amd64" -v $(pwd):/work -w /work docker.io/library/golang:1.18 ./scripts/setup_container.sh
 # Now we can run the lint and test steps
 make fmt lint test
 ```
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index a71da56b..519fd53b 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -93,6 +93,9 @@ type FoundationDBStatusClusterInfo struct {
 	// IncompatibleConnections provides information about processes that try to connect to the cluster with an
 	// incompatible version.
 	IncompatibleConnections []string `json:"incompatible_connections,omitempty"`
+
+	// RecoveryState represents the recovery state.
+	RecoveryState RecoveryState `json:"recovery_state,omitempty"`
 }
 
 // FaultTolerance provides information about the fault tolerance status
@@ -271,11 +274,21 @@ type FoundationDBStatusBackupTag struct {
 	Restorable       bool   `json:"running_backup_is_restorable,omitempty"`
 }
 
-// ProcessRole models the role of a pod
+// ProcessRole models the role of a pod.
 type ProcessRole string
 
 // TODO (johscheuer): add more roles to this list
 const (
-	// ProcessRoleCoordinator model for FDB coordinator role
+	// ProcessRoleCoordinator model for FDB coordinator role.
 	ProcessRoleCoordinator ProcessRole = "coordinator"
 )
+
+// RecoveryState represents the recovery state from the FDB cluster json.
+type RecoveryState struct {
+	// ActiveGenerations represent the current active generations.
+	ActiveGenerations int `json:"active_generations,omitempty"`
+	// Name represent the name of the current recovery state.
+	Name string `json:"name,omitempty"`
+	// SecondsSinceLastRecovered represents the seconds since the last recovery.
+	SecondsSinceLastRecovered float64 `json:"seconds_since_last_recovered,omitempty"`
+}
diff --git a/api/v1beta2/foundationdb_version.go b/api/v1beta2/foundationdb_version.go
index 89e7dd10..bbe9ac14 100644
--- a/api/v1beta2/foundationdb_version.go
+++ b/api/v1beta2/foundationdb_version.go
@@ -202,6 +202,11 @@ func (version Version) SupportsIsPresent() bool {
 	return version.IsAtLeast(Versions.SupportsIsPresent)
 }
 
+// SupportsRecoveryState returns true if the version of FDB supports the recovered since field.
+func (version Version) SupportsRecoveryState() bool {
+	return version.IsAtLeast(Versions.SupportsRecoveryState)
+}
+
 // Versions provides a shorthand for known versions.
 // This is only to be used in testing.
 var Versions = struct {
@@ -213,6 +218,7 @@ var Versions = struct {
 	SupportsShardedRocksDB,
 	IncompatibleVersion,
 	PreviousPatchVersion,
+	SupportsRecoveryState,
 	Default Version
 }{
 	Default:                Version{Major: 6, Minor: 2, Patch: 21},
@@ -224,4 +230,5 @@ var Versions = struct {
 	SupportsRocksDBV1:      Version{Major: 7, Minor: 1, Patch: 0, ReleaseCandidate: 4},
 	SupportsIsPresent:      Version{Major: 7, Minor: 1, Patch: 4},
 	SupportsShardedRocksDB: Version{Major: 7, Minor: 2, Patch: 0},
+	SupportsRecoveryState:  Version{Major: 7, Minor: 1, Patch: 22},
 }
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 97750238..d73d53c7 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -968,6 +968,7 @@ func (in *FoundationDBStatusClusterInfo) DeepCopyInto(out *FoundationDBStatusClu
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	out.RecoveryState = in.RecoveryState
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusClusterInfo.
@@ -1446,6 +1447,21 @@ func (in *ProcessSettings) DeepCopy() *ProcessSettings {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RecoveryState) DeepCopyInto(out *RecoveryState) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecoveryState.
+func (in *RecoveryState) DeepCopy() *RecoveryState {
+	if in == nil {
+		return nil
+	}
+	out := new(RecoveryState)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Region) DeepCopyInto(out *Region) {
 	*out = *in
diff --git a/controllers/bounce_processes.go b/controllers/bounce_processes.go
index 8c5d09cd..0aec4f9e 100644
--- a/controllers/bounce_processes.go
+++ b/controllers/bounce_processes.go
@@ -55,15 +55,9 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		return &requeue{curError: err}
 	}
 
-	minimumUptime := math.Inf(1)
-	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
-
-	for _, process := range status.Cluster.Processes {
-		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
-
-		if process.UptimeSeconds < minimumUptime && !process.Excluded {
-			minimumUptime = process.UptimeSeconds
-		}
+	minimumUptime, addressMap, err := getMinimumUptimeAndAddressMap(cluster, status, r.EnableRecoveryState)
+	if err != nil {
+		return &requeue{curError: err}
 	}
 
 	processesToBounce := fdbv1beta2.FilterByConditions(cluster.Status.ProcessGroups, map[fdbv1beta2.ProcessGroupConditionType]bool{
@@ -122,8 +116,6 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		return &requeue{message: "Waiting for config map to sync to all pods"}
 	}
 
-	upgrading := cluster.Status.RunningVersion != cluster.Spec.Version
-
 	if len(addresses) > 0 {
 		if !pointer.BoolDeref(cluster.Spec.AutomationOptions.KillProcesses, true) {
 			r.Recorder.Event(cluster, corev1.EventTypeNormal, "NeedsBounce",
@@ -166,6 +158,8 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 			return &requeue{curError: err}
 		}
 
+		upgrading := cluster.Status.RunningVersion != cluster.Spec.Version
+
 		if useLocks && upgrading {
 			processGroupIDs := make([]string, 0, len(cluster.Status.ProcessGroups))
 			for _, processGroup := range cluster.Status.ProcessGroups {
@@ -253,3 +247,36 @@ func getAddressesForUpgrade(r *FoundationDBClusterReconciler, databaseStatus *fd
 
 	return addresses, nil
 }
+
+// getMinimumUptimeAndAddressMap returns address map of the processes included the the foundationdb status. The minimum
+// uptime will be either secondsSinceLastRecovered if the recovery state is supported and enabled otherwise we will
+// take the minimum uptime of all processes.
+func getMinimumUptimeAndAddressMap(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, recoveryStateEnabled bool) (float64, map[string][]fdbv1beta2.ProcessAddress, error) {
+	runningVersion, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion())
+	if err != nil {
+		return 0, nil, err
+	}
+
+	useRecoveryState := runningVersion.SupportsRecoveryState() && recoveryStateEnabled
+
+	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
+
+	minimumUptime := math.Inf(1)
+	if useRecoveryState {
+		minimumUptime = status.Cluster.RecoveryState.SecondsSinceLastRecovered
+	}
+
+	for _, process := range status.Cluster.Processes {
+		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
+
+		if useRecoveryState || process.Excluded {
+			continue
+		}
+
+		if process.UptimeSeconds < minimumUptime {
+			minimumUptime = process.UptimeSeconds
+		}
+	}
+
+	return minimumUptime, addressMap, nil
+}
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index b3c3a79e..9bc4e2b0 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -56,13 +56,14 @@ type FoundationDBClusterReconciler struct {
 	Log                                logr.Logger
 	InSimulation                       bool
 	EnableRestartIncompatibleProcesses bool
+	ServerSideApply                    bool
+	EnableRecoveryState                bool
 	PodLifecycleManager                podmanager.PodLifecycleManager
 	PodClientProvider                  func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error)
 	DatabaseClientProvider             DatabaseClientProvider
 	DeprecationOptions                 internal.DeprecationOptions
 	GetTimeout                         time.Duration
 	PostTimeout                        time.Duration
-	ServerSideApply                    bool
 }
 
 // NewFoundationDBClusterReconciler creates a new FoundationDBClusterReconciler with defaults.
diff --git a/distrolesss.dockerfile b/distrolesss.dockerfile
deleted file mode 100644
index 0252d1c5..00000000
--- a/distrolesss.dockerfile
+++ /dev/null
@@ -1,63 +0,0 @@
-# Build the manager binary
-FROM docker.io/library/golang:1.18.5 as builder
-
-# Install FDB this version is only required to compile the fdb operator
-ARG FDB_VERSION=6.2.29
-ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
-ARG TAG="latest"
-
-RUN set -eux && \
-	curl --fail -L ${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb -o fdb.deb && \
-	dpkg -i fdb.deb && \
-    rm fdb.deb
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download -x
-
-# Copy the go source
-COPY main.go main.go
-COPY Makefile Makefile
-COPY api/ api/
-COPY controllers/ controllers/
-COPY setup/ setup/
-COPY fdbclient/ fdbclient/
-COPY internal/ internal/
-COPY pkg/ pkg/
-COPY mock-kubernetes-client/ mock-kubernetes-client/
-
-# Build
-RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on make manager
-
-# Create user and group here since we don't have the tools
-# in distroless
-RUN groupadd --gid 4059 fdb && \
-	useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
-	mkdir -p /var/log/fdb && \
-	touch /var/log/fdb/.keep
-
-FROM gcr.io/distroless/base
-
-VOLUME /usr/lib/fdb
-
-WORKDIR /
-
-COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder /etc/group /etc/group
-COPY --chown=fdb:fdb --from=builder /workspace/bin/manager .
-COPY --from=builder /usr/lib/libfdb_c.so /usr/lib/
-COPY --chown=fdb:fdb --from=builder /var/log/fdb/.keep /var/log/fdb/.keep
-
-# Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
-USER 4059
-
-ENV FDB_NETWORK_OPTION_TRACE_LOG_GROUP=fdb-kubernetes-operator
-ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb
-ENV FDB_BINARY_DIR=/usr/bin/fdb
-ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/usr/bin/fdb
-
-ENTRYPOINT ["/manager"]
diff --git a/setup/setup.go b/setup/setup.go
index df3f558f..b85d82ff 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -55,6 +55,7 @@ type Options struct {
 	PrintVersion                       bool
 	EnableRestartIncompatibleProcesses bool
 	ServerSideApply                    bool
+	EnableRecoveryState                bool
 	MetricsAddr                        string
 	LeaderElectionID                   string
 	LogFile                            string
@@ -92,11 +93,12 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&o.CompressOldFiles, "compress", false, "Defines whether the rotated log files should be compressed using gzip or not.")
 	fs.BoolVar(&o.PrintVersion, "version", false, "Prints the version of the operator and exits.")
 	fs.StringVar(&o.LabelSelector, "label-selector", "", "Defines a label-selector that will be used to select resources.")
-	fs.StringVar(&o.WatchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "Defines which namespace the operator should watch")
-	fs.DurationVar(&o.GetTimeout, "get-timeout", 5*time.Second, "http timeout for get requests to the FDB sidecar")
-	fs.DurationVar(&o.PostTimeout, "post-timeout", 10*time.Second, "http timeout for post requests to the FDB sidecar")
+	fs.StringVar(&o.WatchNamespace, "watch-namespace", os.Getenv("WATCH_NAMESPACE"), "Defines which namespace the operator should watch.")
+	fs.DurationVar(&o.GetTimeout, "get-timeout", 5*time.Second, "http timeout for get requests to the FDB sidecar.")
+	fs.DurationVar(&o.PostTimeout, "post-timeout", 10*time.Second, "http timeout for post requests to the FDB sidecar.")
 	fs.BoolVar(&o.EnableRestartIncompatibleProcesses, "enable-restart-incompatible-processes", true, "This flag enables/disables in the operator to restart incompatible fdbserver processes.")
-	fs.BoolVar(&o.ServerSideApply, "server-side-apply", false, "This flag enables server side apply")
+	fs.BoolVar(&o.ServerSideApply, "server-side-apply", false, "This flag enables server side apply.")
+	fs.BoolVar(&o.EnableRecoveryState, "enable-recovery-state", true, "This flag enables the use of the recovery state for the minimum uptime between bounced if the FDB version supports it.")
 }
 
 // StartManager will start the FoundationDB operator manager.
@@ -185,6 +187,7 @@ func StartManager(
 		clusterReconciler.Log = logr.WithName("controllers").WithName("FoundationDBCluster")
 		clusterReconciler.EnableRestartIncompatibleProcesses = operatorOpts.EnableRestartIncompatibleProcesses
 		clusterReconciler.ServerSideApply = operatorOpts.ServerSideApply
+		clusterReconciler.EnableRecoveryState = operatorOpts.EnableRecoveryState
 
 		if err := clusterReconciler.SetupWithManager(mgr, operatorOpts.MaxConcurrentReconciles, *labelSelector, watchedObjects...); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "FoundationDBCluster")


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index cfb97b79..843a087f 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -453,6 +453,9 @@ var _ = Describe("FoundationDBStatus", func() {
 							},
 						},
 					},
+					RecoveryState: RecoveryState{
+						Name: "fully_recovered",
+					},
 				},
 			}))
 		})
@@ -772,6 +775,11 @@ var _ = Describe("FoundationDBStatus", func() {
 				MaxZoneFailuresWithoutLosingData:         1,
 				MaxZoneFailuresWithoutLosingAvailability: 1,
 			},
+			RecoveryState: RecoveryState{
+				ActiveGenerations:         1,
+				Name:                      "fully_recovered",
+				SecondsSinceLastRecovered: 76.8155,
+			},
 		}
 
 		It("should parse all values correctly", func() {
diff --git a/controllers/bounce_processes_test.go b/controllers/bounce_processes_test.go
index 9b50f89a..fbaccc28 100644
--- a/controllers/bounce_processes_test.go
+++ b/controllers/bounce_processes_test.go
@@ -23,6 +23,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"net"
 	"time"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -381,4 +382,111 @@ var _ = Describe("bounceProcesses", func() {
 			})
 		})
 	})
+
+	DescribeTable("when getting the minimum uptime and the address map", func(cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBStatus, useRecoveryState bool, expectedMinimumUptime float64, expectedAddressMap map[string][]fdbv1beta2.ProcessAddress) {
+		minimumUptime, addressMap, err := getMinimumUptimeAndAddressMap(cluster, status, useRecoveryState)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(minimumUptime).To(BeNumerically("==", expectedMinimumUptime))
+		Expect(len(addressMap)).To(BeNumerically("==", len(expectedAddressMap)))
+		for key, value := range expectedAddressMap {
+			Expect(addressMap).To(HaveKeyWithValue(key, value))
+		}
+	},
+		Entry("when recovered since is not available",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.Default.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			true,
+			30.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+		Entry("when recovered since is enabled and version supports it",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			true,
+			90.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+		Entry("when recovered since is disabled and version supports it",
+			&fdbv1beta2.FoundationDBCluster{
+				Spec: fdbv1beta2.FoundationDBClusterSpec{
+					Version: fdbv1beta2.Versions.SupportsRecoveryState.String(),
+				},
+			}, &fdbv1beta2.FoundationDBStatus{
+				Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+					Processes: map[string]fdbv1beta2.FoundationDBStatusProcessInfo{
+						"test": {
+							Address: fdbv1beta2.ProcessAddress{
+								IPAddress: net.ParseIP("127.0.0.1"),
+							},
+							Locality: map[string]string{
+								fdbv1beta2.FDBLocalityInstanceIDKey: "test",
+							},
+							UptimeSeconds: 30.0,
+						},
+					},
+					RecoveryState: fdbv1beta2.RecoveryState{
+						SecondsSinceLastRecovered: 90.0,
+					},
+				},
+			},
+			false,
+			30.0,
+			map[string][]fdbv1beta2.ProcessAddress{
+				"test": {
+					{
+						IPAddress: net.ParseIP("127.0.0.1"),
+					},
+				},
+			}),
+	)
 })