Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1689 (2023-06-20 19:02:41)

Do fault tolerance checks on status object
# Description

Do fault tolerance checks on FoundationDB status object.

## Type of change

- New feature (non-breaking change which adds functionality)

## Discussion

*Are there any design details that you would like to discuss further?*

No.

## Testing

*Please describe the tests that you ran to verify your changes. Unit tests?
Manual testing?*

Ran operator unit tests locally.

*Do we need to perform additional testing once this is merged, or perform in a larger testing environment?*

Yes, nightly regression tests will need to be run. (But I think nightly tests only run upgrade related tests and we don't test this functionality in those tests, so not sure if running nightly regression tests will help here.)

## Documentation

*Did you update relevant documentation within this repository?*

No (maybe at a later stage).

*If this change is adding new functionality, do we need to describe it in our user manual?*

No (maybe at a later stage).

*If this change is adding or removing subreconcilers, have we updated the core technical design doc to reflect that?*

N/A

*If this change is adding new safety checks or new potential failure modes, have we documented and how to debug potential issues?*

Not sure (maybe we need to document this at a later stage).

## Follow-up

*Are there any follow-up issues that we should pursue in the future?*

No.

*Does this introduce new defaults that we should re-evaluate in the future?*

N/A


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 781f5ff0834324ed2c6df2a06a19a12771c83975
* Duration 2:13:56
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/f5608ac7-036d-4e69-826a-25594958f57c.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/f5608ac7-036d-4e69-826a-25594958f57c/workspace.zip) zip file of the working directory (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr-kind on Linux CentOS 7

* Commit ID: 781f5ff0834324ed2c6df2a06a19a12771c83975
* Duration 4:08:42
* Result: :x: **FAILED**
* Error: `Error while executing command: if $fail_test; then exit 1; fi. Reason: exit status 1`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr-kind/e8ffa3fd-a5fd-49b6-a929-56c3454a2735.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr-kind/e8ffa3fd-a5fd-49b6-a929-56c3454a2735/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index e484557a..4044a14b 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -95,6 +95,9 @@ type FoundationDBStatusClusterInfo struct {
 	// cluster.
 	Layers FoundationDBStatusLayerInfo `json:"layers,omitempty"`
 
+	// Logs provides information about log processes running in the cluster.
+	Logs []FoundationDBStatusLogInfo `json:"logs,omitempty"`
+
 	// FaultTolerance provides information about the fault tolerance status
 	// of the cluster.
 	FaultTolerance FaultTolerance `json:"fault_tolerance,omitempty"`
@@ -185,6 +188,9 @@ type FoundationDBStatusDataStatistics struct {
 
 	// State provides a summary of the state of data distribution.
 	State FoundationDBStatusDataState `json:"state,omitempty"`
+
+	// TeamTrackers provides information about (highest priority) teams.
+	TeamTrackers []FoundationDBStatusTeamTracker `json:"team_trackers,omitempty"`
 }
 
 // FoundationDBStatusDataState provides information about the state of data
@@ -200,6 +206,10 @@ type FoundationDBStatusDataState struct {
 	// Name provides a machine-readable identifier for the data distribution
 	// state.
 	Name string `json:"name,omitempty"`
+
+	// MinReplicasRemaining provides the number of replicas remaining in the
+	// highest priority team.
+	MinReplicasRemaining int `json:"min_replicas_remaining,omitempty"`
 }
 
 // FoundationDBStatusMovingData provides information about the current data
@@ -216,6 +226,16 @@ type FoundationDBStatusMovingData struct {
 	InQueueBytes int `json:"in_queue_bytes,omitempty"`
 }
 
+// FoundationDBStatusTeamTracker provides information about the highest priority team.
+type FoundationDBStatusTeamTracker struct {
+	// Primary indicates whether the team is in the primary region or not.
+	Primary bool `json:"primary,omitempty"`
+
+	// FoundationDBStatusDataState provides information about the state of the
+	// highest priority team.
+	State FoundationDBStatusDataState `json:"state,omitempty"`
+}
+
 // FoundationDBStatusClientDBStatus represents the databaseStatus field in the
 // JSON database status
 type FoundationDBStatusClientDBStatus struct {
@@ -304,6 +324,18 @@ type FoundationDBStatusBackupTag struct {
 	Restorable       bool   `json:"running_backup_is_restorable,omitempty"`
 }
 
+// FoundationDBStatusLogInfo provides information about the fault tolerance metrics
+// of log processes in the cluster.
+type FoundationDBStatusLogInfo struct {
+	Current                       bool `json:"current,omitempty"`
+	LogFaultTolerance             int  `json:"log_fault_tolerance,omitempty"`
+	LogReplicationFactor          int  `json:"log_replication_factor,omitempty"`
+	RemoteLogFaultTolerance       int  `json:"remote_log_fault_tolerance,omitempty"`
+	RemoteLogReplicationFactor    int  `json:"remote_log_replication_factor,omitempty"`
+	SatelliteLogFaultTolerance    int  `json:"satellite_log_fault_tolerance,omitempty"`
+	SatelliteLogReplicationFactor int  `json:"satellite_log_replication_factor,omitempty"`
+}
+
 // ProcessRole models the role of a pod.
 type ProcessRole string
 
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 553daa2d..a399cabf 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -1030,9 +1030,14 @@ func (in *FoundationDBStatusClusterInfo) DeepCopyInto(out *FoundationDBStatusClu
 			(*out)[key] = *val.DeepCopy()
 		}
 	}
-	out.Data = in.Data
+	in.Data.DeepCopyInto(&out.Data)
 	in.Clients.DeepCopyInto(&out.Clients)
 	in.Layers.DeepCopyInto(&out.Layers)
+	if in.Logs != nil {
+		in, out := &in.Logs, &out.Logs
+		*out = make([]FoundationDBStatusLogInfo, len(*in))
+		copy(*out, *in)
+	}
 	out.FaultTolerance = in.FaultTolerance
 	if in.IncompatibleConnections != nil {
 		in, out := &in.IncompatibleConnections, &out.IncompatibleConnections
@@ -1125,6 +1130,11 @@ func (in *FoundationDBStatusDataStatistics) DeepCopyInto(out *FoundationDBStatus
 	*out = *in
 	out.MovingData = in.MovingData
 	out.State = in.State
+	if in.TeamTrackers != nil {
+		in, out := &in.TeamTrackers, &out.TeamTrackers
+		*out = make([]FoundationDBStatusTeamTracker, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusDataStatistics.
@@ -1170,6 +1180,21 @@ func (in *FoundationDBStatusLocalClientInfo) DeepCopy() *FoundationDBStatusLocal
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBStatusLogInfo) DeepCopyInto(out *FoundationDBStatusLogInfo) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusLogInfo.
+func (in *FoundationDBStatusLogInfo) DeepCopy() *FoundationDBStatusLogInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBStatusLogInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FoundationDBStatusMovingData) DeepCopyInto(out *FoundationDBStatusMovingData) {
 	*out = *in
@@ -1273,6 +1298,22 @@ func (in *FoundationDBStatusSupportedVersion) DeepCopy() *FoundationDBStatusSupp
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBStatusTeamTracker) DeepCopyInto(out *FoundationDBStatusTeamTracker) {
+	*out = *in
+	out.State = in.State
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusTeamTracker.
+func (in *FoundationDBStatusTeamTracker) DeepCopy() *FoundationDBStatusTeamTracker {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBStatusTeamTracker)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ImageConfig) DeepCopyInto(out *ImageConfig) {
 	*out = *in
diff --git a/internal/foundationdb_status.go b/internal/foundationdb_status.go
index 670a6c59..6d9ed0cf 100644
--- a/internal/foundationdb_status.go
+++ b/internal/foundationdb_status.go
@@ -95,3 +95,49 @@ func GetMinimumUptimeAndAddressMap(logger logr.Logger, cluster *fdbv1beta2.Found
 
 	return minimumUptime, addressMap, nil
 }
+
+// DoStorageServerFaultDomainCheckOnStatus does a storage server related fault domain check over the given status object.
+func DoStorageServerFaultDomainCheckOnStatus(status *fdbv1beta2.FoundationDBStatus) bool {
+	// Storage server related check.
+	for _, tracker := range status.Cluster.Data.TeamTrackers {
+		if !tracker.State.Healthy {
+			return false
+		}
+	}
+
+	return true
+}
+
+// DoLogServerFaultDomainCheckOnStatus does a log server related fault domain check over the given status object.
+func DoLogServerFaultDomainCheckOnStatus(status *fdbv1beta2.FoundationDBStatus) bool {
+	// Log server related check.
+	for _, log := range status.Cluster.Logs {
+		// @todo do we need to do this check only for the current log server set? Revisit this issue later.
+		if (log.LogReplicationFactor != 0 && log.LogFaultTolerance+1 != log.LogReplicationFactor) ||
+			(log.RemoteLogReplicationFactor != 0 && log.RemoteLogFaultTolerance+1 != log.RemoteLogReplicationFactor) ||
+			(log.SatelliteLogReplicationFactor != 0 && log.SatelliteLogFaultTolerance+1 != log.SatelliteLogReplicationFactor) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// DoCoordinatorFaultDomainCheckOnStatus does a coordinator related fault domain check over the given status object.
+// @note an empty function for now. We will revisit this later.
+func DoCoordinatorFaultDomainCheckOnStatus(status *fdbv1beta2.FoundationDBStatus) bool {
+	// @todo decide if we need to do coordinator related check.
+	return true
+}
+
+// DoFaultDomainChecksOnStatus does the specified fault domain check(s) over the given status object.
+// @note this is a wrapper over the above fault domain related functions.
+func DoFaultDomainChecksOnStatus(status *fdbv1beta2.FoundationDBStatus, storageServerCheck bool, logServerCheck bool, coordinatorCheck bool) bool {
+	if (storageServerCheck && !DoStorageServerFaultDomainCheckOnStatus(status)) ||
+		(logServerCheck && !DoLogServerFaultDomainCheckOnStatus(status)) ||
+		(coordinatorCheck && !DoCoordinatorFaultDomainCheckOnStatus(status)) {
+		return false
+	}
+
+	return true
+}


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index db6c6b4c..97aca83f 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -296,7 +296,13 @@ var _ = Describe("FoundationDBStatus", func() {
 					Data: FoundationDBStatusDataStatistics{
 						KVBytes:    0,
 						MovingData: FoundationDBStatusMovingData{HighestPriority: 0, InFlightBytes: 0, InQueueBytes: 0},
-						State:      FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy"},
+						State:      FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 0},
+						TeamTrackers: []FoundationDBStatusTeamTracker{
+							{
+								Primary: true,
+								State:   FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 0},
+							},
+						},
 					},
 					FullReplication: true,
 					Clients: FoundationDBStatusClusterClientInfo{
@@ -717,7 +723,13 @@ var _ = Describe("FoundationDBStatus", func() {
 			Data: FoundationDBStatusDataStatistics{
 				KVBytes:    0,
 				MovingData: FoundationDBStatusMovingData{HighestPriority: 0, InFlightBytes: 0, InQueueBytes: 0},
-				State:      FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy"},
+				State:      FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 2},
+				TeamTrackers: []FoundationDBStatusTeamTracker{
+					{
+						Primary: true,
+						State:   FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 2},
+					},
+				},
 			},
 			FullReplication: true,
 			Clients: FoundationDBStatusClusterClientInfo{
@@ -842,6 +854,17 @@ var _ = Describe("FoundationDBStatus", func() {
 				Backup: FoundationDBStatusBackupInfo{Paused: false, Tags: nil},
 				Error:  "",
 			},
+			Logs: []FoundationDBStatusLogInfo{
+				{
+					Current:                       true,
+					LogFaultTolerance:             1,
+					LogReplicationFactor:          2,
+					RemoteLogFaultTolerance:       0,
+					RemoteLogReplicationFactor:    0,
+					SatelliteLogFaultTolerance:    0,
+					SatelliteLogReplicationFactor: 0,
+				},
+			},
 			FaultTolerance: FaultTolerance{
 				MaxZoneFailuresWithoutLosingData:         1,
 				MaxZoneFailuresWithoutLosingAvailability: 1,
diff --git a/internal/foundationdb_status_test.go b/internal/foundationdb_status_test.go
index e16656ea..39af19c8 100644
--- a/internal/foundationdb_status_test.go
+++ b/internal/foundationdb_status_test.go
@@ -251,4 +251,213 @@ var _ = Describe("Internal FoundationDBStatus", func() {
 				},
 			}),
 	)
+
+	Context("fault domain checks on status object", func() {
+		var status *fdbv1beta2.FoundationDBStatus
+
+		Context("storage server fault domain checks", func() {
+			It("storage server team healthy", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Data: fdbv1beta2.FoundationDBStatusDataStatistics{
+							TeamTrackers: []fdbv1beta2.FoundationDBStatusTeamTracker{
+								{
+									Primary: true,
+									State:   fdbv1beta2.FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 4},
+								},
+							},
+						},
+					},
+				}
+				Expect(DoStorageServerFaultDomainCheckOnStatus(status)).To(BeTrue())
+			})
+
+			It("remote storage server team unhealthy", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Data: fdbv1beta2.FoundationDBStatusDataStatistics{
+							TeamTrackers: []fdbv1beta2.FoundationDBStatusTeamTracker{
+								{
+									Primary: true,
+									State:   fdbv1beta2.FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 4},
+								},
+								{
+									Primary: false,
+									State:   fdbv1beta2.FoundationDBStatusDataState{Description: "", Healthy: false, Name: "healthy", MinReplicasRemaining: 0},
+								},
+							},
+						},
+					},
+				}
+				Expect(DoStorageServerFaultDomainCheckOnStatus(status)).To(BeFalse())
+			})
+		})
+
+		Context("log server fault domain checks", func() {
+			It("no log server fault tolerance information in status", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Logs: []fdbv1beta2.FoundationDBStatusLogInfo{},
+					},
+				}
+
+				Expect(DoLogServerFaultDomainCheckOnStatus(status)).To(BeTrue())
+			})
+
+			It("all primary log servers available", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Logs: []fdbv1beta2.FoundationDBStatusLogInfo{
+							{
+								Current:                       true,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       0,
+								RemoteLogReplicationFactor:    0,
+								SatelliteLogFaultTolerance:    0,
+								SatelliteLogReplicationFactor: 0,
+							},
+						},
+					},
+				}
+				Expect(DoLogServerFaultDomainCheckOnStatus(status)).To(BeTrue())
+			})
+
+			It("not enough replicas in remote", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Logs: []fdbv1beta2.FoundationDBStatusLogInfo{
+							{
+								Current:                       true,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       1,
+								RemoteLogReplicationFactor:    4,
+								SatelliteLogFaultTolerance:    0,
+								SatelliteLogReplicationFactor: 0,
+							},
+						},
+					},
+				}
+				Expect(DoLogServerFaultDomainCheckOnStatus(status)).To(BeFalse())
+			})
+
+			It("not enough replicas in the satellite", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Logs: []fdbv1beta2.FoundationDBStatusLogInfo{
+							{
+								Current:                       true,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       0,
+								RemoteLogReplicationFactor:    0,
+								SatelliteLogFaultTolerance:    1,
+								SatelliteLogReplicationFactor: 4,
+							},
+						},
+					},
+				}
+				Expect(DoLogServerFaultDomainCheckOnStatus(status)).To(BeFalse())
+			})
+
+			It("multiple log server sets", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Logs: []fdbv1beta2.FoundationDBStatusLogInfo{
+							{
+								Current:                       true,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       0,
+								RemoteLogReplicationFactor:    0,
+								SatelliteLogFaultTolerance:    0,
+								SatelliteLogReplicationFactor: 0,
+							},
+							{
+								Current:                       false,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       0,
+								RemoteLogReplicationFactor:    0,
+								SatelliteLogFaultTolerance:    0,
+								SatelliteLogReplicationFactor: 0,
+							},
+						},
+					},
+				}
+				Expect(DoLogServerFaultDomainCheckOnStatus(status)).To(BeTrue())
+			})
+		})
+
+		Context("coordinator fault domain checks", func() {
+			It("no coordinator related information in status", func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{},
+				}
+				Expect(DoCoordinatorFaultDomainCheckOnStatus(status)).To(BeTrue())
+			})
+		})
+
+		Context("multiple fault domain checks", func() {
+			BeforeEach(func() {
+				status = &fdbv1beta2.FoundationDBStatus{
+					Cluster: fdbv1beta2.FoundationDBStatusClusterInfo{
+						Data: fdbv1beta2.FoundationDBStatusDataStatistics{
+							TeamTrackers: []fdbv1beta2.FoundationDBStatusTeamTracker{
+								{
+									Primary: true,
+									State:   fdbv1beta2.FoundationDBStatusDataState{Description: "", Healthy: true, Name: "healthy", MinReplicasRemaining: 4},
+								},
+								{
+									Primary: false,
+									State:   fdbv1beta2.FoundationDBStatusDataState{Description: "", Healthy: false, Name: "healthy", MinReplicasRemaining: 0},
+								},
+							},
+						},
+						Logs: []fdbv1beta2.FoundationDBStatusLogInfo{
+							{
+								Current:                       true,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       0,
+								RemoteLogReplicationFactor:    0,
+								SatelliteLogFaultTolerance:    0,
+								SatelliteLogReplicationFactor: 0,
+							},
+							{
+								Current:                       false,
+								LogFaultTolerance:             1,
+								LogReplicationFactor:          2,
+								RemoteLogFaultTolerance:       0,
+								RemoteLogReplicationFactor:    0,
+								SatelliteLogFaultTolerance:    0,
+								SatelliteLogReplicationFactor: 0,
+							},
+						},
+					},
+				}
+			})
+
+			It("do storage server fault domain check", func() {
+				Expect(DoFaultDomainChecksOnStatus(status, true, false, false)).To(BeFalse())
+			})
+
+			It("do log server fault domain check", func() {
+				Expect(DoFaultDomainChecksOnStatus(status, false, true, false)).To(BeTrue())
+			})
+
+			It("do coordinator fault domain check", func() {
+				Expect(DoFaultDomainChecksOnStatus(status, false, false, true)).To(BeTrue())
+			})
+
+			It("do storage server and log server fault domain checks", func() {
+				Expect(DoFaultDomainChecksOnStatus(status, true, true, false)).To(BeFalse())
+			})
+
+			It("do all fault domain checks", func() {
+				Expect(DoFaultDomainChecksOnStatus(status, true, true, true)).To(BeFalse())
+			})
+		})
+	})
 })