Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1365 (2022-09-27 09:26:01)

Add support for desired and reconciled process groups in status
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1298

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

Sadly the current CRD setup (not the setup in our repository) doesn't allow "complex" jsonpath queries, so we have to add two different fields in the kubectl output. The `priority: 1` means that those fields are only be present in the output if someone runs `kubectl get fdb ... -o wide`

## Testing

Local testing and unit tests.

## Documentation

-

## Follow-up

-


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 6947fd1f08581c9ccf9a80f3b4ba4d51e9735d7e
* Duration 4:06:05
* Result: :x: **FAILED**
* Error: `Error while executing command: if $(grep -q -- "--- FAIL:" logs/*.log); then echo "TESTS FAILED SEE THESE LOGS:"; echo ; grep -l -- "--- FAIL:" logs/*.log; exit 1; fi. Reason: exit status 1`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/3adf6c9d-07c9-4c2a-ab4b-ec886632713a.gz) (available for 30 days)
* [Build Artifact](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/3adf6c9d-07c9-4c2a-ab4b-ec886632713a/workspace.zip) (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index cc30a78a..84f7d482 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -911,5 +911,20 @@ func (counts ProcessCounts) Diff(currentCounts ProcessCounts) map[ProcessClass]i
 			diff[label] = desired - current
 		}
 	}
+
 	return diff
 }
+
+// Total gets the total number of processes for the cluster configuration.
+func (counts ProcessCounts) Total() int {
+	var total int64
+	desiredValue := reflect.ValueOf(counts)
+	for _, index := range processClassIndices {
+		desired := desiredValue.Field(index).Int()
+		if desired > 0 {
+			total += desired
+		}
+	}
+
+	return int(total)
+}
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index d2af2f3c..e2463915 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -39,6 +39,8 @@ import (
 // +kubebuilder:printcolumn:name="Reconciled",type="integer",JSONPath=".status.generations.reconciled",description="Last reconciled generation of the spec",priority=0
 // +kubebuilder:printcolumn:name="Available",type="boolean",JSONPath=".status.health.available",description="Database available",priority=0
 // +kubebuilder:printcolumn:name="FullReplication",type="boolean",JSONPath=".status.health.fullReplication",description="Database fully replicated",priority=0
+// +kubebuilder:printcolumn:name="ReconciledProcessGroups",type="integer",JSONPath=".status.reconciledProcessGroups",description="Number of reconciled process groups",priority=1
+// +kubebuilder:printcolumn:name="DesiredProcessGroups",type="integer",JSONPath=".status.desiredProcessGroups",description="Desired number of process groups",priority=1
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.runningVersion",description="Running version",priority=0
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 // +kubebuilder:storageversion
@@ -282,6 +284,12 @@ type FoundationDBClusterStatus struct {
 
 	// MaintenenanceModeInfo contains information regarding process groups in maintenance mode
 	MaintenanceModeInfo MaintenanceModeInfo `json:"maintenanceModeInfo,omitempty"`
+
+	// DesiredProcessGroups reflects the number of expected running process groups.
+	DesiredProcessGroups int `json:"desiredProcessGroups,omitempty"`
+
+	// ReconciledProcessGroups reflects the number of process groups that have no condition and are not marked for removal.
+	ReconciledProcessGroups int `json:"reconciledProcessGroups,omitempty"`
 }
 
 // MaintenanceModeInfo contains information regarding the zone and process groups that are put
@@ -1154,7 +1162,6 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 	}
 
 	cluster.Status.Generations = ClusterGenerationStatus{Reconciled: cluster.Status.Generations.Reconciled}
-
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if !processGroup.IsMarkedForRemoval() {
 			continue
@@ -1189,8 +1196,15 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 		}
 	}
 
+	cluster.Status.DesiredProcessGroups = desiredCounts.Total()
+	cluster.Status.ReconciledProcessGroups = 0
+
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		if len(processGroup.ProcessGroupConditions) > 0 && !processGroup.IsMarkedForRemoval() {
+		if processGroup.IsMarkedForRemoval() {
+			continue
+		}
+
+		if len(processGroup.ProcessGroupConditions) > 0 {
 			conditions := make([]ProcessGroupConditionType, 0, len(processGroup.ProcessGroupConditions))
 			for _, condition := range processGroup.ProcessGroupConditions {
 				conditions = append(conditions, condition.ProcessGroupConditionType)
@@ -1199,7 +1213,14 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 			logger.Info("Has unhealthy process group", "processGroupID", processGroup.ProcessGroupID, "state", "HasUnhealthyProcess", "conditions", conditions)
 			cluster.Status.Generations.HasUnhealthyProcess = cluster.ObjectMeta.Generation
 			reconciled = false
+			continue
 		}
+
+		cluster.Status.ReconciledProcessGroups++
+	}
+
+	if cluster.Status.DesiredProcessGroups != cluster.Status.ReconciledProcessGroups {
+		logger.Info("Not all process groups are reconciled", "desiredProcessGroups", cluster.Status.DesiredProcessGroups, "reconciledProcessGroups", cluster.Status.ReconciledProcessGroups)
 	}
 
 	if !cluster.Status.Health.Available {
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 01b2c9df..71c9e105 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -9828,6 +9828,16 @@ spec:
       jsonPath: .status.health.fullReplication
       name: FullReplication
       type: boolean
+    - description: Number of reconciled process groups
+      jsonPath: .status.reconciledProcessGroups
+      name: ReconciledProcessGroups
+      priority: 1
+      type: integer
+    - description: Desired number of process groups
+      jsonPath: .status.desiredProcessGroups
+      name: DesiredProcessGroups
+      priority: 1
+      type: integer
     - description: Running version
       jsonPath: .status.runningVersion
       name: Version
@@ -13678,6 +13688,8 @@ spec:
                   usable_regions:
                     type: integer
                 type: object
+              desiredProcessGroups:
+                type: integer
               generations:
                 properties:
                   hasExtraListeners:
@@ -13798,6 +13810,8 @@ spec:
                       type: string
                   type: object
                 type: array
+              reconciledProcessGroups:
+                type: integer
               requiredAddresses:
                 properties:
                   nonTLS:
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 1d26887b..cf7d36c1 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -258,6 +258,8 @@ FoundationDBClusterStatus defines the observed state of FoundationDBCluster
 | processGroups | ProcessGroups contain information about a process group. This information is used in multiple places to trigger the according action. | []*[ProcessGroupStatus](#processgroupstatus) | false |
 | locks | Locks contains information about the locking system. | [LockSystemStatus](#locksystemstatus) | false |
 | maintenanceModeInfo | MaintenenanceModeInfo contains information regarding process groups in maintenance mode | [MaintenanceModeInfo](#maintenancemodeinfo) | false |
+| desiredProcessGroups | DesiredProcessGroups reflects the number of expected running process groups. | int | false |
+| reconciledProcessGroups | ReconciledProcessGroups reflects the number of process groups that have no condition and are not marked for removal. | int | false |
 
 [Back to TOC](#table-of-contents)
 


Test patch:
diff --git a/api/v1beta2/foundationdb_database_configuration_test.go b/api/v1beta2/foundationdb_database_configuration_test.go
index c404be82..ec63884b 100644
--- a/api/v1beta2/foundationdb_database_configuration_test.go
+++ b/api/v1beta2/foundationdb_database_configuration_test.go
@@ -157,4 +157,37 @@ var _ = Describe("DatabaseConfiguration", func() {
 			})
 		})
 	})
+
+	When("using ProcessCounts", func() {
+		When("calculating the total number of processes", func() {
+			var counts ProcessCounts
+
+			When("no negative process counts are specified", func() {
+				BeforeEach(func() {
+					counts = ProcessCounts{
+						Storage: 5,
+						Log:     5,
+					}
+				})
+
+				It("should print the correct number of total processes", func() {
+					Expect(counts.Total()).To(BeNumerically("==", 10))
+				})
+			})
+
+			When("negative process counts are specified", func() {
+				BeforeEach(func() {
+					counts = ProcessCounts{
+						Storage:   5,
+						Log:       5,
+						Stateless: -1,
+					}
+				})
+
+				It("should print the correct number of total processes", func() {
+					Expect(counts.Total()).To(BeNumerically("==", 10))
+				})
+			})
+		})
+	})
 })