Repository: Azure/ARO-RP
Pull request #1237 (2020-12-24 14:25:29)

Cluster provisioning state metrics
### Which issue this PR addresses:

For work item [№8785023](https://msazure.visualstudio.com/AzureRedHatOpenShift/_workitems/edit/8785023/).


### What this PR does / why we need it:

There is currently no way to build a dashboard showing dynamic of failed clusters (failed creation or failed deletions). We do have required data in the `cluster.summary` metric, but we don't emit it for clusters where we fail to get clsuter version or a list of nodes due to this piece of code:

https://github.com/Azure/ARO-RP/blob/606bcb52e27dfd47e38772d021c5b9a2a6a7bb0f/pkg/monitor/cluster/summary.go#L23-L31

It leaves us only with a small portion of failed clusters which doesn't give us an accurate representation.

This PR changes our metrics:

* Introdueces a new metrics which contains provisioning state and failed provisioning state.

Alternatively we can keep metric as is and use power shell to gather data about failed clusters. But metrics & dashboards will be able to show us trends over time.

### Test plan for issue:

Updated unit tests.

### Is there any documentation that needs to be updated for this PR?

No



Patch:
diff --git a/pkg/monitor/cluster/cluster.go b/pkg/monitor/cluster/cluster.go
index d4d4b3f16..b725ac8fb 100644
--- a/pkg/monitor/cluster/cluster.go
+++ b/pkg/monitor/cluster/cluster.go
@@ -100,6 +100,13 @@ func NewMonitor(ctx context.Context, log *logrus.Entry, restConfig *rest.Config,
 func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
 	mon.log.Debug("monitoring")
 
+	if mon.hourlyRun {
+		mon.emitGauge("cluster.provisioning", 1, map[string]string{
+			"provisioningState":       mon.oc.Properties.ProvisioningState.String(),
+			"failedProvisioningState": mon.oc.Properties.FailedProvisioningState.String(),
+		})
+	}
+
 	// If API is not returning 200, don't need to run the next checks
 	statusCode, err := mon.emitAPIServerHealthzCode(ctx)
 	if err != nil {
diff --git a/pkg/monitor/cluster/summary.go b/pkg/monitor/cluster/summary.go
index 70f296355..caf1e10ca 100644
--- a/pkg/monitor/cluster/summary.go
+++ b/pkg/monitor/cluster/summary.go
@@ -41,12 +41,11 @@ func (mon *Monitor) emitSummary(ctx context.Context) error {
 	}
 
 	mon.emitGauge("cluster.summary", 1, map[string]string{
-		"actualVersion":           actualVersion(cv),
-		"desiredVersion":          desiredVersion(cv),
-		"masterCount":             strconv.Itoa(masterCount),
-		"workerCount":             strconv.Itoa(workerCount),
-		"provisioningState":       mon.oc.Properties.ProvisioningState.String(),
-		"failedProvisioningState": mon.oc.Properties.FailedProvisioningState.String(),
+		"actualVersion":     actualVersion(cv),
+		"desiredVersion":    desiredVersion(cv),
+		"masterCount":       strconv.Itoa(masterCount),
+		"workerCount":       strconv.Itoa(workerCount),
+		"provisioningState": mon.oc.Properties.ProvisioningState.String(),
 	})
 
 	return nil


Test patch:
diff --git a/pkg/monitor/cluster/summary_test.go b/pkg/monitor/cluster/summary_test.go
index 50a9fcb79..bb1a8463c 100644
--- a/pkg/monitor/cluster/summary_test.go
+++ b/pkg/monitor/cluster/summary_test.go
@@ -74,20 +74,18 @@ func TestEmitSummary(t *testing.T) {
 		m:         m,
 		oc: &api.OpenShiftCluster{
 			Properties: api.OpenShiftClusterProperties{
-				ProvisioningState:       api.ProvisioningStateFailed,
-				FailedProvisioningState: api.ProvisioningStateDeleting,
+				ProvisioningState: api.ProvisioningStateFailed,
 			},
 		},
 		hourlyRun: true,
 	}
 
 	m.EXPECT().EmitGauge("cluster.summary", int64(1), map[string]string{
-		"actualVersion":           "4.3.0",
-		"desiredVersion":          "4.3.3",
-		"masterCount":             "1",
-		"workerCount":             "2",
-		"provisioningState":       api.ProvisioningStateFailed.String(),
-		"failedProvisioningState": api.ProvisioningStateDeleting.String(),
+		"actualVersion":     "4.3.0",
+		"desiredVersion":    "4.3.3",
+		"masterCount":       "1",
+		"workerCount":       "2",
+		"provisioningState": api.ProvisioningStateFailed.String(),
 	})
 
 	err := mon.emitSummary(ctx)