Repository: Azure/ARO-RP
Pull request #1367 (2021-03-08 12:23:08)

add count metrics for components
### Which issue this PR addresses:

Fixes our ability to see object counts. This will prevent "exploding object count" cluster issues. 

### What this PR does / why we need it:

Relates to ability detect enormous amounts of objects on the clusters ddos'ing etcd

### Test plan for issue:

unit

### Is there any documentation that needs to be updated for this PR?

no


github-actions[bot]: Please rebase pull request.

Patch:
diff --git a/pkg/monitor/cluster/daemonsetstatuses.go b/pkg/monitor/cluster/daemonsetstatuses.go
index fa4383198..d4523ce81 100644
--- a/pkg/monitor/cluster/daemonsetstatuses.go
+++ b/pkg/monitor/cluster/daemonsetstatuses.go
@@ -14,12 +14,15 @@ import (
 
 func (mon *Monitor) emitDaemonsetStatuses(ctx context.Context) error {
 	var cont string
+	var count int64
 	for {
 		dss, err := mon.cli.AppsV1().DaemonSets("").List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(dss.Items))
+
 		for _, ds := range dss.Items {
 			if !namespace.IsOpenShift(ds.Namespace) {
 				continue
@@ -41,6 +44,10 @@ func (mon *Monitor) emitDaemonsetStatuses(ctx context.Context) error {
 		if cont == "" {
 			break
 		}
+
 	}
+
+	mon.emitGauge("daemonset.count", count, nil)
+
 	return nil
 }
diff --git a/pkg/monitor/cluster/deploymentstatuses.go b/pkg/monitor/cluster/deploymentstatuses.go
index a61dd0212..ed02e290b 100644
--- a/pkg/monitor/cluster/deploymentstatuses.go
+++ b/pkg/monitor/cluster/deploymentstatuses.go
@@ -14,12 +14,15 @@ import (
 
 func (mon *Monitor) emitDeploymentStatuses(ctx context.Context) error {
 	var cont string
+	var count int64
 	for {
 		ds, err := mon.cli.AppsV1().Deployments("").List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(ds.Items))
+
 		for _, d := range ds.Items {
 			if !namespace.IsOpenShift(d.Namespace) {
 				continue
@@ -42,6 +45,6 @@ func (mon *Monitor) emitDeploymentStatuses(ctx context.Context) error {
 			break
 		}
 	}
-
+	mon.emitGauge("deployment.count", count, nil)
 	return nil
 }
diff --git a/pkg/monitor/cluster/jobconditions.go b/pkg/monitor/cluster/jobconditions.go
index b27804612..f45104ca3 100644
--- a/pkg/monitor/cluster/jobconditions.go
+++ b/pkg/monitor/cluster/jobconditions.go
@@ -20,12 +20,15 @@ var jobConditionsExpected = map[batchv1.JobConditionType]corev1.ConditionStatus{
 
 func (mon *Monitor) emitJobConditions(ctx context.Context) error {
 	var cont string
+	var count int64
 	for {
 		jobs, err := mon.cli.BatchV1().Jobs("").List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(jobs.Items))
+
 		for _, job := range jobs.Items {
 			if !namespace.IsOpenShift(job.Namespace) {
 				continue
@@ -56,5 +59,7 @@ func (mon *Monitor) emitJobConditions(ctx context.Context) error {
 		}
 	}
 
+	mon.emitGauge("job.count", count, nil)
+
 	return nil
 }
diff --git a/pkg/monitor/cluster/machineconfigpoolconditions.go b/pkg/monitor/cluster/machineconfigpoolconditions.go
index 411bb0c50..6e310b1e5 100644
--- a/pkg/monitor/cluster/machineconfigpoolconditions.go
+++ b/pkg/monitor/cluster/machineconfigpoolconditions.go
@@ -22,12 +22,15 @@ var machineConfigPoolConditionsExpected = map[mcv1.MachineConfigPoolConditionTyp
 
 func (mon *Monitor) emitMachineConfigPoolConditions(ctx context.Context) error {
 	var cont string
+	var count int64
 	for {
 		mcps, err := mon.mcocli.MachineconfigurationV1().MachineConfigPools().List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(mcps.Items))
+
 		for _, mcp := range mcps.Items {
 			for _, c := range mcp.Status.Conditions {
 				if c.Status == machineConfigPoolConditionsExpected[c.Type] {
@@ -58,5 +61,7 @@ func (mon *Monitor) emitMachineConfigPoolConditions(ctx context.Context) error {
 		}
 	}
 
+	mon.emitGauge("machineconfigpool.count", count, nil)
+
 	return nil
 }
diff --git a/pkg/monitor/cluster/podconditions.go b/pkg/monitor/cluster/podconditions.go
index 013aad93a..681219e0b 100644
--- a/pkg/monitor/cluster/podconditions.go
+++ b/pkg/monitor/cluster/podconditions.go
@@ -23,12 +23,15 @@ var podConditionsExpected = map[corev1.PodConditionType]corev1.ConditionStatus{
 func (mon *Monitor) emitPodConditions(ctx context.Context) error {
 	// to list pods once
 	var cont string
+	var count int64
 	for {
 		ps, err := mon.cli.CoreV1().Pods("").List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(ps.Items))
+
 		mon._emitPodConditions(ps)
 		mon._emitPodContainerStatuses(ps)
 
@@ -37,6 +40,9 @@ func (mon *Monitor) emitPodConditions(ctx context.Context) error {
 			break
 		}
 	}
+
+	mon.emitGauge("pod.count", count, nil)
+
 	return nil
 }
 
diff --git a/pkg/monitor/cluster/prometheusalerts.go b/pkg/monitor/cluster/prometheusalerts.go
index 448e32883..909e80eb5 100644
--- a/pkg/monitor/cluster/prometheusalerts.go
+++ b/pkg/monitor/cluster/prometheusalerts.go
@@ -73,6 +73,8 @@ func (mon *Monitor) emitPrometheusAlerts(ctx context.Context) error {
 		severity string
 	}{}
 
+	mon.emitGauge("prometheus.alerts", int64(len(alerts)), nil)
+
 	for _, alert := range alerts {
 		if !namespace.IsOpenShift(string(alert.Labels["namespace"])) {
 			continue
diff --git a/pkg/monitor/cluster/replicasetstatuses.go b/pkg/monitor/cluster/replicasetstatuses.go
index 5cd726cfe..fc5fc8dc2 100644
--- a/pkg/monitor/cluster/replicasetstatuses.go
+++ b/pkg/monitor/cluster/replicasetstatuses.go
@@ -14,12 +14,15 @@ import (
 
 func (mon *Monitor) emitReplicasetStatuses(ctx context.Context) error {
 	var cont string
+	var count int64
 	for {
 		rss, err := mon.cli.AppsV1().ReplicaSets("").List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(rss.Items))
+
 		for _, rs := range rss.Items {
 			if !namespace.IsOpenShift(rs.Namespace) {
 				continue
@@ -43,5 +46,7 @@ func (mon *Monitor) emitReplicasetStatuses(ctx context.Context) error {
 		}
 	}
 
+	mon.emitGauge("replicaset.count", count, nil)
+
 	return nil
 }
diff --git a/pkg/monitor/cluster/statefulsetstatuses.go b/pkg/monitor/cluster/statefulsetstatuses.go
index 8d0639782..d348eec7d 100644
--- a/pkg/monitor/cluster/statefulsetstatuses.go
+++ b/pkg/monitor/cluster/statefulsetstatuses.go
@@ -14,12 +14,15 @@ import (
 
 func (mon *Monitor) emitStatefulsetStatuses(ctx context.Context) error {
 	var cont string
+	var count int64
 	for {
 		sss, err := mon.cli.AppsV1().StatefulSets("").List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
 		if err != nil {
 			return err
 		}
 
+		count += int64(len(sss.Items))
+
 		for _, ss := range sss.Items {
 			if !namespace.IsOpenShift(ss.Namespace) {
 				continue
@@ -43,5 +46,7 @@ func (mon *Monitor) emitStatefulsetStatuses(ctx context.Context) error {
 		}
 	}
 
+	mon.emitGauge("statefulset.count", count, nil)
+
 	return nil
 }


Test patch:
diff --git a/pkg/monitor/cluster/daemonsetstatuses_test.go b/pkg/monitor/cluster/daemonsetstatuses_test.go
index 83d5c69e3..89354d02e 100644
--- a/pkg/monitor/cluster/daemonsetstatuses_test.go
+++ b/pkg/monitor/cluster/daemonsetstatuses_test.go
@@ -67,6 +67,8 @@ func TestEmitDaemonsetStatuses(t *testing.T) {
 		"numberAvailable":        strconv.Itoa(1),
 	})
 
+	m.EXPECT().EmitGauge("daemonset.count", int64(3), map[string]string{})
+
 	err := mon.emitDaemonsetStatuses(ctx)
 	if err != nil {
 		t.Fatal(err)
diff --git a/pkg/monitor/cluster/deploymentstatuses_test.go b/pkg/monitor/cluster/deploymentstatuses_test.go
index 47c621016..206364af8 100644
--- a/pkg/monitor/cluster/deploymentstatuses_test.go
+++ b/pkg/monitor/cluster/deploymentstatuses_test.go
@@ -60,6 +60,8 @@ func TestEmitDeploymentStatuses(t *testing.T) {
 		m:   m,
 	}
 
+	m.EXPECT().EmitGauge("deployment.count", int64(3), map[string]string{})
+
 	m.EXPECT().EmitGauge("deployment.statuses", int64(1), map[string]string{
 		"availableReplicas": strconv.Itoa(1),
 		"name":              "name1",
diff --git a/pkg/monitor/cluster/jobconditions_test.go b/pkg/monitor/cluster/jobconditions_test.go
index 88194d6c9..c5188b296 100644
--- a/pkg/monitor/cluster/jobconditions_test.go
+++ b/pkg/monitor/cluster/jobconditions_test.go
@@ -80,6 +80,8 @@ func TestEmitJobConditions(t *testing.T) {
 		m:   m,
 	}
 
+	m.EXPECT().EmitGauge("job.count", int64(2), map[string]string{})
+
 	m.EXPECT().EmitGauge("job.conditions", int64(1), map[string]string{
 		"name":      "job-failing",
 		"namespace": "openshift",
diff --git a/pkg/monitor/cluster/machineconfigpoolconditions_test.go b/pkg/monitor/cluster/machineconfigpoolconditions_test.go
index c342bb98e..35cc2466a 100644
--- a/pkg/monitor/cluster/machineconfigpoolconditions_test.go
+++ b/pkg/monitor/cluster/machineconfigpoolconditions_test.go
@@ -59,6 +59,8 @@ func TestEmitMachineConfigPoolConditions(t *testing.T) {
 		m:      m,
 	}
 
+	m.EXPECT().EmitGauge("machineconfigpool.count", int64(1), map[string]string{})
+
 	m.EXPECT().EmitGauge("machineconfigpool.conditions", int64(1), map[string]string{
 		"name":   "machine-config-pool",
 		"type":   "Degraded",
diff --git a/pkg/monitor/cluster/replicasetstatuses_test.go b/pkg/monitor/cluster/replicasetstatuses_test.go
index 6ab1a8901..c12ea9808 100644
--- a/pkg/monitor/cluster/replicasetstatuses_test.go
+++ b/pkg/monitor/cluster/replicasetstatuses_test.go
@@ -60,6 +60,8 @@ func TestEmitReplicasetStatuses(t *testing.T) {
 		m:   m,
 	}
 
+	m.EXPECT().EmitGauge("replicaset.count", int64(3), map[string]string{})
+
 	m.EXPECT().EmitGauge("replicaset.statuses", int64(1), map[string]string{
 		"availableReplicas": strconv.Itoa(1),
 		"name":              "name1",
diff --git a/pkg/monitor/cluster/statefulsetstatuses_test.go b/pkg/monitor/cluster/statefulsetstatuses_test.go
index 199fef8c6..d77663efa 100644
--- a/pkg/monitor/cluster/statefulsetstatuses_test.go
+++ b/pkg/monitor/cluster/statefulsetstatuses_test.go
@@ -60,6 +60,8 @@ func TestEmitStatefulsetStatuses(t *testing.T) {
 		m:   m,
 	}
 
+	m.EXPECT().EmitGauge("statefulset.count", int64(3), map[string]string{})
+
 	m.EXPECT().EmitGauge("statefulset.statuses", int64(1), map[string]string{
 		"name":          "name1",
 		"namespace":     "openshift",