Repository: Azure/ARO-RP
Pull request #1450 (2021-04-19 15:59:40)

Monitor for MCP unmanaged nodes
emit count of nodes which are not managed by MCP
 
```
 =0 is expected normal (all nodes are managed) 
 >0 mcp isn't managing all nodes 
 <0 nodes are missing from mcp
```

### Which issue this PR addresses:

Fixes User Story 9534737 in ADO

### What this PR does / why we need it:

Monitor for unmanaged nodes. Defined as a difference between a count of known nodes (`oc get nodes`) and the total of machineconfigpool.machinecount (`oc get mcp`) 

### Test plan for issue:

1. Create a new machineset with bad labels:

https://docs.openshift.com/container-platform/4.6/machine_management/creating_machinesets/creating-machineset-azure.html

```
    spec:
      metadata:
        labels:
          node-role.kubernetes.io/tester: ""
          node-role.kubernetes.io/worker: "no"
```

2. Note the number of nodes is different from the machineconfigpool.machinecount in `oc get nodes,machines,machineset,mcp -A`

3. Check the number emitted to Geneva is non-zero.

### Is there any documentation that needs to be updated for this PR?

No, documentation will be provided with triggered alerts.

drewandersonnz: Resolved other comments, remaining is unit tests and mocking for `machineconfigpool.go` 
drewandersonnz: /azp run e2e
azure-pipelines[bot]: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>

Patch:
diff --git a/pkg/monitor/cluster/cluster.go b/pkg/monitor/cluster/cluster.go
index 6e72edc97..fce45a0c0 100644
--- a/pkg/monitor/cluster/cluster.go
+++ b/pkg/monitor/cluster/cluster.go
@@ -128,6 +128,7 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
 		mon.emitDaemonsetStatuses,
 		mon.emitDeploymentStatuses,
 		mon.emitMachineConfigPoolConditions,
+		mon.emitMachineConfigPoolUnmanagedNodeCounts,
 		mon.emitNodeConditions,
 		mon.emitPodConditions,
 		mon.emitReplicasetStatuses,
diff --git a/pkg/monitor/cluster/machineconfigpool.go b/pkg/monitor/cluster/machineconfigpool.go
new file mode 100644
index 000000000..9f9e86eea
--- /dev/null
+++ b/pkg/monitor/cluster/machineconfigpool.go
@@ -0,0 +1,77 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (mon *Monitor) getMachineConfigPoolNodeCounts(ctx context.Context) (int64, error) {
+	var cont string
+	var count int64
+
+	for {
+		mcps, err := mon.mcocli.MachineconfigurationV1().MachineConfigPools().List(ctx, metav1.ListOptions{Limit: 500, Continue: cont})
+		if err != nil {
+			return 0, err
+		}
+
+		for _, mcp := range mcps.Items {
+			count += int64(mcp.Status.MachineCount)
+		}
+
+		cont = mcps.Continue
+		if cont == "" {
+			break
+		}
+	}
+
+	return count, nil
+}
+
+func (mon *Monitor) getNodeCounts(ctx context.Context) (int64, error) {
+	ns, err := mon.listNodes(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	return int64(len(ns.Items)), nil
+}
+
+/***
+Count the number of nodes available
+Total the nodes under machineconfigpool control
+Alert if different
+*/
+func (mon *Monitor) emitMachineConfigPoolUnmanagedNodeCounts(ctx context.Context) error {
+	mcpcount, err := mon.getMachineConfigPoolNodeCounts(ctx)
+	if err != nil {
+		return err
+	}
+
+	getnodescount, err := mon.getNodeCounts(ctx)
+	if err != nil {
+		return err
+	}
+
+	// unmanagednodescount of 0 is normal (machineconfigpool nodes == nodes)
+	// also report if there are missing nodes with too many machineconfigs
+	unmanagednodescount := getnodescount - mcpcount
+
+	// emit count of nodes which are not managed by MCP
+	// =0 is expected normal (all nodes are managed)
+	// >0 mcp isn't managing all nodes
+	// <0 nodes are missing from mcp
+	if unmanagednodescount != 0 {
+		mon.emitGauge("machineconfigpool.unmanagednodescount", unmanagednodescount, nil)
+	}
+
+	if mon.hourlyRun {
+		mon.log.Printf("machineconfigpool.unmanagednodescount: %d", unmanagednodescount)
+	}
+
+	return nil
+}


Test patch:
diff --git a/pkg/monitor/cluster/machineconfigpool_test.go b/pkg/monitor/cluster/machineconfigpool_test.go
new file mode 100644
index 000000000..ac4c1565b
--- /dev/null
+++ b/pkg/monitor/cluster/machineconfigpool_test.go
@@ -0,0 +1,88 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	mcofake "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/fake"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+
+	mock_metrics "github.com/Azure/ARO-RP/pkg/util/mocks/metrics"
+)
+
+func TestEmitMachineConfigPoolUnmanagedNodes(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tt := range []struct {
+		name   string
+		mcps   runtime.Object
+		nodes  runtime.Object
+		expect int64
+	}{
+		{
+			name:   "Too Many Nodes",
+			expect: 1,
+			nodes: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "aro-master-0",
+				},
+			},
+			mcps: &mcv1.MachineConfigPool{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "machine-config-pool",
+				},
+				Status: mcv1.MachineConfigPoolStatus{
+					MachineCount: 0,
+				},
+			},
+		},
+		{
+			name:   "Too Few Nodes",
+			expect: -1,
+			nodes: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "aro-master-0",
+				},
+			},
+			mcps: &mcv1.MachineConfigPool{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "machine-config-pool",
+				},
+				Status: mcv1.MachineConfigPoolStatus{
+					MachineCount: 2,
+				},
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			mcocli := mcofake.NewSimpleClientset(tt.mcps)
+			cli := fake.NewSimpleClientset(tt.nodes)
+
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			m := mock_metrics.NewMockInterface(controller)
+
+			mon := &Monitor{
+				mcocli: mcocli,
+				m:      m,
+				cli:    cli,
+			}
+
+			m.EXPECT().EmitGauge("machineconfigpool.unmanagednodescount", int64(tt.expect), map[string]string{})
+
+			err := mon.emitMachineConfigPoolUnmanagedNodeCounts(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}