Repository: metallb/metallb
Pull request #961 (2021-09-22 11:27:41)

Add bgp metrics tests
We extend the e2e tests with one test that validates the presence of the current metrics, create a service and checks the service announcement metrics.

This may require some rework when https://github.com/metallb/metallb/pull/931 lands.

russellb: I rebased this branch on main to get it to run against the updated CI config which includes the IPv6 tests.  I wanted to make sure this test still passed OK
russellb: @fedepaol test failed for some reason after rebasing on the ipv6 test jobs
fedepaol: > @fedepaol test failed for some reason after rebasing on the ipv6 test jobs

Sep 27 19:52:50.025: Expected
        <int>: -9223372036854775808
    to equal
        <int>: 256

almost :-) 

Patch:
diff --git a/dev-env/bgp/config.yaml.tmpl b/dev-env/bgp/config.yaml.tmpl
index 04572b51..0313ba38 100644
--- a/dev-env/bgp/config.yaml.tmpl
+++ b/dev-env/bgp/config.yaml.tmpl
@@ -16,4 +16,4 @@ data:
       protocol: bgp
       addresses:
       - 192.168.10.0/24
-      - fc00:f853:0ccd:e799::/64
+      - fc00:f853:0ccd:e799::/124
diff --git a/go.mod b/go.mod
index bcd25f97..0ae0491f 100644
--- a/go.mod
+++ b/go.mod
@@ -19,8 +19,10 @@ require (
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.0
 	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_model v0.2.0
+	github.com/prometheus/common v0.10.0
 	github.com/spf13/viper v1.8.1 // indirect
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
 	gopkg.in/yaml.v2 v2.4.0


Test patch:
diff --git a/e2etest/bgp_test.go b/e2etest/bgp_test.go
index 04d1be04..92636b6d 100644
--- a/e2etest/bgp_test.go
+++ b/e2etest/bgp_test.go
@@ -23,12 +23,15 @@ import (
 	"fmt"
 	"net"
 	"strconv"
-	"time"
+
+	"go.universe.tf/metallb/e2etest/pkg/metrics"
 
 	"github.com/onsi/ginkgo"
+	dto "github.com/prometheus/client_model/go"
 	"go.universe.tf/metallb/e2etest/pkg/executor"
 	"go.universe.tf/metallb/e2etest/pkg/frr"
 	"go.universe.tf/metallb/e2etest/pkg/routes"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -39,13 +42,10 @@ const frrContainer = "frr"
 
 var _ = ginkgo.Describe("BGP", func() {
 	f := framework.NewDefaultFramework("bgp")
-	var loadBalancerCreateTimeout time.Duration
-
 	var cs clientset.Interface
 
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
-		loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
 	})
 
 	ginkgo.AfterEach(func() {
@@ -55,19 +55,10 @@ var _ = ginkgo.Describe("BGP", func() {
 	})
 
 	ginkgo.It("should work for type=Loadbalancer", func() {
-		namespace := f.Namespace.Name
-		serviceName := "external-local-lb"
-		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-
-		svc, err := jig.CreateLoadBalancerService(loadBalancerCreateTimeout,
-			tweakServicePort())
-		framework.ExpectNoError(err)
-
-		_, err = jig.Run(tweakRCPort())
-		framework.ExpectNoError(err)
+		svc := createServiceWithBackend(cs, f.Namespace.Name)
 
 		defer func() {
-			err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+			err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
 			framework.ExpectNoError(err)
 		}()
 
@@ -86,7 +77,7 @@ var _ = ginkgo.Describe("BGP", func() {
 			ginkgo.By(fmt.Sprintf("checking connectivity to %s with docker", address))
 		}
 
-		err = wgetRetry(address, exc)
+		err := wgetRetry(address, exc)
 		framework.ExpectNoError(err)
 
 		allNodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
@@ -110,4 +101,92 @@ var _ = ginkgo.Describe("BGP", func() {
 		err = frr.RoutesMatchNodes(allNodes.Items, routes)
 		framework.ExpectNoError(err)
 	})
+
+	ginkgo.Context("metrics", func() {
+		var controllerPod *corev1.Pod
+		var speakerPods []*corev1.Pod
+
+		ginkgo.BeforeEach(func() {
+			pods, err := cs.CoreV1().Pods("metallb-system").List(context.Background(), metav1.ListOptions{
+				LabelSelector: "component=controller",
+			})
+			framework.ExpectNoError(err)
+			framework.ExpectEqual(len(pods.Items), 1, "More than one controller found")
+			controllerPod = &pods.Items[0]
+
+			speakers, err := cs.CoreV1().Pods("metallb-system").List(context.Background(), metav1.ListOptions{
+				LabelSelector: "component=speaker",
+			})
+			framework.ExpectNoError(err)
+			speakerPods = make([]*corev1.Pod, 0)
+			for _, item := range speakers.Items {
+				speakerPods = append(speakerPods, &item)
+			}
+		})
+
+		ginkgo.It("should be exposed by the controller", func() {
+			peerAddr := "172.18.0.5:179" // TODO replace this when we create the config locally
+			poolName := "dev-env-bgp"    // TODO replace this when we create the config locally
+			ginkgo.By("checking the metrics when no service is added")
+			controllerMetrics, err := metrics.ForPod(controllerPod, controllerPod)
+			framework.ExpectNoError(err)
+			validateGaugeValue(0, "metallb_allocator_addresses_in_use_total", map[string]string{"pool": poolName}, controllerMetrics)
+			validateGaugeValue(272, "metallb_allocator_addresses_total", map[string]string{"pool": poolName}, controllerMetrics)
+
+			for _, speaker := range speakerPods {
+				ginkgo.By(fmt.Sprintf("checking speaker %s", speaker.Name))
+
+				speakerMetrics, err := metrics.ForPod(controllerPod, speaker)
+				framework.ExpectNoError(err)
+				validateGaugeValue(1, "metallb_bgp_session_up", map[string]string{"peer": peerAddr}, speakerMetrics)
+				validateGaugeValue(0, "metallb_bgp_announced_prefixes_total", map[string]string{"peer": peerAddr}, speakerMetrics)
+			}
+
+			ginkgo.By("creating a service")
+			svc := createServiceWithBackend(cs, f.Namespace.Name) // Is a sleep required here?
+			defer func() {
+				err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+				framework.ExpectNoError(err)
+			}()
+
+			ginkgo.By("checking the metrics when a service is added")
+			controllerMetrics, err = metrics.ForPod(controllerPod, controllerPod)
+			framework.ExpectNoError(err)
+			validateGaugeValue(1, "metallb_allocator_addresses_in_use_total", map[string]string{"pool": poolName}, controllerMetrics)
+
+			for _, speaker := range speakerPods {
+				ginkgo.By(fmt.Sprintf("checking speaker %s", speaker.Name))
+
+				speakerMetrics, err := metrics.ForPod(controllerPod, speaker)
+				framework.ExpectNoError(err)
+				validateGaugeValue(1, "metallb_bgp_session_up", map[string]string{"peer": peerAddr}, speakerMetrics)
+				validateGaugeValue(1, "metallb_bgp_announced_prefixes_total", map[string]string{"peer": peerAddr}, speakerMetrics)
+
+				updatesTotal, err := metrics.CounterForLabels("metallb_bgp_updates_total", map[string]string{"peer": peerAddr}, speakerMetrics)
+				framework.ExpectNoError(err)
+				framework.ExpectEqual(updatesTotal >= 1, true, "expecting ", updatesTotal, "greater than 1")
+
+				validateGaugeValue(1, "metallb_speaker_announced", map[string]string{"node": speaker.Spec.NodeName, "protocol": "bgp", "service": fmt.Sprintf("%s/%s", f.Namespace.Name, svc.Name)}, speakerMetrics)
+			}
+		})
+	})
+
 })
+
+func createServiceWithBackend(cs clientset.Interface, namespace string) *corev1.Service {
+	serviceName := "external-local-lb"
+	jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+	timeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
+
+	svc, err := jig.CreateLoadBalancerService(timeout, tweakServicePort())
+	framework.ExpectNoError(err)
+	_, err = jig.Run(tweakRCPort())
+	framework.ExpectNoError(err)
+	return svc
+}
+
+func validateGaugeValue(expectedValue int, metricName string, labels map[string]string, m map[string]*dto.MetricFamily) {
+	value, err := metrics.GaugeForLabels(metricName, labels, m)
+	framework.ExpectNoError(err)
+	framework.ExpectEqual(value, expectedValue, "invalid value for ", metricName, labels)
+}
diff --git a/e2etest/pkg/frr/parse_test.go b/e2etest/pkg/frr/parse_test.go
index 7f4c59f0..33480616 100644
--- a/e2etest/pkg/frr/parse_test.go
+++ b/e2etest/pkg/frr/parse_test.go
@@ -331,6 +331,10 @@ func TestNeighbours(t *testing.T) {
 	if len(nn) != 3 {
 		t.Fatalf("Expected 3 neighbours, got %d", len(nn))
 	}
+	sort.Slice(nn, func(i, j int) bool {
+		return (bytes.Compare(nn[i].ip, nn[j].ip) < 0)
+	})
+
 	if !nn[0].ip.Equal(net.ParseIP("172.18.0.2")) {
 		t.Fatal("neighbour ip not matching")
 	}
@@ -429,7 +433,6 @@ func TestRoutes(t *testing.T) {
 	sort.Slice(ips, func(i, j int) bool {
 		return (bytes.Compare(ips[i], ips[j]) < 0)
 	})
-	fmt.Println(ips)
 	if !ips[0].Equal(net.ParseIP("172.18.0.2")) {
 		t.Fatal("neighbour ip not matching")
 	}
diff --git a/e2etest/pkg/metrics/metrics.go b/e2etest/pkg/metrics/metrics.go
new file mode 100644
index 00000000..cfbaa963
--- /dev/null
+++ b/e2etest/pkg/metrics/metrics.go
@@ -0,0 +1,79 @@
+package metrics
+
+import (
+	"fmt"
+	"net"
+	"path"
+	"strings"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/pkg/errors"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// MetricsForPod returns the parsed metrics for the given pod, scraping them
+// from the executor pod.
+func ForPod(executor, target *corev1.Pod) (map[string]*dto.MetricFamily, error) {
+	metricsUrl := path.Join(net.JoinHostPort(target.Status.PodIP, "7472"), "metrics")
+	metrics, err := framework.RunKubectl("metallb-system", "exec", executor.Name, "--", "wget", "-qO-", metricsUrl)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to scrape metrics for %s", target.Name)
+	}
+	res, err := metricsFromString(metrics)
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+// GaugeForLabels retrieves the value of the Gauge matching the given set of labels.
+func GaugeForLabels(metricName string, labels map[string]string, metrics map[string]*dto.MetricFamily) (int, error) {
+	return metricForLabels(metricName, labels, metrics, func(m *dto.Metric) int {
+		return int(m.GetGauge().GetValue())
+	})
+}
+
+// CounterForLabels retrieves the value of the Counter matching the given set of labels.
+func CounterForLabels(metricName string, labels map[string]string, metrics map[string]*dto.MetricFamily) (int, error) {
+	return metricForLabels(metricName, labels, metrics, func(m *dto.Metric) int {
+		return int(m.GetCounter().GetValue())
+	})
+}
+
+func metricForLabels(metricName string, labels map[string]string, metrics map[string]*dto.MetricFamily, getValue func(m *dto.Metric) int) (int, error) {
+	mf, ok := metrics[metricName]
+	if !ok {
+		return 0, fmt.Errorf("metric %s not in metrics", metricName)
+	}
+	mm := mf.GetMetric()
+	for _, m := range mm {
+		toMatch := len(labels)
+		label := m.GetLabel()
+		for _, l := range label {
+			v, ok := labels[l.GetName()]
+			if !ok {
+				continue
+			}
+			if v != l.GetValue() {
+				continue
+			}
+			toMatch--
+		}
+		if toMatch == 0 {
+			return getValue(m), nil
+		}
+	}
+	return 0, fmt.Errorf("label %s not found in metrics for %s", labels, metricName)
+}
+
+func metricsFromString(metrics string) (map[string]*dto.MetricFamily, error) {
+	var parser expfmt.TextParser
+	mf, err := parser.TextToMetricFamilies(strings.NewReader(metrics))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse metrics %s", metrics)
+	}
+	return mf, nil
+}
diff --git a/e2etest/pkg/metrics/metrics_test.go b/e2etest/pkg/metrics/metrics_test.go
new file mode 100644
index 00000000..f42bc835
--- /dev/null
+++ b/e2etest/pkg/metrics/metrics_test.go
@@ -0,0 +1,70 @@
+package metrics
+
+import (
+	"testing"
+)
+
+const sample = `# HELP metallb_bgp_announced_prefixes_total Number of prefixes currently being advertised on the BGP session
+# TYPE metallb_bgp_announced_prefixes_total gauge
+metallb_bgp_announced_prefixes_total{peer="172.18.0.5:179"} 1
+# HELP metallb_bgp_pending_prefixes_total Number of prefixes that should be advertised on the BGP session
+# TYPE metallb_bgp_pending_prefixes_total gauge
+metallb_bgp_pending_prefixes_total{peer="172.18.0.5:179"} 1
+# HELP metallb_bgp_session_up BGP session state (1 is up, 0 is down)
+# TYPE metallb_bgp_session_up gauge
+metallb_bgp_session_up{peer="172.18.0.5:179"} 1
+# HELP metallb_bgp_updates_total Number of BGP UPDATE messages sent
+# TYPE metallb_bgp_updates_total counter
+metallb_bgp_updates_total{peer="172.18.0.5:179"} 7
+# HELP metallb_k8s_client_config_loaded_bool 1 if the MetalLB configuration was successfully loaded at least once.
+# TYPE metallb_k8s_client_config_loaded_bool gauge
+metallb_k8s_client_config_loaded_bool 1
+# HELP metallb_k8s_client_config_stale_bool 1 if running on a stale configuration, because the latest config failed to load.
+# TYPE metallb_k8s_client_config_stale_bool gauge
+metallb_k8s_client_config_stale_bool 0
+# HELP metallb_k8s_client_update_errors_total Number of k8s object updates that failed for some reason.
+# TYPE metallb_k8s_client_update_errors_total counter
+metallb_k8s_client_update_errors_total 0
+# HELP metallb_k8s_client_updates_total Number of k8s object updates that have been processed.
+# TYPE metallb_k8s_client_updates_total counter
+metallb_k8s_client_updates_total 56
+# HELP metallb_speaker_announced Services being announced from this node. This is desired state, it does not guarantee that the routing protocols have converged.
+# TYPE metallb_speaker_announced gauge
+metallb_speaker_announced{ip="192.168.10.0",node="kind-worker",protocol="bgp",service="bgp-8081/external-local-lb"} 1
+`
+
+func TestParsing(t *testing.T) {
+	m, err := metricsFromString(sample)
+	if err != nil {
+		t.Fail()
+	}
+	updatesTotal, err := CounterForLabels("metallb_bgp_updates_total", map[string]string{"peer": "172.18.0.5:179"}, m)
+	if err != nil {
+		t.Fail()
+	}
+	if updatesTotal != 7 {
+		t.Fatalf("expected value was 7, got %d", updatesTotal)
+	}
+	speakerAnnounced, err := GaugeForLabels("metallb_speaker_announced", map[string]string{
+		"ip":       "192.168.10.0",
+		"node":     "kind-worker",
+		"protocol": "bgp",
+		"service":  "bgp-8081/external-local-lb",
+	}, m)
+	if err != nil {
+		t.Fail()
+	}
+	if speakerAnnounced != 1 {
+		t.Fatalf("expected speakerAnnounced was 1, got %d", speakerAnnounced)
+	}
+	_, err = GaugeForLabels("metallb_speaker_announced", map[string]string{
+		"ip":          "192.168.10.0",
+		"node":        "kind-worker",
+		"protocol":    "bgp",
+		"service":     "bgp-8081/external-local-lb",
+		"unnecessary": "label",
+	}, m)
+	if err == nil {
+		t.Fail()
+	}
+}