Repository: metallb/metallb
Pull request #1088 (2021-11-29 14:21:47)

Add BFD metrics collector 
The new BFD collector creates BFD metrics based on BFD peers counters.

Adding unit tests to verify BFD metrics collector functionality and extending the BFD E2E test to check BFD metrics.

fedepaol: LGTM
Let's wait CI
oribon: LGTM

Patch:
diff --git a/frr-metrics/collector/bfd.go b/frr-metrics/collector/bfd.go
new file mode 100644
index 00000000..e284f9a3
--- /dev/null
+++ b/frr-metrics/collector/bfd.go
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier:Apache-2.0
+
+package collector
+
+import (
+	"encoding/json"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+
+	bgpfrr "go.universe.tf/metallb/internal/bgp/frr"
+	bgpstats "go.universe.tf/metallb/internal/bgp/native"
+)
+
+type bfdPeerCounters struct {
+	Peer                string `json:"peer"`
+	ControlPacketInput  int    `json:"control-packet-input"`
+	ControlPacketOutput int    `json:"control-packet-output"`
+	EchoPacketInput     int    `json:"echo-packet-input"`
+	EchoPacketOutput    int    `json:"echo-packet-output"`
+	SessionUpEvents     int    `json:"session-up"`
+	SessionDownEvents   int    `json:"session-down"`
+	ZebraNotifications  int    `json:"zebra-notifications"`
+}
+
+const subsystem = "bfd"
+
+var (
+	bfdSessionUpDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, bgpstats.SessionUp.Name),
+		"BFD session state (1 is up, 0 is down)",
+		bgpstats.Labels,
+		nil,
+	)
+
+	controlPacketInputDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "control_packet_input"),
+		"Number of received BFD control packets",
+		bgpstats.Labels,
+		nil,
+	)
+
+	controlPacketOutputDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "control_packet_output"),
+		"Number of sent BFD control packets",
+		bgpstats.Labels,
+		nil,
+	)
+
+	echoPacketInputDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "echo_packet_input"),
+		"Number of received BFD echo packets",
+		bgpstats.Labels,
+		nil,
+	)
+
+	echoPacketOutputDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "echo_packet_output"),
+		"Number of sent BFD echo packets",
+		bgpstats.Labels,
+		nil,
+	)
+
+	sessionUpEventsDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "session_up_events"),
+		"Number of BFD session up events",
+		bgpstats.Labels,
+		nil,
+	)
+
+	sessionDownEventsDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "session_down_events"),
+		"Number of BFD session down events",
+		bgpstats.Labels,
+		nil,
+	)
+
+	zebraNotificationsDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(bgpstats.Namespace, subsystem, "zebra_notifications"),
+		"Number of BFD zebra notifications",
+		bgpstats.Labels,
+		nil,
+	)
+)
+
+type bfd struct {
+	Log    log.Logger
+	frrCli func(args ...string) (string, error)
+}
+
+func NewBFD(l log.Logger) *bfd {
+	log := log.With(l, "collector", subsystem)
+	return &bfd{Log: log, frrCli: runVtysh}
+}
+
+func (c *bfd) Describe(ch chan<- *prometheus.Desc) {
+	ch <- bfdSessionUpDesc
+	ch <- controlPacketInputDesc
+	ch <- controlPacketOutputDesc
+	ch <- echoPacketInputDesc
+	ch <- echoPacketOutputDesc
+	ch <- sessionUpEventsDesc
+	ch <- sessionDownEventsDesc
+	ch <- zebraNotificationsDesc
+}
+
+func (c *bfd) Collect(ch chan<- prometheus.Metric) {
+	peers, err := getBFDPeers(c.frrCli)
+	if err != nil {
+		level.Error(c.Log).Log("error", err, "msg", "failed to fetch BFD peers from FRR")
+		return
+	}
+
+	updatePeersMetrics(ch, peers)
+
+	peersCounters, err := getBFDPeersCounters(c.frrCli)
+	if err != nil {
+		level.Error(c.Log).Log("error", err, "msg", "failed to fetch BFD peers counters from FRR")
+		return
+	}
+
+	updatePeersCountersMetrics(ch, peersCounters)
+}
+
+func updatePeersMetrics(ch chan<- prometheus.Metric, peers map[string]bgpfrr.BFDPeer) {
+	for _, p := range peers {
+		sessionUp := 1
+		if p.Status == "down" {
+			sessionUp = 0
+		}
+
+		ch <- prometheus.MustNewConstMetric(bfdSessionUpDesc, prometheus.GaugeValue, float64(sessionUp), p.Peer)
+	}
+}
+
+func updatePeersCountersMetrics(ch chan<- prometheus.Metric, peersCounters []bfdPeerCounters) {
+	for _, p := range peersCounters {
+		ch <- prometheus.MustNewConstMetric(controlPacketInputDesc, prometheus.CounterValue, float64(p.ControlPacketInput), p.Peer)
+		ch <- prometheus.MustNewConstMetric(controlPacketOutputDesc, prometheus.CounterValue, float64(p.ControlPacketOutput), p.Peer)
+		ch <- prometheus.MustNewConstMetric(echoPacketInputDesc, prometheus.CounterValue, float64(p.EchoPacketInput), p.Peer)
+		ch <- prometheus.MustNewConstMetric(echoPacketOutputDesc, prometheus.CounterValue, float64(p.EchoPacketOutput), p.Peer)
+		ch <- prometheus.MustNewConstMetric(sessionUpEventsDesc, prometheus.CounterValue, float64(p.SessionUpEvents), p.Peer)
+		ch <- prometheus.MustNewConstMetric(sessionDownEventsDesc, prometheus.CounterValue, float64(p.SessionDownEvents), p.Peer)
+		ch <- prometheus.MustNewConstMetric(zebraNotificationsDesc, prometheus.CounterValue, float64(p.ZebraNotifications), p.Peer)
+	}
+}
+
+func getBFDPeers(frrCli func(args ...string) (string, error)) (map[string]bgpfrr.BFDPeer, error) {
+	res, err := frrCli("show bfd peers json")
+	if err != nil {
+		return nil, err
+	}
+
+	return bgpfrr.ParseBFDPeers(res)
+}
+
+func getBFDPeersCounters(frrCli func(args ...string) (string, error)) ([]bfdPeerCounters, error) {
+	res, err := frrCli("show bfd peers counters json")
+	if err != nil {
+		return nil, err
+	}
+
+	parseRes := []bfdPeerCounters{}
+	err = json.Unmarshal([]byte(res), &parseRes)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseRes, nil
+}
diff --git a/frr-metrics/collector/bgp.go b/frr-metrics/collector/bgp.go
index dd089def..2b70b20a 100644
--- a/frr-metrics/collector/bgp.go
+++ b/frr-metrics/collector/bgp.go
@@ -4,7 +4,6 @@ package collector
 
 import (
 	"fmt"
-	"os/exec"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
@@ -85,9 +84,3 @@ func getBGPNeighbors(frrCli func(args ...string) (string, error)) ([]*bgpfrr.Nei
 
 	return bgpfrr.ParseNeighbours(res)
 }
-
-func runVtysh(args ...string) (string, error) {
-	newArgs := append([]string{"-c"}, args...)
-	out, err := exec.Command("/usr/bin/vtysh", newArgs...).CombinedOutput()
-	return string(out), err
-}
diff --git a/frr-metrics/collector/vtysh.go b/frr-metrics/collector/vtysh.go
new file mode 100644
index 00000000..f839cc9f
--- /dev/null
+++ b/frr-metrics/collector/vtysh.go
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier:Apache-2.0
+
+package collector
+
+import (
+	"os/exec"
+)
+
+func runVtysh(args ...string) (string, error) {
+	newArgs := append([]string{"-c"}, args...)
+	out, err := exec.Command("/usr/bin/vtysh", newArgs...).CombinedOutput()
+	return string(out), err
+}
diff --git a/frr-metrics/exporter.go b/frr-metrics/exporter.go
index 60a234fe..3964b704 100644
--- a/frr-metrics/exporter.go
+++ b/frr-metrics/exporter.go
@@ -28,9 +28,11 @@ var (
 
 func metricsHandler(logger log.Logger) http.Handler {
 	BGPCollector := collector.NewBGP(logger)
+	BFDCollector := collector.NewBFD(logger)
 
 	registry := prometheus.NewRegistry()
 	registry.MustRegister(BGPCollector)
+	registry.MustRegister(BFDCollector)
 
 	gatherers := prometheus.Gatherers{
 		prometheus.DefaultGatherer,


Test patch:
diff --git a/e2etest/bgp_test.go b/e2etest/bgp_test.go
index 9a768a0c..35667bc5 100644
--- a/e2etest/bgp_test.go
+++ b/e2etest/bgp_test.go
@@ -598,6 +598,177 @@ var _ = ginkgo.Describe("BGP", func() {
 					testservice.DualStack(svc)
 				}),
 		)
+
+		table.DescribeTable("metrics", func(bfd bfdProfile, pairingFamily string, poolAddresses []string) {
+			configData := configFile{
+				Pools: []addressPool{
+					{
+						Name:      "bfd-test",
+						Protocol:  BGP,
+						Addresses: poolAddresses,
+					},
+				},
+				Peers:       withBFD(peersForContainers(frrContainers, pairingFamily), bfd.Name),
+				BFDProfiles: []bfdProfile{bfd},
+			}
+			err := updateConfigMap(cs, configData)
+			framework.ExpectNoError(err)
+
+			for _, c := range frrContainers {
+				pairExternalFRRWithNodes(cs, c, pairingFamily, func(container *frrcontainer.FRR) {
+					container.NeighborConfig.BFDEnabled = true
+				})
+			}
+
+			for _, c := range frrContainers {
+				validateFRRPeeredWithNodes(cs, c, pairingFamily)
+			}
+
+			ginkgo.By("checking metrics")
+			pods, err := cs.CoreV1().Pods(testNameSpace).List(context.Background(), metav1.ListOptions{
+				LabelSelector: "component=controller",
+			})
+			framework.ExpectNoError(err)
+			framework.ExpectEqual(len(pods.Items), 1, "Expected one controller pod")
+			controllerPod := &pods.Items[0]
+			speakerPods := getSpeakerPods(cs)
+
+			var peerAddrs []string
+			for _, c := range frrContainers {
+				address := c.Ipv4
+				if pairingFamily == "ipv6" {
+					address = c.Ipv6
+				}
+				peerAddrs = append(peerAddrs, address)
+			}
+
+			for _, speaker := range speakerPods {
+				ginkgo.By(fmt.Sprintf("checking speaker %s", speaker.Name))
+
+				Eventually(func() error {
+					speakerMetrics, err := metrics.ForPod(controllerPod, speaker, testNameSpace)
+					if err != nil {
+						return err
+					}
+
+					for _, addr := range peerAddrs {
+						err = validateGaugeValue(1, "metallb_bfd_session_up", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						err = validateCounterValue(1, "metallb_bfd_control_packet_input", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						err = validateCounterValue(1, "metallb_bfd_control_packet_output", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						err = validateGaugeValue(0, "metallb_bfd_session_down_events", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						err = validateCounterValue(1, "metallb_bfd_session_up_events", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						err = validateCounterValue(1, "metallb_bfd_zebra_notifications", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						if bfd.EchoMode != nil && *bfd.EchoMode {
+							err = validateCounterValue(1, "metallb_bfd_echo_packet_input", map[string]string{"peer": addr}, speakerMetrics)
+							if err != nil {
+								return err
+							}
+
+							err = validateCounterValue(1, "metallb_bfd_echo_packet_output", map[string]string{"peer": addr}, speakerMetrics)
+							if err != nil {
+								return err
+							}
+						}
+					}
+					return nil
+				}, 2*time.Minute, 1*time.Second).Should(BeNil())
+			}
+
+			ginkgo.By("disabling BFD in external FRR containers")
+			for _, c := range frrContainers {
+				pairExternalFRRWithNodes(cs, c, pairingFamily, func(container *frrcontainer.FRR) {
+					container.NeighborConfig.BFDEnabled = false
+				})
+			}
+
+			ginkgo.By("validating session down metrics")
+			for _, speaker := range speakerPods {
+				ginkgo.By(fmt.Sprintf("checking speaker %s", speaker.Name))
+
+				Eventually(func() error {
+					speakerMetrics, err := metrics.ForPod(controllerPod, speaker, testNameSpace)
+					if err != nil {
+						return err
+					}
+
+					for _, addr := range peerAddrs {
+						err = validateGaugeValue(0, "metallb_bfd_session_up", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+
+						err = validateCounterValue(1, "metallb_bfd_session_down_events", map[string]string{"peer": addr}, speakerMetrics)
+						if err != nil {
+							return err
+						}
+					}
+					return nil
+				}, 2*time.Minute, 1*time.Second).Should(BeNil())
+			}
+		},
+			table.Entry("IPV4 - default",
+				bfdProfile{
+					Name: "bar",
+				}, "ipv4", []string{v4PoolAddresses}),
+			table.Entry("IPV4 - echo mode enabled",
+				bfdProfile{
+					Name:             "echo",
+					ReceiveInterval:  uint32Ptr(80),
+					TransmitInterval: uint32Ptr(81),
+					EchoInterval:     uint32Ptr(82),
+					EchoMode:         boolPtr(true),
+					PassiveMode:      boolPtr(false),
+					MinimumTTL:       uint32Ptr(254),
+				}, "ipv4", []string{v4PoolAddresses}),
+			table.Entry("IPV6 - default",
+				bfdProfile{
+					Name: "bar",
+				}, "ipv6", []string{v6PoolAddresses}),
+			table.Entry("IPV6 - echo mode enabled",
+				bfdProfile{
+					Name:             "echo",
+					ReceiveInterval:  uint32Ptr(80),
+					TransmitInterval: uint32Ptr(81),
+					EchoInterval:     uint32Ptr(82),
+					EchoMode:         boolPtr(true),
+					PassiveMode:      boolPtr(false),
+					MinimumTTL:       uint32Ptr(254),
+				}, "ipv6", []string{v6PoolAddresses}),
+			table.Entry("DUALSTACK - full params",
+				bfdProfile{
+					Name:             "full1",
+					ReceiveInterval:  uint32Ptr(60),
+					TransmitInterval: uint32Ptr(61),
+					EchoInterval:     uint32Ptr(62),
+					EchoMode:         boolPtr(false),
+					PassiveMode:      boolPtr(false),
+					MinimumTTL:       uint32Ptr(254),
+				}, "dual", []string{v4PoolAddresses, v6PoolAddresses}),
+		)
 	})
 
 	ginkgo.Context("validate configuration changes", func() {
diff --git a/frr-metrics/collector/bfd_test.go b/frr-metrics/collector/bfd_test.go
new file mode 100644
index 00000000..f45f784d
--- /dev/null
+++ b/frr-metrics/collector/bfd_test.go
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier:Apache-2.0
+
+package collector
+
+import (
+	"bytes"
+	"testing"
+	"text/template"
+
+	"github.com/go-kit/kit/log"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+var (
+	bfdMetricsTmpl = `
+	# HELP metallb_bfd_session_up BFD session state (1 is up, 0 is down)
+	# TYPE metallb_bfd_session_up gauge
+	metallb_bfd_session_up{peer="{{ .Peer }}"} {{ .SessionUp }}
+	# HELP metallb_bfd_control_packet_input Number of received BFD control packets
+	# TYPE metallb_bfd_control_packet_input counter
+	metallb_bfd_control_packet_input{peer="{{ .Peer }}"} {{ .ControlPacketInput }}
+	# HELP metallb_bfd_control_packet_output Number of sent BFD control packets
+	# TYPE metallb_bfd_control_packet_output counter
+	metallb_bfd_control_packet_output{peer="{{ .Peer }}"} {{ .ControlPacketOutput }}
+	# HELP metallb_bfd_echo_packet_input Number of received BFD echo packets
+	# TYPE metallb_bfd_echo_packet_input counter
+	metallb_bfd_echo_packet_input{peer="{{ .Peer }}"} {{ .EchoPacketInput }}
+	# HELP metallb_bfd_echo_packet_output Number of sent BFD echo packets
+	# TYPE metallb_bfd_echo_packet_output counter
+	metallb_bfd_echo_packet_output{peer="{{ .Peer }}"} {{ .EchoPacketOutput }}
+	# HELP metallb_bfd_session_down_events Number of BFD session down events
+	# TYPE metallb_bfd_session_down_events counter
+	metallb_bfd_session_down_events{peer="{{ .Peer }}"} {{ .SessionDownEvents }}
+	# HELP metallb_bfd_session_up_events Number of BFD session up events
+	# TYPE metallb_bfd_session_up_events counter
+	metallb_bfd_session_up_events{peer="{{ .Peer }}"} {{ .SessionUpEvents }}
+	# HELP metallb_bfd_zebra_notifications Number of BFD zebra notifications
+	# TYPE metallb_bfd_zebra_notifications counter
+	metallb_bfd_zebra_notifications{peer="{{ .Peer }}"} {{ .ZebraNotifications }}
+	`
+
+	bfdTests = []struct {
+		desc                     string
+		vtyshPeersOutput         string
+		vtyshPeersCountersOutput string
+		peer                     string
+		sessionUp                int
+		controlPacketInput       int
+		controlPacketOutput      int
+		echoPacketInput          int
+		echoPacketOutput         int
+		sessionUpEvents          int
+		sessionDownEvents        int
+		zebraNotifications       int
+	}{
+		{
+			desc:                     "Output contains IPv4",
+			vtyshPeersOutput:         peersIPv4,
+			vtyshPeersCountersOutput: peersCountersIPv4,
+			peer:                     "172.18.0.4",
+			sessionUp:                1,
+			controlPacketInput:       5,
+			controlPacketOutput:      5,
+			echoPacketInput:          0,
+			echoPacketOutput:         0,
+			sessionUpEvents:          1,
+			sessionDownEvents:        0,
+			zebraNotifications:       4,
+		},
+		{
+			desc:                     "Output contains IPv6",
+			vtyshPeersOutput:         peersIPv6,
+			vtyshPeersCountersOutput: peersCountersIPv6,
+			peer:                     "fc00:f853:ccd:e793::4",
+			sessionUp:                0,
+			controlPacketInput:       10,
+			controlPacketOutput:      10,
+			echoPacketInput:          0,
+			echoPacketOutput:         0,
+			sessionUpEvents:          1,
+			sessionDownEvents:        0,
+			zebraNotifications:       4,
+		},
+	}
+	peersIPv4 = `
+	[
+		{
+			"multihop":false,
+			"peer":"172.18.0.4",
+			"vrf":"default",
+			"interface":"eth0",
+			"id":2508913041,
+			"remote-id":3444899611,
+			"passive-mode":false,
+			"status":"up",
+			"uptime":13,
+			"diagnostic":"ok",
+			"remote-diagnostic":"ok",
+			"receive-interval":300,
+			"transmit-interval":300,
+			"echo-interval":0,
+			"detect-multiplier":3,
+			"remote-receive-interval":300,
+			"remote-transmit-interval":300,
+			"remote-echo-interval":50,
+			"remote-detect-multiplier":3
+		}
+	]
+	`
+	peersIPv6 = `
+	[
+		{
+			"multihop":false,
+			"peer":"fc00:f853:ccd:e793::4",
+			"local":"fc00:f853:ccd:e793::6",
+			"vrf":"default",
+			"interface":"eth0",
+			"id":1975516641,
+			"remote-id":505304921,
+			"passive-mode":false,
+			"status":"down",
+			"uptime":33,
+			"diagnostic":"ok",
+			"remote-diagnostic":"ok",
+			"receive-interval":300,
+			"transmit-interval":300,
+			"echo-interval":0,
+			"detect-multiplier":3,
+			"remote-receive-interval":300,
+			"remote-transmit-interval":300,
+			"remote-echo-interval":50,
+			"remote-detect-multiplier":3
+		}
+	]
+	`
+	peersCountersIPv4 = `
+	[
+		{
+			"multihop":false,
+			"peer":"172.18.0.4",
+			"vrf":"default",
+			"interface":"eth0",
+			"control-packet-input":5,
+			"control-packet-output":5,
+			"echo-packet-input":0,
+			"echo-packet-output":0,
+			"session-up":1,
+			"session-down":0,
+			"zebra-notifications":4
+		}
+	]
+	`
+	peersCountersIPv6 = `
+	[
+		{
+			"multihop":false,
+			"peer":"fc00:f853:ccd:e793::4",
+			"local":"fc00:f853:ccd:e793::6",
+			"vrf":"default",
+			"interface":"eth0",
+			"control-packet-input":10,
+			"control-packet-output":10,
+			"echo-packet-input":0,
+			"echo-packet-output":0,
+			"session-up":1,
+			"session-down":0,
+			"zebra-notifications":4
+		}
+	]
+	`
+)
+
+func TestBFDCollect(t *testing.T) {
+	for _, test := range bfdTests {
+		t.Run(test.desc, func(t *testing.T) {
+			tmpl, err := template.New(test.desc).Parse(bfdMetricsTmpl)
+			if err != nil {
+				t.Errorf("expected no error but got %s", err)
+			}
+
+			var w bytes.Buffer
+			err = tmpl.Execute(&w, map[string]interface{}{
+				"Peer":                test.peer,
+				"SessionUp":           test.sessionUp,
+				"ControlPacketInput":  test.controlPacketInput,
+				"ControlPacketOutput": test.controlPacketOutput,
+				"EchoPacketInput":     test.echoPacketInput,
+				"EchoPacketOutput":    test.echoPacketOutput,
+				"SessionUpEvents":     test.sessionUpEvents,
+				"SessionDownEvents":   test.sessionDownEvents,
+				"ZebraNotifications":  test.zebraNotifications,
+			})
+
+			if err != nil {
+				t.Errorf("expected no error but got %s", err)
+			}
+
+			l := log.NewNopLogger()
+			collector := NewBFD(l)
+			collector.frrCli = func(args ...string) (string, error) {
+				if args[0] == "show bfd peers json" {
+					return test.vtyshPeersOutput, nil
+				}
+				return test.vtyshPeersCountersOutput, nil
+			}
+			buf := bytes.NewReader(w.Bytes())
+			err = testutil.CollectAndCompare(collector, buf)
+			if err != nil {
+				t.Errorf("expected no error but got %s", err)
+			}
+		})
+
+	}
+}