Repository: metallb/metallb
Pull request #943 (2021-09-10 14:55:16)

Extend BGP test to check routes and session state
Currently, BGP test only verifies that the remote peer can hit the service. We don't have guarantee that all the ip is advertised from all the nodes and that the BGP sessions are established.

Here we add helper packages to check both, providing a base for adding more testcases.

The frr parsing utilities are in a package of their own, and may be useful if parsing the FRR output is needed to provide status feedback or metrics.



fedepaol: cc @markdgray @oribon  @sabinaaledort , please take a look
fedepaol: (hopefully CI doesn't fail :D )
fedepaol: > @fedepaol this one has a conflict now

Rebased, thanks!

Patch:
diff --git a/go.mod b/go.mod
index a753269e..bcd25f97 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.0
 	github.com/osrg/gobgp v2.0.0+incompatible
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.7.1
 	github.com/spf13/viper v1.8.1 // indirect
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007


Test patch:
diff --git a/e2etest/bgp_test.go b/e2etest/bgp_test.go
index a9b68c60..04d1be04 100644
--- a/e2etest/bgp_test.go
+++ b/e2etest/bgp_test.go
@@ -26,12 +26,17 @@ import (
 	"time"
 
 	"github.com/onsi/ginkgo"
+	"go.universe.tf/metallb/e2etest/pkg/executor"
+	"go.universe.tf/metallb/e2etest/pkg/frr"
+	"go.universe.tf/metallb/e2etest/pkg/routes"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
 )
 
+const frrContainer = "frr"
+
 var _ = ginkgo.Describe("BGP", func() {
 	f := framework.NewDefaultFramework("bgp")
 	var loadBalancerCreateTimeout time.Duration
@@ -73,15 +78,36 @@ var _ = ginkgo.Describe("BGP", func() {
 		hostport := net.JoinHostPort(ingressIP, port)
 		address := fmt.Sprintf("http://%s/", hostport)
 
-		useDocker := true
+		exc := executor.ForContainer(frrContainer)
 		if skipDockerCmd {
 			ginkgo.By(fmt.Sprintf("checking connectivity to %s", address))
-			useDocker = false
+			exc = executor.Host
 		} else {
 			ginkgo.By(fmt.Sprintf("checking connectivity to %s with docker", address))
 		}
-		err = wgetRetry(useDocker, address)
+
+		err = wgetRetry(address, exc)
+		framework.ExpectNoError(err)
+
+		allNodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+		framework.ExpectNoError(err)
+
+		advertised := routes.ForIP(ingressIP, exc)
+		err = routes.MatchNodes(allNodes.Items, advertised)
 		framework.ExpectNoError(err)
-	})
 
+		neighbors, err := frr.NeighborsInfo(exc)
+		framework.ExpectNoError(err)
+
+		err = frr.NeighborsMatchNodes(allNodes.Items, neighbors)
+		framework.ExpectNoError(err)
+
+		frrRoutes, _, err := frr.Routes(exc)
+		framework.ExpectNoError(err)
+
+		routes, ok := frrRoutes[ingressIP]
+		framework.ExpectEqual(ok, true, ingressIP, "not found in frr routes")
+		err = frr.RoutesMatchNodes(allNodes.Items, routes)
+		framework.ExpectNoError(err)
+	})
 })
diff --git a/e2etest/layer2_test.go b/e2etest/layer2_test.go
index efd5c8ad..66c551e5 100644
--- a/e2etest/layer2_test.go
+++ b/e2etest/layer2_test.go
@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/onsi/ginkgo"
+	"go.universe.tf/metallb/e2etest/pkg/executor"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -73,7 +74,7 @@ var _ = ginkgo.Describe("L2", func() {
 
 		hostport := net.JoinHostPort(ingressIP, port)
 		address := fmt.Sprintf("http://%s/", hostport)
-		err = wgetRetry(false, address)
+		err = wgetRetry(address, executor.Host)
 		framework.ExpectNoError(err)
 	})
 
diff --git a/e2etest/pkg/executor/executor.go b/e2etest/pkg/executor/executor.go
new file mode 100644
index 00000000..f8b3ee0d
--- /dev/null
+++ b/e2etest/pkg/executor/executor.go
@@ -0,0 +1,30 @@
+package executor
+
+import "os/exec"
+
+type Executor interface {
+	Exec(cmd string, args ...string) (string, error)
+}
+
+type hostExecutor struct{}
+
+var Host hostExecutor
+
+func (hostExecutor) Exec(cmd string, args ...string) (string, error) {
+	out, err := exec.Command(cmd, args...).CombinedOutput()
+	return string(out), err
+}
+
+func ForContainer(containerName string) Executor {
+	return &containerExecutor{container: containerName}
+}
+
+type containerExecutor struct {
+	container string
+}
+
+func (e *containerExecutor) Exec(cmd string, args ...string) (string, error) {
+	newArgs := append([]string{"exec", e.container, cmd}, args...)
+	out, err := exec.Command("docker", newArgs...).CombinedOutput()
+	return string(out), err
+}
diff --git a/e2etest/pkg/frr/bgp.go b/e2etest/pkg/frr/bgp.go
new file mode 100644
index 00000000..8c4695fa
--- /dev/null
+++ b/e2etest/pkg/frr/bgp.go
@@ -0,0 +1,63 @@
+package frr
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"go.universe.tf/metallb/e2etest/pkg/executor"
+)
+
+// TODO: Leaving this package "test unaware" on purpose, since we may find it
+// useful for fetching informations from FRR (such as metrics) and we may need to move it
+// to metallb.
+
+// NeighborForContainer returns informations for the given neighbor in the given
+// executor.
+func NeighborInfo(neighborName, exec executor.Executor) (*Neighbor, error) {
+	res, err := exec.Exec("vtysh", "-c", fmt.Sprintf("show bgp neighbor %s json", neighborName))
+
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed to query neighbour %s", neighborName)
+	}
+	neighbor, err := parseNeighbour(res)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed to parse neighbour %s", neighborName)
+	}
+	return neighbor, nil
+}
+
+// NeighborsForContainer returns informations for the all the neighbors in the given
+// executor.
+func NeighborsInfo(exec executor.Executor) ([]*Neighbor, error) {
+	res, err := exec.Exec("vtysh", "-c", "show bgp neighbor json")
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed to query neighbours")
+	}
+	neighbors, err := parseNeighbours(res)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed to parse neighbours %s", res)
+	}
+	return neighbors, nil
+}
+
+// Routes returns informations about routes in the given executor
+// first for ipv4 routes and then for ipv6 routes.
+func Routes(exec executor.Executor) (map[string]Route, map[string]Route, error) {
+	res, err := exec.Exec("vtysh", "-c", "show bgp ipv4 json")
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "Failed to query routes")
+	}
+	v4Routes, err := parseRoutes(res)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "Failed to parse routes %s", res)
+	}
+	res, err = exec.Exec("vtysh", "-c", "show bgp ipv6 json")
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "Failed to query routes")
+	}
+	v6Routes, err := parseRoutes(res)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "Failed to parse routes %s", res)
+	}
+	return v4Routes, v6Routes, nil
+}
diff --git a/e2etest/pkg/frr/parse.go b/e2etest/pkg/frr/parse.go
new file mode 100644
index 00000000..b4aa7a36
--- /dev/null
+++ b/e2etest/pkg/frr/parse.go
@@ -0,0 +1,139 @@
+package frr
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+type Neighbor struct {
+	ip        net.IP
+	connected bool
+	localAS   string
+	remoteAS  string
+}
+
+type Route struct {
+	Destination *net.IPNet
+	NextHops    []net.IP
+}
+
+const bgpConnected = "Established"
+
+type FRRNeighbor struct {
+	RemoteAs   int    `json:"remoteAs"`
+	LocalAs    int    `json:"localAs"`
+	BgpVersion int    `json:"bgpVersion"`
+	BgpState   string `json:"bgpState"`
+}
+
+type IPInfo struct {
+	Routes map[string][]FRRRoute `json:"routes"`
+}
+
+type FRRRoute struct {
+	Valid    bool   `json:"valid"`
+	PeerID   string `json:"peerId"`
+	Nexthops []struct {
+		IP string `json:"ip"`
+	} `json:"nexthops"`
+}
+
+// parseNeighbour takes the result of a show bgp neighbor x.y.w.z
+// and parses the informations related to the neighbour.
+func parseNeighbour(vtyshRes string) (*Neighbor, error) {
+	res := map[string]FRRNeighbor{}
+	err := json.Unmarshal([]byte(vtyshRes), &res)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse vtysh response")
+	}
+	if len(res) > 1 {
+		return nil, errors.New("more than one peer were returned")
+	}
+	if len(res) == 0 {
+		return nil, errors.New("no peers were returned")
+	}
+	for k, n := range res {
+		ip := net.ParseIP(k)
+		if ip == nil {
+			return nil, fmt.Errorf("failed to parse %s as ip", ip)
+		}
+		connected := true
+		if n.BgpState != bgpConnected {
+			connected = false
+		}
+		return &Neighbor{
+			ip:        ip,
+			connected: connected,
+			localAS:   strconv.Itoa(n.LocalAs),
+			remoteAS:  strconv.Itoa(n.RemoteAs),
+		}, nil
+	}
+	return nil, errors.New("no peers were returned")
+}
+
+// parseNeighbour takes the result of a show bgp neighbor
+// and parses the informations related to all the neighbours.
+func parseNeighbours(vtyshRes string) ([]*Neighbor, error) {
+	toParse := map[string]FRRNeighbor{}
+	err := json.Unmarshal([]byte(vtyshRes), &toParse)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse vtysh response")
+	}
+
+	res := make([]*Neighbor, 0)
+	for k, n := range toParse {
+		ip := net.ParseIP(k)
+		if ip == nil {
+			return nil, fmt.Errorf("failed to parse %s as ip", ip)
+		}
+		connected := true
+		if n.BgpState != bgpConnected {
+			connected = false
+		}
+		res = append(res, &Neighbor{
+			ip:        ip,
+			connected: connected,
+			localAS:   strconv.Itoa(n.LocalAs),
+			remoteAS:  strconv.Itoa(n.RemoteAs),
+		})
+	}
+	return res, nil
+}
+
+// parseRoute takes the result of a show bgp neighbor
+// and parses the informations related to all the neighbours.
+func parseRoutes(vtyshRes string) (map[string]Route, error) {
+	toParse := IPInfo{}
+	err := json.Unmarshal([]byte(vtyshRes), &toParse)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse vtysh response")
+	}
+
+	res := make(map[string]Route)
+	for k, frrRoutes := range toParse.Routes {
+		destIP, dest, err := net.ParseCIDR(k)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse cidr for %s", k)
+		}
+
+		r := Route{
+			Destination: dest,
+			NextHops:    make([]net.IP, 0),
+		}
+		for _, n := range frrRoutes {
+			for _, h := range n.Nexthops {
+				ip := net.ParseIP(h.IP)
+				if ip == nil {
+					return nil, fmt.Errorf("failed to parse ip %s", h.IP)
+				}
+				r.NextHops = append(r.NextHops, ip)
+			}
+		}
+		res[destIP.String()] = r
+	}
+	return res, nil
+}
diff --git a/e2etest/pkg/frr/parse_test.go b/e2etest/pkg/frr/parse_test.go
new file mode 100644
index 00000000..7f4c59f0
--- /dev/null
+++ b/e2etest/pkg/frr/parse_test.go
@@ -0,0 +1,442 @@
+package frr
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"sort"
+	"testing"
+)
+
+func TestNeighbour(t *testing.T) {
+	sample := `{
+    "%s":{
+      "remoteAs":%s,
+      "localAs":%s,
+      "nbrInternalLink":true,
+      "bgpVersion":4,
+      "remoteRouterId":"0.0.0.0",
+      "localRouterId":"172.18.0.5",
+      "bgpState":"%s",
+      "bgpTimerLastRead":253000,
+      "bgpTimerLastWrite":3405000,
+      "bgpInUpdateElapsedTimeMsecs":3405000,
+      "bgpTimerHoldTimeMsecs":180000,
+      "bgpTimerKeepAliveIntervalMsecs":60000,
+      "gracefulRestartInfo":{
+        "endOfRibSend":{
+        },
+        "endOfRibRecv":{
+        },
+        "localGrMode":"Helper*",
+        "remoteGrMode":"NotApplicable",
+        "rBit":false,
+        "timers":{
+          "configuredRestartTimer":120,
+          "receivedRestartTimer":0
+        }
+      },
+      "messageStats":{
+        "depthInq":0,
+        "depthOutq":0,
+        "opensSent":0,
+        "opensRecv":0,
+        "notificationsSent":0,
+        "notificationsRecv":0,
+        "updatesSent":0,
+        "updatesRecv":0,
+        "keepalivesSent":0,
+        "keepalivesRecv":0,
+        "routeRefreshSent":0,
+        "routeRefreshRecv":0,
+        "capabilitySent":0,
+        "capabilityRecv":0,
+        "totalSent":0,
+        "totalRecv":0
+      },
+      "minBtwnAdvertisementRunsTimerMsecs":0,
+      "addressFamilyInfo":{
+        "ipv4Unicast":{
+          "routerAlwaysNextHop":true,
+          "commAttriSentToNbr":"extendedAndStandard",
+          "acceptedPrefixCounter":0
+        }
+      },
+      "connectionsEstablished":0,
+      "connectionsDropped":0,
+      "lastResetTimerMsecs":253000,
+      "lastResetDueTo":"Waiting for peer OPEN",
+      "lastResetCode":32,
+      "connectRetryTimer":120,
+      "nextConnectTimerDueInMsecs":107000,
+      "readThread":"off",
+      "writeThread":"off"
+    }
+  }`
+
+	tests := []struct {
+		name          string
+		neighborIP    string
+		remoteAS      string
+		localAS       string
+		status        string
+		expectedError string
+	}{
+		{
+			"ipv4, connected",
+			"172.18.0.5",
+			"64512",
+			"64512",
+			"Established",
+			"",
+		},
+		{
+			"ipv4, connected",
+			"172.18.0.5",
+			"64512",
+			"64512",
+			"Active",
+			"",
+		},
+		{
+			"ipv6, connected",
+			"2620:52:0:1302::8af5",
+			"64512",
+			"64512",
+			"Established",
+			"",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			n, err := parseNeighbour(fmt.Sprintf(sample, tt.neighborIP, tt.remoteAS, tt.localAS, tt.status))
+			if err != nil {
+				t.Fatal("Failed to parse ", err)
+			}
+			if !n.ip.Equal(net.ParseIP(tt.neighborIP)) {
+				t.Fatal("Expected neighbour ip", tt.neighborIP, "got", n.ip.String())
+			}
+			if n.remoteAS != tt.remoteAS {
+				t.Fatal("Expected remote as", tt.remoteAS, "got", n.remoteAS)
+			}
+			if n.localAS != tt.localAS {
+				t.Fatal("Expected local as", tt.localAS, "got", n.localAS)
+			}
+			if tt.status == "Established" && n.connected != true {
+				t.Fatal("Expected connected", true, "got", n.connected)
+			}
+			if tt.status != "Established" && n.connected == true {
+				t.Fatal("Expected connected", false, "got", n.connected)
+			}
+		})
+	}
+}
+
+const threeNeighbours = `
+{
+  "172.18.0.2":{
+    "remoteAs":64512,
+    "localAs":64512,
+    "nbrInternalLink":true,
+    "bgpVersion":4,
+    "remoteRouterId":"0.0.0.0",
+    "localRouterId":"172.18.0.5",
+    "bgpState":"Active",
+    "bgpTimerLastRead":14000,
+    "bgpTimerLastWrite":3166000,
+    "bgpInUpdateElapsedTimeMsecs":3166000,
+    "bgpTimerHoldTimeMsecs":180000,
+    "bgpTimerKeepAliveIntervalMsecs":60000,
+    "gracefulRestartInfo":{
+      "endOfRibSend":{
+      },
+      "endOfRibRecv":{
+      },
+      "localGrMode":"Helper*",
+      "remoteGrMode":"NotApplicable",
+      "rBit":false,
+      "timers":{
+        "configuredRestartTimer":120,
+        "receivedRestartTimer":0
+      }
+    },
+    "messageStats":{
+      "depthInq":0,
+      "depthOutq":0,
+      "opensSent":0,
+      "opensRecv":0,
+      "notificationsSent":0,
+      "notificationsRecv":0,
+      "updatesSent":0,
+      "updatesRecv":0,
+      "keepalivesSent":0,
+      "keepalivesRecv":0,
+      "routeRefreshSent":0,
+      "routeRefreshRecv":0,
+      "capabilitySent":0,
+      "capabilityRecv":0,
+      "totalSent":0,
+      "totalRecv":0
+    },
+    "minBtwnAdvertisementRunsTimerMsecs":0,
+    "addressFamilyInfo":{
+      "ipv4Unicast":{
+        "routerAlwaysNextHop":true,
+        "commAttriSentToNbr":"extendedAndStandard",
+        "acceptedPrefixCounter":0
+      }
+    },
+    "connectionsEstablished":0,
+    "connectionsDropped":0,
+    "lastResetTimerMsecs":14000,
+    "lastResetDueTo":"Waiting for peer OPEN",
+    "lastResetCode":32,
+    "connectRetryTimer":120,
+    "nextConnectTimerDueInMsecs":107000,
+    "readThread":"off",
+    "writeThread":"off"
+  },
+  "172.18.0.3":{
+    "remoteAs":64512,
+    "localAs":64512,
+    "nbrInternalLink":true,
+    "bgpVersion":4,
+    "remoteRouterId":"0.0.0.0",
+    "localRouterId":"172.18.0.5",
+    "bgpState":"Active",
+    "bgpTimerLastRead":14000,
+    "bgpTimerLastWrite":3166000,
+    "bgpInUpdateElapsedTimeMsecs":3166000,
+    "bgpTimerHoldTimeMsecs":180000,
+    "bgpTimerKeepAliveIntervalMsecs":60000,
+    "gracefulRestartInfo":{
+      "endOfRibSend":{
+      },
+      "endOfRibRecv":{
+      },
+      "localGrMode":"Helper*",
+      "remoteGrMode":"NotApplicable",
+      "rBit":false,
+      "timers":{
+        "configuredRestartTimer":120,
+        "receivedRestartTimer":0
+      }
+    },
+    "messageStats":{
+      "depthInq":0,
+      "depthOutq":0,
+      "opensSent":0,
+      "opensRecv":0,
+      "notificationsSent":0,
+      "notificationsRecv":0,
+      "updatesSent":0,
+      "updatesRecv":0,
+      "keepalivesSent":0,
+      "keepalivesRecv":0,
+      "routeRefreshSent":0,
+      "routeRefreshRecv":0,
+      "capabilitySent":0,
+      "capabilityRecv":0,
+      "totalSent":0,
+      "totalRecv":0
+    },
+    "minBtwnAdvertisementRunsTimerMsecs":0,
+    "addressFamilyInfo":{
+      "ipv4Unicast":{
+        "routerAlwaysNextHop":true,
+        "commAttriSentToNbr":"extendedAndStandard",
+        "acceptedPrefixCounter":0
+      }
+    },
+    "connectionsEstablished":0,
+    "connectionsDropped":0,
+    "lastResetTimerMsecs":14000,
+    "lastResetDueTo":"Waiting for peer OPEN",
+    "lastResetCode":32,
+    "connectRetryTimer":120,
+    "nextConnectTimerDueInMsecs":107000,
+    "readThread":"off",
+    "writeThread":"off"
+  },
+  "172.18.0.4":{
+    "remoteAs":64512,
+    "localAs":64512,
+    "nbrInternalLink":true,
+    "bgpVersion":4,
+    "remoteRouterId":"0.0.0.0",
+    "localRouterId":"172.18.0.5",
+    "bgpState":"Active",
+    "bgpTimerLastRead":14000,
+    "bgpTimerLastWrite":3166000,
+    "bgpInUpdateElapsedTimeMsecs":3166000,
+    "bgpTimerHoldTimeMsecs":180000,
+    "bgpTimerKeepAliveIntervalMsecs":60000,
+    "gracefulRestartInfo":{
+      "endOfRibSend":{
+      },
+      "endOfRibRecv":{
+      },
+      "localGrMode":"Helper*",
+      "remoteGrMode":"NotApplicable",
+      "rBit":false,
+      "timers":{
+        "configuredRestartTimer":120,
+        "receivedRestartTimer":0
+      }
+    },
+    "messageStats":{
+      "depthInq":0,
+      "depthOutq":0,
+      "opensSent":0,
+      "opensRecv":0,
+      "notificationsSent":0,
+      "notificationsRecv":0,
+      "updatesSent":0,
+      "updatesRecv":0,
+      "keepalivesSent":0,
+      "keepalivesRecv":0,
+      "routeRefreshSent":0,
+      "routeRefreshRecv":0,
+      "capabilitySent":0,
+      "capabilityRecv":0,
+      "totalSent":0,
+      "totalRecv":0
+    },
+    "minBtwnAdvertisementRunsTimerMsecs":0,
+    "addressFamilyInfo":{
+      "ipv4Unicast":{
+        "routerAlwaysNextHop":true,
+        "commAttriSentToNbr":"extendedAndStandard",
+        "acceptedPrefixCounter":0
+      }
+    },
+    "connectionsEstablished":0,
+    "connectionsDropped":0,
+    "lastResetTimerMsecs":14000,
+    "lastResetDueTo":"Waiting for peer OPEN",
+    "lastResetCode":32,
+    "connectRetryTimer":120,
+    "nextConnectTimerDueInMsecs":107000,
+    "readThread":"off",
+    "writeThread":"off"
+  }
+}`
+
+func TestNeighbours(t *testing.T) {
+	nn, err := parseNeighbours(threeNeighbours)
+	if err != nil {
+		t.Fatalf("Failed to parse %s", err)
+	}
+	if len(nn) != 3 {
+		t.Fatalf("Expected 3 neighbours, got %d", len(nn))
+	}
+	if !nn[0].ip.Equal(net.ParseIP("172.18.0.2")) {
+		t.Fatal("neighbour ip not matching")
+	}
+	if !nn[1].ip.Equal(net.ParseIP("172.18.0.3")) {
+		t.Fatal("neighbour ip not matching")
+	}
+	if !nn[2].ip.Equal(net.ParseIP("172.18.0.4")) {
+		t.Fatal("neighbour ip not matching")
+	}
+}
+
+const routes = `{
+  "vrfId": 0,
+  "vrfName": "default",
+  "tableVersion": 7,
+  "routerId": "172.18.0.5",
+  "defaultLocPrf": 100,
+  "localAS": 64512,
+  "routes": { "192.168.10.0/32": [
+   {
+     "valid":true,
+     "multipath":true,
+     "pathFrom":"internal",
+     "prefix":"192.168.10.0",
+     "prefixLen":32,
+     "network":"192.168.10.0\/32",
+     "locPrf":0,
+     "weight":0,
+     "peerId":"172.18.0.4",
+     "path":"",
+     "origin":"incomplete",
+     "nexthops":[
+       {
+         "ip":"172.18.0.4",
+         "afi":"ipv4",
+         "used":true
+       }
+     ]
+   },
+   {
+     "valid":true,
+     "bestpath":true,
+     "pathFrom":"internal",
+     "prefix":"192.168.10.0",
+     "prefixLen":32,
+     "network":"192.168.10.0\/32",
+     "locPrf":0,
+     "weight":0,
+     "peerId":"172.18.0.2",
+     "path":"",
+     "origin":"incomplete",
+     "nexthops":[
+       {
+         "ip":"172.18.0.2",
+         "afi":"ipv4",
+         "used":true
+       }
+     ]
+   },
+   {
+     "valid":true,
+     "multipath":true,
+     "pathFrom":"internal",
+     "prefix":"192.168.10.0",
+     "prefixLen":32,
+     "network":"192.168.10.0\/32",
+     "locPrf":0,
+     "weight":0,
+     "peerId":"172.18.0.3",
+     "path":"",
+     "origin":"incomplete",
+     "nexthops":[
+       {
+         "ip":"172.18.0.3",
+         "afi":"ipv4",
+         "used":true
+       }
+     ]
+   }
+ ] }  }`
+
+func TestRoutes(t *testing.T) {
+	rr, err := parseRoutes(routes)
+	if err != nil {
+		t.Fatalf("Failed to parse %s", err)
+	}
+
+	ipRoutes, ok := rr["192.168.10.0"]
+	if !ok {
+		t.Fatalf("Routes for 192.168.10.0/32 not found")
+	}
+
+	ips := make([]net.IP, 0)
+	ips = append(ips, ipRoutes.NextHops...)
+
+	sort.Slice(ips, func(i, j int) bool {
+		return (bytes.Compare(ips[i], ips[j]) < 0)
+	})
+	fmt.Println(ips)
+	if !ips[0].Equal(net.ParseIP("172.18.0.2")) {
+		t.Fatal("neighbour ip not matching")
+	}
+	if !ips[1].Equal(net.ParseIP("172.18.0.3")) {
+		t.Fatal("neighbour ip not matching")
+	}
+	if !ips[2].Equal(net.ParseIP("172.18.0.4")) {
+		t.Fatal("neighbour ip not matching")
+	}
+}
diff --git a/e2etest/pkg/frr/validation.go b/e2etest/pkg/frr/validation.go
new file mode 100644
index 00000000..6df5ec4b
--- /dev/null
+++ b/e2etest/pkg/frr/validation.go
@@ -0,0 +1,61 @@
+package frr
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+// NeighborsMatchNodes tells if ALL the given nodes are peered with the
+// frr instance. We only care about established connections, as the
+// frr instance may be configured with more nodes than are currently
+// paired.
+func NeighborsMatchNodes(nodes []v1.Node, neighbors []*Neighbor) error {
+	nodesIPs := map[string]struct{}{}
+
+	for _, n := range nodes {
+		for _, a := range n.Status.Addresses {
+			if a.Type == v1.NodeInternalIP {
+				nodesIPs[a.Address] = struct{}{}
+			}
+		}
+	}
+	for _, n := range neighbors {
+		if _, ok := nodesIPs[n.ip.String()]; !ok { // skipping neighbors that are not nodes
+			continue
+		}
+		if !n.connected {
+			return fmt.Errorf("node %s BGP session not established", n.ip.String())
+		}
+		delete(nodesIPs, n.ip.String())
+	}
+	if len(nodesIPs) != 0 { // some leftover, meaning more nodes than neighbors
+		return fmt.Errorf("IP %v found in nodes but not in neighbors", nodesIPs)
+	}
+	return nil
+}
+
+// RoutesMatchNodes tells if ALL the given nodes are exposed as
+// destinations for the given address.
+func RoutesMatchNodes(nodes []v1.Node, route Route) error {
+	nodesIPs := map[string]struct{}{}
+
+	for _, n := range nodes {
+		for _, a := range n.Status.Addresses {
+			if a.Type == v1.NodeInternalIP {
+				nodesIPs[a.Address] = struct{}{}
+			}
+		}
+	}
+	for _, h := range route.NextHops {
+		if _, ok := nodesIPs[h.String()]; !ok { // skipping neighbors that are not nodes
+			return fmt.Errorf("%s not found in nodes ips", h.String())
+		}
+
+		delete(nodesIPs, h.String())
+	}
+	if len(nodesIPs) != 0 { // some leftover, meaning more nodes than routes
+		return fmt.Errorf("IP %v found in nodes but not in next hops", nodesIPs)
+	}
+	return nil
+}
diff --git a/e2etest/pkg/routes/routes.go b/e2etest/pkg/routes/routes.go
new file mode 100644
index 00000000..ed9ed8a9
--- /dev/null
+++ b/e2etest/pkg/routes/routes.go
@@ -0,0 +1,78 @@
+package routes
+
+import (
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+
+	"go.universe.tf/metallb/e2etest/pkg/executor"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var (
+	ipv4Re *regexp.Regexp
+	ipv6Re *regexp.Regexp
+)
+
+func init() {
+	ipv4Re = regexp.MustCompile(`(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|)){4})`)
+	ipv6Re = regexp.MustCompile(`(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))`)
+}
+
+// For IP returns the list of routes in the given container
+// (or in the current host) to reach the service ip.
+func ForIP(target string, exec executor.Executor) []net.IP {
+	res, err := exec.Exec("ip", []string{"route", "show", target}...)
+	framework.ExpectNoError(err)
+
+	routes := make([]net.IP, 0)
+
+	dst := net.ParseIP(target)
+	framework.ExpectNotEqual(dst, nil, "Failed to convert", target, "to ip")
+
+	re := ipv4Re
+	if dst.To4() == nil { // assuming it's an ipv6 address
+		re = ipv6Re
+	}
+	rows := strings.Split(res, "\n")
+	for _, r := range rows {
+		if !strings.Contains(r, "nexthop via") {
+			continue
+		}
+		ip := re.FindString(r)
+		if ip == "" {
+			continue
+		}
+		netIP := net.ParseIP(ip)
+		framework.ExpectNotEqual(netIP, nil, "Failed to convert", ip, "to ip")
+		routes = append(routes, netIP)
+	}
+
+	return routes
+}
+
+// MatchNodes tells whether the given list of destination ips
+// matches the expected list of nodes.
+func MatchNodes(nodes []v1.Node, ips []net.IP) error {
+	nodesIPs := map[string]struct{}{}
+
+	for _, n := range nodes {
+		for _, a := range n.Status.Addresses {
+			if a.Type == v1.NodeInternalIP {
+				nodesIPs[a.Address] = struct{}{}
+			}
+		}
+	}
+	for _, ip := range ips {
+		if _, ok := nodesIPs[ip.String()]; !ok {
+			return fmt.Errorf("IP %s found in routes but not in nodes", ip.String())
+		}
+		delete(nodesIPs, ip.String())
+	}
+	if len(nodesIPs) != 0 { // some leftover, meaning more nodes than routes
+		return fmt.Errorf("IP %v found in nodes but not in routes", nodesIPs)
+	}
+	return nil
+}
diff --git a/e2etest/utils.go b/e2etest/utils.go
index 1945a371..34a88223 100644
--- a/e2etest/utils.go
+++ b/e2etest/utils.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os/exec"
 
+	"go.universe.tf/metallb/e2etest/pkg/executor"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -22,25 +23,14 @@ func DescribeSvc(ns string) {
 	framework.Logf(desc)
 }
 
-func runCommand(bgp bool, name string, args ...string) (string, error) {
-	if bgp {
-		// prepend "docker exec frr"
-		cmd := []string{"exec", "frr", name}
-		name = "docker"
-		args = append(cmd, args...)
-	}
-	out, err := exec.Command(name, args...).CombinedOutput()
-	return string(out), err
-}
-
-func wgetRetry(bgp bool, address string) error {
+func wgetRetry(address string, exc executor.Executor) error {
 	retrycnt := 0
 	code := 0
 	var err error
 
 	// Retry loop to handle wget NetworkFailure errors
 	for {
-		_, err = runCommand(bgp, "wget", "-O-", "-q", address, "-T", "60")
+		_, err = exc.Exec("wget", "-O-", "-q", address, "-T", "60")
 		if exitErr, ok := err.(*exec.ExitError); err != nil && ok {
 			code = exitErr.ExitCode()
 		} else {