Repository: metallb/metallb
Pull request #1334 (2022-04-21 12:16:08)

E2E: add a test failure reporter
Triaging test failures / flakes is hard.

With this commit we:
- Dump all the frr related information in a file, named after the
container or the pod, in a folder named after the failed test.

- Add a kubernetes reporter that dumps all the relevant information, such
as crds, services, nodes in the same path.

Closes https://github.com/metallb/metallb/issues/1200

liornoy: Besides 2 comments this PR looks good and is really a long-awaited addition
that will make our developing work on MetalLB much easier!
liornoy: lgtm. Also tried it on my env to see the output and it's very comprehensive!

Patch:
diff --git a/go.mod b/go.mod
index 9bef7098..adb37840 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
+	github.com/openshift-kni/k8sreporter v0.1.0
 	github.com/ory/dockertest/v3 v3.8.1
 	github.com/osrg/gobgp v2.0.0+incompatible
 	github.com/pkg/errors v0.9.1
@@ -60,6 +61,7 @@ require (
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
@@ -94,6 +96,8 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/opencontainers/runc v1.0.2 // indirect
+	github.com/openshift/api v0.0.0-20200829102639-8a3a835f1acf // indirect
+	github.com/openshift/client-go v0.0.0-20200827190008-3062137373b5 // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
diff --git a/go.sum b/go.sum
index 148b07e7..cbab5abe 100644
--- a/go.sum
+++ b/go.sum
@@ -265,6 +265,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -428,6 +429,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -551,6 +553,14 @@ github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/openshift-kni/k8sreporter v0.1.0 h1:EbmAn7QAtBV+wu/JI5j9Jh2HRv/Lrsz8WJIEQ9PXPUs=
+github.com/openshift-kni/k8sreporter v0.1.0/go.mod h1:Bc075FJEln4XadhV+fPRNnow1bbpFQgOqTlwX6q/oLI=
+github.com/openshift/api v0.0.0-20200827090112-c05698d102cf/go.mod h1:M3xexPhgM8DISzzRpuFUy+jfPjQPIcs9yqEYj17mXV8=
+github.com/openshift/api v0.0.0-20200829102639-8a3a835f1acf h1:KP/v5AGCaq1Sbe6QrlFGZM1fHBswiHuniMTC4/hgbVw=
+github.com/openshift/api v0.0.0-20200829102639-8a3a835f1acf/go.mod h1:M3xexPhgM8DISzzRpuFUy+jfPjQPIcs9yqEYj17mXV8=
+github.com/openshift/build-machinery-go v0.0.0-20200819073603-48aa266c95f7/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
+github.com/openshift/client-go v0.0.0-20200827190008-3062137373b5 h1:E6WhVL5p3rfjtc+o+jVG/29Aclnf3XIF7akxXvadwR0=
+github.com/openshift/client-go v0.0.0-20200827190008-3062137373b5/go.mod h1:5rGmrkQ8DJEUXA+AR3rEjfH+HFyg4/apY9iCQFgvPfE=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/ory/dockertest/v3 v3.8.1 h1:vU/8d1We4qIad2YM0kOwRVtnyue7ExvacPiw1yDm17g=
 github.com/ory/dockertest/v3 v3.8.1/go.mod h1:wSRQ3wmkz+uSARYMk7kVJFDBGm8x5gSxIhI7NDc+BAQ=
@@ -1051,6 +1061,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
@@ -1269,6 +1280,7 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
@@ -1307,6 +1319,7 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25 h1:DEQ12ZRxJjsglk5JIi5bLgpKaHihGervKmg5uryaEHw=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
+sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
 sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA=
 sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
@@ -1317,6 +1330,7 @@ sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLja
 sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
diff --git a/tasks.py b/tasks.py
index 746f2300..f9a48b6d 100644
--- a/tasks.py
+++ b/tasks.py
@@ -744,8 +744,16 @@ def e2etest(ctx, name="kind", export=None, kubeconfig=None, system_namespaces="k
         ipv6_service_range, ips = get_available_ips(6)
         ips_for_containers_v6 = "--ips-for-containers-v6=" + ips
 
+    if export != None:
+        report_path = export
+    else:
+        report_path = "/tmp/metallbreport{}".format(time.time())
+
+    print("Writing reports to {}".format(report_path))
+    os.makedirs(report_path, exist_ok=True)
+
     testrun = run("cd `git rev-parse --show-toplevel`/e2etest &&"
-            "KUBECONFIG={} go test -timeout 2h {} {} --provider=local --kubeconfig={} --service-pod-port={} {} {} -ipv4-service-range={} -ipv6-service-range={} {}".format(kubeconfig, ginkgo_focus, ginkgo_skip, kubeconfig, service_pod_port, ips_for_containers_v4, ips_for_containers_v6, ipv4_service_range, ipv6_service_range, opt_skip_docker), warn="True")
+            "KUBECONFIG={} go test -timeout 2h {} {} --provider=local --kubeconfig={} --service-pod-port={} {} {} -ipv4-service-range={} -ipv6-service-range={} {} --report-path {}".format(kubeconfig, ginkgo_focus, ginkgo_skip, kubeconfig, service_pod_port, ips_for_containers_v4, ips_for_containers_v6, ipv4_service_range, ipv6_service_range, opt_skip_docker, report_path), warn="True")
 
     if export != None:
         run("kind export logs {}".format(export))


Test patch:
diff --git a/e2etest/bgptests/bgp.go b/e2etest/bgptests/bgp.go
index f785593a..7a6eb3e3 100644
--- a/e2etest/bgptests/bgp.go
+++ b/e2etest/bgptests/bgp.go
@@ -38,6 +38,7 @@ import (
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	"github.com/openshift-kni/k8sreporter"
 
 	"go.universe.tf/metallb/e2etest/pkg/frr"
 	frrconfig "go.universe.tf/metallb/e2etest/pkg/frr/config"
@@ -60,7 +61,11 @@ const (
 	SpeakerContainerName  = "speaker"
 )
 
-var ConfigUpdater config.Updater
+var (
+	ConfigUpdater config.Updater
+	Reporter      *k8sreporter.KubernetesReporter
+	ReportPath    string
+)
 
 var _ = ginkgo.Describe("BGP", func() {
 	var cs clientset.Interface
@@ -84,7 +89,8 @@ var _ = ginkgo.Describe("BGP", func() {
 
 	ginkgo.AfterEach(func() {
 		if ginkgo.CurrentGinkgoTestDescription().Failed {
-			dumpBGPInfo(cs, f)
+			dumpBGPInfo(ReportPath, ginkgo.CurrentGinkgoTestDescription().TestText, cs, f)
+			k8s.DumpInfo(Reporter, ginkgo.CurrentGinkgoTestDescription().TestText)
 		}
 	})
 
diff --git a/e2etest/bgptests/bgppeer_selector.go b/e2etest/bgptests/bgppeer_selector.go
index dce1d08b..1d8277df 100644
--- a/e2etest/bgptests/bgppeer_selector.go
+++ b/e2etest/bgptests/bgppeer_selector.go
@@ -10,6 +10,7 @@ import (
 
 	metallbv1beta1 "go.universe.tf/metallb/api/v1beta1"
 	"go.universe.tf/metallb/api/v1beta2"
+	"go.universe.tf/metallb/e2etest/pkg/k8s"
 	"go.universe.tf/metallb/e2etest/pkg/metallb"
 	testservice "go.universe.tf/metallb/e2etest/pkg/service"
 	metallbconfig "go.universe.tf/metallb/internal/config"
@@ -33,7 +34,8 @@ var _ = ginkgo.Describe("BGP Peer Selector", func() {
 
 	ginkgo.AfterEach(func() {
 		if ginkgo.CurrentGinkgoTestDescription().Failed {
-			dumpBGPInfo(cs, f)
+			dumpBGPInfo(ReportPath, ginkgo.CurrentGinkgoTestDescription().TestText, cs, f)
+			k8s.DumpInfo(Reporter, ginkgo.CurrentGinkgoTestDescription().TestText)
 		}
 	})
 
diff --git a/e2etest/bgptests/dump.go b/e2etest/bgptests/dump.go
index da73b7e0..b08ea9e9 100644
--- a/e2etest/bgptests/dump.go
+++ b/e2etest/bgptests/dump.go
@@ -3,7 +3,11 @@
 package bgptests
 
 import (
+	"errors"
 	"fmt"
+	"os"
+	"path"
+	"strings"
 
 	"go.universe.tf/metallb/e2etest/pkg/executor"
 	"go.universe.tf/metallb/e2etest/pkg/frr"
@@ -13,15 +17,31 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
-func dumpBGPInfo(cs clientset.Interface, f *framework.Framework) {
+func dumpBGPInfo(basePath, testName string, cs clientset.Interface, f *framework.Framework) {
+	testPath := path.Join(basePath, strings.Replace(testName, " ", "-", -1))
+	err := os.Mkdir(testPath, 0755)
+	if err != nil && !errors.Is(err, os.ErrExist) {
+		fmt.Fprintf(os.Stderr, "failed to create test dir: %v\n", err)
+		return
+	}
+
 	for _, c := range FRRContainers {
-		address := c.Ipv4
-		if address == "" {
-			address = c.Ipv6
-		}
-		peerAddr := address + fmt.Sprintf(":%d", c.RouterConfig.BGPPort)
 		dump, err := frr.RawDump(c, "/etc/frr/bgpd.conf", "/tmp/frr.log", "/etc/frr/daemons")
-		framework.Logf("External frr dump for %s:%s\n%s\nerrors:%v", c.Name, peerAddr, dump, err)
+		if err != nil {
+			framework.Logf("External frr dump for container %s failed %v", c.Name, err)
+			continue
+		}
+		f, err := logFileFor(testPath, fmt.Sprintf("frrdump-%s", c.Name))
+		if err != nil {
+			framework.Logf("External frr dump for container %s, failed to open file %v", c.Name, err)
+			continue
+		}
+		fmt.Fprintf(f, "Dumping information for %s, local addresses: ipv4 - %s, ipv6 - %s\n", c.Name, c.Ipv4, c.Ipv6)
+		_, err = fmt.Fprint(f, dump)
+		if err != nil {
+			framework.Logf("External frr dump for container %s, failed to write to file %v", c.Name, err)
+			continue
+		}
 	}
 
 	speakerPods, err := metallb.SpeakerPods(cs)
@@ -32,7 +52,30 @@ func dumpBGPInfo(cs clientset.Interface, f *framework.Framework) {
 		}
 		podExec := executor.ForPod(pod.Namespace, pod.Name, "frr")
 		dump, err := frr.RawDump(podExec, "/etc/frr/frr.conf", "/etc/frr/frr.log")
-		framework.Logf("External frr dump for pod %s\n%s %v", pod.Name, dump, err)
+		if err != nil {
+			framework.Logf("External frr dump for pod %s failed %v", pod.Name, err)
+			continue
+		}
+		f, err := logFileFor(testPath, fmt.Sprintf("frrdump-%s", pod.Name))
+		if err != nil {
+			framework.Logf("External frr dump for pod %s, failed to open file %v", pod.Name, err)
+			continue
+		}
+		fmt.Fprintf(f, "Dumping information for %s, local addresses: %s\n", pod.Name, pod.Status.PodIPs)
+		_, err = fmt.Fprint(f, dump)
+		if err != nil {
+			framework.Logf("External frr dump for pod %s, failed to write to file %v", pod.Name, err)
+			continue
+		}
 	}
 	k8s.DescribeSvc(f.Namespace.Name)
 }
+
+func logFileFor(base string, kind string) (*os.File, error) {
+	path := path.Join(base, kind) + ".log"
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
diff --git a/e2etest/bgptests/node_selector.go b/e2etest/bgptests/node_selector.go
index 8e7185ad..93b7fcf4 100644
--- a/e2etest/bgptests/node_selector.go
+++ b/e2etest/bgptests/node_selector.go
@@ -39,7 +39,8 @@ var _ = ginkgo.Describe("BGP Node Selector", func() {
 		}
 
 		if ginkgo.CurrentGinkgoTestDescription().Failed {
-			dumpBGPInfo(cs, f)
+			dumpBGPInfo(ReportPath, ginkgo.CurrentGinkgoTestDescription().TestText, cs, f)
+			k8s.DumpInfo(Reporter, ginkgo.CurrentGinkgoTestDescription().TestText)
 		}
 	})
 
diff --git a/e2etest/e2etest_suite_test.go b/e2etest/e2etest_suite_test.go
index fde1818a..846dffe3 100644
--- a/e2etest/e2etest_suite_test.go
+++ b/e2etest/e2etest_suite_test.go
@@ -14,7 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-// https://github.com/ovn-org/ovn-kubernetes/blob/a99cc892576be4f15caceca62a87557572e0a447/test/e2e/e2e_suite_test.go
 
 package e2e
 
@@ -33,6 +32,7 @@ import (
 	"go.universe.tf/metallb/e2etest/bgptests"
 	"go.universe.tf/metallb/e2etest/l2tests"
 	testsconfig "go.universe.tf/metallb/e2etest/pkg/config"
+	"go.universe.tf/metallb/e2etest/pkg/k8s"
 	"go.universe.tf/metallb/e2etest/pkg/metallb"
 	"go.universe.tf/metallb/e2etest/pkg/service"
 	"go.universe.tf/metallb/e2etest/webhookstests"
@@ -48,6 +48,7 @@ var (
 	ipv4ForContainers string
 	ipv6ForContainers string
 	useOperator       bool
+	reportPath        string
 )
 
 // handleFlags sets up all flags and parses the command line.
@@ -71,6 +72,7 @@ func handleFlags() {
 	flag.StringVar(&l2tests.IPV4ServiceRange, "ipv4-service-range", "0", "a range of IPv4 addresses for MetalLB to use when running in layer2 mode")
 	flag.StringVar(&l2tests.IPV6ServiceRange, "ipv6-service-range", "0", "a range of IPv6 addresses for MetalLB to use when running in layer2 mode")
 	flag.BoolVar(&useOperator, "use-operator", false, "set this to true to run the tests using operator custom resources")
+	flag.StringVar(&reportPath, "report-path", "/tmp/report", "the path to be used to dump test failure information")
 	flag.Parse()
 }
 
@@ -131,9 +133,15 @@ var _ = ginkgo.BeforeSuite(func() {
 	updater, err := testsconfig.UpdaterForCRs(clientconfig, metallb.Namespace)
 	framework.ExpectNoError(err)
 
+	reporter := k8s.InitReporter(framework.TestContext.KubeConfig, reportPath, metallb.Namespace)
+
 	bgptests.ConfigUpdater = updater
 	l2tests.ConfigUpdater = updater
 	webhookstests.ConfigUpdater = updater
+	bgptests.Reporter = reporter
+	bgptests.ReportPath = reportPath
+	l2tests.Reporter = reporter
+	webhookstests.Reporter = reporter
 })
 
 var _ = ginkgo.AfterSuite(func() {
diff --git a/e2etest/l2tests/l2.go b/e2etest/l2tests/l2.go
index 47e8e121..167b2c77 100644
--- a/e2etest/l2tests/l2.go
+++ b/e2etest/l2tests/l2.go
@@ -29,6 +29,7 @@ import (
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/extensions/table"
 	"github.com/onsi/gomega"
+	"github.com/openshift-kni/k8sreporter"
 	metallbv1beta1 "go.universe.tf/metallb/api/v1beta1"
 	"go.universe.tf/metallb/e2etest/pkg/config"
 	"go.universe.tf/metallb/e2etest/pkg/executor"
@@ -51,12 +52,13 @@ import (
 
 var (
 	ConfigUpdater    config.Updater
+	Reporter         *k8sreporter.KubernetesReporter
 	IPV4ServiceRange string
 	IPV6ServiceRange string
 )
 
 var _ = ginkgo.Describe("L2", func() {
-	f := framework.NewDefaultFramework("l2")
+	var f *framework.Framework
 	var loadBalancerCreateTimeout time.Duration
 	var cs clientset.Interface
 
@@ -66,6 +68,18 @@ var _ = ginkgo.Describe("L2", func() {
 		},
 	}
 
+	ginkgo.AfterEach(func() {
+		// Clean previous configuration.
+		err := ConfigUpdater.Clean()
+		framework.ExpectNoError(err)
+
+		if ginkgo.CurrentGinkgoTestDescription().Failed {
+			k8s.DumpInfo(Reporter, ginkgo.CurrentGinkgoTestDescription().TestText)
+		}
+	})
+
+	f = framework.NewDefaultFramework("l2")
+
 	ginkgo.BeforeEach(func() {
 		cs = f.ClientSet
 		loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
@@ -76,16 +90,6 @@ var _ = ginkgo.Describe("L2", func() {
 		framework.ExpectNoError(err)
 	})
 
-	ginkgo.AfterEach(func() {
-		// Clean previous configuration.
-		err := ConfigUpdater.Clean()
-		framework.ExpectNoError(err)
-
-		if ginkgo.CurrentGinkgoTestDescription().Failed {
-			k8s.DescribeSvc(f.Namespace.Name)
-		}
-	})
-
 	ginkgo.Context("type=Loadbalancer", func() {
 		ginkgo.BeforeEach(func() {
 			resources := internalconfig.ClusterResources{
diff --git a/e2etest/l2tests/node_selector.go b/e2etest/l2tests/node_selector.go
index ddd7094f..fde28e52 100644
--- a/e2etest/l2tests/node_selector.go
+++ b/e2etest/l2tests/node_selector.go
@@ -29,31 +29,33 @@ import (
 )
 
 var _ = ginkgo.Describe("L2", func() {
-	f := framework.NewDefaultFramework("l2")
 	var cs clientset.Interface
 	var nodeToLabel *v1.Node
 
-	ginkgo.BeforeEach(func() {
-		cs = f.ClientSet
-
-		ginkgo.By("Clearing any previous configuration")
-
-		err := ConfigUpdater.Clean()
-		framework.ExpectNoError(err)
-	})
-
+	var f *framework.Framework
 	ginkgo.AfterEach(func() {
 		if nodeToLabel != nil {
 			k8s.RemoveLabelFromNode(nodeToLabel.Name, "bgp-node-selector-test", cs)
 		}
 
+		if ginkgo.CurrentGinkgoTestDescription().Failed {
+			k8s.DumpInfo(Reporter, ginkgo.CurrentGinkgoTestDescription().TestText)
+		}
+
 		// Clean previous configuration.
 		err := ConfigUpdater.Clean()
 		framework.ExpectNoError(err)
+	})
 
-		if ginkgo.CurrentGinkgoTestDescription().Failed {
-			k8s.DescribeSvc(f.Namespace.Name)
-		}
+	f = framework.NewDefaultFramework("l2")
+
+	ginkgo.BeforeEach(func() {
+		cs = f.ClientSet
+
+		ginkgo.By("Clearing any previous configuration")
+
+		err := ConfigUpdater.Clean()
+		framework.ExpectNoError(err)
 	})
 
 	ginkgo.Context("Node Selector", func() {
diff --git a/e2etest/pkg/k8s/reporter.go b/e2etest/pkg/k8s/reporter.go
new file mode 100644
index 00000000..b41961e3
--- /dev/null
+++ b/e2etest/pkg/k8s/reporter.go
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier:Apache-2.0
+
+package k8s
+
+import (
+	"log"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/openshift-kni/k8sreporter"
+	metallbv1beta1 "go.universe.tf/metallb/api/v1beta1"
+	metallbv1beta2 "go.universe.tf/metallb/api/v1beta2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func InitReporter(kubeconfig, path, namespace string) *k8sreporter.KubernetesReporter {
+	// When using custom crds, we need to add them to the scheme
+	addToScheme := func(s *runtime.Scheme) error {
+		err := metallbv1beta1.AddToScheme(s)
+		if err != nil {
+			return err
+		}
+		err = metallbv1beta2.AddToScheme(s)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// The namespaces we want to dump resources for (including pods and pod logs)
+	dumpNamespace := func(ns string) bool {
+		switch {
+		case ns == namespace:
+			return true
+		case strings.HasPrefix(ns, "l2"):
+			return true
+		case strings.HasPrefix(ns, "bgp"):
+			return true
+		}
+		return false
+	}
+
+	// The list of CRDs we want to dump
+	crds := []k8sreporter.CRData{
+		{Cr: &metallbv1beta1.IPAddressPoolList{}},
+		{Cr: &metallbv1beta1.AddressPoolList{}},
+		{Cr: &metallbv1beta2.BGPPeerList{}},
+		{Cr: &metallbv1beta1.L2AdvertisementList{}},
+		{Cr: &metallbv1beta1.BGPAdvertisementList{}},
+		{Cr: &metallbv1beta1.BFDProfileList{}},
+		{Cr: &metallbv1beta1.CommunityList{}},
+		{Cr: &corev1.ServiceList{}},
+	}
+
+	reporter, err := k8sreporter.New(kubeconfig, addToScheme, dumpNamespace, path, crds...)
+	if err != nil {
+		log.Fatalf("Failed to initialize the reporter %s", err)
+	}
+	return reporter
+}
+
+func DumpInfo(reporter *k8sreporter.KubernetesReporter, testName string) {
+	testNameNoSpaces := strings.Replace(ginkgo.CurrentGinkgoTestDescription().TestText, " ", "-", -1)
+	reporter.Dump(10*time.Minute, testNameNoSpaces)
+}
diff --git a/e2etest/webhookstests/webhooks.go b/e2etest/webhookstests/webhooks.go
index 07e63f91..86705589 100644
--- a/e2etest/webhookstests/webhooks.go
+++ b/e2etest/webhookstests/webhooks.go
@@ -20,9 +20,11 @@ package webhookstests
 
 import (
 	"go.universe.tf/metallb/e2etest/pkg/config"
+	"go.universe.tf/metallb/e2etest/pkg/k8s"
 
 	"github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/openshift-kni/k8sreporter"
 
 	metallbv1beta1 "go.universe.tf/metallb/api/v1beta1"
 	metallbv1beta2 "go.universe.tf/metallb/api/v1beta2"
@@ -32,7 +34,10 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
-var ConfigUpdater config.Updater
+var (
+	ConfigUpdater config.Updater
+	Reporter      *k8sreporter.KubernetesReporter
+)
 
 var _ = ginkgo.Describe("Webhooks", func() {
 	ginkgo.BeforeEach(func() {
@@ -42,6 +47,10 @@ var _ = ginkgo.Describe("Webhooks", func() {
 	})
 
 	ginkgo.AfterEach(func() {
+		if ginkgo.CurrentGinkgoTestDescription().Failed {
+			k8s.DumpInfo(Reporter, ginkgo.CurrentGinkgoTestDescription().TestText)
+		}
+
 		// Clean previous configuration.
 		err := ConfigUpdater.Clean()
 		framework.ExpectNoError(err)