Repository: prometheus/blackbox_exporter
Pull request #659 (2020-07-01 07:50:19)

Log http version string we compare with, rather than float we put in metric.
See #658

Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index e47cc11..e21b05c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -13,7 +13,7 @@ jobs:
     # Whenever the Go version is updated here, .travis.yml and .promu.yml
     # should also be updated.
     environment:
-      DOCKER_TEST_IMAGE_NAME: quay.io/prometheus/golang-builder:1.14-base
+      DOCKER_TEST_IMAGE_NAME: quay.io/prometheus/golang-builder:1.15-base
 
     steps:
     - checkout
diff --git a/.promu.yml b/.promu.yml
index 4d28c06..396c1a3 100644
--- a/.promu.yml
+++ b/.promu.yml
@@ -1,7 +1,7 @@
 go:
     # Whenever the Go version is updated here, .travis.yml and
     # .circle/config.yml should also be updated.
-    version: 1.14
+    version: 1.15
 repository:
     path: github.com/prometheus/blackbox_exporter
 build:
@@ -22,7 +22,6 @@ crossbuild:
         - linux/amd64
         - linux/386
         - darwin/amd64
-        - darwin/386
         - windows/amd64
         - windows/386
         - freebsd/amd64
diff --git a/.travis.yml b/.travis.yml
index 71e2bbd..c5d12c4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ language: go
 # Whenever the Go version is updated here, .circleci/config.yml and .promu.yml
 # should also be updated.
 go:
-- 1.13.x
+- 1.15.x
 
 go_import_path: github.com/prometheus/blackbox_exporter
 
diff --git a/README.md b/README.md
index 1fb8f3b..9323f81 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,7 @@ HTTP, HTTPS (via the `http` prober), DNS, TCP socket and ICMP (see permissions s
 Additional modules can be defined to meet your needs.
 
 The timeout of each probe is automatically determined from the `scrape_timeout` in the [Prometheus config](https://prometheus.io/docs/operating/configuration/#configuration-file), slightly reduced to allow for network delays. 
-This can be further limited by the `timeout` in the Blackbox exporter config file. If neither is specified, it defaults to 10 seconds.
+This can be further limited by the `timeout` in the Blackbox exporter config file. If neither is specified, it defaults to 120 seconds.
 
 ## Prometheus Configuration
 
diff --git a/prober/dns.go b/prober/dns.go
index 5ecb6f0..4b3aeac 100644
--- a/prober/dns.go
+++ b/prober/dns.go
@@ -126,6 +126,10 @@ func validRcode(rcode int, valid []string, logger log.Logger) bool {
 
 func ProbeDNS(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger log.Logger) bool {
 	var dialProtocol string
+	probeDNSDurationGaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "probe_dns_duration_seconds",
+		Help: "Duration of DNS request by phase",
+	}, []string{"phase"})
 	probeDNSAnswerRRSGauge := prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "probe_dns_answer_rrs",
 		Help: "Returns number of entries in the answer resource record list",
@@ -138,6 +142,12 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry
 		Name: "probe_dns_additional_rrs",
 		Help: "Returns number of entries in the additional resource record list",
 	})
+
+	for _, lv := range []string{"resolve", "connect", "request"} {
+		probeDNSDurationGaugeVec.WithLabelValues(lv)
+	}
+
+	registry.MustRegister(probeDNSDurationGaugeVec)
 	registry.MustRegister(probeDNSAnswerRRSGauge)
 	registry.MustRegister(probeDNSAuthorityRRSGauge)
 	registry.MustRegister(probeDNSAdditionalRRSGauge)
@@ -182,11 +192,12 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry
 		}
 		targetAddr = target
 	}
-	ip, _, err = chooseProtocol(ctx, module.DNS.IPProtocol, module.DNS.IPProtocolFallback, targetAddr, registry, logger)
+	ip, lookupTime, err := chooseProtocol(ctx, module.DNS.IPProtocol, module.DNS.IPProtocolFallback, targetAddr, registry, logger)
 	if err != nil {
 		level.Error(logger).Log("msg", "Error resolving address", "err", err)
 		return false
 	}
+	probeDNSDurationGaugeVec.WithLabelValues("resolve").Add(lookupTime)
 	targetIP := net.JoinHostPort(ip.String(), port)
 
 	if ip.IP.To4() == nil {
@@ -246,7 +257,14 @@ func ProbeDNS(ctx context.Context, target string, module config.Module, registry
 	level.Info(logger).Log("msg", "Making DNS query", "target", targetIP, "dial_protocol", dialProtocol, "query", module.DNS.QueryName, "type", qt, "class", qc)
 	timeoutDeadline, _ := ctx.Deadline()
 	client.Timeout = time.Until(timeoutDeadline)
-	response, _, err := client.Exchange(msg, targetIP)
+	requestStart := time.Now()
+	response, rtt, err := client.Exchange(msg, targetIP)
+	// The rtt value returned from client.Exchange includes only the time to
+	// exchange messages with the server _after_ the connection is created.
+	// We compute the connection time as the total time for the operation
+	// minus the time for the actual request rtt.
+	probeDNSDurationGaugeVec.WithLabelValues("connect").Set((time.Since(requestStart) - rtt).Seconds())
+	probeDNSDurationGaugeVec.WithLabelValues("request").Set(rtt.Seconds())
 	if err != nil {
 		level.Error(logger).Log("msg", "Error while sending a DNS query", "err", err)
 		return false
diff --git a/prober/http.go b/prober/http.go
index 7fa6c9c..125b508 100644
--- a/prober/http.go
+++ b/prober/http.go
@@ -226,6 +226,19 @@ func (t *transport) GotFirstResponseByte() {
 	t.current.responseStart = time.Now()
 }
 
+// byteCounter implements an io.ReadCloser that keeps track of the total
+// number of bytes it has read.
+type byteCounter struct {
+	io.ReadCloser
+	n int64
+}
+
+func (bc *byteCounter) Read(p []byte) (int, error) {
+	n, err := bc.ReadCloser.Read(p)
+	bc.n += int64(n)
+	return n, err
+}
+
 func ProbeHTTP(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger log.Logger) (success bool) {
 	var redirects int
 	var (
@@ -266,6 +279,14 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			Help: "Returns last SSL chain expiry in timestamp seconds",
 		})
 
+		probeSSLLastInformation = prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "probe_ssl_last_chain_info",
+				Help: "Contains SSL leaf certificate information",
+			},
+			[]string{"fingerprint_sha256"},
+		)
+
 		probeTLSVersion = prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "probe_tls_version_info",
@@ -450,8 +471,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			}
 		}
 
+		byteCounter := &byteCounter{ReadCloser: resp.Body}
+
 		if success && (len(httpConfig.FailIfBodyMatchesRegexp) > 0 || len(httpConfig.FailIfBodyNotMatchesRegexp) > 0) {
-			success = matchRegularExpressions(resp.Body, httpConfig, logger)
+			success = matchRegularExpressions(byteCounter, httpConfig, logger)
 			if success {
 				probeFailedDueToRegex.Set(0)
 			} else {
@@ -460,13 +483,19 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 		}
 
 		if resp != nil && !requestErrored {
-			respBodyBytes, err = io.Copy(ioutil.Discard, resp.Body)
+			_, err = io.Copy(ioutil.Discard, byteCounter)
 			if err != nil {
 				level.Info(logger).Log("msg", "Failed to read HTTP response body", "err", err)
 				success = false
 			}
 
-			resp.Body.Close()
+			respBodyBytes = byteCounter.n
+
+			if err := byteCounter.Close(); err != nil {
+				// We have already read everything we could from the server. The error here might be a
+				// TCP error. Log it in case it contains useful information as to what's the problem.
+				level.Info(logger).Log("msg", "Error while closing response from server", "error", err.Error())
+			}
 		}
 
 		// At this point body is fully read and we can write end time.
@@ -494,7 +523,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 				}
 			}
 			if !found {
-				level.Error(logger).Log("msg", "Invalid HTTP version number", "version", httpVersionNumber)
+				level.Error(logger).Log("msg", "Invalid HTTP version number", "version", resp.Proto)
 				success = false
 			}
 		}
@@ -549,10 +578,11 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 
 	if resp.TLS != nil {
 		isSSLGauge.Set(float64(1))
-		registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
+		registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds, probeSSLLastInformation)
 		probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(resp.TLS).Unix()))
 		probeTLSVersion.WithLabelValues(getTLSVersion(resp.TLS)).Set(1)
 		probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(resp.TLS).Unix()))
+		probeSSLLastInformation.WithLabelValues(getFingerprint(resp.TLS)).Set(1)
 		if httpConfig.FailIfSSL {
 			level.Error(logger).Log("msg", "Final request was over SSL")
 			success = false
diff --git a/prober/icmp.go b/prober/icmp.go
index 1a5668e..1e3410a 100644
--- a/prober/icmp.go
+++ b/prober/icmp.go
@@ -100,7 +100,7 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 	setupStart := time.Now()
 	level.Info(logger).Log("msg", "Creating socket")
 
-	unprivileged := false
+	privileged := true
 	// Unprivileged sockets are supported on Darwin and Linux only.
 	tryUnprivileged := runtime.GOOS == "darwin" || runtime.GOOS == "linux"
 
@@ -119,11 +119,11 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 			if err != nil {
 				level.Debug(logger).Log("msg", "Unable to do unprivileged listen on socket, will attempt privileged", "err", err)
 			} else {
-				unprivileged = true
+				privileged = false
 			}
 		}
 
-		if !unprivileged {
+		if privileged {
 			icmpConn, err = icmp.ListenPacket("ip6:ipv6-icmp", srcIP.String())
 			if err != nil {
 				level.Error(logger).Log("msg", "Error listening to socket", "err", err)
@@ -140,27 +140,15 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 			srcIP = net.ParseIP("0.0.0.0")
 		}
 
-		var icmpConn *icmp.PacketConn
-		// If the user has set the don't fragment option we cannot use unprivileged
-		// sockets as it is not possible to set IP header level options.
-		if tryUnprivileged && !module.ICMP.DontFragment {
-			icmpConn, err = icmp.ListenPacket("udp4", srcIP.String())
-			if err != nil {
-				level.Debug(logger).Log("msg", "Unable to do unprivileged listen on socket, will attempt privileged", "err", err)
-			} else {
-				unprivileged = true
-			}
-		}
-
-		if !unprivileged {
-			icmpConn, err = icmp.ListenPacket("ip4:icmp", srcIP.String())
+		if module.ICMP.DontFragment {
+			// If the user has set the don't fragment option we cannot use unprivileged
+			// sockets as it is not possible to set IP header level options.
+			icmpConn, err := net.ListenPacket("ip4:icmp", srcIP.String())
 			if err != nil {
 				level.Error(logger).Log("msg", "Error listening to socket", "err", err)
 				return
 			}
-		}
 
-		if module.ICMP.DontFragment {
 			rc, err := ipv4.NewRawConn(icmpConn)
 			if err != nil {
 				level.Error(logger).Log("msg", "Error creating raw connection", "err", err)
@@ -168,6 +156,25 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 			}
 			socket = &v4Conn{c: rc, df: true}
 		} else {
+			var icmpConn *icmp.PacketConn
+
+			if tryUnprivileged {
+				icmpConn, err = icmp.ListenPacket("udp4", srcIP.String())
+				if err != nil {
+					level.Debug(logger).Log("msg", "Unable to do unprivileged listen on socket, will attempt privileged", "err", err)
+				} else {
+					privileged = false
+				}
+			}
+
+			if privileged {
+				icmpConn, err = icmp.ListenPacket("ip4:icmp", srcIP.String())
+				if err != nil {
+					level.Error(logger).Log("msg", "Error listening to socket", "err", err)
+					return
+				}
+			}
+
 			socket = icmpConn
 		}
 	}
@@ -175,7 +182,7 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 	defer socket.Close()
 
 	var dst net.Addr = ip
-	if unprivileged {
+	if !privileged {
 		dst = &net.UDPAddr{IP: ip.IP, Zone: ip.Zone}
 	}
 
@@ -217,7 +224,7 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 	// unprivileged sockets were used and the kernel used its own.
 	wm.Type = replyType
 	// Unprivileged cannot set IDs on Linux.
-	idUnknown := unprivileged && runtime.GOOS == "linux"
+	idUnknown := !privileged && runtime.GOOS == "linux"
 	if idUnknown {
 		body.ID = 0
 	}
diff --git a/prober/tcp.go b/prober/tcp.go
index b5f5add..3e8339b 100644
--- a/prober/tcp.go
+++ b/prober/tcp.go
@@ -98,6 +98,13 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 		Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
 		Help: "Returns last SSL chain expiry in unixtime",
 	})
+	probeSSLLastInformation := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "probe_ssl_last_chain_info",
+			Help: "Contains SSL leaf certificate information",
+		},
+		[]string{"fingerprint_sha256"},
+	)
 	probeTLSVersion := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "probe_tls_version_info",
@@ -129,10 +136,11 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 	}
 	if module.TCP.TLS {
 		state := conn.(*tls.Conn).ConnectionState()
-		registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
+		registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds, probeSSLLastInformation)
 		probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
 		probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
 		probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
+		probeSSLLastInformation.WithLabelValues(getFingerprint(&state)).Set(1)
 	}
 	scanner := bufio.NewScanner(conn)
 	for i, qr := range module.TCP.QueryResponse {
@@ -203,6 +211,7 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 			probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
 			probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
 			probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
+			probeSSLLastInformation.WithLabelValues(getFingerprint(&state)).Set(1)
 		}
 	}
 	return true
diff --git a/prober/tls.go b/prober/tls.go
index a4ef0f9..38d5533 100644
--- a/prober/tls.go
+++ b/prober/tls.go
@@ -14,7 +14,9 @@
 package prober
 
 import (
+	"crypto/sha256"
 	"crypto/tls"
+	"encoding/hex"
 	"time"
 )
 
@@ -28,6 +30,12 @@ func getEarliestCertExpiry(state *tls.ConnectionState) time.Time {
 	return earliest
 }
 
+func getFingerprint(state *tls.ConnectionState) string {
+	cert := state.PeerCertificates[0]
+	fingerprint := sha256.Sum256(cert.Raw)
+	return hex.EncodeToString(fingerprint[:])
+}
+
 func getLastChainExpiry(state *tls.ConnectionState) time.Time {
 	lastChainExpiry := time.Time{}
 	for _, chain := range state.VerifiedChains {
@@ -37,7 +45,7 @@ func getLastChainExpiry(state *tls.ConnectionState) time.Time {
 				earliestCertExpiry = cert.NotAfter
 			}
 		}
-		if lastChainExpiry.IsZero() || lastChainExpiry.After(earliestCertExpiry) {
+		if lastChainExpiry.IsZero() || lastChainExpiry.Before(earliestCertExpiry) {
 			lastChainExpiry = earliestCertExpiry
 		}
 


Test patch:
diff --git a/prober/dns_test.go b/prober/dns_test.go
index 75dc7c3..ef2d2b0 100644
--- a/prober/dns_test.go
+++ b/prober/dns_test.go
@@ -570,3 +570,49 @@ func TestDNSProtocol(t *testing.T) {
 
 	}
 }
+
+// TestDNSMetrics checks that calling ProbeDNS populates the expected
+// set of metrics for a DNS probe, but it does not test that those
+// metrics contain specific values.
+func TestDNSMetrics(t *testing.T) {
+	server, addr := startDNSServer("udp", recursiveDNSHandler)
+	defer server.Shutdown()
+
+	_, port, _ := net.SplitHostPort(addr.String())
+
+	module := config.Module{
+		Timeout: time.Second,
+		DNS: config.DNSProbe{
+			IPProtocol:         "ip4",
+			IPProtocolFallback: true,
+			QueryName:          "example.com",
+		},
+	}
+	registry := prometheus.NewRegistry()
+	testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	result := ProbeDNS(testCTX, net.JoinHostPort("localhost", port), module, registry, log.NewNopLogger())
+	if !result {
+		t.Fatalf("DNS test connection failed, expected success.")
+	}
+	mfs, err := registry.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedMetrics := map[string]map[string]map[string]struct{}{
+		"probe_dns_lookup_time_seconds": nil,
+		"probe_dns_duration_seconds": {
+			"phase": {
+				"resolve": {},
+				"connect": {},
+				"request": {},
+			},
+		},
+		"probe_dns_answer_rrs":     nil,
+		"probe_dns_authority_rrs":  nil,
+		"probe_dns_additional_rrs": nil,
+	}
+
+	checkMetrics(expectedMetrics, mfs, t)
+}
diff --git a/prober/http_test.go b/prober/http_test.go
index 3d3f63e..3ebb536 100644
--- a/prober/http_test.go
+++ b/prober/http_test.go
@@ -14,6 +14,7 @@
 package prober
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
@@ -97,6 +98,100 @@ func TestValidHTTPVersion(t *testing.T) {
 	}
 }
 
+func TestContentLength(t *testing.T) {
+	type testdata struct {
+		contentLength          int
+		uncompressedBodyLength int
+		handler                http.HandlerFunc
+		expectFailure          bool
+	}
+
+	testmsg := []byte(strings.Repeat("hello world", 10))
+
+	notfoundMsg := []byte("not found")
+
+	testcases := map[string]testdata{
+		"identity": {
+			contentLength:          len(testmsg),
+			uncompressedBodyLength: len(testmsg),
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Add("Content-Encoding", "identity")
+				w.WriteHeader(http.StatusOK)
+				w.Write(testmsg)
+			},
+		},
+
+		"no content-encoding": {
+			contentLength:          len(testmsg),
+			uncompressedBodyLength: len(testmsg),
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write(testmsg)
+			},
+		},
+
+		// Unknown Content-Encoding, we should let this pass thru.
+		"unknown content-encoding": {
+			contentLength:          len(testmsg),
+			uncompressedBodyLength: len(testmsg),
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Add("Content-Encoding", "xxx")
+				w.WriteHeader(http.StatusOK)
+				w.Write(bytes.Repeat([]byte{'x'}, len(testmsg)))
+			},
+		},
+
+		// 401 response, verify that the content-length is still computed correctly.
+		"401": {
+			expectFailure:          true,
+			contentLength:          len(notfoundMsg),
+			uncompressedBodyLength: len(notfoundMsg),
+			handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusNotFound)
+				// Send something in the body to make sure that this get reported as the content length.
+				w.Write(notfoundMsg)
+			},
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			ts := httptest.NewServer(tc.handler)
+			defer ts.Close()
+
+			testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			registry := prometheus.NewRegistry()
+			var logbuf bytes.Buffer
+			result := ProbeHTTP(testCTX,
+				ts.URL,
+				config.Module{
+					Timeout: time.Second,
+					HTTP:    config.HTTPProbe{IPProtocolFallback: true},
+				},
+				registry,
+				log.NewLogfmtLogger(&logbuf))
+			if !tc.expectFailure && !result {
+				t.Fatalf("probe failed unexpectedly: %s", logbuf.String())
+			} else if tc.expectFailure && result {
+				t.Fatalf("probe succeeded unexpectedly: %s", logbuf.String())
+			}
+
+			mfs, err := registry.Gather()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			expectedResults := map[string]float64{
+				"probe_http_content_length":           float64(tc.contentLength),
+				"probe_http_uncompressed_body_length": float64(tc.uncompressedBodyLength),
+			}
+			checkRegistryResults(expectedResults, mfs, t)
+		})
+	}
+}
+
 func TestRedirectFollowed(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
@@ -237,80 +332,70 @@ func TestFailIfNotSSL(t *testing.T) {
 }
 
 func TestFailIfBodyMatchesRegexp(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "Bad news: could not connect to database server")
-	}))
-	defer ts.Close()
-
-	recorder := httptest.NewRecorder()
-	registry := prometheus.NewRegistry()
-	testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	result := ProbeHTTP(testCTX, ts.URL,
-		config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyMatchesRegexp: []string{"could not connect to database"}}}, registry, log.NewNopLogger())
-	body := recorder.Body.String()
-	if result {
-		t.Fatalf("Regexp test succeeded unexpectedly, got %s", body)
-	}
-	mfs, err := registry.Gather()
-	if err != nil {
-		t.Fatal(err)
-	}
-	expectedResults := map[string]float64{
-		"probe_failed_due_to_regex": 1,
-	}
-	checkRegistryResults(expectedResults, mfs, t)
-
-	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "Download the latest version here")
-	}))
-	defer ts.Close()
+	testcases := map[string]struct {
+		respBody       string
+		regexps        []string
+		expectedResult bool
+	}{
+		"one regex, match": {
+			respBody:       "Bad news: could not connect to database server",
+			regexps:        []string{"could not connect to database"},
+			expectedResult: false,
+		},
 
-	recorder = httptest.NewRecorder()
-	registry = prometheus.NewRegistry()
-	result = ProbeHTTP(testCTX, ts.URL,
-		config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyMatchesRegexp: []string{"could not connect to database"}}}, registry, log.NewNopLogger())
-	body = recorder.Body.String()
-	if !result {
-		t.Fatalf("Regexp test failed unexpectedly, got %s", body)
-	}
-	mfs, err = registry.Gather()
-	if err != nil {
-		t.Fatal(err)
-	}
-	expectedResults = map[string]float64{
-		"probe_failed_due_to_regex": 0,
-	}
-	checkRegistryResults(expectedResults, mfs, t)
+		"one regex, no match": {
+			respBody:       "Download the latest version here",
+			regexps:        []string{"could not connect to database"},
+			expectedResult: true,
+		},
 
-	// With multiple regexps configured, verify that any matching regexp causes
-	// the probe to fail, but probes succeed when no regexp matches.
-	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "internal error")
-	}))
-	defer ts.Close()
+		"multiple regexes, match": {
+			respBody:       "internal error",
+			regexps:        []string{"could not connect to database", "internal error"},
+			expectedResult: false,
+		},
 
-	recorder = httptest.NewRecorder()
-	registry = prometheus.NewRegistry()
-	result = ProbeHTTP(testCTX, ts.URL,
-		config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyMatchesRegexp: []string{"could not connect to database", "internal error"}}}, registry, log.NewNopLogger())
-	body = recorder.Body.String()
-	if result {
-		t.Fatalf("Regexp test succeeded unexpectedly, got %s", body)
+		"multiple regexes, no match": {
+			respBody:       "hello world",
+			regexps:        []string{"could not connect to database", "internal error"},
+			expectedResult: true,
+		},
 	}
 
-	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "hello world")
-	}))
-	defer ts.Close()
-
-	recorder = httptest.NewRecorder()
-	registry = prometheus.NewRegistry()
-	result = ProbeHTTP(testCTX, ts.URL,
-		config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyMatchesRegexp: []string{"could not connect to database", "internal error"}}}, registry, log.NewNopLogger())
-	body = recorder.Body.String()
-	if !result {
-		t.Fatalf("Regexp test failed unexpectedly, got %s", body)
+	for name, testcase := range testcases {
+		t.Run(name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				fmt.Fprintf(w, testcase.respBody)
+			}))
+			defer ts.Close()
+
+			recorder := httptest.NewRecorder()
+			registry := prometheus.NewRegistry()
+			testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			result := ProbeHTTP(testCTX, ts.URL, config.Module{Timeout: time.Second, HTTP: config.HTTPProbe{IPProtocolFallback: true, FailIfBodyMatchesRegexp: testcase.regexps}}, registry, log.NewNopLogger())
+			if testcase.expectedResult && !result {
+				t.Fatalf("Regexp test failed unexpectedly, got %s", recorder.Body.String())
+			} else if !testcase.expectedResult && result {
+				t.Fatalf("Regexp test succeeded unexpectedly, got %s", recorder.Body.String())
+			}
+			mfs, err := registry.Gather()
+			if err != nil {
+				t.Fatal(err)
+			}
+			boolToFloat := func(v bool) float64 {
+				if v {
+					return 1
+				}
+				return 0
+			}
+			expectedResults := map[string]float64{
+				"probe_failed_due_to_regex":           boolToFloat(!testcase.expectedResult),
+				"probe_http_content_length":           float64(len(testcase.respBody)), // Issue #673: check that this is correctly populated when using regex validations.
+				"probe_http_uncompressed_body_length": float64(len(testcase.respBody)), // Issue #673, see above.
+			}
+			checkRegistryResults(expectedResults, mfs, t)
+		})
 	}
 }
 
@@ -702,42 +787,19 @@ func TestHTTPPhases(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	found := false
-	foundLabel := map[string]bool{
-		"connect":    false,
-		"processing": false,
-		"resolve":    false,
-		"transfer":   false,
-		"tls":        false,
-	}
-	for _, mf := range mfs {
-		if mf.GetName() == "probe_http_duration_seconds" {
-			found = true
-			for _, metric := range mf.GetMetric() {
-				for _, lp := range metric.Label {
-					if lp.GetName() == "phase" {
-						f, ok := foundLabel[lp.GetValue()]
-						if !ok {
-							t.Fatalf("Unexpected label phase=%s", lp.GetValue())
-						}
-						if f {
-							t.Fatalf("Label phase=%s duplicated", lp.GetValue())
-						}
-						foundLabel[lp.GetValue()] = true
-					}
-				}
-			}
-
-		}
-	}
-	if !found {
-		t.Fatal("probe_http_duration_seconds not found")
-	}
-	for lv, found := range foundLabel {
-		if !found {
-			t.Fatalf("Label phase=%s not found", lv)
-		}
+	expectedMetrics := map[string]map[string]map[string]struct{}{
+		"probe_http_duration_seconds": {
+			"phase": {
+				"connect":    {},
+				"processing": {},
+				"resolve":    {},
+				"transfer":   {},
+				"tls":        {},
+			},
+		},
 	}
+
+	checkMetrics(expectedMetrics, mfs, t)
 }
 
 func TestCookieJar(t *testing.T) {
diff --git a/prober/tcp_test.go b/prober/tcp_test.go
index d05f317..6b4fdb3 100644
--- a/prober/tcp_test.go
+++ b/prober/tcp_test.go
@@ -196,6 +196,7 @@ func TestTCPConnectionWithTLS(t *testing.T) {
 	// Check values
 	expectedResults := map[string]float64{
 		"probe_ssl_earliest_cert_expiry": float64(certExpiry.Unix()),
+		"probe_ssl_last_chain_info":      1,
 		"probe_tls_version_info":         1,
 	}
 	checkRegistryResults(expectedResults, mfs, t)
@@ -216,33 +217,34 @@ func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) {
 	testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	// From here prepare two certificate chains where one is expired
+	// From here prepare two certificate chains where one expires before the
+	// other
 
 	rootPrivatekey, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
 		panic(fmt.Sprintf("Error creating rsa key: %s", err))
 	}
 
-	rootCertExpiry := time.Now().AddDate(0, 0, 2)
+	rootCertExpiry := time.Now().AddDate(0, 0, 3)
 	rootCertTmpl := generateCertificateTemplate(rootCertExpiry, false)
 	rootCertTmpl.IsCA = true
 	_, rootCertPem := generateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivatekey)
 
-	oldRootCertExpiry := time.Now().AddDate(0, 0, -1)
-	expiredRootCertTmpl := generateCertificateTemplate(oldRootCertExpiry, false)
-	expiredRootCertTmpl.IsCA = true
-	expiredRootCert, expiredRootCertPem := generateSelfSignedCertificateWithPrivateKey(expiredRootCertTmpl, rootPrivatekey)
+	olderRootCertExpiry := time.Now().AddDate(0, 0, 1)
+	olderRootCertTmpl := generateCertificateTemplate(olderRootCertExpiry, false)
+	olderRootCertTmpl.IsCA = true
+	olderRootCert, olderRootCertPem := generateSelfSignedCertificateWithPrivateKey(olderRootCertTmpl, rootPrivatekey)
 
-	serverCertExpiry := time.Now().AddDate(0, 0, 1)
+	serverCertExpiry := time.Now().AddDate(0, 0, 2)
 	serverCertTmpl := generateCertificateTemplate(serverCertExpiry, false)
-	_, serverCertPem, serverKey := generateSignedCertificate(serverCertTmpl, expiredRootCert, rootPrivatekey)
+	_, serverCertPem, serverKey := generateSignedCertificate(serverCertTmpl, olderRootCert, rootPrivatekey)
 
 	// CAFile must be passed via filesystem, use a tempfile.
 	tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
 	if err != nil {
 		t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
 	}
-	if _, err := tmpCaFile.Write(bytes.Join([][]byte{rootCertPem, expiredRootCertPem}, []byte("\n"))); err != nil {
+	if _, err := tmpCaFile.Write(bytes.Join([][]byte{rootCertPem, olderRootCertPem}, []byte("\n"))); err != nil {
 		t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
 	}
 	if err := tmpCaFile.Close(); err != nil {
@@ -262,7 +264,8 @@ func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) {
 
 		serverKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)})
 
-		keypair, err := tls.X509KeyPair(serverCertPem, serverKeyPem)
+		// Include the older root cert in the chain
+		keypair, err := tls.X509KeyPair(append(serverCertPem, olderRootCertPem...), serverKeyPem)
 		if err != nil {
 			panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
 		}
@@ -315,8 +318,9 @@ func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) {
 
 	// Check values
 	expectedResults := map[string]float64{
-		"probe_ssl_earliest_cert_expiry":                float64(serverCertExpiry.Unix()),
+		"probe_ssl_earliest_cert_expiry":                float64(olderRootCertExpiry.Unix()),
 		"probe_ssl_last_chain_expiry_timestamp_seconds": float64(serverCertExpiry.Unix()),
+		"probe_ssl_last_chain_info":                     1,
 		"probe_tls_version_info":                        1,
 	}
 	checkRegistryResults(expectedResults, mfs, t)
diff --git a/prober/utils_test.go b/prober/utils_test.go
index 2ac38ee..a53ee49 100644
--- a/prober/utils_test.go
+++ b/prober/utils_test.go
@@ -171,3 +171,83 @@ func TestChooseProtocol(t *testing.T) {
 		t.Error("without fallback it should not answer")
 	}
 }
+
+func checkMetrics(expected map[string]map[string]map[string]struct{}, mfs []*dto.MetricFamily, t *testing.T) {
+	type (
+		valueValidation struct {
+			found bool
+		}
+		labelValidation struct {
+			found  bool
+			values map[string]valueValidation
+		}
+		metricValidation struct {
+			found  bool
+			labels map[string]labelValidation
+		}
+	)
+
+	foundMetrics := map[string]metricValidation{}
+
+	for mname, labels := range expected {
+		var mv metricValidation
+		if labels != nil {
+			mv.labels = map[string]labelValidation{}
+			for lname, values := range labels {
+				var lv labelValidation
+				if values != nil {
+					lv.values = map[string]valueValidation{}
+					for vname, _ := range values {
+						lv.values[vname] = valueValidation{}
+					}
+				}
+				mv.labels[lname] = lv
+			}
+		}
+		foundMetrics[mname] = mv
+	}
+
+	for _, mf := range mfs {
+		info, wanted := foundMetrics[mf.GetName()]
+		if !wanted {
+			continue
+		}
+		info.found = true
+		for _, metric := range mf.GetMetric() {
+			if info.labels == nil {
+				continue
+			}
+			for _, lp := range metric.Label {
+				if label, labelWanted := info.labels[lp.GetName()]; labelWanted {
+					label.found = true
+					if label.values != nil {
+						if value, wanted := label.values[lp.GetValue()]; !wanted {
+							t.Fatalf("Unexpected label %s=%s", lp.GetName(), lp.GetValue())
+						} else if value.found {
+							t.Fatalf("Label %s=%s duplicated", lp.GetName(), lp.GetValue())
+						}
+						label.values[lp.GetValue()] = valueValidation{found: true}
+					}
+					info.labels[lp.GetName()] = label
+				}
+			}
+		}
+		foundMetrics[mf.GetName()] = info
+	}
+
+	for mname, m := range foundMetrics {
+		if !m.found {
+			t.Fatalf("metric %s wanted, not found", mname)
+		}
+		for lname, label := range m.labels {
+			if !label.found {
+				t.Fatalf("metric %s, label %s wanted, not found", mname, lname)
+			}
+			for vname, value := range label.values {
+				if !value.found {
+					t.Fatalf("metric %s, label %s, value %s wanted, not found", mname, lname, vname)
+				}
+			}
+		}
+	}
+}