Repository: ClusterLabs/ha_cluster_exporter
Pull request #51 (2019-10-16 08:16:42)

add remote disk metric
# desc:

1) create a metric like

```
# HELP ha_cluster_drbd_resources_remote_connection show per remote connection resource name, its role, the volume and disk_state (Diskless,Attaching, Failed, Negotiating, Inconsistent, Outdated, DUnknown, Consistent, UpToDate)
# TYPE ha_cluster_drbd_resources_remote_connection gauge
ha_cluster_drbd_resources_remote_connection{peer_disk_state="dunknown",peer_node_id="0",peer_role="Unknown",resource_name="drbd_passive",volume="0"} 1
```

2) fix some error handling things:
 - use warn instead of error
 - other minor refactor and improvements

MalloZup: @stefanotorresi  feel free to review PR https://github.com/ClusterLabs/ha_cluster_exporter/pull/53

I have split the refactor. I'm labeling this PR as blocked because we need a DRBD env. for testing it. I can rebase it then on tthe latest change when time it will come

Patch:
diff --git a/Makefile b/Makefile
index 373f9c1..7378564 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,7 @@ default: build
 
 build: fmt-check  vet-check test
 	go build .
+	go mod tidy
 
 install:
 	go install
diff --git a/corosync_metrics.go b/corosync_metrics.go
index 9d11d98..26a19e6 100644
--- a/corosync_metrics.go
+++ b/corosync_metrics.go
@@ -1,8 +1,8 @@
 package main
 
 import (
-	"errors"
 	"fmt"
+	"os"
 	"os/exec"
 	"regexp"
 	"strconv"
@@ -12,11 +12,15 @@ import (
 // Quorum metrics
 
 // return the output of quorum in raw format
-func getQuoromClusterInfo() []byte {
+func getQuoromClusterInfo() ([]byte, error) {
 	// ignore error because If any interfaces are faulty, 1 is returned by the binary. If all interfaces
 	// are active 0 is returned to the shell.
+	if _, err := os.Stat("/usr/sbin/corosync-quorumtool"); os.IsNotExist(err) {
+		return nil, fmt.Errorf("could not find corosync-quromtool binary")
+	}
+
 	quorumInfoRaw, _ := exec.Command("/usr/sbin/corosync-quorumtool").Output()
-	return quorumInfoRaw
+	return quorumInfoRaw, nil
 }
 
 func parseQuoromStatus(quoromStatus []byte) (map[string]int, string, error) {
@@ -36,6 +40,16 @@ func parseQuoromStatus(quoromStatus []byte) (map[string]int, string, error) {
 	// and convert it to integer type
 	numberOnly := regexp.MustCompile("[0-9]+")
 	wordOnly := regexp.MustCompile("[a-zA-Z]+")
+	quoratePresent := regexp.MustCompile("Quorate:")
+
+	// In case of error, the binary is there but execution was erroring out, check output for quorate string.
+	quorateWordPresent := quoratePresent.FindString(string(quoromRaw))
+
+	// check the case there is an sbd_config but the SBD_DEVICE is not set
+
+	if quorateWordPresent == "" {
+		return nil, "", fmt.Errorf("the quorum status output is not in parsable format as expected")
+	}
 
 	quorateRaw := wordOnly.FindString(strings.SplitAfterN(quoromRaw, "Quorate:", 2)[1])
 	quorate := strings.ToLower(quorateRaw)
@@ -52,7 +66,7 @@ func parseQuoromStatus(quoromStatus []byte) (map[string]int, string, error) {
 	}
 
 	if len(voteQuorumInfo) == 0 {
-		return voteQuorumInfo, quorate, errors.New("could not retrieve any quorum information")
+		return voteQuorumInfo, quorate, fmt.Errorf("could not retrieve any quorum information")
 	}
 
 	return voteQuorumInfo, quorate, nil
diff --git a/drbd_metrics.go b/drbd_metrics.go
index 2f3e36a..9e3fb62 100644
--- a/drbd_metrics.go
+++ b/drbd_metrics.go
@@ -13,6 +13,14 @@ type drbdStatus struct {
 		Volume    int    `json:"volume"`
 		DiskState string `json:"disk-state"`
 	} `json:"devices"`
+	Connections []struct {
+		PeerNodeID  int    `json:"peer-node-id"`
+		PeerRole    string `json:"peer-role"`
+		PeerDevices []struct {
+			Volume        int    `json:"volume"`
+			PeerDiskState string `json:"peer-disk-state"`
+		} `json:"peer_devices"`
+	} `json:"connections"`
 }
 
 // return drbd status in byte raw json
diff --git a/go.sum b/go.sum
index 15d9185..b7a3adc 100644
--- a/go.sum
+++ b/go.sum
@@ -5,6 +5,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -32,6 +33,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -53,6 +55,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
diff --git a/ha_cluster_exporter.go b/ha_cluster_exporter.go
index 3b7b0bb..e85cade 100644
--- a/ha_cluster_exporter.go
+++ b/ha_cluster_exporter.go
@@ -73,6 +73,12 @@ var (
 			Name: "ha_cluster_drbd_resource",
 			Help: "show per resource name, its role, the volume and disk_state (Diskless,Attaching, Failed, Negotiating, Inconsistent, Outdated, DUnknown, Consistent, UpToDate)",
 		}, []string{"resource_name", "role", "volume", "disk_state"})
+
+	remoteDrbdDiskState = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "ha_cluster_drbd_resources_remote_connection",
+			Help: "show per remote connection resource name, its role, peer-id , the volume and disk_state (Diskless,Attaching, Failed, Negotiating, Inconsistent, Outdated, DUnknown, Consistent, UpToDate)",
+		}, []string{"resource_name", "peer_node_id", "peer_role", "volume", "peer_disk_state"})
 )
 
 func init() {
@@ -85,6 +91,7 @@ func init() {
 	prometheus.MustRegister(corosyncQuorate)
 	prometheus.MustRegister(sbdDevStatus)
 	prometheus.MustRegister(drbdDiskState)
+	prometheus.MustRegister(remoteDrbdDiskState)
 }
 
 // this function is for some cluster metrics which have resource as labels.
@@ -134,10 +141,30 @@ func resetDrbdMetrics() error {
 	if err != nil {
 		return errors.Wrap(err, "failed to register DRBD disk state metric. Perhaps another exporter is already running?")
 	}
-
+	prometheus.Unregister(remoteDrbdDiskState)
+	remoteDrbdDiskState = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "ha_cluster_drbd_resources_remote_connection",
+			Help: "show per remote connection resource name, its role, peer-id , the volume and disk_state (Diskless,Attaching, Failed, Negotiating, Inconsistent, Outdated, DUnknown, Consistent, UpToDate)",
+		}, []string{"resource_name", "peer_node_id", "peer_role", "volume", "peer_disk_state"})
+	err = prometheus.Register(remoteDrbdDiskState)
+	if err != nil {
+		return errors.Wrap(err, "failed to register DRBD remote disk state metric. Perhaps another exporter is already running?")
+	}
 	return nil
 }
 
+// helper function for setting timeout for goroutines
+// we skip the timeout by init apply each time later
+func sleepDefaultTimeout(firstTime *bool) {
+	// by the initialization of exporter we don't wait timeout but serve metrics.
+	if *firstTime {
+		*firstTime = false
+		return
+	}
+	time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
+}
+
 var portNumber = flag.String("port", ":9002", "The port number to listen on for HTTP requests.")
 var timeoutSeconds = flag.Int("timeout", 5, "timeout seconds for exporter to wait to fetch new data")
 
@@ -149,55 +176,64 @@ func main() {
 	// for each different metrics, handle it in differents gorutines, and use same timeout.
 
 	// set DRBD metrics
-
 	go func() {
 		log.Infoln("Starting DRBD metrics collector...")
-
-		started := false
-
+		firstTime := true
 		for {
-			if started {
-				time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
-			} else {
-				started = true
-			}
-
-			time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
-
+			sleepDefaultTimeout(&firstTime)
 			log.Infoln("Reading DRBD status...")
 
 			// retrieve drbdInfos calling its binary
 			drbdStatusJSONRaw, err := getDrbdInfo()
 			if err != nil {
-				log.Errorln(err)
+				log.Warnf("Error by retrieving drbd infos %s", err)
 				continue
 			}
 			// populate structs and parse relevant info we will expose via metrics
 			drbdDev, err := parseDrbdStatus(drbdStatusJSONRaw)
 			if err != nil {
-				log.Errorln(err)
+				log.Warnf("Error by parsing drbd json: %s", err)
 				continue
 			}
 
 			// reset metrics before setting news to remove any state information
 			err = resetDrbdMetrics()
 			if err != nil {
-				log.Errorln(err)
+				log.Warnf("Error by resetting drbd metrics %s", err)
 				continue
 			}
 
-			// create a metric like : ha_cluster_drbd_resource{resource_name="1-single-0", role="primary", volume="0",  disk_state="uptodate"} 1
+			// 1) ha_cluster_drbd_resource{resource_name="1-single-0", role="primary", volume="0",  disk_state="uptodate"} 1
 			// the metric is always set to 1 or is absent
 			for _, resource := range drbdDev {
 				for _, device := range resource.Devices {
 					drbdDiskState.WithLabelValues(resource.Name, resource.Role, strconv.Itoa(device.Volume), strings.ToLower(resource.Devices[device.Volume].DiskState)).Set(float64(1))
 				}
+				// 2) ha_cluster_drbd_resource_remote_connection{resource_name="1-single-0", peer_node_id="1", role="primary", volume="0",  disk_state="uptodate"} 1
+				// a resource could not have any connection
+				if len(resource.Connections) == 0 {
+					log.Warnln("could not retrieve any remote disk state connection info")
+					continue
+				}
+				// a Resource can have multiple connection with different nodes
+				for _, conn := range resource.Connections {
+					// []string{"resource_name", "peer_node_id", "peer_role", "volume", "peer_disk_state"})
+					// pro resource go through the volume of peer and its peer state
+					if len(conn.PeerDevices) == 0 {
+						log.Warnln("could not retrieve any peer Devices metric")
+						continue
+					}
+					for _, peerDev := range conn.PeerDevices {
+						remoteDrbdDiskState.WithLabelValues(resource.Name, strconv.Itoa(conn.PeerNodeID), conn.PeerRole, strconv.Itoa(peerDev.Volume), strings.ToLower(peerDev.PeerDiskState)).Set(float64(1))
+					}
+				}
 			}
 		}
 	}()
 
 	// set SBD device metrics
 	go func() {
+		firstTime := true
 		if _, err := os.Stat("/etc/sysconfig/sbd"); os.IsNotExist(err) {
 			log.Warnln("SBD configuration not available, SBD metrics won't be collected")
 			return
@@ -205,33 +241,26 @@ func main() {
 
 		log.Infoln("Starting SBD metrics collector...")
 
-		started := false
-
 		for {
-			if started {
-				time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
-			} else {
-				started = true
-			}
-
+			sleepDefaultTimeout(&firstTime)
 			// read configuration of SBD
 			sbdConfiguration, err := readSdbFile()
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 			// retrieve a list of sbd devices
 			sbdDevices, err := getSbdDevices(sbdConfiguration)
 			// mostly, the sbd_device were not set in conf file for returning an error
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 
 			// set and return a map of sbd devices with true if healthy, false if not
 			sbdStatus, err := getSbdDeviceHealth(sbdDevices)
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 			for sbdDev, sbdStatusBool := range sbdStatus {
@@ -248,21 +277,16 @@ func main() {
 	// set corosync metrics: ring errors
 	go func() {
 		log.Infoln("Starting corosync ring errors collector...")
-
-		started := false
-
+		firstTime := true
 		for {
-			if started {
-				time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
-			} else {
-				started = true
-			}
+
+			sleepDefaultTimeout(&firstTime)
 
 			log.Infoln("Reading ring status...")
 			ringStatus := getCorosyncRingStatus()
 			ringErrorsTotal, err := parseRingStatus(ringStatus)
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 			corosyncRingErrorsTotal.Set(float64(ringErrorsTotal))
@@ -272,21 +296,19 @@ func main() {
 	// set corosync metrics: quorum metrics
 	go func() {
 		log.Infoln("Starting corosync quorum metrics collector...")
-
-		started := false
-
+		firstTime := true
 		for {
-			if started {
-				time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
-			} else {
-				started = true
-			}
+			sleepDefaultTimeout(&firstTime)
 
 			log.Infoln("Reading quorum status...")
-			quoromStatus := getQuoromClusterInfo()
+			quoromStatus, err := getQuoromClusterInfo()
+			if err != nil {
+				log.Warnln(err)
+				continue
+			}
 			voteQuorumInfo, quorate, err := parseQuoromStatus(quoromStatus)
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 
@@ -313,20 +335,14 @@ func main() {
 	// set cluster pacemaker metrics
 	go func() {
 		log.Infoln("Starting pacemaker metrics collector...")
-
-		started := false
-
+		firstTime := true
 		for {
-			if started {
-				time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
-			} else {
-				started = true
-			}
+			sleepDefaultTimeout(&firstTime)
 
 			// remove all global state contained by metrics
 			err := resetClusterMetrics()
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 
@@ -334,14 +350,14 @@ func main() {
 			log.Infoln("Reading cluster configuration with crm_mon..")
 			pacemakerXMLRaw, err := exec.Command("/usr/sbin/crm_mon", "-1", "--as-xml", "--group-by-node", "--inactive").Output()
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 
 			// parse raw XML returned from crm_mon and populate structs for metrics
 			status, err := parsePacemakerStatus(pacemakerXMLRaw)
 			if err != nil {
-				log.Errorln(err)
+				log.Warnln(err)
 				continue
 			}
 


Test patch:
diff --git a/drbd_metrics_test.go b/drbd_metrics_test.go
index 75e2c83..93ff34c 100644
--- a/drbd_metrics_test.go
+++ b/drbd_metrics_test.go
@@ -123,6 +123,14 @@ func TestDrbdParsing(t *testing.T) {
 		t.Errorf("disk-states doesn't correspond! fail got %s", drbdDevs[0].Devices[0].DiskState)
 	}
 
+	if 1 != drbdDevs[0].Connections[0].PeerNodeID {
+		t.Errorf("peerNodeID doesn't correspond! fail got %d", drbdDevs[0].Connections[0].PeerNodeID)
+	}
+
+	if "UpToDate" != drbdDevs[0].Connections[0].PeerDevices[0].PeerDiskState {
+		t.Errorf("peerDiskState doesn't correspond! fail got %s", drbdDevs[0].Connections[0].PeerDevices[0].PeerDiskState)
+	}
+
 	if 0 != drbdDevs[0].Devices[0].Volume {
 		t.Errorf("volumes should be 0")
 	}