Repository: ClusterLabs/ha_cluster_exporter
Pull request #15 (2019-09-30 10:31:02)

Add corosync metrics (ringFailures and quorum)
Add corosync metrics (ringFailures and quorum)

add tests


Patch:
diff --git a/Makefile b/Makefile
index 3760113..5a9fbfd 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 default: build
 
-build: fmt-check  vet-check 
+build: fmt-check  vet-check test
 	go build .
 
 install:
@@ -11,7 +11,8 @@ vet-check:
 
 fmt-check:
 	go fmt  .
-
+test:
+	go test 
 # This deploy the binary to a node of cluster in devel mode port :9002. 
 # you need to change the IP adress in the script. 
 # TODO: (In future we can add an arg var..)
diff --git a/README.md b/README.md
index ebe2621..0e9d4a2 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,8 @@
 # ha_cluster_exporter
 
+[![Build Status](https://travis-ci.org/ClusterLabs/ha_cluster_exporter.svg?branch=master)](https://travis-ci.org/ClusterLabs/ha_cluster_exporter)
+
+
 This prometheus exporter is used to serve metrics for pacemaker https://github.com/ClusterLabs/pacemaker
 
 It should run inside a node of the cluster or both.
@@ -8,7 +11,7 @@ It should run inside a node of the cluster or both.
 
 - expose cluster node and resource metrics via `crm_mon` (pacemaker data xml)
 
-- expose corosync metrics **not done yet WIP**
+- expose corosync metrics (ring errors, quorum metrics)
 
 # Devel:
 
diff --git a/corosync_metrics.go b/corosync_metrics.go
new file mode 100644
index 0000000..d23143d
--- /dev/null
+++ b/corosync_metrics.go
@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Quorum metrics
+
+// return the output of quorum in raw format
+func getQuoromClusterInfo() []byte {
+	// get ringStatus
+	log.Println("[INFO]: Reading quorum status with corosync-quorumtool...")
+	// ignore error because If any interfaces are faulty, 1 is returned by the binary. If all interfaces
+	// are active 0 is returned to the shell.
+	quorumInfoRaw, _ := exec.Command("/usr/sbin/corosync-quorumtool").Output()
+	return quorumInfoRaw
+}
+
+func parseQuoromStatus(quoromStatus []byte) (map[string]int, string) {
+	quoromRaw := string(quoromStatus)
+	// Quorate:          Yes
+
+	// Votequorum information
+	// ----------------------
+	// Expected votes:   2
+	// Highest expected: 2
+	// Total votes:      2
+	// Quorum:           1
+
+	// We apply the same method for all the metrics/data:
+	// first split the string for finding the word , e.g "Expected votes:", and get it via regex
+	// only the number   2,
+	// and convert it to integer type
+	numberOnly := regexp.MustCompile("[0-9]+")
+	wordOnly := regexp.MustCompile("[a-zA-Z]+")
+
+	quorateRaw := wordOnly.FindString(strings.SplitAfterN(quoromRaw, "Quorate:", 2)[1])
+	quorate := strings.ToLower(quorateRaw)
+	expVotes, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Expected votes:", 2)[1]))
+	highVotes, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Highest expected:", 2)[1]))
+	totalVotes, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Total votes:", 2)[1]))
+	quorum, _ := strconv.Atoi(numberOnly.FindString(strings.SplitAfterN(quoromRaw, "Quorum:", 2)[1]))
+
+	voteQuorumInfo := map[string]int{
+		"expectedVotes":   expVotes,
+		"highestExpected": highVotes,
+		"totalVotes":      totalVotes,
+		"quorum":          quorum,
+	}
+
+	return voteQuorumInfo, quorate
+}
+
+// RING metrics
+
+// get status ring and return it as bytes
+// this function can return also just an malformed output in case of error, we don't check.
+// It is the parser that will check the status
+func getCorosyncRingStatus() []byte {
+	// get ringStatus
+	log.Println("[INFO]: Reading ring status with corosync-cfgtool...")
+	// ignore error because If any interfaces are faulty, 1 is returned by the binary. If all interfaces
+	// are active 0 is returned to the shell.
+	ringStatusRaw, _ := exec.Command("/usr/sbin/corosync-cfgtool", "-s").Output()
+	return ringStatusRaw
+}
+
+// return the number of RingError that we will use as gauge, and error if somethings unexpected happens
+func parseRingStatus(ringStatus []byte) (int, error) {
+	statusRaw := string(ringStatus)
+	// check if there is a ring ERROR first
+	ringErrorsTotal := strings.Count(statusRaw, "FAULTY")
+
+	// in case there is no error we need to check that the output is not
+	if ringErrorsTotal == 0 {
+		// if there is no RING ID word, the command corosync-cfgtool went wrong/error out
+		if strings.Count(statusRaw, "RING ID") == 0 {
+			return 0, fmt.Errorf("[ERROR]: corosync-cfgtool command returned an unexpected error %s", statusRaw)
+		}
+
+		return 0, nil
+	}
+
+	// there is a ringError
+	return ringErrorsTotal, nil
+}
diff --git a/ha_cluster_exporter.go b/ha_cluster_exporter.go
index 1b6012f..f947b46 100644
--- a/ha_cluster_exporter.go
+++ b/ha_cluster_exporter.go
@@ -68,6 +68,19 @@ type resource struct {
 }
 
 var (
+	// corosync metrics
+
+	corosyncRingErrorsTotal = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "corosync_ring_errors_total",
+		Help: "Total number of ring errors in corosync",
+	})
+
+	corosyncQuorate = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "corosync_quorate",
+		Help: "shows if the cluster is quorate. 1 cluster is quorate, 0 not",
+	})
+
+	// cluster metrics
 	clusterNodesConf = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "cluster_nodes_configured_total",
 		Help: "Number of nodes configured in ha cluster",
@@ -79,6 +92,15 @@ var (
 	})
 
 	// metrics with labels. (prefer these always as guideline)
+
+	// corosync quorum
+	corosyncQuorum = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "corosync_quorum",
+			Help: "cluster quorum information",
+		}, []string{"type"})
+
+	// cluster metrics
 	clusterNodes = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "cluster_nodes",
@@ -92,14 +114,21 @@ var (
 		}, []string{"node", "resource_name", "role", "managed", "status"})
 )
 
-func initMetrics() {
+func init() {
 	prometheus.MustRegister(clusterNodes)
 	prometheus.MustRegister(nodeResources)
 	prometheus.MustRegister(clusterResourcesConf)
 	prometheus.MustRegister(clusterNodesConf)
+	prometheus.MustRegister(corosyncRingErrorsTotal)
+	prometheus.MustRegister(corosyncQuorum)
+	prometheus.MustRegister(corosyncQuorate)
 }
 
-func resetMetrics() {
+// this function is for some cluster metrics which have resource as labels.
+// since we cannot be sure a resource exists always, we need to destroy the metrics at each iteration
+// otherwise we will have wrong metrics ( thinking a resource exist when not)
+
+func resetClusterMetrics() {
 	// We want to reset certains metrics to 0 each time for removing the state.
 	// since we have complex/nested metrics with multiples labels, unregistering/re-registering is the cleanest way.
 	prometheus.Unregister(nodeResources)
@@ -137,16 +166,55 @@ var timeoutSeconds = flag.Int("timeout", 5, "timeout seconds for exporter to wai
 func main() {
 	// read cli option and setup initial stat
 	flag.Parse()
-	initMetrics()
 	http.Handle("/metrics", promhttp.Handler())
 
-	// parse each X seconds the cluster configuration and update the metrics accordingly
-	// this is done in a goroutine async. we update in this way each 2 second the metrics. (the second will be a parameter in future)
+	// for each different metrics, handle it in differents gorutines, and use same timeout.
+
+	// 1a) set corosync metrics: Ring errors
+	go func() {
+		for {
+			ringStatus := getCorosyncRingStatus()
+			ringErrorsTotal, err := parseRingStatus(ringStatus)
+			// todo: reflect if we want to error out. We could just ignore error for resiliance
+			if err != nil {
+				log.Println("[ERROR]: could not execute command: usr/sbin/corosync-cfgtool -s")
+				log.Panic(err)
+			}
+			corosyncRingErrorsTotal.Set(float64(ringErrorsTotal))
+			time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
+		}
+	}()
+	// 1b) set corosync metrics: quorum metrics
+	go func() {
+		for {
+			quoromStatus := getQuoromClusterInfo()
+			voteQuorumInfo, quorate := parseQuoromStatus(quoromStatus)
+
+			// set metrics relative to quorum infos
+			corosyncQuorum.WithLabelValues("expected_votes").Set(float64(voteQuorumInfo["expectedVotes"]))
+			corosyncQuorum.WithLabelValues("highest_expected").Set(float64(voteQuorumInfo["highestExpected"]))
+			corosyncQuorum.WithLabelValues("total_votes").Set(float64(voteQuorumInfo["totalVotes"]))
+			corosyncQuorum.WithLabelValues("quorum").Set(float64(voteQuorumInfo["quorum"]))
+
+			// set metric if we have a quorate or not
+			// 1 means we have it
+			if quorate == "yes" {
+				corosyncQuorate.Set(float64(1))
+			}
+
+			if quorate == "no" {
+				corosyncQuorate.Set(float64(0))
+			}
+
+			time.Sleep(time.Duration(int64(*timeoutSeconds)) * time.Second)
+		}
+	}()
+	// 2) set cluster pacemaker metrics
 	go func() {
 		for {
 
 			// remove all global state contained by metrics
-			resetMetrics()
+			resetClusterMetrics()
 
 			// get cluster status xml
 			log.Println("[INFO]: Reading cluster configuration with crm_mon..")
diff --git a/tools/deploy-to-cluster.sh b/tools/deploy-to-cluster.sh
index c450df4..7458829 100755
--- a/tools/deploy-to-cluster.sh
+++ b/tools/deploy-to-cluster.sh
@@ -4,11 +4,9 @@
 
 # this script is just for deploying the binary to the cluster. Nothing else.
 
-node="root@10.162.31.39"
+node="root@10.162.29.106"
 
 ssh $node "rm /root/ha_cluster_exporter"
 echo "copying binary"
 scp ha_cluster_exporter  $node:
 
-echo "run exporter"
-ssh $node "/root/ha_cluster_exporter"


Test patch:
diff --git a/corosync_metrics_test.go b/corosync_metrics_test.go
new file mode 100644
index 0000000..f85e688
--- /dev/null
+++ b/corosync_metrics_test.go
@@ -0,0 +1,161 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TEST group quorum metrics
+func TestQuoromMetricParsing(t *testing.T) {
+	// the data is fake data
+	fmt.Println("=== Testing quorum info retrivial")
+	quoromStatus := `
+	Quorum information
+	------------------
+	Date:             Sun Sep 29 16:10:37 2019
+	Quorum provider:  corosync_votequorum
+	Nodes:            2
+	Node ID:          1084780051
+	Ring ID:          1084780051/44
+	Quorate:          Yes
+	
+	Votequorum information
+	----------------------
+	Expected votes:   232
+	Highest expected: 22
+	Total votes:      21
+	Quorum:           421  
+	Flags:            2Node Quorate WaitForAll 
+	
+	Membership information
+	----------------------
+		Nodeid      Votes Name
+	1084780051          1 dma-dog-hana01 (local)
+	1084780052          1 dma-dog-hana02
+	dma-dog-hana01:~ # 
+	`
+	getQuoromClusterInfo()
+	voteQuorumInfo, quorate := parseQuoromStatus([]byte(quoromStatus))
+
+	if voteQuorumInfo["expectedVotes"] != 232 {
+		t.Errorf("expectedVotes should be 232 got instead: %d", voteQuorumInfo["expectedVotes"])
+	}
+	if voteQuorumInfo["highestExpected"] != 22 {
+		t.Errorf("expectedVotes should be 232 got instead: %d", voteQuorumInfo["highestExpected"])
+	}
+
+	if voteQuorumInfo["totalVotes"] != 21 {
+		t.Errorf("expectedVotes should be 232 got instead: %d", voteQuorumInfo["totalVotes"])
+	}
+
+	if voteQuorumInfo["quorum"] != 421 {
+		t.Errorf("expectedVotes should be 421 got instead: %d", voteQuorumInfo["quorum"])
+	}
+
+	if quorate != "yes" {
+		t.Errorf("quorate should be set to Yes, got %s", quorate)
+	}
+
+}
+
+// TEST group RING metrics
+// test that we recognize 1 error (for increasing metric later)
+func TestOneRingError(t *testing.T) {
+	fmt.Println("=== Test one ring error")
+	ringStatusWithOneError := `Printing ring status.
+	Local node ID 16777226
+	RING ID 0
+			id      = 10.0.0.1
+			status  = Marking ringid 0 interface 10.0.0.1 FAULTY
+	RING ID 1
+			id      = 172.16.0.1
+			status  = ring 1 active with no faults																				   
+			`
+
+	getCorosyncRingStatus()
+	ringErrorsTotal, err := parseRingStatus([]byte(ringStatusWithOneError))
+	RingExpectedErrors := 1
+	if ringErrorsTotal != RingExpectedErrors {
+		t.Errorf("ringErrors was incorrect, got: %d, expected: %d.", ringErrorsTotal, RingExpectedErrors)
+	}
+	if err != nil {
+		t.Errorf("error should be nil got instead: %s", err)
+	}
+}
+
+func TestZeroRingErrors(t *testing.T) {
+	fmt.Println("=== Test zero Ring errors")
+	ringStatusWithOneError := `Printing ring status.
+	Local node ID 16777226
+	RING ID 0
+			id      = 10.0.0.1
+			status  = Marking ringid 0 interface 10.0.0.1 
+	RING ID 1
+			id      = 172.16.0.1
+			status  = ring 1 active with no faults																				   
+			`
+
+	getCorosyncRingStatus()
+	ringErrorsTotal, err := parseRingStatus([]byte(ringStatusWithOneError))
+	RingExpectedErrors := 0
+	if ringErrorsTotal != RingExpectedErrors {
+		t.Errorf("ringErrors was incorrect, got: %d, expected: %d.", ringErrorsTotal, RingExpectedErrors)
+	}
+	if err != nil {
+		t.Errorf("error should be nil got instead: %s", err)
+	}
+}
+
+// test that we recognize 3 rings error (for increasing metric later)
+func TestMultipleRingErrors(t *testing.T) {
+	fmt.Println("=== Test multiples ring error")
+	ringStatusWithOneError := `Printing ring status.
+	Local node ID 16777226
+	RING ID 0
+			id      = 10.0.0.1
+			status  = Marking ringid 0 interface 10.0.0.1 FAULTY
+	RING ID 1
+			id      = 172.16.0.1
+			status  = ring 1 active with no faults
+	RING ID 2
+			id      = 10.0.0.1
+			status  = Marking ringid 1 interface 10.0.0.1 FAULTY
+	RING ID 3
+			id      = 172.16.0.1
+			status  = ring 1 active with no faults
+	RING ID 4
+			id      = 10.0.0.1
+			status  = Marking ringid 1 interface 10.0.0.1 FAULTY
+	RING ID 5
+			id      = 172.16.0.1
+			status  = ring 1 active with no faults
+																											   
+			`
+
+	getCorosyncRingStatus()
+	ringErrorsTotal, err := parseRingStatus([]byte(ringStatusWithOneError))
+	RingExpectedErrors := 3
+	if ringErrorsTotal != RingExpectedErrors {
+		t.Errorf("ringErrors was incorrect, got: %d, expected: %d.", ringErrorsTotal, RingExpectedErrors)
+	}
+	if err != nil {
+		t.Errorf("error should be nil got instead: %s", err)
+	}
+}
+
+// test that in case of system unexpected error we detect this
+func TestSystemUnexpectedError(t *testing.T) {
+	fmt.Println("=== Test unexpected error")
+	// since there is no cluster in a Test env. this will return an error
+	ringStatusError := getCorosyncRingStatus()
+	parseRingStatus([]byte(ringStatusError))
+	ringErrorsTotal, err := parseRingStatus([]byte(ringStatusError))
+	RingExpectedErrors := 0
+	if ringErrorsTotal != RingExpectedErrors {
+		t.Errorf("ringErrors was incorrect, got: %d, expected: %d.", ringErrorsTotal, RingExpectedErrors)
+	}
+	if err == nil {
+		t.Errorf("error should not be nil got !!")
+	}
+
+}