Repository: ClusterLabs/ha_cluster_exporter
Pull request #71 (2019-10-23 15:02:04)

Implement Fail count metric and Migration threshold metric
# Desc:

- implement failCount metric
- implement migration_threshold metric

## done

- [x] implement structs for new metric and add test for parsing 
- add new test for metrics expected.
- implement new metric structure
- populate metric


Patch:
diff --git a/doc/metric_spec.md b/doc/metric_spec.md
index ef7a9b4..27f488c 100644
--- a/doc/metric_spec.md
+++ b/doc/metric_spec.md
@@ -29,6 +29,8 @@ The Pacemaker subsystem collects an atomic snapshot of the HA cluster directly f
 3. [`ha_cluster_pacemaker_resources`](#ha_cluster_pacemaker_resources)
 4. [`ha_cluster_pacemaker_resources_total`](#ha_cluster_pacemaker_resources_total)
 5. [`ha_cluster_pacemaker_stonith_enabled`](#ha_cluster_pacemaker_stonith_enabled)
+6. [`ha_cluster_pacemaker_fail_count`](#ha_cluster_pacemaker_fail_count)
+7. [`ha_cluster_pacemaker_migration_threshold`](#ha_cluster_pacemaker_migration_threshold)
 
 
 ### `ha_cluster_pacemaker_nodes`
@@ -86,6 +88,21 @@ The total number of *configured* resources in the cluster. This value is mostly
 Whether or not stonith is enabled in the cluster.  
 Value is either `1` or `0`.
 
+### `ha_cluster_pacemaker_fail_count`
+
+#### Description
+
+The number of fail count per node and resource ID.
+The value can vary from 0, 1 , 5 etc to `+Inf`, that  correspond to the infinity fail count. For more information about fail count refer to pacemaker upstream documentation.
+
+### `ha_cluster_pacemaker_migration_threshold`
+
+#### Description
+
+The number of migration threshold pro node and resource ID set by a pacemaker cluster. 
+Possible values are positive numbers.
+
+
 ## Corosync
 
 The Corosync subsystem collects cluster quorum votes and ring status by parsing the output of `corosync-quorumtool` and `corosync-cfgtool`.
diff --git a/pacemaker_metrics.go b/pacemaker_metrics.go
index d476cb8..aabbf6d 100644
--- a/pacemaker_metrics.go
+++ b/pacemaker_metrics.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/xml"
+	"math"
 	"os"
 	"os/exec"
 	"strconv"
@@ -14,33 +15,39 @@ import (
 
 // *** crm_mon XML unserialization structures
 type pacemakerStatus struct {
-	Version string  `xml:"version,attr"`
-	Summary summary `xml:"summary"`
-	Nodes   nodes   `xml:"nodes"`
+	Version     string  `xml:"version,attr"`
+	Summary     summary `xml:"summary"`
+	Nodes       nodes   `xml:"nodes"`
+	NodeHistory struct {
+		Node []struct {
+			Name            string `xml:"name,attr"`
+			ResourceHistory []struct {
+				Name               string `xml:"id,attr"`
+				MigrationThreshold int    `xml:"migration-threshold,attr"`
+				FailCount          int    `xml:"fail-count,attr"`
+			} `xml:"resource_history"`
+		} `xml:"node"`
+	} `xml:"node_history"`
 }
 
 type summary struct {
 	Nodes struct {
 		Number int `xml:"number,attr"`
 	} `xml:"nodes_configured"`
-	Resources      resourcesConfigured `xml:"resources_configured"`
-	ClusterOptions clusterOptions      `xml:"cluster_options"`
-}
-
-type resourcesConfigured struct {
-	Number   int `xml:"number,attr"`
-	Disabled int `xml:"disabled,attr"`
-	Blocked  int `xml:"blocked,attr"`
+	Resources struct {
+		Number   int `xml:"number,attr"`
+		Disabled int `xml:"disabled,attr"`
+		Blocked  int `xml:"blocked,attr"`
+	} `xml:"resources_configured"`
+	ClusterOptions struct {
+		StonithEnabled bool `xml:"stonith-enabled,attr"`
+	} `xml:"cluster_options"`
 }
 
 type nodes struct {
 	Node []node `xml:"node"`
 }
 
-type clusterOptions struct {
-	StonithEnabled bool `xml:"stonith-enabled,attr"`
-}
-
 type node struct {
 	Name             string     `xml:"name,attr"`
 	ID               string     `xml:"id,attr"`
@@ -71,17 +78,17 @@ type resource struct {
 	NodesRunningOn int    `xml:"nodes_running_on,attr"`
 }
 
-// ***
-
 var (
 	pacemakerMetrics = metricDescriptors{
 		// the map key will function as an identifier of the metric throughout the rest of the code;
 		// it is arbitrary, but by convention we use the actual metric name
-		"nodes":           NewMetricDesc("pacemaker", "nodes", "The nodes in the cluster; one line per name, per status", []string{"name", "type", "status"}),
-		"nodes_total":     NewMetricDesc("pacemaker", "nodes_total", "Total number of nodes in the cluster", nil),
-		"resources":       NewMetricDesc("pacemaker", "resources", "The resources in the cluster; one line per id, per status", []string{"node", "id", "role", "managed", "status"}),
-		"resources_total": NewMetricDesc("pacemaker", "resources_total", "Total number of resources in the cluster", nil),
-		"stonith_enabled": NewMetricDesc("pacemaker", "stonith_enabled", "Whether or not stonith is enabled", nil),
+		"nodes":               NewMetricDesc("pacemaker", "nodes", "The nodes in the cluster; one line per name, per status", []string{"name", "type", "status"}),
+		"nodes_total":         NewMetricDesc("pacemaker", "nodes_total", "Total number of nodes in the cluster", nil),
+		"resources":           NewMetricDesc("pacemaker", "resources", "The resources in the cluster; one line per id, per status", []string{"node", "id", "role", "managed", "status"}),
+		"resources_total":     NewMetricDesc("pacemaker", "resources_total", "Total number of resources in the cluster", nil),
+		"stonith_enabled":     NewMetricDesc("pacemaker", "stonith_enabled", "Whether or not stonith is enabled", nil),
+		"fail_count":          NewMetricDesc("pacemaker", "fail_count", "The Fail count number per node and resource id", []string{"node", "resource"}),
+		"migration_threshold": NewMetricDesc("pacemaker", "migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"}),
 	}
 
 	crmMonPath = "/usr/sbin/crm_mon"
@@ -129,6 +136,8 @@ func (c *pacemakerCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- c.makeGaugeMetric("stonith_enabled", stonithEnabled)
 
 	c.recordNodeMetrics(pacemakerStatus, ch)
+	c.recordFailCountMetrics(pacemakerStatus, ch)
+	c.recordMigrationThresholdMetrics(pacemakerStatus, ch)
 }
 
 func getPacemakerStatus() (pacemakerStatus, error) {
@@ -211,3 +220,27 @@ func (c *pacemakerCollector) recordResourcesMetrics(node node, ch chan<- prometh
 		}
 	}
 }
+
+func (c *pacemakerCollector) recordFailCountMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
+	for _, node := range pacemakerStatus.NodeHistory.Node {
+		for _, resHistory := range node.ResourceHistory {
+			failCount := float64(resHistory.FailCount)
+
+			// if value is 1000000 this is a special value in pacemaker which is infinity fail count
+			if resHistory.FailCount >= 1000000 {
+				failCount = math.Inf(1)
+			}
+
+			ch <- c.makeGaugeMetric("fail_count", failCount, node.Name, resHistory.Name)
+
+		}
+	}
+}
+
+func (c *pacemakerCollector) recordMigrationThresholdMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
+	for _, node := range pacemakerStatus.NodeHistory.Node {
+		for _, resHistory := range node.ResourceHistory {
+			ch <- c.makeGaugeMetric("migration_threshold", float64(resHistory.MigrationThreshold), node.Name, resHistory.Name)
+		}
+	}
+}


Test patch:
diff --git a/pacemaker_metrics_test.go b/pacemaker_metrics_test.go
index ccd6cf2..157f87d 100644
--- a/pacemaker_metrics_test.go
+++ b/pacemaker_metrics_test.go
@@ -47,7 +47,7 @@ func TestParsePacemakerXML(t *testing.T) {
 				</resource_history>
 			</node>
 			<node name="Hawk3-1">
-				<resource_history id="d1" orphan="false" migration-threshold="3">
+				<resource_history id="d1" orphan="false" migration-threshold="3" fail-count="300" last-failure="Tue Jan 15 22:20:00 2019">
 					<operation_history call="10" task="start" last-rc-change="Tue Jan 15 22:19:15 2019" last-run="Tue Jan 15 22:19:15 2019" exec-time="23ms" queue-time="0ms" rc="0" rc_text="ok" />
 				</resource_history>
 				<resource_history id="ddd" orphan="false" migration-threshold="3" fail-count="1000000" last-failure="Tue Jan 15 22:19:59 2019">
@@ -121,7 +121,21 @@ func TestParsePacemakerXML(t *testing.T) {
 	if status.Nodes.Node[1].Online != true {
 		t.Errorf("node should be online got instead: %t", status.Nodes.Node[1].Online)
 	}
+	if status.NodeHistory.Node[0].Name != "Hawk3-2" {
+		t.Errorf("node should be called Hawk3-2 got instead: %s", status.NodeHistory.Node[0].Name)
+	}
+
+	if status.NodeHistory.Node[0].ResourceHistory[0].MigrationThreshold != 3 {
+		t.Errorf("migration-treshold should be 3 got instead: %d", status.NodeHistory.Node[0].ResourceHistory[0].MigrationThreshold)
+	}
 
+	if status.NodeHistory.Node[0].ResourceHistory[1].FailCount != 1000000 {
+		t.Errorf("fail-count should be 1000000 got instead: %d", status.NodeHistory.Node[0].ResourceHistory[1].FailCount)
+	}
+
+	if status.NodeHistory.Node[0].ResourceHistory[0].Name != "vip1" {
+		t.Errorf("resource should be called vip1 got instead: %s", status.NodeHistory.Node[0].ResourceHistory[0].Name)
+	}
 }
 
 func TestNewPacemakerCollector(t *testing.T) {
diff --git a/test/fake_crm_mon.sh b/test/fake_crm_mon.sh
index b4c8ec4..7318e98 100755
--- a/test/fake_crm_mon.sh
+++ b/test/fake_crm_mon.sh
@@ -54,29 +54,29 @@ cat <<EOF
     </node_attributes>
     <node_history>
         <node name="hana01">
-            <resource_history id="rsc_SAPHana_PRD_HDB00" orphan="false" migration-threshold="5000">
+            <resource_history id="rsc_SAPHana_PRD_HDB00" orphan="false" migration-threshold="5000" fail-count="1000000" last-failure="Wed Oct 23 12:37:22 2019">
                 <operation_history call="15" task="probe" last-rc-change="Thu Oct 10 12:57:33 2019" last-run="Thu Oct 10 12:57:33 2019" exec-time="4140ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="31" task="promote" last-rc-change="Thu Oct 10 12:57:57 2019" last-run="Thu Oct 10 12:57:57 2019" exec-time="2015ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="32" task="monitor" interval="60000ms" last-rc-change="Thu Oct 10 12:58:03 2019" exec-time="3589ms" queue-time="0ms" rc="8" rc_text="master" />
             </resource_history>
-            <resource_history id="rsc_ip_PRD_HDB00" orphan="false" migration-threshold="5000">
+            <resource_history id="rsc_ip_PRD_HDB00" orphan="false" migration-threshold="5000" fail-count="2" last-failure="Wed Oct 23 12:37:22 2019">
                 <operation_history call="21" task="start" last-rc-change="Thu Oct 10 12:57:33 2019" last-run="Thu Oct 10 12:57:33 2019" exec-time="130ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="22" task="monitor" interval="10000ms" last-rc-change="Thu Oct 10 12:57:33 2019" exec-time="78ms" queue-time="0ms" rc="0" rc_text="ok" />
             </resource_history>
             <resource_history id="stonith-sbd" orphan="false" migration-threshold="5000">
                 <operation_history call="6" task="start" last-rc-change="Thu Oct 10 12:57:31 2019" last-run="Thu Oct 10 12:57:31 2019" exec-time="2201ms" queue-time="0ms" rc="0" rc_text="ok" />
             </resource_history>
-            <resource_history id="rsc_SAPHanaTopology_PRD_HDB00" orphan="false" migration-threshold="5000">
+            <resource_history id="rsc_SAPHanaTopology_PRD_HDB00" orphan="false" migration-threshold="1">
                 <operation_history call="24" task="start" last-rc-change="Thu Oct 10 12:57:39 2019" last-run="Thu Oct 10 12:57:39 2019" exec-time="4538ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="26" task="monitor" interval="10000ms" last-rc-change="Thu Oct 10 12:57:46 2019" exec-time="4220ms" queue-time="0ms" rc="0" rc_text="ok" />
             </resource_history>
         </node>
         <node name="hana02">
-            <resource_history id="rsc_SAPHana_PRD_HDB00" orphan="false" migration-threshold="5000">
+            <resource_history id="rsc_SAPHana_PRD_HDB00" orphan="false" migration-threshold="50" fail-count="300" last-failure="Wed Oct 23 12:37:22 2019">
                 <operation_history call="22" task="start" last-rc-change="Thu Oct 17 15:22:40 2019" last-run="Thu Oct 17 15:22:40 2019" exec-time="44083ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="23" task="monitor" interval="61000ms" last-rc-change="Thu Oct 17 15:23:24 2019" exec-time="2605ms" queue-time="0ms" rc="0" rc_text="ok" />
             </resource_history>
-            <resource_history id="rsc_SAPHanaTopology_PRD_HDB00" orphan="false" migration-threshold="5000">
+            <resource_history id="rsc_SAPHanaTopology_PRD_HDB00" orphan="false" migration-threshold="3">
                 <operation_history call="20" task="start" last-rc-change="Thu Oct 17 15:22:37 2019" last-run="Thu Oct 17 15:22:37 2019" exec-time="2905ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="21" task="monitor" interval="10000ms" last-rc-change="Thu Oct 17 15:22:40 2019" exec-time="3347ms" queue-time="0ms" rc="0" rc_text="ok" />
             </resource_history>
diff --git a/test/pacemaker.metrics b/test/pacemaker.metrics
index 6f2e286..578a88e 100644
--- a/test/pacemaker.metrics
+++ b/test/pacemaker.metrics
@@ -1,3 +1,19 @@
+# HELP ha_cluster_pacemaker_fail_count The Fail count number per node and resource id
+# TYPE ha_cluster_pacemaker_fail_count gauge
+ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0 1234
+ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} +Inf 1234
+ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_ip_PRD_HDB00"} 2 1234
+ha_cluster_pacemaker_fail_count{node="hana01",resource="stonith-sbd"} 0 1234
+ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0 1234
+ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 300 1234
+# HELP ha_cluster_pacemaker_migration_threshold The migration_threshold number per node and resource id
+# TYPE ha_cluster_pacemaker_migration_threshold gauge
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 1 1234
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} 5000 1234
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_ip_PRD_HDB00"} 5000 1234
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="stonith-sbd"} 5000 1234
+ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 3 1234
+ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 50 1234
 # HELP ha_cluster_pacemaker_nodes The nodes in the cluster; one line per name, per status
 # TYPE ha_cluster_pacemaker_nodes gauge
 ha_cluster_pacemaker_nodes{name="hana01",status="dc",type="member"} 1 1234
@@ -22,3 +38,4 @@ ha_cluster_pacemaker_resources_total 6 1234
 # HELP ha_cluster_pacemaker_stonith_enabled Whether or not stonith is enabled
 # TYPE ha_cluster_pacemaker_stonith_enabled gauge
 ha_cluster_pacemaker_stonith_enabled 1 1234
+