Repository: ClusterLabs/ha_cluster_exporter
Pull request #152 (2020-04-27 11:08:33)

avoid disappearing metrics for pacemaker nodes and resources
This PR implements what we discussed at the end last week, following up some feedback we received from MSFT.

`ha_cluster_pacemaker_nodes` and `ha_cluster_pacemaker_resources` now will report a `0` value instead of a missing metric line for those status flags that are not `true` in the CIB.
This way, we correctly report all the possible combinations, including the ones where all the statuses are off. This will result in a more verbose metric set, but it's much clearer in intent and usable from a consumption perspective.

This PR also removes:
- the previously introduced `unknown` node type, which is not needed because the upstream data source has `member` as a default value.
- the artificial empty resource `status=""` label value, previously used to work around the same issue the nodes metric had, now fixed properly.


Patch:
diff --git a/collector/pacemaker/pacemaker.go b/collector/pacemaker/pacemaker.go
index 849953d..eec61ee 100644
--- a/collector/pacemaker/pacemaker.go
+++ b/collector/pacemaker/pacemaker.go
@@ -26,8 +26,8 @@ func NewCollector(crmMonPath string, cibAdminPath string) (*pacemakerCollector,
 		crmmon.NewCrmMonParser(crmMonPath),
 		cib.NewCibAdminParser(cibAdminPath),
 	}
-	c.SetDescriptor("nodes", "The nodes in the cluster; one line per name, per status", []string{"node", "type", "status"})
-	c.SetDescriptor("resources", "The resources in the cluster; one line per id, per status", []string{"node", "resource", "role", "managed", "status", "agent", "group", "clone"})
+	c.SetDescriptor("nodes", "The status of each node in the cluster; 1 means the node is in that status, 0 otherwise", []string{"node", "type", "status"})
+	c.SetDescriptor("resources", "The status of each resource in the cluster; 1 means the resource is in that status, 0 otherwise", []string{"node", "resource", "role", "managed", "status", "agent", "group", "clone"})
 	c.SetDescriptor("stonith_enabled", "Whether or not stonith is enabled", nil)
 	c.SetDescriptor("fail_count", "The Fail count number per node and resource id", []string{"node", "resource"})
 	c.SetDescriptor("migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"})
@@ -82,14 +82,6 @@ func (c *pacemakerCollector) recordStonithStatus(crmMon crmmon.Root, ch chan<- p
 
 func (c *pacemakerCollector) recordNodes(crmMon crmmon.Root, ch chan<- prometheus.Metric) {
 	for _, node := range crmMon.Nodes {
-		var nodeType string
-		switch node.Type {
-		case "member", "ping", "remote":
-			nodeType = node.Type
-			break
-		default:
-			nodeType = "unknown"
-		}
 
 		// this is a map of boolean flags for each possible status of the node
 		nodeStatuses := map[string]bool{
@@ -105,11 +97,13 @@ func (c *pacemakerCollector) recordNodes(crmMon crmmon.Root, ch chan<- prometheu
 		}
 
 		// since we have a combined cardinality of node * status, we cycle through all the possible statuses
-		// and we record a new metric if the flag for that status is on
+		// and we record a metric for each one
 		for nodeStatus, flag := range nodeStatuses {
+			var statusValue float64
 			if flag {
-				ch <- c.MakeGaugeMetric("nodes", float64(1), node.Name, nodeType, nodeStatus)
+				statusValue = 1
 			}
+			ch <- c.MakeGaugeMetric("nodes", statusValue, node.Name, node.Type, nodeStatus)
 		}
 	}
 }
@@ -139,15 +133,14 @@ func (c *pacemakerCollector) recordResource(resource crmmon.Resource, group stri
 		"blocked":         resource.Blocked,
 		"failed":          resource.Failed,
 		"failure_ignored": resource.FailureIgnored,
-		// if no status flag is active we record an empty status; probably a stopped resource, which is tracked in the "role" label instead
-		"": !(resource.Active || resource.Orphaned || resource.Blocked || resource.Failed || resource.FailureIgnored),
 	}
 
 	// since we have a combined cardinality of resource * status, we cycle through all the possible statuses
 	// and we record a new metric if the flag for that status is on
 	for resourceStatus, flag := range resourceStatuses {
-		if !flag {
-			continue
+		var statusValue float64
+		if flag {
+			statusValue = 1
 		}
 		var nodeName string
 		if resource.Node != nil {
@@ -155,7 +148,7 @@ func (c *pacemakerCollector) recordResource(resource crmmon.Resource, group stri
 		}
 		ch <- c.MakeGaugeMetric(
 			"resources",
-			float64(1),
+			statusValue,
 			nodeName,
 			resource.Id,
 			strings.ToLower(resource.Role),
diff --git a/doc/metrics.md b/doc/metrics.md
index c0a2087..3f7ca03 100644
--- a/doc/metrics.md
+++ b/doc/metrics.md
@@ -5,9 +5,6 @@ This document describes the metrics exposed by `ha_cluster_exporter`.
 General notes:
 - All the metrics are _namespaced_ with the prefix `ha_cluster`, which is followed by a _subsystem_, and both are in turn composed into a _Fully Qualified Name_ (FQN) of each metrics.
 - All the metrics and labels _names_ are in snake_case, as conventional with Prometheus. That said, as much as we'll try to keep this consistent throughout the project, the label _values_ may not actually follow this convention, though (e.g. value is a hostname).
-- Some metrics, like `ha_cluster_pacemaker_nodes`, `ha_cluster_pacemaker_resources`, share common traits:
-  - their labels contain the relevant data you may want to track or use for aggregation and filtering;
-  - either their value is `1`, or the line is absent altogether; this is because each line represents one entity of the cluster, but the exporter itself is stateless, i.e. we don't track the life-cycle of entities that do not exist anymore in the cluster.
 
 - If the `enable-timestamps` option is on, all the metrics will be timestamped with the Unix epoch time in milliseconds.
 
@@ -78,8 +75,8 @@ Possible values are positive numbers.
 
 #### Description
 
-The nodes in the cluster; one line per `node`, per `status`.  
-Either the value is `1`, or the line is absent altogether.
+The status of each node in the cluster; it will have one line for each possible `status` of each `node`.
+A value of `1` means the node is in the status specified by the `status` label, a value of `0` means it is not.
 
 #### Labels
 
@@ -87,15 +84,13 @@ Either the value is `1`, or the line is absent altogether.
 - `status`: one of `online|standby|standby_onfail|maintanance|pending|unclean|shutdown|expected_up|dc`. 
 - `type`: one of `member|ping|remote`.
 
-The total number of lines for this metric will be the cardinality of `name` times the cardinality of `status`.
-
 
 ### `ha_cluster_pacemaker_resources` 
 
 #### Description
 
-The resources in the cluster; one line per `id`, per `status`.  
-Either the value is `1`, or the line is absent altogether.
+The status of each resource in the cluster; it will have one line for each possible `status` of each `resource`.  
+A value of `1` means the resource is in the status specified by the `status` label, a value of `0` means it is not.
 
 #### Labels
 
@@ -106,8 +101,6 @@ Either the value is `1`, or the line is absent altogether.
 - `role`:  one of `started|stopped|master|slave` or one of `starting|stopping|migrating|promoting|demoting`.
 - `status`: one of `active|orphaned|blocked|failed|failure_ignored`.
 
-The total number of lines for this metric will be the cardinality of `id` times the cardinality of `status`.
-
 
 ### `ha_cluster_pacemaker_stonith_enabled`
 


Test patch:
diff --git a/test/pacemaker.metrics b/test/pacemaker.metrics
index 04e09da..f23137b 100644
--- a/test/pacemaker.metrics
+++ b/test/pacemaker.metrics
@@ -27,29 +27,98 @@ ha_cluster_pacemaker_migration_threshold{node="node02",resource="rsc_SAPHanaTopo
 ha_cluster_pacemaker_migration_threshold{node="node02",resource="rsc_SAPHana_PRD_HDB00"} 50
 ha_cluster_pacemaker_migration_threshold{node="node02",resource="test"} 5000
 ha_cluster_pacemaker_migration_threshold{node="node02",resource="test-stop"} 5000
-# HELP ha_cluster_pacemaker_nodes The nodes in the cluster; one line per name, per status
+# HELP ha_cluster_pacemaker_nodes The status of each node in the cluster; 1 means the node is in that status, 0 otherwise
 # TYPE ha_cluster_pacemaker_nodes gauge
 ha_cluster_pacemaker_nodes{node="node01",status="dc",type="member"} 1
 ha_cluster_pacemaker_nodes{node="node01",status="expected_up",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node01",status="maintenance",type="member"} 0
 ha_cluster_pacemaker_nodes{node="node01",status="online",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node01",status="pending",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node01",status="shutdown",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node01",status="standby",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node01",status="standby_onfail",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node01",status="unclean",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node02",status="dc",type="member"} 0
 ha_cluster_pacemaker_nodes{node="node02",status="expected_up",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node02",status="maintenance",type="member"} 0
 ha_cluster_pacemaker_nodes{node="node02",status="online",type="member"} 1
-# HELP ha_cluster_pacemaker_resources The resources in the cluster; one line per id, per status
+ha_cluster_pacemaker_nodes{node="node02",status="pending",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node02",status="shutdown",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node02",status="standby",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node02",status="standby_onfail",type="member"} 0
+ha_cluster_pacemaker_nodes{node="node02",status="unclean",type="member"} 0
+# HELP ha_cluster_pacemaker_resources The status of each resource in the cluster; 1 means the resource is in that status, 0 otherwise
 # TYPE ha_cluster_pacemaker_resources gauge
-ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status=""} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status="active"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="node02",resource="test",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="node02",resource="test",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="node02",resource="test",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="node02",resource="test",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="node02",resource="test",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_fs_HA1_ASCS00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_fs_HA1_ASCS00",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_fs_HA1_ASCS00",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_fs_HA1_ASCS00",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_fs_HA1_ASCS00",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_fs_HA1_ERS10",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_fs_HA1_ERS10",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_fs_HA1_ERS10",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_fs_HA1_ERS10",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_fs_HA1_ERS10",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="",managed="true",node="node01",resource="rsc_ip_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="",managed="true",node="node01",resource="rsc_ip_PRD_HDB00",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="",managed="true",node="node01",resource="rsc_ip_PRD_HDB00",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="",managed="true",node="node01",resource="rsc_ip_PRD_HDB00",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="",managed="true",node="node01",resource="rsc_ip_PRD_HDB00",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_ip_HA1_ASCS00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_ip_HA1_ASCS00",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_ip_HA1_ASCS00",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_ip_HA1_ASCS00",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_ip_HA1_ASCS00",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_ip_HA1_ERS10",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_ip_HA1_ERS10",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_ip_HA1_ERS10",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_ip_HA1_ERS10",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_ip_HA1_ERS10",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_sap_HA1_ASCS00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_sap_HA1_ASCS00",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_sap_HA1_ASCS00",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_sap_HA1_ASCS00",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_sap_HA1_ASCS00",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_sap_HA1_ERS10",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_sap_HA1_ERS10",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_sap_HA1_ERS10",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_sap_HA1_ERS10",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_sap_HA1_ERS10",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="orphaned"} 0
 ha_cluster_pacemaker_resources{agent="stonith:external/sbd",clone="",group="",managed="true",node="node01",resource="stonith-sbd",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="stonith:external/sbd",clone="",group="",managed="true",node="node01",resource="stonith-sbd",role="started",status="blocked"} 0
+ha_cluster_pacemaker_resources{agent="stonith:external/sbd",clone="",group="",managed="true",node="node01",resource="stonith-sbd",role="started",status="failed"} 0
+ha_cluster_pacemaker_resources{agent="stonith:external/sbd",clone="",group="",managed="true",node="node01",resource="stonith-sbd",role="started",status="failure_ignored"} 0
+ha_cluster_pacemaker_resources{agent="stonith:external/sbd",clone="",group="",managed="true",node="node01",resource="stonith-sbd",role="started",status="orphaned"} 0
 # HELP ha_cluster_pacemaker_stonith_enabled Whether or not stonith is enabled
 # TYPE ha_cluster_pacemaker_stonith_enabled gauge
 ha_cluster_pacemaker_stonith_enabled 1