Repository: ClusterLabs/ha_cluster_exporter
Pull request #145 (2020-04-02 11:33:04)

Fix grouped stopped resources
follow up of #134 


Patch:
diff --git a/collector/pacemaker/crmmon/data.go b/collector/pacemaker/crmmon/data.go
index a125cc2..280a57a 100644
--- a/collector/pacemaker/crmmon/data.go
+++ b/collector/pacemaker/crmmon/data.go
@@ -32,23 +32,24 @@ type Root struct {
 		} `xml:"node"`
 	} `xml:"node_history"`
 	Resources []Resource `xml:"resources>resource"`
+	Clones    []Clone    `xml:"resources>clone"`
+	Groups    []Group    `xml:"resources>group"`
 }
 
 type Node struct {
-	Name             string     `xml:"name,attr"`
-	Id               string     `xml:"id,attr"`
-	Online           bool       `xml:"online,attr"`
-	Standby          bool       `xml:"standby,attr"`
-	StandbyOnFail    bool       `xml:"standby_onfail,attr"`
-	Maintenance      bool       `xml:"maintenance,attr"`
-	Pending          bool       `xml:"pending,attr"`
-	Unclean          bool       `xml:"unclean,attr"`
-	Shutdown         bool       `xml:"shutdown,attr"`
-	ExpectedUp       bool       `xml:"expected_up,attr"`
-	DC               bool       `xml:"is_dc,attr"`
-	ResourcesRunning int        `xml:"resources_running,attr"`
-	Type             string     `xml:"type,attr"`
-	Resources        []Resource `xml:"resource"`
+	Name             string `xml:"name,attr"`
+	Id               string `xml:"id,attr"`
+	Online           bool   `xml:"online,attr"`
+	Standby          bool   `xml:"standby,attr"`
+	StandbyOnFail    bool   `xml:"standby_onfail,attr"`
+	Maintenance      bool   `xml:"maintenance,attr"`
+	Pending          bool   `xml:"pending,attr"`
+	Unclean          bool   `xml:"unclean,attr"`
+	Shutdown         bool   `xml:"shutdown,attr"`
+	ExpectedUp       bool   `xml:"expected_up,attr"`
+	DC               bool   `xml:"is_dc,attr"`
+	ResourcesRunning int    `xml:"resources_running,attr"`
+	Type             string `xml:"type,attr"`
 }
 
 type Resource struct {
@@ -62,4 +63,24 @@ type Resource struct {
 	Failed         bool   `xml:"failed,attr"`
 	FailureIgnored bool   `xml:"failure_ignored,attr"`
 	NodesRunningOn int    `xml:"nodes_running_on,attr"`
+	Node           *struct {
+		Name   string `xml:"name,attr"`
+		Id     string `xml:"id,attr"`
+		Cached bool   `xml:"cached,attr"`
+	} `xml:"node,omitempty"`
+}
+
+type Clone struct {
+	Id             string     `xml:"id,attr"`
+	MultiState     bool       `xml:"multi_state,attr"`
+	Managed        bool       `xml:"managed,attr"`
+	Failed         bool       `xml:"failed,attr"`
+	FailureIgnored bool       `xml:"failure_ignored,attr"`
+	Unique         bool       `xml:"unique,attr"`
+	Resources      []Resource `xml:"resource"`
+}
+
+type Group struct {
+	Id        string     `xml:"id,attr"`
+	Resources []Resource `xml:"resource"`
 }
diff --git a/collector/pacemaker/crmmon/parser.go b/collector/pacemaker/crmmon/parser.go
index f9fd1e3..195e31b 100644
--- a/collector/pacemaker/crmmon/parser.go
+++ b/collector/pacemaker/crmmon/parser.go
@@ -16,7 +16,7 @@ type crmMonParser struct {
 }
 
 func (c *crmMonParser) Parse() (crmMon Root, err error) {
-	crmMonXML, err := exec.Command(c.crmMonPath, "-X", "--group-by-node", "--inactive").Output()
+	crmMonXML, err := exec.Command(c.crmMonPath, "-X", "--inactive").Output()
 	if err != nil {
 		return crmMon, errors.Wrap(err, "error while executing crm_mon")
 	}
diff --git a/collector/pacemaker/pacemaker.go b/collector/pacemaker/pacemaker.go
index 9ca1a88..9b0b7e8 100644
--- a/collector/pacemaker/pacemaker.go
+++ b/collector/pacemaker/pacemaker.go
@@ -27,7 +27,7 @@ func NewCollector(crmMonPath string, cibAdminPath string) (*pacemakerCollector,
 		cib.NewCibAdminParser(cibAdminPath),
 	}
 	c.SetDescriptor("nodes", "The nodes in the cluster; one line per name, per status", []string{"node", "type", "status"})
-	c.SetDescriptor("resources", "The resources in the cluster; one line per id, per status", []string{"node", "resource", "role", "managed", "status", "agent"})
+	c.SetDescriptor("resources", "The resources in the cluster; one line per id, per status", []string{"node", "resource", "role", "managed", "status", "agent", "group", "clone"})
 	c.SetDescriptor("stonith_enabled", "Whether or not stonith is enabled", nil)
 	c.SetDescriptor("fail_count", "The Fail count number per node and resource id", []string{"node", "resource"})
 	c.SetDescriptor("migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"})
@@ -66,7 +66,7 @@ func (c *pacemakerCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- c.MakeGaugeMetric("stonith_enabled", stonithEnabled)
 
 	c.recordNodes(crmMon, ch)
-	c.recordUngroupedResources(crmMon, ch)
+	c.recordResources(crmMon, ch)
 	c.recordFailCounts(crmMon, ch)
 	c.recordMigrationThresholds(crmMon, ch)
 	c.recordResourceAgentsChanges(crmMon, ch)
@@ -104,19 +104,26 @@ func (c *pacemakerCollector) recordNodes(crmMon crmmon.Root, ch chan<- prometheu
 				ch <- c.MakeGaugeMetric("nodes", float64(1), node.Name, nodeType, nodeStatus)
 			}
 		}
-
-		c.recordNodeResources(node, ch)
 	}
 }
 
-func (c *pacemakerCollector) recordNodeResources(node crmmon.Node, ch chan<- prometheus.Metric) {
-	nodeName := node.Name
-	for _, resource := range node.Resources {
-		c.recordResource(resource, nodeName, ch)
+func (c *pacemakerCollector) recordResources(crmMon crmmon.Root, ch chan<- prometheus.Metric) {
+	for _, resource := range crmMon.Resources {
+		c.recordResource(resource, "", "", ch)
+	}
+	for _, clone := range crmMon.Clones {
+		for _, resource := range clone.Resources {
+			c.recordResource(resource, "", clone.Id, ch)
+		}
+	}
+	for _, group := range crmMon.Groups {
+		for _, resource := range group.Resources {
+			c.recordResource(resource, group.Id, "", ch)
+		}
 	}
 }
 
-func (c *pacemakerCollector) recordResource(resource crmmon.Resource, nodeName string, ch chan<- prometheus.Metric) {
+func (c *pacemakerCollector) recordResource(resource crmmon.Resource, group string, clone string, ch chan<- prometheus.Metric) {
 
 	// this is a map of boolean flags for each possible status of the resource
 	resourceStatuses := map[string]bool{
@@ -135,6 +142,10 @@ func (c *pacemakerCollector) recordResource(resource crmmon.Resource, nodeName s
 		if !flag {
 			continue
 		}
+		var nodeName string
+		if resource.Node != nil {
+			nodeName = resource.Node.Name
+		}
 		ch <- c.MakeGaugeMetric(
 			"resources",
 			float64(1),
@@ -143,13 +154,9 @@ func (c *pacemakerCollector) recordResource(resource crmmon.Resource, nodeName s
 			strings.ToLower(resource.Role),
 			strconv.FormatBool(resource.Managed),
 			resourceStatus,
-			resource.Agent)
-	}
-}
-
-func (c *pacemakerCollector) recordUngroupedResources(crmMon crmmon.Root, ch chan<- prometheus.Metric) {
-	for _, resource := range crmMon.Resources {
-		c.recordResource(resource, "", ch)
+			resource.Agent,
+			group,
+			clone)
 	}
 }
 


Test patch:
diff --git a/collector/pacemaker/cib/parser_test.go b/collector/pacemaker/cib/parser_test.go
index a7f69a8..1fec2fd 100644
--- a/collector/pacemaker/cib/parser_test.go
+++ b/collector/pacemaker/cib/parser_test.go
@@ -16,8 +16,8 @@ func TestParse(t *testing.T) {
 	data, err := p.Parse()
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(data.Configuration.Nodes))
-	assert.Equal(t, "hana01", data.Configuration.Nodes[0].Uname)
-	assert.Equal(t, "hana02", data.Configuration.Nodes[1].Uname)
+	assert.Equal(t, "node01", data.Configuration.Nodes[0].Uname)
+	assert.Equal(t, "node02", data.Configuration.Nodes[1].Uname)
 	assert.Equal(t, 4, len(data.Configuration.Resources.Primitives))
 	assert.Equal(t, 1, len(data.Configuration.Resources.Masters))
 	assert.Equal(t, 1, len(data.Configuration.Resources.Clones))
diff --git a/collector/pacemaker/crmmon/parser_test.go b/collector/pacemaker/crmmon/parser_test.go
index 29c6216..1c1153d 100644
--- a/collector/pacemaker/crmmon/parser_test.go
+++ b/collector/pacemaker/crmmon/parser_test.go
@@ -21,7 +21,7 @@ func TestParse(t *testing.T) {
 	assert.Equal(t, 0, data.Summary.Resources.Blocked)
 	assert.Equal(t, "Fri Oct 18 11:48:22 2019", data.Summary.LastChange.Time)
 	assert.Equal(t, 2, data.Summary.Nodes.Number)
-	assert.Equal(t, "hana01", data.Nodes[0].Name)
+	assert.Equal(t, "node01", data.Nodes[0].Name)
 	assert.Equal(t, "1084783375", data.Nodes[0].Id)
 	assert.Equal(t, true, data.Nodes[0].Online)
 	assert.Equal(t, true, data.Nodes[0].ExpectedUp)
@@ -32,7 +32,7 @@ func TestParse(t *testing.T) {
 	assert.Equal(t, false, data.Nodes[0].Maintenance)
 	assert.Equal(t, false, data.Nodes[0].Pending)
 	assert.Equal(t, false, data.Nodes[0].Standby)
-	assert.Equal(t, "hana02", data.Nodes[1].Name)
+	assert.Equal(t, "node02", data.Nodes[1].Name)
 	assert.Equal(t, "1084783376", data.Nodes[1].Id)
 	assert.Equal(t, true, data.Nodes[1].Online)
 	assert.Equal(t, true, data.Nodes[1].ExpectedUp)
@@ -43,12 +43,46 @@ func TestParse(t *testing.T) {
 	assert.Equal(t, false, data.Nodes[1].Maintenance)
 	assert.Equal(t, false, data.Nodes[1].Pending)
 	assert.Equal(t, false, data.Nodes[1].Standby)
-	assert.Equal(t, "hana01", data.NodeHistory.Node[0].Name)
+	assert.Equal(t, "node01", data.NodeHistory.Node[0].Name)
 	assert.Equal(t, 5000, data.NodeHistory.Node[0].ResourceHistory[0].MigrationThreshold)
 	assert.Equal(t, 2, data.NodeHistory.Node[0].ResourceHistory[1].FailCount)
 	assert.Equal(t, "rsc_SAPHana_PRD_HDB00", data.NodeHistory.Node[0].ResourceHistory[0].Name)
-	assert.Equal(t, 1, len(data.Resources))
+	assert.Equal(t, 4, len(data.Resources))
 	assert.Equal(t, "test-stop", data.Resources[0].Id)
 	assert.Equal(t, false, data.Resources[0].Active)
 	assert.Equal(t, "Stopped", data.Resources[0].Role)
 }
+
+func TestParseClones(t *testing.T) {
+	p := NewCrmMonParser("../../../test/fake_crm_mon.sh")
+	data, err := p.Parse()
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(data.Clones))
+	assert.Equal(t, "msl_SAPHana_PRD_HDB00", data.Clones[0].Id)
+	assert.Equal(t, "cln_SAPHanaTopology_PRD_HDB00", data.Clones[1].Id)
+	assert.Equal(t, 2, len(data.Clones[0].Resources))
+	assert.Equal(t, 2, len(data.Clones[1].Resources))
+	assert.Equal(t, "rsc_SAPHana_PRD_HDB00", data.Clones[0].Resources[0].Id)
+	assert.Equal(t, "Master", data.Clones[0].Resources[0].Role)
+	assert.Equal(t, "rsc_SAPHana_PRD_HDB00", data.Clones[0].Resources[1].Id)
+	assert.Equal(t, "Slave", data.Clones[0].Resources[1].Role)
+}
+
+func TestParseGroups(t *testing.T) {
+	p := NewCrmMonParser("../../../test/fake_crm_mon.sh")
+	data, err := p.Parse()
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(data.Groups))
+
+	assert.Equal(t, "grp_HA1_ASCS00", data.Groups[0].Id)
+	assert.Equal(t, 3, len(data.Groups[0].Resources))
+	assert.Equal(t, "rsc_ip_HA1_ASCS00", data.Groups[0].Resources[0].Id)
+	assert.Equal(t, "rsc_fs_HA1_ASCS00", data.Groups[0].Resources[1].Id)
+	assert.Equal(t, "rsc_sap_HA1_ASCS00", data.Groups[0].Resources[2].Id)
+
+	assert.Equal(t, "grp_HA1_ERS10", data.Groups[1].Id)
+	assert.Equal(t, 3, len(data.Groups[1].Resources))
+	assert.Equal(t, "rsc_ip_HA1_ERS10", data.Groups[1].Resources[0].Id)
+	assert.Equal(t, "rsc_fs_HA1_ERS10", data.Groups[1].Resources[1].Id)
+	assert.Equal(t, "rsc_sap_HA1_ERS10", data.Groups[1].Resources[2].Id)
+}
diff --git a/test/fake_cibadmin.sh b/test/fake_cibadmin.sh
index 855583f..476a42b 100755
--- a/test/fake_cibadmin.sh
+++ b/test/fake_cibadmin.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 cat <<EOF
-<cib crm_feature_set="3.1.0" validate-with="pacemaker-3.0" epoch="6881" num_updates="0" admin_epoch="0" cib-last-written="Mon Nov 18 17:48:21 2019" update-origin="hana01" update-client="crm_attribute" update-user="root" have-quorum="1" dc-uuid="1084783375">
+<cib crm_feature_set="3.1.0" validate-with="pacemaker-3.0" epoch="6881" num_updates="0" admin_epoch="0" cib-last-written="Mon Nov 18 17:48:21 2019" update-origin="node01" update-client="crm_attribute" update-user="root" have-quorum="1" dc-uuid="1084783375">
   <configuration>
     <crm_config>
       <cluster_property_set id="cib-bootstrap-options">
@@ -14,22 +14,22 @@ cat <<EOF
       </cluster_property_set>
     </crm_config>
     <nodes>
-      <node id="1084783375" uname="hana01">
+      <node id="1084783375" uname="node01">
         <instance_attributes id="nodes-1084783375">
           <nvpair id="nodes-1084783375-lpa_prd_lpt" name="lpa_prd_lpt" value="1574095701"/>
-          <nvpair id="nodes-1084783375-hana_prd_vhost" name="hana_prd_vhost" value="hana01"/>
+          <nvpair id="nodes-1084783375-hana_prd_vhost" name="hana_prd_vhost" value="node01"/>
           <nvpair id="nodes-1084783375-hana_prd_site" name="hana_prd_site" value="PRIMARY_SITE_NAME"/>
           <nvpair id="nodes-1084783375-hana_prd_op_mode" name="hana_prd_op_mode" value="logreplay"/>
           <nvpair id="nodes-1084783375-hana_prd_srmode" name="hana_prd_srmode" value="sync"/>
-          <nvpair id="nodes-1084783375-hana_prd_remoteHost" name="hana_prd_remoteHost" value="hana02"/>
+          <nvpair id="nodes-1084783375-hana_prd_remoteHost" name="hana_prd_remoteHost" value="node02"/>
         </instance_attributes>
       </node>
-      <node id="1084783376" uname="hana02">
+      <node id="1084783376" uname="node02">
         <instance_attributes id="nodes-1084783376">
           <nvpair id="nodes-1084783376-lpa_prd_lpt" name="lpa_prd_lpt" value="30"/>
           <nvpair id="nodes-1084783376-hana_prd_op_mode" name="hana_prd_op_mode" value="logreplay"/>
-          <nvpair id="nodes-1084783376-hana_prd_vhost" name="hana_prd_vhost" value="hana02"/>
-          <nvpair id="nodes-1084783376-hana_prd_remoteHost" name="hana_prd_remoteHost" value="hana01"/>
+          <nvpair id="nodes-1084783376-hana_prd_vhost" name="hana_prd_vhost" value="node02"/>
+          <nvpair id="nodes-1084783376-hana_prd_remoteHost" name="hana_prd_remoteHost" value="node01"/>
           <nvpair id="nodes-1084783376-hana_prd_site" name="hana_prd_site" value="SECONDARY_SITE_NAME"/>
           <nvpair id="nodes-1084783376-hana_prd_srmode" name="hana_prd_srmode" value="sync"/>
         </instance_attributes>
@@ -107,10 +107,10 @@ cat <<EOF
     <constraints>
       <rsc_colocation id="col_saphana_ip_PRD_HDB00" score="2000" rsc="rsc_ip_PRD_HDB00" rsc-role="Started" with-rsc="msl_SAPHana_PRD_HDB00" with-rsc-role="Master"/>
       <rsc_order id="ord_SAPHana_PRD_HDB00" kind="Optional" first="cln_SAPHanaTopology_PRD_HDB00" then="msl_SAPHana_PRD_HDB00"/>
-      <rsc_location id="cli-prefer-msl_SAPHana_PRD_HDB00" rsc="msl_SAPHana_PRD_HDB00" role="Started" node="hana01" score="INFINITY"/>
-      <rsc_location id="cli-prefer-cln_SAPHanaTopology_PRD_HDB00" rsc="cln_SAPHanaTopology_PRD_HDB00" role="Started" node="hana01" score="INFINITY"/>
-      <rsc_location id="cli-ban-msl_SAPHana_PRD_HDB00-on-hana01" rsc="msl_SAPHana_PRD_HDB00" role="Started" node="hana01" score="-INFINITY"/>
-      <rsc_location id="test" rsc="test" role="Started" node="hana02" score="666"/>
+      <rsc_location id="cli-prefer-msl_SAPHana_PRD_HDB00" rsc="msl_SAPHana_PRD_HDB00" role="Started" node="node01" score="INFINITY"/>
+      <rsc_location id="cli-prefer-cln_SAPHanaTopology_PRD_HDB00" rsc="cln_SAPHanaTopology_PRD_HDB00" role="Started" node="node01" score="INFINITY"/>
+      <rsc_location id="cli-ban-msl_SAPHana_PRD_HDB00-on-node01" rsc="msl_SAPHana_PRD_HDB00" role="Started" node="node01" score="-INFINITY"/>
+      <rsc_location id="test" rsc="test" role="Started" node="node02" score="666"/>
     </constraints>
     <rsc_defaults>
       <meta_attributes id="rsc-options">
@@ -126,7 +126,7 @@ cat <<EOF
     </op_defaults>
   </configuration>
   <status>
-    <node_state id="1084783375" uname="hana01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+    <node_state id="1084783375" uname="node01" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
       <transient_attributes id="1084783375">
         <instance_attributes id="status-1084783375">
           <nvpair id="status-1084783375-master-rsc_SAPHana_PRD_HDB00" name="master-rsc_SAPHana_PRD_HDB00" value="150"/>
@@ -139,53 +139,53 @@ cat <<EOF
       <lrm id="1084783375">
         <lrm_resources>
           <lrm_resource id="rsc_SAPHana_PRD_HDB00" type="SAPHana" class="ocf" provider="suse">
-            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_failure_0" operation_key="rsc_SAPHana_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="3:3:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;3:3:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1573663876" last-rc-change="1573663876" exec-time="3450" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1"/>
-            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_0" operation_key="rsc_SAPHana_PRD_HDB00_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="12:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;12:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1573663898" last-rc-change="1573663898" exec-time="2257" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1" op-force-restart=" INSTANCE_PROFILE " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
-            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_monitor_60000" operation_key="rsc_SAPHana_PRD_HDB00_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="14:9:8:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:8;14:9:8:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="32" rc-code="8" op-status="0" interval="60000" last-rc-change="1573663906" exec-time="3586" queue-time="0" op-digest="05b857e482ebd46019d347fd55ebbcdb"/>
+            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_failure_0" operation_key="rsc_SAPHana_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="3:3:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;3:3:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1573663876" last-rc-change="1573663876" exec-time="3450" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1"/>
+            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_0" operation_key="rsc_SAPHana_PRD_HDB00_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="12:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;12:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1573663898" last-rc-change="1573663898" exec-time="2257" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1" op-force-restart=" INSTANCE_PROFILE " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_monitor_60000" operation_key="rsc_SAPHana_PRD_HDB00_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="14:9:8:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:8;14:9:8:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="32" rc-code="8" op-status="0" interval="60000" last-rc-change="1573663906" exec-time="3586" queue-time="0" op-digest="05b857e482ebd46019d347fd55ebbcdb"/>
           </lrm_resource>
           <lrm_resource id="rsc_ip_PRD_HDB00" type="IPaddr2" class="ocf" provider="heartbeat">
-            <lrm_rsc_op id="rsc_ip_PRD_HDB00_last_0" operation_key="rsc_ip_PRD_HDB00_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="7:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="21" rc-code="0" op-status="0" interval="0" last-run="1573663876" last-rc-change="1573663876" exec-time="136" queue-time="0" op-digest="a6da6959be1e15c2f9f5e88476e82ba4"/>
-            <lrm_rsc_op id="rsc_ip_PRD_HDB00_monitor_10000" operation_key="rsc_ip_PRD_HDB00_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="8:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;8:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="22" rc-code="0" op-status="0" interval="10000" last-rc-change="1573663876" exec-time="85" queue-time="0" op-digest="c7df6e2194c50ed86aa98b66e909fe11"/>
+            <lrm_rsc_op id="rsc_ip_PRD_HDB00_last_0" operation_key="rsc_ip_PRD_HDB00_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="7:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="21" rc-code="0" op-status="0" interval="0" last-run="1573663876" last-rc-change="1573663876" exec-time="136" queue-time="0" op-digest="a6da6959be1e15c2f9f5e88476e82ba4"/>
+            <lrm_rsc_op id="rsc_ip_PRD_HDB00_monitor_10000" operation_key="rsc_ip_PRD_HDB00_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="8:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;8:3:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="22" rc-code="0" op-status="0" interval="10000" last-rc-change="1573663876" exec-time="85" queue-time="0" op-digest="c7df6e2194c50ed86aa98b66e909fe11"/>
           </lrm_resource>
           <lrm_resource id="stonith-sbd" type="external/sbd" class="stonith">
-            <lrm_rsc_op id="stonith-sbd_last_0" operation_key="stonith-sbd_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="3:2:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;3:2:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1573663874" last-rc-change="1573663874" exec-time="2238" queue-time="0" op-digest="265be3215da5e5037d35e7fe1bcc5ae0"/>
+            <lrm_rsc_op id="stonith-sbd_last_0" operation_key="stonith-sbd_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="3:2:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;3:2:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1573663874" last-rc-change="1573663874" exec-time="2238" queue-time="0" op-digest="265be3215da5e5037d35e7fe1bcc5ae0"/>
           </lrm_resource>
           <lrm_resource id="rsc_SAPHanaTopology_PRD_HDB00" type="SAPHanaTopology" class="ocf" provider="suse">
-            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_last_0" operation_key="rsc_SAPHanaTopology_PRD_HDB00_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="19:4:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;19:4:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="24" rc-code="0" op-status="0" interval="0" last-run="1573663881" last-rc-change="1573663881" exec-time="4355" queue-time="0" op-digest="2d8d79c3726afb91c33d406d5af79b53" op-force-restart="" op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
-            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation_key="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="22:5:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;22:5:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1573663885" exec-time="4949" queue-time="0" op-digest="64db68ca3e12e0d41eb98ce63b9610d2"/>
+            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_last_0" operation_key="rsc_SAPHanaTopology_PRD_HDB00_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="19:4:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;19:4:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="24" rc-code="0" op-status="0" interval="0" last-run="1573663881" last-rc-change="1573663881" exec-time="4355" queue-time="0" op-digest="2d8d79c3726afb91c33d406d5af79b53" op-force-restart="" op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation_key="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.1.0" transition-key="22:5:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;22:5:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="26" rc-code="0" op-status="0" interval="10000" last-rc-change="1573663885" exec-time="4949" queue-time="0" op-digest="64db68ca3e12e0d41eb98ce63b9610d2"/>
           </lrm_resource>
           <lrm_resource id="test" type="Dummy" class="ocf" provider="heartbeat">
-            <lrm_rsc_op id="test_last_0" operation_key="test_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="8:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;8:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana01" call-id="37" rc-code="0" op-status="0" interval="0" last-run="1574095329" last-rc-change="1574095329" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="test_last_0" operation_key="test_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="8:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;8:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node01" call-id="37" rc-code="0" op-status="0" interval="0" last-run="1574095329" last-rc-change="1574095329" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
           </lrm_resource>
           <lrm_resource id="test-stop" type="Dummy" class="ocf" provider="heartbeat">
-            <lrm_rsc_op id="test-stop_last_0" operation_key="test-stop_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:13662:7:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" transition-magic="0:7;7:13662:7:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" exit-reason="" on_node="stefanotorresi2-hana01" call-id="40" rc-code="7" op-status="0" interval="0" last-run="1582534010" last-rc-change="1582534010" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="test-stop_last_0" operation_key="test-stop_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:13662:7:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" transition-magic="0:7;7:13662:7:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" exit-reason="" on_node="stefanotorresi2-node01" call-id="40" rc-code="7" op-status="0" interval="0" last-run="1582534010" last-rc-change="1582534010" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
           </lrm_resource>
         </lrm_resources>
       </lrm>
     </node_state>
-    <node_state id="1084783376" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" uname="hana02" join="member" expected="member">
+    <node_state id="1084783376" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" uname="node02" join="member" expected="member">
       <lrm id="1084783376">
         <lrm_resources>
           <lrm_resource id="stonith-sbd" type="external/sbd" class="stonith">
-            <lrm_rsc_op id="stonith-sbd_last_0" operation_key="stonith-sbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="5:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:7;5:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="1" queue-time="0" op-digest="265be3215da5e5037d35e7fe1bcc5ae0"/>
+            <lrm_rsc_op id="stonith-sbd_last_0" operation_key="stonith-sbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="5:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:7;5:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="1" queue-time="0" op-digest="265be3215da5e5037d35e7fe1bcc5ae0"/>
           </lrm_resource>
           <lrm_resource id="rsc_ip_PRD_HDB00" type="IPaddr2" class="ocf" provider="heartbeat">
-            <lrm_rsc_op id="rsc_ip_PRD_HDB00_last_0" operation_key="rsc_ip_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="6:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:7;6:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="56" queue-time="0" op-digest="a6da6959be1e15c2f9f5e88476e82ba4"/>
+            <lrm_rsc_op id="rsc_ip_PRD_HDB00_last_0" operation_key="rsc_ip_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="6:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:7;6:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="56" queue-time="0" op-digest="a6da6959be1e15c2f9f5e88476e82ba4"/>
           </lrm_resource>
           <lrm_resource id="rsc_SAPHana_PRD_HDB00" type="SAPHana" class="ocf" provider="suse">
-            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_0" operation_key="rsc_SAPHana_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="3515" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1" op-force-restart=" INSTANCE_PROFILE " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
-            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_failure_0" operation_key="rsc_SAPHana_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="3515" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1"/>
-            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_monitor_61000" operation_key="rsc_SAPHana_PRD_HDB00_monitor_61000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="13:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;13:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="20" rc-code="0" op-status="0" interval="61000" last-rc-change="1573663895" exec-time="3225" queue-time="0" op-digest="05b857e482ebd46019d347fd55ebbcdb"/>
+            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_0" operation_key="rsc_SAPHana_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="3515" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1" op-force-restart=" INSTANCE_PROFILE " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_last_failure_0" operation_key="rsc_SAPHana_PRD_HDB00_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:6:7:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1573663890" last-rc-change="1573663890" exec-time="3515" queue-time="0" op-digest="ff4ff123bc6f906497ef0ef5e44dffd1"/>
+            <lrm_rsc_op id="rsc_SAPHana_PRD_HDB00_monitor_61000" operation_key="rsc_SAPHana_PRD_HDB00_monitor_61000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="13:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;13:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="20" rc-code="0" op-status="0" interval="61000" last-rc-change="1573663895" exec-time="3225" queue-time="0" op-digest="05b857e482ebd46019d347fd55ebbcdb"/>
           </lrm_resource>
           <lrm_resource id="rsc_SAPHanaTopology_PRD_HDB00" type="SAPHanaTopology" class="ocf" provider="suse">
-            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_last_0" operation_key="rsc_SAPHanaTopology_PRD_HDB00_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="24:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;24:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="21" rc-code="0" op-status="0" interval="0" last-run="1573663895" last-rc-change="1573663895" exec-time="3650" queue-time="0" op-digest="2d8d79c3726afb91c33d406d5af79b53" op-force-restart="" op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
-            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation_key="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="28:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;28:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="22" rc-code="0" op-status="0" interval="10000" last-rc-change="1573663898" exec-time="3978" queue-time="0" op-digest="64db68ca3e12e0d41eb98ce63b9610d2"/>
+            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_last_0" operation_key="rsc_SAPHanaTopology_PRD_HDB00_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="24:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;24:7:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="21" rc-code="0" op-status="0" interval="0" last-run="1573663895" last-rc-change="1573663895" exec-time="3650" queue-time="0" op-digest="2d8d79c3726afb91c33d406d5af79b53" op-force-restart="" op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation_key="rsc_SAPHanaTopology_PRD_HDB00_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="28:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;28:8:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="22" rc-code="0" op-status="0" interval="10000" last-rc-change="1573663898" exec-time="3978" queue-time="0" op-digest="64db68ca3e12e0d41eb98ce63b9610d2"/>
           </lrm_resource>
           <lrm_resource id="test" type="Dummy" class="ocf" provider="heartbeat">
-            <lrm_rsc_op id="test_last_0" operation_key="test_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="hana02" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1574095329" last-rc-change="1574095329" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="test_last_0" operation_key="test_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="7:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" transition-magic="0:0;7:6863:0:70ea6528-73ad-48be-9eb7-583ee933f216" exit-reason="" on_node="node02" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1574095329" last-rc-change="1574095329" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
           </lrm_resource>
           <lrm_resource id="test-stop" type="Dummy" class="ocf" provider="heartbeat">
-            <lrm_rsc_op id="test-stop_last_0" operation_key="test-stop_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="35:13663:0:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" transition-magic="0:0;35:13663:0:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" exit-reason="" on_node="stefanotorresi2-hana02" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1582534018" last-rc-change="1582534018" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="test-stop_last_0" operation_key="test-stop_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.1.0" transition-key="35:13663:0:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" transition-magic="0:0;35:13663:0:5a2e7427-7cbd-4bd9-8e8c-fd633866c4a9" exit-reason="" on_node="stefanotorresi2-node02" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1582534018" last-rc-change="1582534018" exec-time="12" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
           </lrm_resource>
         </lrm_resources>
       </lrm>
diff --git a/test/fake_crm_mon.sh b/test/fake_crm_mon.sh
index 04a81c6..3a85220 100755
--- a/test/fake_crm_mon.sh
+++ b/test/fake_crm_mon.sh
@@ -5,59 +5,97 @@ cat <<EOF
 <crm_mon version="2.0.0">
     <summary>
         <stack type="corosync" />
-        <current_dc present="true" version="1.1.18+20180430.b12c320f5-3.15.1-b12c320f5" name="hana01" id="1084783375" with_quorum="true" />
+        <current_dc present="true" version="1.1.18+20180430.b12c320f5-3.15.1-b12c320f5" name="node01" id="1084783375" with_quorum="true" />
         <last_update time="Fri Oct 18 11:48:54 2019" />
-        <last_change time="Fri Oct 18 11:48:22 2019" user="root" client="crm_attribute" origin="hana01" />
+        <last_change time="Fri Oct 18 11:48:22 2019" user="root" client="crm_attribute" origin="node01" />
         <nodes_configured number="2" />
         <resources_configured number="8" disabled="1" blocked="0" />
         <cluster_options stonith-enabled="true" symmetric-cluster="true" no-quorum-policy="stop" maintenance-mode="false" />
     </summary>
     <nodes>
-        <node name="hana01" id="1084783375" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="4" type="member" >
-            <resource id="rsc_SAPHana_PRD_HDB00" resource_agent="ocf::suse:SAPHana" role="Master" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-            <resource id="rsc_ip_PRD_HDB00" resource_agent="ocf::heartbeat:IPaddr2" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-            <resource id="stonith-sbd" resource_agent="stonith:external/sbd" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-            <resource id="rsc_SAPHanaTopology_PRD_HDB00" resource_agent="ocf::suse:SAPHanaTopology" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-        </node>
-        <node name="hana02" id="1084783376" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="2" type="member" >
-            <resource id="rsc_SAPHana_PRD_HDB00" resource_agent="ocf::suse:SAPHana" role="Slave" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-            <resource id="rsc_SAPHanaTopology_PRD_HDB00" resource_agent="ocf::suse:SAPHanaTopology" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-            <resource id="test" resource_agent="ocf::heartbeat:Dummy" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" />
-        </node>
+        <node name="node01" id="1084783375" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="true" resources_running="7" type="member" />
+        <node name="node02" id="1084783376" online="true" standby="false" standby_onfail="false" maintenance="false" pending="false" unclean="false" shutdown="false" expected_up="true" is_dc="false" resources_running="5" type="member" />
     </nodes>
     <resources>
         <resource id="test-stop" resource_agent="ocf::heartbeat:Dummy" role="Stopped" target_role="Stopped" active="false" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="0" />
+        <resource id="test" resource_agent="ocf::heartbeat:Dummy" role="Started" target_role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1">
+            <node name="node02" id="1084783376" cached="false"/>
+        </resource>
+        <resource id="stonith-sbd" resource_agent="stonith:external/sbd" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+            <node name="node01" id="1084783375" cached="false"/>
+        </resource>
+        <resource id="rsc_ip_PRD_HDB00" resource_agent="ocf::heartbeat:IPaddr2" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+            <node name="node01" id="1084783375" cached="false"/>
+        </resource>
+        <clone id="msl_SAPHana_PRD_HDB00" multi_state="true" unique="false" managed="true" failed="false" failure_ignored="false" >
+            <resource id="rsc_SAPHana_PRD_HDB00" resource_agent="ocf::suse:SAPHana" role="Master" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                <node name="node01" id="1084783375" cached="false"/>
+            </resource>
+            <resource id="rsc_SAPHana_PRD_HDB00" resource_agent="ocf::suse:SAPHana" role="Slave" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" pending="Monitoring" >
+                <node name="node02" id="1084783376" cached="false"/>
+            </resource>
+        </clone>
+        <clone id="cln_SAPHanaTopology_PRD_HDB00" multi_state="false" unique="false" managed="true" failed="false" failure_ignored="false" >
+            <resource id="rsc_SAPHanaTopology_PRD_HDB00" resource_agent="ocf::suse:SAPHanaTopology" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                <node name="node01" id="1084783375" cached="false"/>
+            </resource>
+            <resource id="rsc_SAPHanaTopology_PRD_HDB00" resource_agent="ocf::suse:SAPHanaTopology" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                <node name="node02" id="1084783376" cached="false"/>
+            </resource>
+        </clone>
+        <group id="grp_HA1_ASCS00" number_resources="3" >
+             <resource id="rsc_ip_HA1_ASCS00" resource_agent="ocf::heartbeat:IPaddr2" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                 <node name="node01" id="1084783375" cached="false"/>
+             </resource>
+             <resource id="rsc_fs_HA1_ASCS00" resource_agent="ocf::heartbeat:Filesystem" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                 <node name="node01" id="1084783375" cached="false"/>
+             </resource>
+             <resource id="rsc_sap_HA1_ASCS00" resource_agent="ocf::heartbeat:SAPInstance" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                 <node name="node01" id="1084783375" cached="false"/>
+             </resource>
+        </group>
+        <group id="grp_HA1_ERS10" number_resources="3" >
+             <resource id="rsc_ip_HA1_ERS10" resource_agent="ocf::heartbeat:IPaddr2" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                 <node name="node02" id="1084783376" cached="false"/>
+             </resource>
+             <resource id="rsc_fs_HA1_ERS10" resource_agent="ocf::heartbeat:Filesystem" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                 <node name="node02" id="1084783376" cached="false"/>
+             </resource>
+             <resource id="rsc_sap_HA1_ERS10" resource_agent="ocf::heartbeat:SAPInstance" role="Started" active="true" orphaned="false" blocked="false" managed="true" failed="false" failure_ignored="false" nodes_running_on="1" >
+                 <node name="node02" id="1084783376" cached="false"/>
+             </resource>
+        </group>
     </resources>
     <node_attributes>
-        <node name="hana01">
+        <node name="node01">
             <attribute name="hana_prd_clone_state" value="PROMOTED" />
             <attribute name="hana_prd_op_mode" value="logreplay" />
-            <attribute name="hana_prd_remoteHost" value="hana02" />
+            <attribute name="hana_prd_remoteHost" value="node02" />
             <attribute name="hana_prd_roles" value="4:P:master1:master:worker:master" />
             <attribute name="hana_prd_site" value="PRIMARY_SITE_NAME" />
             <attribute name="hana_prd_srmode" value="sync" />
             <attribute name="hana_prd_sync_state" value="PRIM" />
             <attribute name="hana_prd_version" value="2.00.040.00.1553674765" />
-            <attribute name="hana_prd_vhost" value="hana01" />
+            <attribute name="hana_prd_vhost" value="node01" />
             <attribute name="lpa_prd_lpt" value="1571392102" />
             <attribute name="master-rsc_SAPHana_PRD_HDB00" value="150" />
         </node>
-        <node name="hana02">
+        <node name="node02">
             <attribute name="hana_prd_clone_state" value="DEMOTED" />
             <attribute name="hana_prd_op_mode" value="logreplay" />
-            <attribute name="hana_prd_remoteHost" value="hana01" />
+            <attribute name="hana_prd_remoteHost" value="node01" />
             <attribute name="hana_prd_roles" value="4:S:master1:master:worker:master" />
             <attribute name="hana_prd_site" value="SECONDARY_SITE_NAME" />
             <attribute name="hana_prd_srmode" value="sync" />
             <attribute name="hana_prd_sync_state" value="SOK" />
             <attribute name="hana_prd_version" value="2.00.040.00.1553674765" />
-            <attribute name="hana_prd_vhost" value="hana02" />
+            <attribute name="hana_prd_vhost" value="node02" />
             <attribute name="lpa_prd_lpt" value="30" />
             <attribute name="master-rsc_SAPHana_PRD_HDB00" value="100" />
         </node>
     </node_attributes>
     <node_history>
-        <node name="hana01">
+        <node name="node01">
             <resource_history id="rsc_SAPHana_PRD_HDB00" orphan="false" migration-threshold="5000" fail-count="1000000" last-failure="Wed Oct 23 12:37:22 2019">
                 <operation_history call="15" task="probe" last-rc-change="Thu Oct 10 12:57:33 2019" last-run="Thu Oct 10 12:57:33 2019" exec-time="4140ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="31" task="promote" last-rc-change="Thu Oct 10 12:57:57 2019" last-run="Thu Oct 10 12:57:57 2019" exec-time="2015ms" queue-time="0ms" rc="0" rc_text="ok" />
@@ -75,7 +113,7 @@ cat <<EOF
                 <operation_history call="26" task="monitor" interval="10000ms" last-rc-change="Thu Oct 10 12:57:46 2019" exec-time="4220ms" queue-time="0ms" rc="0" rc_text="ok" />
             </resource_history>
         </node>
-        <node name="hana02">
+        <node name="node02">
             <resource_history id="rsc_SAPHana_PRD_HDB00" orphan="false" migration-threshold="50" fail-count="300" last-failure="Wed Oct 23 12:37:22 2019">
                 <operation_history call="22" task="start" last-rc-change="Thu Oct 17 15:22:40 2019" last-run="Thu Oct 17 15:22:40 2019" exec-time="44083ms" queue-time="0ms" rc="0" rc_text="ok" />
                 <operation_history call="23" task="monitor" interval="61000ms" last-rc-change="Thu Oct 17 15:23:24 2019" exec-time="2605ms" queue-time="0ms" rc="0" rc_text="ok" />
diff --git a/test/pacemaker.metrics b/test/pacemaker.metrics
index 7733953..04e09da 100644
--- a/test/pacemaker.metrics
+++ b/test/pacemaker.metrics
@@ -3,47 +3,53 @@
 ha_cluster_pacemaker_config_last_change 1.571399302e+09
 # HELP ha_cluster_pacemaker_fail_count The Fail count number per node and resource id
 # TYPE ha_cluster_pacemaker_fail_count gauge
-ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0
-ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} +Inf
-ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_ip_PRD_HDB00"} 2
-ha_cluster_pacemaker_fail_count{node="hana01",resource="stonith-sbd"} 0
-ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0
-ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 300
-ha_cluster_pacemaker_fail_count{node="hana02",resource="test"} 0
-ha_cluster_pacemaker_fail_count{node="hana02",resource="test-stop"} 0
+ha_cluster_pacemaker_fail_count{node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0
+ha_cluster_pacemaker_fail_count{node="node01",resource="rsc_SAPHana_PRD_HDB00"} +Inf
+ha_cluster_pacemaker_fail_count{node="node01",resource="rsc_ip_PRD_HDB00"} 2
+ha_cluster_pacemaker_fail_count{node="node01",resource="stonith-sbd"} 0
+ha_cluster_pacemaker_fail_count{node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0
+ha_cluster_pacemaker_fail_count{node="node02",resource="rsc_SAPHana_PRD_HDB00"} 300
+ha_cluster_pacemaker_fail_count{node="node02",resource="test"} 0
+ha_cluster_pacemaker_fail_count{node="node02",resource="test-stop"} 0
 # HELP ha_cluster_pacemaker_location_constraints Resource location constraints. The value indicates the score.
 # TYPE ha_cluster_pacemaker_location_constraints gauge
-ha_cluster_pacemaker_location_constraints{constraint="cli-ban-msl_SAPHana_PRD_HDB00-on-hana01",node="hana01",resource="msl_SAPHana_PRD_HDB00",role="started"} -Inf
-ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-cln_SAPHanaTopology_PRD_HDB00",node="hana01",resource="cln_SAPHanaTopology_PRD_HDB00",role="started"} +Inf
-ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-msl_SAPHana_PRD_HDB00",node="hana01",resource="msl_SAPHana_PRD_HDB00",role="started"} +Inf
-ha_cluster_pacemaker_location_constraints{constraint="test",node="hana02",resource="test",role="started"} 666
+ha_cluster_pacemaker_location_constraints{constraint="cli-ban-msl_SAPHana_PRD_HDB00-on-node01",node="node01",resource="msl_SAPHana_PRD_HDB00",role="started"} -Inf
+ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-cln_SAPHanaTopology_PRD_HDB00",node="node01",resource="cln_SAPHanaTopology_PRD_HDB00",role="started"} +Inf
+ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-msl_SAPHana_PRD_HDB00",node="node01",resource="msl_SAPHana_PRD_HDB00",role="started"} +Inf
+ha_cluster_pacemaker_location_constraints{constraint="test",node="node02",resource="test",role="started"} 666
 # HELP ha_cluster_pacemaker_migration_threshold The migration_threshold number per node and resource id
 # TYPE ha_cluster_pacemaker_migration_threshold gauge
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 1
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} 5000
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_ip_PRD_HDB00"} 5000
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="stonith-sbd"} 5000
-ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 3
-ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 50
-ha_cluster_pacemaker_migration_threshold{node="hana02",resource="test"} 5000
-ha_cluster_pacemaker_migration_threshold{node="hana02",resource="test-stop"} 5000
+ha_cluster_pacemaker_migration_threshold{node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 1
+ha_cluster_pacemaker_migration_threshold{node="node01",resource="rsc_SAPHana_PRD_HDB00"} 5000
+ha_cluster_pacemaker_migration_threshold{node="node01",resource="rsc_ip_PRD_HDB00"} 5000
+ha_cluster_pacemaker_migration_threshold{node="node01",resource="stonith-sbd"} 5000
+ha_cluster_pacemaker_migration_threshold{node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 3
+ha_cluster_pacemaker_migration_threshold{node="node02",resource="rsc_SAPHana_PRD_HDB00"} 50
+ha_cluster_pacemaker_migration_threshold{node="node02",resource="test"} 5000
+ha_cluster_pacemaker_migration_threshold{node="node02",resource="test-stop"} 5000
 # HELP ha_cluster_pacemaker_nodes The nodes in the cluster; one line per name, per status
 # TYPE ha_cluster_pacemaker_nodes gauge
-ha_cluster_pacemaker_nodes{node="hana01",status="dc",type="member"} 1
-ha_cluster_pacemaker_nodes{node="hana01",status="expected_up",type="member"} 1
-ha_cluster_pacemaker_nodes{node="hana01",status="online",type="member"} 1
-ha_cluster_pacemaker_nodes{node="hana02",status="expected_up",type="member"} 1
-ha_cluster_pacemaker_nodes{node="hana02",status="online",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node01",status="dc",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node01",status="expected_up",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node01",status="online",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node02",status="expected_up",type="member"} 1
+ha_cluster_pacemaker_nodes{node="node02",status="online",type="member"} 1
 # HELP ha_cluster_pacemaker_resources The resources in the cluster; one line per id, per status
 # TYPE ha_cluster_pacemaker_resources gauge
-ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",managed="true",node="",resource="test-stop",role="stopped",status=""} 1
-ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",managed="true",node="hana02",resource="test",role="started",status="active"} 1
-ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",managed="true",node="hana01",resource="rsc_ip_PRD_HDB00",role="started",status="active"} 1
-ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",managed="true",node="hana01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="active"} 1
-ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",managed="true",node="hana02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="active"} 1
-ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",managed="true",node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
-ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",managed="true",node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
-ha_cluster_pacemaker_resources{agent="stonith:external/sbd",managed="true",node="hana01",resource="stonith-sbd",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="",resource="test-stop",role="stopped",status=""} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Dummy",clone="",group="",managed="true",node="node02",resource="test",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_fs_HA1_ASCS00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:Filesystem",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_fs_HA1_ERS10",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="",managed="true",node="node01",resource="rsc_ip_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_ip_HA1_ASCS00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:IPaddr2",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_ip_HA1_ERS10",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ASCS00",managed="true",node="node01",resource="rsc_sap_HA1_ASCS00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::heartbeat:SAPInstance",clone="",group="grp_HA1_ERS10",managed="true",node="node02",resource="rsc_sap_HA1_ERS10",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHana",clone="msl_SAPHana_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="ocf::suse:SAPHanaTopology",clone="cln_SAPHanaTopology_PRD_HDB00",group="",managed="true",node="node02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{agent="stonith:external/sbd",clone="",group="",managed="true",node="node01",resource="stonith-sbd",role="started",status="active"} 1
 # HELP ha_cluster_pacemaker_stonith_enabled Whether or not stonith is enabled
 # TYPE ha_cluster_pacemaker_stonith_enabled gauge
 ha_cluster_pacemaker_stonith_enabled 1