Repository: ClusterLabs/ha_cluster_exporter (go)

Issue:
Make timestamps opt-in
Follow-up to SUSE/ha-sap-terraform-deployments#298

The solution would be to add a CLI flag, i.e. `--enable-timestamps`, so that we accommodate for the current use case as an opt-in, and revert to Prometheus best-practices by default.

Pull request description:
Make metrics timestamping optional
This PR adds the `--enable-timestamps` flag (or config option).
Metrics are now **not** timestamped by default, as suggested by Prometheus best practices.

closes #113

Patch:
diff --git a/doc/metric_spec.md b/doc/metric_spec.md
index 4ddce21..989d140 100644
--- a/doc/metric_spec.md
+++ b/doc/metric_spec.md
@@ -5,11 +5,11 @@ This document describes the metrics exposed by `ha_cluster_exporter`.
 General notes:
 - All the metrics are _namespaced_ with the prefix `ha_cluster`, which is followed by a _subsystem_, and both are in turn composed into a _Fully Qualified Name_ (FQN) of each metrics.
 - All the metrics and labels _names_ are in snake_case, as conventional with Prometheus. That said, as much as we'll try to keep this consistent throughout the project, the label _values_ may not actually follow this convention, though (e.g. value is a hostname).
-- All the metrics are timestamped with the Unix epoch time in milliseconds; in the provided examples, this value will always be `1234`.
 - Some metrics, like `ha_cluster_pacemaker_nodes`, `ha_cluster_pacemaker_resources`, share common traits:
   - their labels contain the relevant data you may want to track or use for aggregation and filtering;
   - either their value is `1`, or the line is absent altogether; this is because each line represents one entity of the cluster, but the exporter itself is stateless, i.e. we don't track the life-cycle of entities that do not exist anymore in the cluster.
 
+- If the `enable-timestamps` option is on, all the metrics will be timestamped with the Unix epoch time in milliseconds.
 
 These are the currently implemented subsystems.
 
diff --git a/go.mod b/go.mod
index d89f1b9..4b02f61 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,9 @@ go 1.12
 require (
 	github.com/pkg/errors v0.8.0
 	github.com/prometheus/client_golang v1.1.0
+	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.5.0
+	github.com/stretchr/testify v1.3.0
 )
diff --git a/ha_cluster_exporter.go b/ha_cluster_exporter.go
index 27d2314..51f17d1 100644
--- a/ha_cluster_exporter.go
+++ b/ha_cluster_exporter.go
@@ -41,12 +41,24 @@ func (c *DefaultCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (c *DefaultCollector) makeGaugeMetric(metricKey string, value float64, labelValues ...string) prometheus.Metric {
+	return c.makeMetric(metricKey, value, prometheus.GaugeValue, labelValues...)
+}
+
+func (c *DefaultCollector) makeCounterMetric(metricKey string, value float64, labelValues ...string) prometheus.Metric {
+	return c.makeMetric(metricKey, value, prometheus.CounterValue, labelValues...)
+}
+
+func (c *DefaultCollector) makeMetric(metricKey string, value float64, valueType prometheus.ValueType, labelValues ...string) prometheus.Metric {
 	desc, ok := c.metrics[metricKey]
 	if !ok {
 		// we hard panic on this because it's most certainly a coding error
 		panic(errors.Errorf("undeclared metric '%s'", metricKey))
 	}
-	return prometheus.NewMetricWithTimestamp(clock.Now(), prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, value, labelValues...))
+	metric := prometheus.MustNewConstMetric(desc, valueType, value, labelValues...)
+	if config.GetBool("enable-timestamps") {
+		metric = prometheus.NewMetricWithTimestamp(clock.Now(), metric)
+	}
+	return metric
 }
 
 // Convenience wrapper around Prometheus constructors.
@@ -127,6 +139,7 @@ func init() {
 	flag.String("sbd-config-path", "/etc/sysconfig/sbd", "path to sbd configuration")
 	flag.String("drbdsetup-path", "/sbin/drbdsetup", "path to drbdsetup executable")
 	flag.String("drbdsplitbrain-path", "/var/run/drbd/splitbrain", "path to drbd splitbrain hooks temporary files")
+	flag.Bool("enable-timestamps", false, "Add the timestamp to every metric line (hint: don't do this unless you really know what you are doing)")
 
 	err := config.BindPFlags(flag.CommandLine)
 	if err != nil {
diff --git a/ha_cluster_exporter.yaml b/ha_cluster_exporter.yaml
index e253633..1b8b360 100644
--- a/ha_cluster_exporter.yaml
+++ b/ha_cluster_exporter.yaml
@@ -9,3 +9,4 @@ corosync-quorumtool-path: "/usr/sbin/corosync-quorumtool"
 sbd-path: "/usr/sbin/sbd"
 sbd-config-path: "/etc/sysconfig/sbd"
 drbdsetup-path: "/sbin/drbdsetup"
+enable-timestamps: false
diff --git a/pacemaker_metrics.go b/pacemaker_metrics.go
index 6754c1f..7e3909b 100644
--- a/pacemaker_metrics.go
+++ b/pacemaker_metrics.go
@@ -267,8 +267,8 @@ func (c *pacemakerCollector) recordResourceAgentsChanges(pacemakerStatus pacemak
 		log.Warnln(err)
 		return
 	}
-	// we record the timestamp of the last change as a float counter metric, which is in turn timestamped with the time it was checked
-	ch <- prometheus.NewMetricWithTimestamp(clock.Now(), prometheus.MustNewConstMetric(c.metrics["config_last_change"], prometheus.CounterValue, float64(t.Unix())))
+	// we record the timestamp of the last change as a float counter metric
+	ch <- c.makeCounterMetric("config_last_change", float64(t.Unix()))
 }
 
 func (c *pacemakerCollector) recordMigrationThresholdMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {


Test patch:
diff --git a/corosync_metrics_test.go b/corosync_metrics_test.go
index 49034ce..8f4d71c 100644
--- a/corosync_metrics_test.go
+++ b/corosync_metrics_test.go
@@ -191,8 +191,6 @@ func TestNewCorosyncCollectorChecksQuorumtoolExecutableBits(t *testing.T) {
 }
 
 func TestCorosyncCollector(t *testing.T) {
-	clock = StoppedClock{}
-
 	collector, _ := NewCorosyncCollector("test/fake_corosync-cfgtool.sh", "test/fake_corosync-quorumtool.sh")
 	expectMetrics(t, collector, "corosync.metrics")
 }
diff --git a/drbd_metrics_test.go b/drbd_metrics_test.go
index 1db6052..2f02320 100644
--- a/drbd_metrics_test.go
+++ b/drbd_metrics_test.go
@@ -231,15 +231,11 @@ func TestNewDrbdCollectorChecksDrbdsetupExecutableBits(t *testing.T) {
 }
 
 func TestDRBDCollector(t *testing.T) {
-	clock = StoppedClock{}
-
 	collector, _ := NewDrbdCollector("test/fake_drbdsetup.sh", "fake")
 	expectMetrics(t, collector, "drbd.metrics")
-
 }
 
 func TestDRBDSplitbrainCollector(t *testing.T) {
-	clock = StoppedClock{}
 	splitBrainDir := "/var/tmp/drbd/splitbrain"
 	testFiles := [3]string{
 		"drbd-split-brain-detected-resource01-vol01",
@@ -264,8 +260,8 @@ func TestDRBDSplitbrainCollector(t *testing.T) {
 	expect := `
 	# HELP ha_cluster_drbd_split_brain Whether a split brain has been detected; 1 line per resource, per volume.
 	# TYPE ha_cluster_drbd_split_brain gauge
-	ha_cluster_drbd_split_brain{resource="resource01",volume="vol01"} 1 1234
-	ha_cluster_drbd_split_brain{resource="resource02",volume="vol02"} 1 1234
+	ha_cluster_drbd_split_brain{resource="resource01",volume="vol01"} 1
+	ha_cluster_drbd_split_brain{resource="resource02",volume="vol02"} 1
 	`
 
 	if err := testutil.CollectAndCompare(collector, strings.NewReader(expect)); err != nil {
diff --git a/ha_cluster_exporter_test.go b/ha_cluster_exporter_test.go
index 95734b7..288e218 100644
--- a/ha_cluster_exporter_test.go
+++ b/ha_cluster_exporter_test.go
@@ -8,15 +8,20 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
+	config "github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
 )
 
 type StoppedClock struct{}
 
+const TEST_TIMESTAMP = 1234
+
 func (StoppedClock) Now() time.Time {
-	ms := 1234 * time.Millisecond
+	ms := TEST_TIMESTAMP * time.Millisecond
 	return time.Date(1970, 1, 1, 0, 0, 0, int(ms.Nanoseconds()), time.UTC)
 	// 1234 millisecond after Unix epoch (1970-01-01 00:00:01.234 +0000 UTC)
-	// this will allow us to assert that all the metrics are timestamped with "1234"
+	// this will allow us to use a fixed timestamped when running assertions
 }
 
 // borrowed from haproxy_exporter
@@ -30,3 +35,38 @@ func expectMetrics(t *testing.T, c prometheus.Collector, fixture string) {
 		t.Fatal("Unexpected metrics returned:", err)
 	}
 }
+
+func TestMetricFactory(t *testing.T) {
+	SUT := &DefaultCollector{
+		metrics: metricDescriptors{
+			"test_metric": NewMetricDesc("test", "metric", "", nil),
+		},
+	}
+
+	metric := SUT.makeGaugeMetric("test_metric", 1)
+
+	assert.Equal(t, SUT.metrics["test_metric"], metric.Desc())
+}
+
+func TestMetricFactoryWithTimestamp(t *testing.T) {
+	config.Set("enable-timestamps", true)
+	defer func() {
+		config.Set("enable-timestamps", false)
+		clock = SystemClock{}
+	}()
+
+	clock = StoppedClock{}
+	SUT := &DefaultCollector{
+		metrics: metricDescriptors{
+			"test_metric": NewMetricDesc("test", "metric", "", nil),
+		},
+	}
+
+	metric := SUT.makeGaugeMetric("test_metric", 1)
+	metricDto := &dto.Metric{}
+	err := metric.Write(metricDto)
+
+	assert.Nil(t, err, "Unexpected error")
+
+	assert.Equal(t, int64(TEST_TIMESTAMP), *metricDto.TimestampMs)
+}
diff --git a/pacemaker_metrics_test.go b/pacemaker_metrics_test.go
index e2d3381..0e7c864 100644
--- a/pacemaker_metrics_test.go
+++ b/pacemaker_metrics_test.go
@@ -168,7 +168,6 @@ func TestNewPacemakerCollectorChecksCrmMonExecutableBits(t *testing.T) {
 }
 
 func TestPacemakerCollector(t *testing.T) {
-	clock = StoppedClock{}
 	collector, err := NewPacemakerCollector("test/fake_crm_mon.sh", "test/fake_cibadmin.sh")
 	if err != nil {
 		t.Fatal(err)
diff --git a/sbd_metrics_test.go b/sbd_metrics_test.go
index 760c070..6790ed1 100644
--- a/sbd_metrics_test.go
+++ b/sbd_metrics_test.go
@@ -267,8 +267,6 @@ func TestNewSbdCollectorChecksSbdExecutableBits(t *testing.T) {
 }
 
 func TestSBDCollector(t *testing.T) {
-	clock = StoppedClock{}
-
 	collector, _ := NewSbdCollector("test/fake_sbd.sh", "test/fake_sbdconfig")
 	expectMetrics(t, collector, "sbd.metrics")
 }
diff --git a/test/corosync.metrics b/test/corosync.metrics
index 1e0a1b6..5c765e2 100644
--- a/test/corosync.metrics
+++ b/test/corosync.metrics
@@ -1,12 +1,12 @@
 # HELP ha_cluster_corosync_quorate Whether or not the cluster is quorate
 # TYPE ha_cluster_corosync_quorate gauge
-ha_cluster_corosync_quorate 1 1234
+ha_cluster_corosync_quorate 1
 # HELP ha_cluster_corosync_quorum_votes Cluster quorum votes; one line per type
 # TYPE ha_cluster_corosync_quorum_votes gauge
-ha_cluster_corosync_quorum_votes{type="expected_votes"} 2 1234
-ha_cluster_corosync_quorum_votes{type="highest_expected"} 2 1234
-ha_cluster_corosync_quorum_votes{type="quorum"} 1 1234
-ha_cluster_corosync_quorum_votes{type="total_votes"} 2 1234
+ha_cluster_corosync_quorum_votes{type="expected_votes"} 2
+ha_cluster_corosync_quorum_votes{type="highest_expected"} 2
+ha_cluster_corosync_quorum_votes{type="quorum"} 1
+ha_cluster_corosync_quorum_votes{type="total_votes"} 2
 # HELP ha_cluster_corosync_ring_errors_total Total number of corosync ring errors
 # TYPE ha_cluster_corosync_ring_errors_total gauge
-ha_cluster_corosync_ring_errors_total 1 1234
+ha_cluster_corosync_ring_errors_total 1
diff --git a/test/drbd.metrics b/test/drbd.metrics
index f6bbc92..ac6424b 100644
--- a/test/drbd.metrics
+++ b/test/drbd.metrics
@@ -1,56 +1,56 @@
 # HELP ha_cluster_drbd_al_writes Writes to activity log; 1 line per res, per volume
 # TYPE ha_cluster_drbd_al_writes gauge
-ha_cluster_drbd_al_writes{resource="1-single-0",volume="0"} 123 1234
-ha_cluster_drbd_al_writes{resource="1-single-1",volume="0"} 123 1234
+ha_cluster_drbd_al_writes{resource="1-single-0",volume="0"} 123
+ha_cluster_drbd_al_writes{resource="1-single-1",volume="0"} 123
 # HELP ha_cluster_drbd_bm_writes Writes to bitmap; 1 line per res, per volume
 # TYPE ha_cluster_drbd_bm_writes gauge
-ha_cluster_drbd_bm_writes{resource="1-single-0",volume="0"} 321 1234
-ha_cluster_drbd_bm_writes{resource="1-single-1",volume="0"} 321 1234
+ha_cluster_drbd_bm_writes{resource="1-single-0",volume="0"} 321
+ha_cluster_drbd_bm_writes{resource="1-single-1",volume="0"} 321
 # HELP ha_cluster_drbd_connections The DRBD resource connections; 1 line per per resource, per peer_node_id
 # TYPE ha_cluster_drbd_connections gauge
-ha_cluster_drbd_connections{peer_disk_state="uptodate",peer_node_id="1",peer_role="Primary",resource="1-single-0",volume="0"} 1 1234
-ha_cluster_drbd_connections{peer_disk_state="uptodate",peer_node_id="1",peer_role="Primary",resource="1-single-1",volume="0"} 1 1234
+ha_cluster_drbd_connections{peer_disk_state="uptodate",peer_node_id="1",peer_role="Primary",resource="1-single-0",volume="0"} 1
+ha_cluster_drbd_connections{peer_disk_state="uptodate",peer_node_id="1",peer_role="Primary",resource="1-single-1",volume="0"} 1
 # HELP ha_cluster_drbd_connections_pending Pending value per connection
 # TYPE ha_cluster_drbd_connections_pending gauge
-ha_cluster_drbd_connections_pending{peer_node_id="1",resource="1-single-0",volume="0"} 3 1234
-ha_cluster_drbd_connections_pending{peer_node_id="1",resource="1-single-1",volume="0"} 3 1234
+ha_cluster_drbd_connections_pending{peer_node_id="1",resource="1-single-0",volume="0"} 3
+ha_cluster_drbd_connections_pending{peer_node_id="1",resource="1-single-1",volume="0"} 3
 # HELP ha_cluster_drbd_connections_received KiB received per connection
 # TYPE ha_cluster_drbd_connections_received gauge
-ha_cluster_drbd_connections_received{peer_node_id="1",resource="1-single-0",volume="0"} 456 1234
-ha_cluster_drbd_connections_received{peer_node_id="1",resource="1-single-1",volume="0"} 456 1234
+ha_cluster_drbd_connections_received{peer_node_id="1",resource="1-single-0",volume="0"} 456
+ha_cluster_drbd_connections_received{peer_node_id="1",resource="1-single-1",volume="0"} 456
 # HELP ha_cluster_drbd_connections_sent KiB sent per connection
 # TYPE ha_cluster_drbd_connections_sent gauge
-ha_cluster_drbd_connections_sent{peer_node_id="1",resource="1-single-0",volume="0"} 654 1234
-ha_cluster_drbd_connections_sent{peer_node_id="1",resource="1-single-1",volume="0"} 654 1234
+ha_cluster_drbd_connections_sent{peer_node_id="1",resource="1-single-0",volume="0"} 654
+ha_cluster_drbd_connections_sent{peer_node_id="1",resource="1-single-1",volume="0"} 654
 # HELP ha_cluster_drbd_connections_sync The in sync percentage value for DRBD resource connections
 # TYPE ha_cluster_drbd_connections_sync gauge
-ha_cluster_drbd_connections_sync{peer_node_id="1",resource="1-single-0",volume="0"} 100 1234
-ha_cluster_drbd_connections_sync{peer_node_id="1",resource="1-single-1",volume="0"} 100 1234
+ha_cluster_drbd_connections_sync{peer_node_id="1",resource="1-single-0",volume="0"} 100
+ha_cluster_drbd_connections_sync{peer_node_id="1",resource="1-single-1",volume="0"} 100
 # HELP ha_cluster_drbd_connections_unacked Unacked value per connection
 # TYPE ha_cluster_drbd_connections_unacked gauge
-ha_cluster_drbd_connections_unacked{peer_node_id="1",resource="1-single-0",volume="0"} 4 1234
-ha_cluster_drbd_connections_unacked{peer_node_id="1",resource="1-single-1",volume="0"} 4 1234
+ha_cluster_drbd_connections_unacked{peer_node_id="1",resource="1-single-0",volume="0"} 4
+ha_cluster_drbd_connections_unacked{peer_node_id="1",resource="1-single-1",volume="0"} 4
 # HELP ha_cluster_drbd_lower_pending Lower pending; 1 line per res, per volume
 # TYPE ha_cluster_drbd_lower_pending gauge
-ha_cluster_drbd_lower_pending{resource="1-single-0",volume="0"} 2 1234
-ha_cluster_drbd_lower_pending{resource="1-single-1",volume="0"} 2 1234
+ha_cluster_drbd_lower_pending{resource="1-single-0",volume="0"} 2
+ha_cluster_drbd_lower_pending{resource="1-single-1",volume="0"} 2
 # HELP ha_cluster_drbd_quorum Quorum status per resource and per volume
 # TYPE ha_cluster_drbd_quorum gauge
-ha_cluster_drbd_quorum{resource="1-single-0",volume="0"} 1 1234
-ha_cluster_drbd_quorum{resource="1-single-1",volume="0"} 0 1234
+ha_cluster_drbd_quorum{resource="1-single-0",volume="0"} 1
+ha_cluster_drbd_quorum{resource="1-single-1",volume="0"} 0
 # HELP ha_cluster_drbd_read KiB read from DRBD; 1 line per res, per volume
 # TYPE ha_cluster_drbd_read gauge
-ha_cluster_drbd_read{resource="1-single-0",volume="0"} 654321 1234
-ha_cluster_drbd_read{resource="1-single-1",volume="0"} 654321 1234
+ha_cluster_drbd_read{resource="1-single-0",volume="0"} 654321
+ha_cluster_drbd_read{resource="1-single-1",volume="0"} 654321
 # HELP ha_cluster_drbd_resources The DRBD resources; 1 line per name, per volume
 # TYPE ha_cluster_drbd_resources gauge
-ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-0",role="Secondary",volume="0"} 1 1234
-ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-1",role="Secondary",volume="0"} 1 1234
+ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-0",role="Secondary",volume="0"} 1
+ha_cluster_drbd_resources{disk_state="uptodate",resource="1-single-1",role="Secondary",volume="0"} 1
 # HELP ha_cluster_drbd_upper_pending Upper pending; 1 line per res, per volume
 # TYPE ha_cluster_drbd_upper_pending gauge
-ha_cluster_drbd_upper_pending{resource="1-single-0",volume="0"} 1 1234
-ha_cluster_drbd_upper_pending{resource="1-single-1",volume="0"} 1 1234
+ha_cluster_drbd_upper_pending{resource="1-single-0",volume="0"} 1
+ha_cluster_drbd_upper_pending{resource="1-single-1",volume="0"} 1
 # HELP ha_cluster_drbd_written KiB written to DRBD; 1 line per res, per volume
 # TYPE ha_cluster_drbd_written gauge
-ha_cluster_drbd_written{resource="1-single-0",volume="0"} 123456 1234
-ha_cluster_drbd_written{resource="1-single-1",volume="0"} 123456 1234
+ha_cluster_drbd_written{resource="1-single-0",volume="0"} 123456
+ha_cluster_drbd_written{resource="1-single-1",volume="0"} 123456
diff --git a/test/pacemaker.metrics b/test/pacemaker.metrics
index d0dbc77..d51dcab 100644
--- a/test/pacemaker.metrics
+++ b/test/pacemaker.metrics
@@ -1,49 +1,49 @@
 # HELP ha_cluster_pacemaker_config_last_change The timestamp of the last change of the cluster configuration
 # TYPE ha_cluster_pacemaker_config_last_change counter
-ha_cluster_pacemaker_config_last_change 1.571399302e+09 1234
+ha_cluster_pacemaker_config_last_change 1.571399302e+09
 # HELP ha_cluster_pacemaker_fail_count The Fail count number per node and resource id
 # TYPE ha_cluster_pacemaker_fail_count gauge
-ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0 1234
-ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} +Inf 1234
-ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_ip_PRD_HDB00"} 2 1234
-ha_cluster_pacemaker_fail_count{node="hana01",resource="stonith-sbd"} 0 1234
-ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0 1234
-ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 300 1234
+ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0
+ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} +Inf
+ha_cluster_pacemaker_fail_count{node="hana01",resource="rsc_ip_PRD_HDB00"} 2
+ha_cluster_pacemaker_fail_count{node="hana01",resource="stonith-sbd"} 0
+ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 0
+ha_cluster_pacemaker_fail_count{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 300
 # HELP ha_cluster_pacemaker_location_constraints Resource location constraints. The value indicates the score.
 # TYPE ha_cluster_pacemaker_location_constraints gauge
-ha_cluster_pacemaker_location_constraints{constraint="cli-ban-msl_SAPHana_PRD_HDB00-on-hana01",node="hana01",resource="msl_SAPHana_PRD_HDB00",role="started"} -Inf 1234
-ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-cln_SAPHanaTopology_PRD_HDB00",node="hana01",resource="cln_SAPHanaTopology_PRD_HDB00",role="started"} +Inf 1234
-ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-msl_SAPHana_PRD_HDB00",node="hana01",resource="msl_SAPHana_PRD_HDB00",role="started"} +Inf 1234
-ha_cluster_pacemaker_location_constraints{constraint="test",node="hana02",resource="test",role="started"} 666 1234
+ha_cluster_pacemaker_location_constraints{constraint="cli-ban-msl_SAPHana_PRD_HDB00-on-hana01",node="hana01",resource="msl_SAPHana_PRD_HDB00",role="started"} -Inf
+ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-cln_SAPHanaTopology_PRD_HDB00",node="hana01",resource="cln_SAPHanaTopology_PRD_HDB00",role="started"} +Inf
+ha_cluster_pacemaker_location_constraints{constraint="cli-prefer-msl_SAPHana_PRD_HDB00",node="hana01",resource="msl_SAPHana_PRD_HDB00",role="started"} +Inf
+ha_cluster_pacemaker_location_constraints{constraint="test",node="hana02",resource="test",role="started"} 666
 # HELP ha_cluster_pacemaker_migration_threshold The migration_threshold number per node and resource id
 # TYPE ha_cluster_pacemaker_migration_threshold gauge
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 1 1234
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} 5000 1234
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_ip_PRD_HDB00"} 5000 1234
-ha_cluster_pacemaker_migration_threshold{node="hana01",resource="stonith-sbd"} 5000 1234
-ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 3 1234
-ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 50 1234
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00"} 1
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_SAPHana_PRD_HDB00"} 5000
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="rsc_ip_PRD_HDB00"} 5000
+ha_cluster_pacemaker_migration_threshold{node="hana01",resource="stonith-sbd"} 5000
+ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00"} 3
+ha_cluster_pacemaker_migration_threshold{node="hana02",resource="rsc_SAPHana_PRD_HDB00"} 50
 # HELP ha_cluster_pacemaker_nodes The nodes in the cluster; one line per name, per status
 # TYPE ha_cluster_pacemaker_nodes gauge
-ha_cluster_pacemaker_nodes{node="hana01",status="dc",type="member"} 1 1234
-ha_cluster_pacemaker_nodes{node="hana01",status="expected_up",type="member"} 1 1234
-ha_cluster_pacemaker_nodes{node="hana01",status="online",type="member"} 1 1234
-ha_cluster_pacemaker_nodes{node="hana02",status="expected_up",type="member"} 1 1234
-ha_cluster_pacemaker_nodes{node="hana02",status="online",type="member"} 1 1234
+ha_cluster_pacemaker_nodes{node="hana01",status="dc",type="member"} 1
+ha_cluster_pacemaker_nodes{node="hana01",status="expected_up",type="member"} 1
+ha_cluster_pacemaker_nodes{node="hana01",status="online",type="member"} 1
+ha_cluster_pacemaker_nodes{node="hana02",status="expected_up",type="member"} 1
+ha_cluster_pacemaker_nodes{node="hana02",status="online",type="member"} 1
 # HELP ha_cluster_pacemaker_nodes_total Total number of nodes in the cluster
 # TYPE ha_cluster_pacemaker_nodes_total gauge
-ha_cluster_pacemaker_nodes_total 2 1234
+ha_cluster_pacemaker_nodes_total 2
 # HELP ha_cluster_pacemaker_resources The resources in the cluster; one line per id, per status
 # TYPE ha_cluster_pacemaker_resources gauge
-ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="rsc_ip_PRD_HDB00",role="started",status="active"} 1 1234
-ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="active"} 1 1234
-ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1 1234
-ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="stonith-sbd",role="started",status="active"} 1 1234
-ha_cluster_pacemaker_resources{managed="true",node="hana02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="active"} 1 1234
-ha_cluster_pacemaker_resources{managed="true",node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1 1234
+ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="rsc_ip_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="rsc_SAPHana_PRD_HDB00",role="master",status="active"} 1
+ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{managed="true",node="hana01",resource="stonith-sbd",role="started",status="active"} 1
+ha_cluster_pacemaker_resources{managed="true",node="hana02",resource="rsc_SAPHana_PRD_HDB00",role="slave",status="active"} 1
+ha_cluster_pacemaker_resources{managed="true",node="hana02",resource="rsc_SAPHanaTopology_PRD_HDB00",role="started",status="active"} 1
 # HELP ha_cluster_pacemaker_resources_total Total number of resources in the cluster
 # TYPE ha_cluster_pacemaker_resources_total gauge
-ha_cluster_pacemaker_resources_total 6 1234
+ha_cluster_pacemaker_resources_total 6
 # HELP ha_cluster_pacemaker_stonith_enabled Whether or not stonith is enabled
 # TYPE ha_cluster_pacemaker_stonith_enabled gauge
-ha_cluster_pacemaker_stonith_enabled 1 1234
+ha_cluster_pacemaker_stonith_enabled 1
diff --git a/test/sbd.metrics b/test/sbd.metrics
index 5000ff0..d8a812d 100644
--- a/test/sbd.metrics
+++ b/test/sbd.metrics
@@ -1,7 +1,7 @@
 # HELP ha_cluster_sbd_device_status Whether or not an SBD device is healthy; one line per device
 # TYPE ha_cluster_sbd_device_status gauge
-ha_cluster_sbd_device_status{device="/dev/vdc"} 1 1234
-ha_cluster_sbd_device_status{device="/dev/vdd"} 0 1234
+ha_cluster_sbd_device_status{device="/dev/vdc"} 1
+ha_cluster_sbd_device_status{device="/dev/vdd"} 0
 # HELP ha_cluster_sbd_devices_total Total count of configured SBD devices
 # TYPE ha_cluster_sbd_devices_total gauge
-ha_cluster_sbd_devices_total 2 1234
+ha_cluster_sbd_devices_total 2