Repository: ClusterLabs/ha_cluster_exporter
Pull request #89 (2019-11-04 17:08:42)

fix config_last_change pacemaker metric due to Prometheus limitations
TIL [Prometheus won't import any metric timestamped with a time older than "5 minutes ago"](https://stackoverflow.com/questions/43890629/prometheus-timestamps), and they don't intend supporting this.

Because of this, trying to record `ha_cluster_pacemaker_config_last_change` will cause this warning, as shown in Prometheus logs:
```
Nov 04 17:31:14 stefanotorresi-monitoring prometheus[12076]: level=warn ts=2019-11-04T16:31:14.298897787Z caller=scrape.go:1094 component="scrape manager" scrape_pool=cluster target=http://192.168.123.15:9002/metrics msg="Error on ingesting samples that are too old or are too far into the future" num_dropped=1
````

As a matter of fact, we should not rely on timestamps in the future, because apparently they are discussing deprecating them in the future! My bad for thinking that using them was the proper thing to do.

I reverted the implementation of this metric to your original one, converting the time scraped from `crm_mon` to a float value.


Patch:
diff --git a/doc/metric_spec.md b/doc/metric_spec.md
index fb3f4cc..c38d701 100644
--- a/doc/metric_spec.md
+++ b/doc/metric_spec.md
@@ -24,15 +24,15 @@ These are the currently implemented subsystems.
 The Pacemaker subsystem collects an atomic snapshot of the HA cluster directly from the XML CIB of Pacemaker via `crm_mon`.
 
 0. [Sample](../test/pacemaker.metrics)
-1. [`ha_cluster_pacemaker_nodes`](#ha_cluster_pacemaker_nodes)
-2. [`ha_cluster_pacemaker_nodes_total`](#ha_cluster_pacemaker_nodes_total)
-3. [`ha_cluster_pacemaker_resources`](#ha_cluster_pacemaker_resources)
-4. [`ha_cluster_pacemaker_resources_total`](#ha_cluster_pacemaker_resources_total)
-5. [`ha_cluster_pacemaker_stonith_enabled`](#ha_cluster_pacemaker_stonith_enabled)
-6. [`ha_cluster_pacemaker_fail_count`](#ha_cluster_pacemaker_fail_count)
-7. [`ha_cluster_pacemaker_migration_threshold`](#ha_cluster_pacemaker_migration_threshold)
-8. [`ha_cluster_pacemaker_config_last_change`](#ha_cluster_pacemaker_config_last_change)
-8. [`ha_cluster_pacemaker_constraints`](#ha_cluster_pacemaker_constraints)
+1. [`ha_cluster_pacemaker_config_last_change`](#ha_cluster_pacemaker_config_last_change)
+2. [`ha_cluster_pacemaker_constraints`](#ha_cluster_pacemaker_constraints)
+3. [`ha_cluster_pacemaker_fail_count`](#ha_cluster_pacemaker_fail_count)
+4. [`ha_cluster_pacemaker_migration_threshold`](#ha_cluster_pacemaker_migration_threshold)
+5. [`ha_cluster_pacemaker_nodes_total`](#ha_cluster_pacemaker_nodes_total)
+6. [`ha_cluster_pacemaker_nodes`](#ha_cluster_pacemaker_nodes)
+7. [`ha_cluster_pacemaker_resources_total`](#ha_cluster_pacemaker_resources_total)
+8. [`ha_cluster_pacemaker_resources`](#ha_cluster_pacemaker_resources)
+9. [`ha_cluster_pacemaker_stonith_enabled`](#ha_cluster_pacemaker_stonith_enabled)
 
 ### `ha_cluster_pacemaker_nodes`
 
@@ -107,8 +107,8 @@ Possible values are positive numbers.
 
 #### Description
 
-The relevant part of this metric is its timestamp, which corresponds to the last time Pacemaker configuration changed.
-The actual metric value will always be 1 and can be ignored.
+The value of this metric is a Unix timestamp in seconds, converted to a float, corresponding to the last time Pacemaker configuration changed.
+The metric is in turn timestamped with the time it was last checked.
 
 
 ### `ha_cluster_pacemaker_constraints`
diff --git a/pacemaker_metrics.go b/pacemaker_metrics.go
index 945f9c4..2ad4cdc 100644
--- a/pacemaker_metrics.go
+++ b/pacemaker_metrics.go
@@ -97,7 +97,7 @@ var (
 		"stonith_enabled":     NewMetricDesc("pacemaker", "stonith_enabled", "Whether or not stonith is enabled", nil),
 		"fail_count":          NewMetricDesc("pacemaker", "fail_count", "The Fail count number per node and resource id", []string{"node", "resource"}),
 		"migration_threshold": NewMetricDesc("pacemaker", "migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"}),
-		"config_last_change":  NewMetricDesc("pacemaker", "config_last_change", "Indicate if a configuration of resource has changed in cluster", nil),
+		"config_last_change":  NewMetricDesc("pacemaker", "config_last_change", "The timestamp of the last change of the cluster configuration", nil),
 		"constraints":         NewMetricDesc("pacemaker", "constraints", "Indicate if a constraints of specific type is present per ID and per resource", []string{"type", "id", "resource"}),
 	}
 
@@ -260,8 +260,8 @@ func (c *pacemakerCollector) recordResourceAgentsChanges(pacemakerStatus pacemak
 		log.Warnln(err)
 		return
 	}
-	// is the resource have changed we set a different timeout from pacemaker
-	ch <- prometheus.NewMetricWithTimestamp(t, prometheus.MustNewConstMetric(c.metrics["config_last_change"], prometheus.CounterValue, 1))
+	// we record the timestamp of the last change as a float counter metric, which is in turn timestamped with the time it was checked
+	ch <- prometheus.NewMetricWithTimestamp(clock.Now(), prometheus.MustNewConstMetric(c.metrics["config_last_change"], prometheus.CounterValue, float64(t.Unix())))
 }
 
 func (c *pacemakerCollector) recordMigrationThresholdMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {


Test patch:
diff --git a/test/pacemaker.metrics b/test/pacemaker.metrics
index 330aaa1..f765e98 100644
--- a/test/pacemaker.metrics
+++ b/test/pacemaker.metrics
@@ -1,6 +1,6 @@
-# HELP ha_cluster_pacemaker_config_last_change Indicate if a configuration of resource has changed in cluster
+# HELP ha_cluster_pacemaker_config_last_change The timestamp of the last change of the cluster configuration
 # TYPE ha_cluster_pacemaker_config_last_change counter
-ha_cluster_pacemaker_config_last_change 1 1571399302000
+ha_cluster_pacemaker_config_last_change 1.571399302e+09 1234
 # HELP ha_cluster_pacemaker_constraints Indicate if a constraints of specific type is present per ID and per resource
 # TYPE ha_cluster_pacemaker_constraints gauge
 ha_cluster_pacemaker_constraints{id="cli-ban-msl_SAPHana_PRD_HDB00-on-damadog-hana01",resource="msl_SAPHana_PRD_HDB00",type="ban"} 1 1234