Repository: ClusterLabs/ha_cluster_exporter
Pull request #131 (2020-02-18 13:23:16)

Go packaging
This PR introduces a more proper separation of the various parts of the project using Go packages.
Every collector now has a dedicated sub-package inside the `collector` package, and common generic code is stored in dedicated `internal` packages. All of this was done according to various Go best practices.

The only code in the `main` package is the one that creates the binary.

This is propedeutic to further development where some structs are very likely to eventually occur into name clashes (e.g. `Resource`, `Device`, `Node` are all entities that may be present in different collectors), so they needed to be namespaced in different packages.


Patch:
diff --git a/Makefile b/Makefile
index dc6d192..e249933 100644
--- a/Makefile
+++ b/Makefile
@@ -33,10 +33,10 @@ install:
 static-checks: vet-check fmt-check
 
 vet-check: download
-	go vet .
+	go vet ./...
 
 fmt:
-	go fmt
+	go fmt ./...
 
 mod-tidy:
 	go mod tidy
@@ -45,11 +45,11 @@ fmt-check:
 	.ci/go_lint.sh
 
 test: download
-	go test -v
+	go test -v ./...
 
 coverage: coverage.out
 coverage.out:
-	go test -cover -coverprofile=coverage.out
+	go test -cover -coverprofile=coverage.out ./...
 	go tool cover -html=coverage.out
 
 clean: clean-bin clean-obs
@@ -64,6 +64,7 @@ clean-obs:
 
 obs-workdir: build/obs
 build/obs:
+	@mkdir -p build/obs/$(OBS_PACKAGE)
 	osc checkout $(OBS_PROJECT)/$(OBS_PACKAGE) -o build/obs
 	rm build/obs/*.tar.gz
 	cp -rv .ci/obs/* build/obs/
diff --git a/corosync_metrics.go b/collector/corosync/corosync.go
similarity index 85%
rename from corosync_metrics.go
rename to collector/corosync/corosync.go
index 5cd321b..bef2892 100644
--- a/corosync_metrics.go
+++ b/collector/corosync/corosync.go
@@ -1,4 +1,4 @@
-package main
+package corosync
 
 import (
 	"fmt"
@@ -10,30 +10,30 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/ClusterLabs/ha_cluster_exporter/collector"
 )
 
-func NewCorosyncCollector(cfgToolPath string, quorumToolPath string) (*corosyncCollector, error) {
-	err := CheckExecutables(cfgToolPath, quorumToolPath)
+func NewCollector(cfgToolPath string, quorumToolPath string) (*corosyncCollector, error) {
+	err := collector.CheckExecutables(cfgToolPath, quorumToolPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize Corosync collector")
 	}
 
-	collector := &corosyncCollector{
-		DefaultCollector{
-			subsystem: "corosync",
-		},
+	c := &corosyncCollector{
+		collector.NewDefaultCollector("corosync"),
 		cfgToolPath,
 		quorumToolPath,
 	}
-	collector.setDescriptor("quorate", "Whether or not the cluster is quorate", nil)
-	collector.setDescriptor("ring_errors", "The number of corosync ring errors", nil)
-	collector.setDescriptor("quorum_votes", "Cluster quorum votes; one line per type", []string{"type"})
+	c.SetDescriptor("quorate", "Whether or not the cluster is quorate", nil)
+	c.SetDescriptor("ring_errors", "The number of corosync ring errors", nil)
+	c.SetDescriptor("quorum_votes", "Cluster quorum votes; one line per type", []string{"type"})
 
-	return collector, nil
+	return c, nil
 }
 
 type corosyncCollector struct {
-	DefaultCollector
+	collector.DefaultCollector
 	cfgToolPath    string
 	quorumToolPath string
 }
@@ -53,10 +53,10 @@ func (c *corosyncCollector) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	ch <- c.makeGaugeMetric("quorate", quorate)
+	ch <- c.MakeGaugeMetric("quorate", quorate)
 
 	for voteType, value := range quorumStatus {
-		ch <- c.makeGaugeMetric("quorum_votes", float64(value), voteType)
+		ch <- c.MakeGaugeMetric("quorum_votes", float64(value), voteType)
 	}
 }
 
@@ -67,7 +67,7 @@ func (c *corosyncCollector) collectRingErrorsTotal(ch chan<- prometheus.Metric)
 		return errors.Wrap(err, "cannot parse ring status")
 	}
 
-	ch <- c.makeGaugeMetric("ring_errors", float64(ringErrorsTotal))
+	ch <- c.MakeGaugeMetric("ring_errors", float64(ringErrorsTotal))
 
 	return nil
 }
diff --git a/collector/default_collector.go b/collector/default_collector.go
new file mode 100644
index 0000000..6f04ec7
--- /dev/null
+++ b/collector/default_collector.go
@@ -0,0 +1,83 @@
+package collector
+
+import (
+	"github.com/ClusterLabs/ha_cluster_exporter/internal/clock"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	config "github.com/spf13/viper"
+	"os"
+)
+
+const NAMESPACE = "ha_cluster"
+
+type DefaultCollector struct {
+	subsystem   string
+	descriptors map[string]*prometheus.Desc
+	Clock       clock.Clock
+}
+
+func NewDefaultCollector(subsystem string) DefaultCollector {
+	return DefaultCollector{
+		subsystem,
+		make(map[string]*prometheus.Desc),
+		&clock.SystemClock{},
+	}
+}
+
+func (c *DefaultCollector) GetDescriptor(name string) *prometheus.Desc {
+	desc, ok := c.descriptors[name]
+	if !ok {
+		// we hard panic on this because it's most certainly a coding error
+		panic(errors.Errorf("undeclared metric '%s'", name))
+	}
+	return desc
+}
+
+// Convenience wrapper around prometheus.NewDesc constructor.
+// Stores a metric descriptor with a fully qualified name like `NAMESPACE_subsystem_name`.
+// `name` is the last and most relevant part of the metrics Full Qualified Name;
+// `help` is the message displayed in the HELP line
+// `variableLabels` is a list of labels to declare. Use `nil` to declare no labels.
+func (c *DefaultCollector) SetDescriptor(name, help string, variableLabels []string) {
+	c.descriptors[name] = prometheus.NewDesc(prometheus.BuildFQName(NAMESPACE, c.subsystem, name), help, variableLabels, nil)
+}
+
+func (c *DefaultCollector) Describe(ch chan<- *prometheus.Desc) {
+	for _, descriptor := range c.descriptors {
+		ch <- descriptor
+	}
+}
+
+func (c *DefaultCollector) MakeGaugeMetric(name string, value float64, labelValues ...string) prometheus.Metric {
+	return c.makeMetric(name, value, prometheus.GaugeValue, labelValues...)
+}
+
+func (c *DefaultCollector) MakeCounterMetric(name string, value float64, labelValues ...string) prometheus.Metric {
+	return c.makeMetric(name, value, prometheus.CounterValue, labelValues...)
+}
+
+func (c *DefaultCollector) makeMetric(name string, value float64, valueType prometheus.ValueType, labelValues ...string) prometheus.Metric {
+	desc := c.GetDescriptor(name)
+	metric := prometheus.MustNewConstMetric(desc, valueType, value, labelValues...)
+	if config.GetBool("enable-timestamps") {
+		metric = prometheus.NewMetricWithTimestamp(c.Clock.Now(), metric)
+	}
+	return metric
+}
+
+// check that all the given paths exist and are executable files
+func CheckExecutables(paths ...string) error {
+	for _, path := range paths {
+		fileInfo, err := os.Stat(path)
+		if err != nil || os.IsNotExist(err) {
+			return errors.Errorf("'%s' does not exist", path)
+		}
+		if fileInfo.IsDir() {
+			return errors.Errorf("'%s' is a directory", path)
+		}
+		if (fileInfo.Mode() & 0111) == 0 {
+			return errors.Errorf("'%s' is not executable", path)
+		}
+	}
+	return nil
+}
diff --git a/drbd_metrics.go b/collector/drbd/drbd.go
similarity index 59%
rename from drbd_metrics.go
rename to collector/drbd/drbd.go
index c62005e..8b3702b 100644
--- a/drbd_metrics.go
+++ b/collector/drbd/drbd.go
@@ -1,4 +1,4 @@
-package main
+package drbd
 
 import (
 	"encoding/json"
@@ -11,6 +11,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/ClusterLabs/ha_cluster_exporter/collector"
 )
 
 // drbdStatus is for parsing relevant data we want to convert to metrics
@@ -43,41 +45,39 @@ type drbdStatus struct {
 	} `json:"connections"`
 }
 
-func NewDrbdCollector(drbdSetupPath string, drbdSplitBrainPath string) (*drbdCollector, error) {
-	err := CheckExecutables(drbdSetupPath)
+func NewCollector(drbdSetupPath string, drbdSplitBrainPath string) (*drbdCollector, error) {
+	err := collector.CheckExecutables(drbdSetupPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize DRBD collector")
 	}
 
-	collector := &drbdCollector{
-		DefaultCollector{
-			subsystem: "drbd",
-		},
+	c := &drbdCollector{
+		collector.NewDefaultCollector("drbd"),
 		drbdSetupPath,
 		drbdSplitBrainPath,
 	}
 
-	collector.setDescriptor("resources", "The DRBD resources; 1 line per name, per volume", []string{"resource", "role", "volume", "disk_state"})
-	collector.setDescriptor("written", "KiB written to DRBD; 1 line per res, per volume", []string{"resource", "volume"})
-	collector.setDescriptor("read", "KiB read from DRBD; 1 line per res, per volume", []string{"resource", "volume"})
-	collector.setDescriptor("al_writes", "Writes to activity log; 1 line per res, per volume", []string{"resource", "volume"})
-	collector.setDescriptor("bm_writes", "Writes to bitmap; 1 line per res, per volume", []string{"resource", "volume"})
-	collector.setDescriptor("upper_pending", "Upper pending; 1 line per res, per volume", []string{"resource", "volume"})
-	collector.setDescriptor("lower_pending", "Lower pending; 1 line per res, per volume", []string{"resource", "volume"})
-	collector.setDescriptor("quorum", "Quorum status per resource and per volume", []string{"resource", "volume"})
-	collector.setDescriptor("connections", "The DRBD resource connections; 1 line per per resource, per peer_node_id", []string{"resource", "peer_node_id", "peer_role", "volume", "peer_disk_state"})
-	collector.setDescriptor("connections_sync", "The in sync percentage value for DRBD resource connections", []string{"resource", "peer_node_id", "volume"})
-	collector.setDescriptor("connections_received", "KiB received per connection", []string{"resource", "peer_node_id", "volume"})
-	collector.setDescriptor("connections_sent", "KiB sent per connection", []string{"resource", "peer_node_id", "volume"})
-	collector.setDescriptor("connections_pending", "Pending value per connection", []string{"resource", "peer_node_id", "volume"})
-	collector.setDescriptor("connections_unacked", "Unacked value per connection", []string{"resource", "peer_node_id", "volume"})
-	collector.setDescriptor("split_brain", "Whether a split brain has been detected; 1 line per resource, per volume.", []string{"resource", "volume"})
-
-	return collector, nil
+	c.SetDescriptor("resources", "The DRBD resources; 1 line per name, per volume", []string{"resource", "role", "volume", "disk_state"})
+	c.SetDescriptor("written", "KiB written to DRBD; 1 line per res, per volume", []string{"resource", "volume"})
+	c.SetDescriptor("read", "KiB read from DRBD; 1 line per res, per volume", []string{"resource", "volume"})
+	c.SetDescriptor("al_writes", "Writes to activity log; 1 line per res, per volume", []string{"resource", "volume"})
+	c.SetDescriptor("bm_writes", "Writes to bitmap; 1 line per res, per volume", []string{"resource", "volume"})
+	c.SetDescriptor("upper_pending", "Upper pending; 1 line per res, per volume", []string{"resource", "volume"})
+	c.SetDescriptor("lower_pending", "Lower pending; 1 line per res, per volume", []string{"resource", "volume"})
+	c.SetDescriptor("quorum", "Quorum status per resource and per volume", []string{"resource", "volume"})
+	c.SetDescriptor("connections", "The DRBD resource connections; 1 line per per resource, per peer_node_id", []string{"resource", "peer_node_id", "peer_role", "volume", "peer_disk_state"})
+	c.SetDescriptor("connections_sync", "The in sync percentage value for DRBD resource connections", []string{"resource", "peer_node_id", "volume"})
+	c.SetDescriptor("connections_received", "KiB received per connection", []string{"resource", "peer_node_id", "volume"})
+	c.SetDescriptor("connections_sent", "KiB sent per connection", []string{"resource", "peer_node_id", "volume"})
+	c.SetDescriptor("connections_pending", "Pending value per connection", []string{"resource", "peer_node_id", "volume"})
+	c.SetDescriptor("connections_unacked", "Unacked value per connection", []string{"resource", "peer_node_id", "volume"})
+	c.SetDescriptor("split_brain", "Whether a split brain has been detected; 1 line per resource, per volume.", []string{"resource", "volume"})
+
+	return c, nil
 }
 
 type drbdCollector struct {
-	DefaultCollector
+	collector.DefaultCollector
 	drbdsetupPath      string
 	drbdSplitBrainPath string
 }
@@ -102,24 +102,24 @@ func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, resource := range drbdDev {
 		for _, device := range resource.Devices {
 			// the `resources` metric value is always 1, otherwise it's absent
-			ch <- c.makeGaugeMetric("resources", float64(1), resource.Name, resource.Role, strconv.Itoa(device.Volume), strings.ToLower(device.DiskState))
+			ch <- c.MakeGaugeMetric("resources", float64(1), resource.Name, resource.Role, strconv.Itoa(device.Volume), strings.ToLower(device.DiskState))
 
-			ch <- c.makeGaugeMetric("written", float64(device.Written), resource.Name, strconv.Itoa(device.Volume))
+			ch <- c.MakeGaugeMetric("written", float64(device.Written), resource.Name, strconv.Itoa(device.Volume))
 
-			ch <- c.makeGaugeMetric("read", float64(device.Read), resource.Name, strconv.Itoa(device.Volume))
+			ch <- c.MakeGaugeMetric("read", float64(device.Read), resource.Name, strconv.Itoa(device.Volume))
 
-			ch <- c.makeGaugeMetric("al_writes", float64(device.AlWrites), resource.Name, strconv.Itoa(device.Volume))
+			ch <- c.MakeGaugeMetric("al_writes", float64(device.AlWrites), resource.Name, strconv.Itoa(device.Volume))
 
-			ch <- c.makeGaugeMetric("bm_writes", float64(device.BmWrites), resource.Name, strconv.Itoa(device.Volume))
+			ch <- c.MakeGaugeMetric("bm_writes", float64(device.BmWrites), resource.Name, strconv.Itoa(device.Volume))
 
-			ch <- c.makeGaugeMetric("upper_pending", float64(device.UpPending), resource.Name, strconv.Itoa(device.Volume))
+			ch <- c.MakeGaugeMetric("upper_pending", float64(device.UpPending), resource.Name, strconv.Itoa(device.Volume))
 
-			ch <- c.makeGaugeMetric("lower_pending", float64(device.LoPending), resource.Name, strconv.Itoa(device.Volume))
+			ch <- c.MakeGaugeMetric("lower_pending", float64(device.LoPending), resource.Name, strconv.Itoa(device.Volume))
 
 			if bool(device.Quorum) == true {
-				ch <- c.makeGaugeMetric("quorum", float64(1), resource.Name, strconv.Itoa(device.Volume))
+				ch <- c.MakeGaugeMetric("quorum", float64(1), resource.Name, strconv.Itoa(device.Volume))
 			} else {
-				ch <- c.makeGaugeMetric("quorum", float64(0), resource.Name, strconv.Itoa(device.Volume))
+				ch <- c.MakeGaugeMetric("quorum", float64(0), resource.Name, strconv.Itoa(device.Volume))
 			}
 		}
 		if len(resource.Connections) == 0 {
@@ -133,18 +133,18 @@ func (c *drbdCollector) Collect(ch chan<- prometheus.Metric) {
 				continue
 			}
 			for _, peerDev := range conn.PeerDevices {
-				ch <- c.makeGaugeMetric("connections", float64(1), resource.Name, strconv.Itoa(conn.PeerNodeID),
+				ch <- c.MakeGaugeMetric("connections", float64(1), resource.Name, strconv.Itoa(conn.PeerNodeID),
 					conn.PeerRole, strconv.Itoa(peerDev.Volume), strings.ToLower(peerDev.PeerDiskState))
 
-				ch <- c.makeGaugeMetric("connections_sync", float64(peerDev.PercentInSync), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
+				ch <- c.MakeGaugeMetric("connections_sync", float64(peerDev.PercentInSync), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
 
-				ch <- c.makeGaugeMetric("connections_received", float64(peerDev.Received), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
+				ch <- c.MakeGaugeMetric("connections_received", float64(peerDev.Received), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
 
-				ch <- c.makeGaugeMetric("connections_sent", float64(peerDev.Sent), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
+				ch <- c.MakeGaugeMetric("connections_sent", float64(peerDev.Sent), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
 
-				ch <- c.makeGaugeMetric("connections_pending", float64(peerDev.Pending), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
+				ch <- c.MakeGaugeMetric("connections_pending", float64(peerDev.Pending), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
 
-				ch <- c.makeGaugeMetric("connections_unacked", float64(peerDev.Unacked), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
+				ch <- c.MakeGaugeMetric("connections_unacked", float64(peerDev.Unacked), resource.Name, strconv.Itoa(conn.PeerNodeID), strconv.Itoa(peerDev.Volume))
 
 			}
 		}
@@ -183,7 +183,7 @@ func (c *drbdCollector) recordDrbdSplitBrainMetric(ch chan<- prometheus.Metric)
 		//Resource (0) volume (1) place in slice
 		resourceAndVolume := strings.Split(resAndVolume, "-")
 
-		ch <- c.makeGaugeMetric("split_brain", float64(1), resourceAndVolume[0], resourceAndVolume[1])
+		ch <- c.MakeGaugeMetric("split_brain", float64(1), resourceAndVolume[0], resourceAndVolume[1])
 
 	}
 }
diff --git a/pacemaker_metrics.go b/collector/pacemaker/pacemaker.go
similarity index 83%
rename from pacemaker_metrics.go
rename to collector/pacemaker/pacemaker.go
index 2f95b08..30a632a 100644
--- a/pacemaker_metrics.go
+++ b/collector/pacemaker/pacemaker.go
@@ -1,4 +1,4 @@
-package main
+package pacemaker
 
 import (
 	"encoding/xml"
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ClusterLabs/ha_cluster_exporter/collector"
+
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
@@ -94,30 +96,30 @@ type CIB struct {
 	} `xml:"configuration"`
 }
 
-func NewPacemakerCollector(crmMonPath string, cibAdminPath string) (*pacemakerCollector, error) {
-	err := CheckExecutables(crmMonPath, cibAdminPath)
+func NewCollector(crmMonPath string, cibAdminPath string) (*pacemakerCollector, error) {
+	err := collector.CheckExecutables(crmMonPath, cibAdminPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize Pacemaker collector")
 	}
 
-	collector := &pacemakerCollector{
-		DefaultCollector{subsystem: "pacemaker"},
+	c := &pacemakerCollector{
+		collector.NewDefaultCollector("pacemaker"),
 		crmMonPath,
 		cibAdminPath,
 	}
-	collector.setDescriptor("nodes", "The nodes in the cluster; one line per name, per status", []string{"node", "type", "status"})
-	collector.setDescriptor("resources", "The resources in the cluster; one line per id, per status", []string{"node", "resource", "role", "managed", "status"})
-	collector.setDescriptor("stonith_enabled", "Whether or not stonith is enabled", nil)
-	collector.setDescriptor("fail_count", "The Fail count number per node and resource id", []string{"node", "resource"})
-	collector.setDescriptor("migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"})
-	collector.setDescriptor("config_last_change", "The timestamp of the last change of the cluster configuration", nil)
-	collector.setDescriptor("location_constraints", "Resource location constraints. The value indicates the score.", []string{"constraint", "node", "resource", "role"})
-
-	return collector, nil
+	c.SetDescriptor("nodes", "The nodes in the cluster; one line per name, per status", []string{"node", "type", "status"})
+	c.SetDescriptor("resources", "The resources in the cluster; one line per id, per status", []string{"node", "resource", "role", "managed", "status"})
+	c.SetDescriptor("stonith_enabled", "Whether or not stonith is enabled", nil)
+	c.SetDescriptor("fail_count", "The Fail count number per node and resource id", []string{"node", "resource"})
+	c.SetDescriptor("migration_threshold", "The migration_threshold number per node and resource id", []string{"node", "resource"})
+	c.SetDescriptor("config_last_change", "The timestamp of the last change of the cluster configuration", nil)
+	c.SetDescriptor("location_constraints", "Resource location constraints. The value indicates the score.", []string{"constraint", "node", "resource", "role"})
+
+	return c, nil
 }
 
 type pacemakerCollector struct {
-	DefaultCollector
+	collector.DefaultCollector
 	crmMonPath   string
 	cibAdminPath string
 }
@@ -142,7 +144,7 @@ func (c *pacemakerCollector) Collect(ch chan<- prometheus.Metric) {
 		stonithEnabled = 1
 	}
 
-	ch <- c.makeGaugeMetric("stonith_enabled", stonithEnabled)
+	ch <- c.MakeGaugeMetric("stonith_enabled", stonithEnabled)
 
 	c.recordNodeMetrics(pacemakerStatus, ch)
 	c.recordFailCountMetrics(pacemakerStatus, ch)
@@ -165,6 +167,7 @@ func (c *pacemakerCollector) getPacemakerStatus() (pacemakerStatus, error) {
 
 	return pacemakerStatus, nil
 }
+
 func parsePacemakerStatus(pacemakerXMLRaw []byte) (pacemakerStatus, error) {
 	var pacemakerStatus pacemakerStatus
 	err := xml.Unmarshal(pacemakerXMLRaw, &pacemakerStatus)
@@ -199,7 +202,7 @@ func (c *pacemakerCollector) recordNodeMetrics(pacemakerStatus pacemakerStatus,
 
 		for nodeStatus, isActive := range nodeStatuses {
 			if isActive {
-				ch <- c.makeGaugeMetric("nodes", float64(1), node.Name, nodeType, nodeStatus)
+				ch <- c.MakeGaugeMetric("nodes", float64(1), node.Name, nodeType, nodeStatus)
 			}
 		}
 
@@ -218,7 +221,7 @@ func (c *pacemakerCollector) recordResourcesMetrics(node node, ch chan<- prometh
 		}
 		for resourceStatus, isActive := range resourceStatuses {
 			if isActive {
-				ch <- c.makeGaugeMetric(
+				ch <- c.MakeGaugeMetric(
 					"resources",
 					float64(1),
 					node.Name,
@@ -241,7 +244,7 @@ func (c *pacemakerCollector) recordFailCountMetrics(pacemakerStatus pacemakerSta
 				failCount = math.Inf(1)
 			}
 
-			ch <- c.makeGaugeMetric("fail_count", failCount, node.Name, resHistory.Name)
+			ch <- c.MakeGaugeMetric("fail_count", failCount, node.Name, resHistory.Name)
 
 		}
 	}
@@ -254,13 +257,13 @@ func (c *pacemakerCollector) recordResourceAgentsChanges(pacemakerStatus pacemak
 		return
 	}
 	// we record the timestamp of the last change as a float counter metric
-	ch <- c.makeCounterMetric("config_last_change", float64(t.Unix()))
+	ch <- c.MakeCounterMetric("config_last_change", float64(t.Unix()))
 }
 
 func (c *pacemakerCollector) recordMigrationThresholdMetrics(pacemakerStatus pacemakerStatus, ch chan<- prometheus.Metric) {
 	for _, node := range pacemakerStatus.NodeHistory.Node {
 		for _, resHistory := range node.ResourceHistory {
-			ch <- c.makeGaugeMetric("migration_threshold", float64(resHistory.MigrationThreshold), node.Name, resHistory.Name)
+			ch <- c.MakeGaugeMetric("migration_threshold", float64(resHistory.MigrationThreshold), node.Name, resHistory.Name)
 		}
 	}
 }
@@ -293,6 +296,6 @@ func (c *pacemakerCollector) recordConstraintsMetrics(cib CIB, ch chan<- prometh
 			constraintScore = float64(s)
 		}
 
-		ch <- c.makeGaugeMetric("location_constraints", constraintScore, constraint.Id, constraint.Node, constraint.Resource, strings.ToLower(constraint.Role))
+		ch <- c.MakeGaugeMetric("location_constraints", constraintScore, constraint.Id, constraint.Node, constraint.Resource, strings.ToLower(constraint.Role))
 	}
 }
diff --git a/sbd_metrics.go b/collector/sbd/sbd.go
similarity index 86%
rename from sbd_metrics.go
rename to collector/sbd/sbd.go
index 68fd3cd..56cde98 100644
--- a/sbd_metrics.go
+++ b/collector/sbd/sbd.go
@@ -1,4 +1,4 @@
-package main
+package sbd
 
 import (
 	"fmt"
@@ -11,13 +11,15 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/ClusterLabs/ha_cluster_exporter/collector"
 )
 
 const SBD_STATUS_UNHEALTHY = "unhealthy"
 const SBD_STATUS_HEALTHY = "healthy"
 
-func NewSbdCollector(sbdPath string, sbdConfigPath string) (*sbdCollector, error) {
-	err := CheckExecutables(sbdPath)
+func NewCollector(sbdPath string, sbdConfigPath string) (*sbdCollector, error) {
+	err := collector.CheckExecutables(sbdPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not initialize SBD collector")
 	}
@@ -26,21 +28,19 @@ func NewSbdCollector(sbdPath string, sbdConfigPath string) (*sbdCollector, error
 		return nil, errors.Errorf("could not initialize SBD collector: '%s' does not exist", sbdConfigPath)
 	}
 
-	collector := &sbdCollector{
-		DefaultCollector{
-			subsystem: "sbd",
-		},
+	c := &sbdCollector{
+		collector.NewDefaultCollector("sbd"),
 		sbdPath,
 		sbdConfigPath,
 	}
 
-	collector.setDescriptor("devices", "SBD devices; one line per device", []string{"device", "status"})
+	c.SetDescriptor("devices", "SBD devices; one line per device", []string{"device", "status"})
 
-	return collector, nil
+	return c, nil
 }
 
 type sbdCollector struct {
-	DefaultCollector
+	collector.DefaultCollector
 	sbdPath       string
 	sbdConfigPath string
 }
@@ -58,7 +58,7 @@ func (c *sbdCollector) Collect(ch chan<- prometheus.Metric) {
 
 	sbdStatuses := c.getSbdDeviceStatuses(sbdDevices)
 	for sbdDev, sbdStatus := range sbdStatuses {
-		ch <- c.makeGaugeMetric("devices", 1, sbdDev, sbdStatus)
+		ch <- c.MakeGaugeMetric("devices", 1, sbdDev, sbdStatus)
 	}
 }
 
diff --git a/ha_cluster_exporter.go b/ha_cluster_exporter.go
index 93e714b..0d90b7e 100644
--- a/ha_cluster_exporter.go
+++ b/ha_cluster_exporter.go
@@ -3,109 +3,35 @@ package main
 import (
 	"fmt"
 	"net/http"
-	"os"
-	"time"
 
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
 	config "github.com/spf13/viper"
-)
-
-const NAMESPACE = "ha_cluster"
-
-type Clock interface {
-	Now() time.Time
-}
-
-type SystemClock struct{}
-
-func (SystemClock) Now() time.Time {
-	return time.Now()
-}
-
-type DefaultCollector struct {
-	subsystem   string
-	descriptors map[string]*prometheus.Desc
-}
-
-func (c *DefaultCollector) getDescriptor(name string) *prometheus.Desc {
-	desc, ok := c.descriptors[name]
-	if !ok {
-		// we hard panic on this because it's most certainly a coding error
-		panic(errors.Errorf("undeclared metric '%s'", name))
-	}
-	return desc
-}
-
-// Convenience wrapper around prometheus.NewDesc constructor.
-// Stores a metric descriptor with a fully qualified name like `NAMESPACE_subsystem_name`.
-// `name` is the last and most relevant part of the metrics Full Qualified Name;
-// `help` is the message displayed in the HELP line
-// `variableLabels` is a list of labels to declare. Use `nil` to declare no labels.
-func (c *DefaultCollector) setDescriptor(name, help string, variableLabels []string) {
-	if c.descriptors == nil {
-		c.descriptors = make(map[string]*prometheus.Desc)
-	}
-	c.descriptors[name] = prometheus.NewDesc(prometheus.BuildFQName(NAMESPACE, c.subsystem, name), help, variableLabels, nil)
-}
 
-func (c *DefaultCollector) Describe(ch chan<- *prometheus.Desc) {
-	for _, descriptor := range c.descriptors {
-		ch <- descriptor
-	}
-}
-
-func (c *DefaultCollector) makeGaugeMetric(name string, value float64, labelValues ...string) prometheus.Metric {
-	return c.makeMetric(name, value, prometheus.GaugeValue, labelValues...)
-}
-
-func (c *DefaultCollector) makeCounterMetric(name string, value float64, labelValues ...string) prometheus.Metric {
-	return c.makeMetric(name, value, prometheus.CounterValue, labelValues...)
-}
-
-func (c *DefaultCollector) makeMetric(name string, value float64, valueType prometheus.ValueType, labelValues ...string) prometheus.Metric {
-	desc := c.getDescriptor(name)
-	metric := prometheus.MustNewConstMetric(desc, valueType, value, labelValues...)
-	if config.GetBool("enable-timestamps") {
-		metric = prometheus.NewMetricWithTimestamp(clock.Now(), metric)
-	}
-	return metric
-}
-
-// check that all the given paths exist and are executable files
-func CheckExecutables(paths ...string) error {
-	for _, path := range paths {
-		fileInfo, err := os.Stat(path)
-		if err != nil || os.IsNotExist(err) {
-			return errors.Errorf("'%s' does not exist", path)
-		}
-		if fileInfo.IsDir() {
-			return errors.Errorf("'%s' is a directory", path)
-		}
-		if (fileInfo.Mode() & 0111) == 0 {
-			return errors.Errorf("'%s' is not executable", path)
-		}
-	}
-	return nil
-}
+	"github.com/ClusterLabs/ha_cluster_exporter/collector/corosync"
+	"github.com/ClusterLabs/ha_cluster_exporter/collector/drbd"
+	"github.com/ClusterLabs/ha_cluster_exporter/collector/pacemaker"
+	"github.com/ClusterLabs/ha_cluster_exporter/collector/sbd"
+)
 
 // Landing Page (for /)
 func landingpage(w http.ResponseWriter, r *http.Request) {
-	w.Write([]byte(`<html>
-		<head>
-			<title>HACluster Exporter</title>
-		</head>
-		<body>
-			<h1>HACluster Exporter</h1>
-			<p><a href="metrics">Metrics</a></p>
-			<br />
-			<h2>More information:</h2>
-			<p><a href="https://github.com/ClusterLabs/ha_cluster_exporter">github.com/ClusterLabs/ha_cluster_exporter</a></p>
-		</body>
-		</html>`))
+	body := []byte(`<html>
+<head>
+	<title>ClusterLabs Linux HA Cluster Exporter</title>
+</head>
+<body>
+	<h1>ClusterLabs Linux HA Cluster </h1>
+	<p><a href="metrics">Metrics</a></p>
+	<br />
+	<h2>More information:</h2>
+	<p><a href="https://github.com/ClusterLabs/ha_cluster_exporter">github.com/ClusterLabs/ha_cluster_exporter</a></p>
+</body>
+</html>
+`)
+	w.Write(body)
 }
 
 func loglevel(level string) {
@@ -123,8 +49,6 @@ func loglevel(level string) {
 	}
 }
 
-var clock Clock = &SystemClock{}
-
 func init() {
 	config.SetConfigName("ha_cluster_exporter")
 	config.AddConfigPath("./")
@@ -166,7 +90,7 @@ func main() {
 
 	loglevel(config.GetString("log-level"))
 
-	pacemakerCollector, err := NewPacemakerCollector(
+	pacemakerCollector, err := pacemaker.NewCollector(
 		config.GetString("crm-mon-path"),
 		config.GetString("cibadmin-path"),
 	)
@@ -177,7 +101,7 @@ func main() {
 		log.Info("Pacemaker collector registered")
 	}
 
-	corosyncCollector, err := NewCorosyncCollector(
+	corosyncCollector, err := corosync.NewCollector(
 		config.GetString("corosync-cfgtoolpath-path"),
 		config.GetString("corosync-quorumtool-path"),
 	)
@@ -188,7 +112,7 @@ func main() {
 		log.Info("Corosync collector registered")
 	}
 
-	sbdCollector, err := NewSbdCollector(
+	sbdCollector, err := sbd.NewCollector(
 		config.GetString("sbd-path"),
 		config.GetString("sbd-config-path"),
 	)
@@ -199,7 +123,7 @@ func main() {
 		log.Info("SBD collector registered")
 	}
 
-	drbdCollector, err := NewDrbdCollector(config.GetString("drbdsetup-path"), config.GetString("drbdsplitbrain-path"))
+	drbdCollector, err := drbd.NewCollector(config.GetString("drbdsetup-path"), config.GetString("drbdsplitbrain-path"))
 	if err != nil {
 		log.Warn(err)
 	} else {
diff --git a/internal/assert/assertions.go b/internal/assert/assertions.go
new file mode 100644
index 0000000..8c80041
--- /dev/null
+++ b/internal/assert/assertions.go
@@ -0,0 +1,22 @@
+package assert
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+// borrowed from haproxy_exporter
+// https://github.com/prometheus/haproxy_exporter/blob/0ddc4bc5cb4074ba95d57257f63ab82ab451a45b/haproxy_exporter_test.go
+func Metrics(t *testing.T, c prometheus.Collector, fixture string) {
+	exp, err := os.Open(path.Join("../../test", fixture))
+	if err != nil {
+		t.Fatalf("Error opening fixture file %q: %v", fixture, err)
+	}
+	if err := testutil.CollectAndCompare(c, exp); err != nil {
+		t.Fatal("Unexpected metrics returned:", err)
+	}
+}
diff --git a/internal/clock/clock.go b/internal/clock/clock.go
new file mode 100644
index 0000000..cdfb7ab
--- /dev/null
+++ b/internal/clock/clock.go
@@ -0,0 +1,7 @@
+package clock
+
+import "time"
+
+type Clock interface {
+	Now() time.Time
+}
diff --git a/internal/clock/stop_clock.go b/internal/clock/stop_clock.go
new file mode 100644
index 0000000..3946ba0
--- /dev/null
+++ b/internal/clock/stop_clock.go
@@ -0,0 +1,14 @@
+package clock
+
+import "time"
+
+type StoppedClock struct{}
+
+const TEST_TIMESTAMP = 1234
+
+func (StoppedClock) Now() time.Time {
+	ms := TEST_TIMESTAMP * time.Millisecond
+	return time.Date(1970, 1, 1, 0, 0, 0, int(ms.Nanoseconds()), time.UTC)
+	// 1234 millisecond after Unix epoch (1970-01-01 00:00:01.234 +0000 UTC)
+	// this will allow us to use a fixed timestamped when running assertions
+}
diff --git a/internal/clock/system_clock.go b/internal/clock/system_clock.go
new file mode 100644
index 0000000..4fad6ff
--- /dev/null
+++ b/internal/clock/system_clock.go
@@ -0,0 +1,9 @@
+package clock
+
+import "time"
+
+type SystemClock struct{}
+
+func (SystemClock) Now() time.Time {
+	return time.Now()
+}


Test patch:
diff --git a/corosync_metrics_test.go b/collector/corosync/corosync_test.go
similarity index 78%
rename from corosync_metrics_test.go
rename to collector/corosync/corosync_test.go
index 9c52453..55449c3 100644
--- a/corosync_metrics_test.go
+++ b/collector/corosync/corosync_test.go
@@ -1,9 +1,11 @@
-package main
+package corosync
 
 import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	assertcustom "github.com/ClusterLabs/ha_cluster_exporter/internal/assert"
 )
 
 // TEST group quorum metrics
@@ -117,39 +119,39 @@ func TestRingStatusParsingError(t *testing.T) {
 }
 
 func TestNewCorosyncCollector(t *testing.T) {
-	_, err := NewCorosyncCollector("test/fake_corosync-cfgtool.sh", "test/fake_corosync-quorumtool.sh")
+	_, err := NewCollector("../../test/fake_corosync-cfgtool.sh", "../../test/fake_corosync-quorumtool.sh")
 	assert.Nil(t, err)
 }
 
 func TestNewCorosyncCollectorChecksCfgtoolExistence(t *testing.T) {
-	_, err := NewCorosyncCollector("test/nonexistent", "test/fake_corosync-quorumtool.sh")
+	_, err := NewCollector("../../test/nonexistent", "../../test/fake_corosync-quorumtool.sh")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/nonexistent' does not exist")
+	assert.Contains(t, err.Error(), "'../../test/nonexistent' does not exist")
 }
 
 func TestNewCorosyncCollectorChecksQuorumtoolExistence(t *testing.T) {
-	_, err := NewCorosyncCollector("test/fake_corosync-cfgtool.sh", "test/nonexistent")
+	_, err := NewCollector("../../test/fake_corosync-cfgtool.sh", "../../test/nonexistent")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/nonexistent' does not exist")
+	assert.Contains(t, err.Error(), "'../../test/nonexistent' does not exist")
 }
 
 func TestNewCorosyncCollectorChecksCfgtoolExecutableBits(t *testing.T) {
-	_, err := NewCorosyncCollector("test/dummy", "test/fake_corosync-quorumtool.sh")
+	_, err := NewCollector("../../test/dummy", "../../test/fake_corosync-quorumtool.sh")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/dummy' is not executable")
+	assert.Contains(t, err.Error(), "'../../test/dummy' is not executable")
 }
 
 func TestNewCorosyncCollectorChecksQuorumtoolExecutableBits(t *testing.T) {
-	_, err := NewCorosyncCollector("test/fake_corosync-cfgtool.sh", "test/dummy")
+	_, err := NewCollector("../../test/fake_corosync-cfgtool.sh", "../../test/dummy")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/dummy' is not executable")
+	assert.Contains(t, err.Error(), "'../../test/dummy' is not executable")
 }
 
 func TestCorosyncCollector(t *testing.T) {
-	collector, _ := NewCorosyncCollector("test/fake_corosync-cfgtool.sh", "test/fake_corosync-quorumtool.sh")
-	expectMetrics(t, collector, "corosync.metrics")
+	collector, _ := NewCollector("../../test/fake_corosync-cfgtool.sh", "../../test/fake_corosync-quorumtool.sh")
+	assertcustom.Metrics(t, collector, "corosync.metrics")
 }
diff --git a/collector/default_collector_test.go b/collector/default_collector_test.go
new file mode 100644
index 0000000..3c8bc0f
--- /dev/null
+++ b/collector/default_collector_test.go
@@ -0,0 +1,37 @@
+package collector
+
+import (
+	"testing"
+
+	dto "github.com/prometheus/client_model/go"
+	config "github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/ClusterLabs/ha_cluster_exporter/internal/clock"
+)
+
+func TestMetricFactory(t *testing.T) {
+	SUT := NewDefaultCollector("test")
+	SUT.SetDescriptor("test_metric", "", nil)
+
+	metric := SUT.MakeGaugeMetric("test_metric", 1)
+
+	assert.Equal(t, SUT.GetDescriptor("test_metric"), metric.Desc())
+}
+
+func TestMetricFactoryWithTimestamp(t *testing.T) {
+	config.Set("enable-timestamps", true)
+	defer config.Set("enable-timestamps", false)
+
+	SUT := NewDefaultCollector("test")
+	SUT.Clock = &clock.StoppedClock{}
+	SUT.SetDescriptor("test_metric", "", nil)
+
+	metric := SUT.MakeGaugeMetric("test_metric", 1)
+	metricDto := &dto.Metric{}
+	err := metric.Write(metricDto)
+
+	assert.Nil(t, err, "Unexpected error")
+
+	assert.Equal(t, int64(clock.TEST_TIMESTAMP), *metricDto.TimestampMs)
+}
diff --git a/drbd_metrics_test.go b/collector/drbd/drbd_test.go
similarity index 88%
rename from drbd_metrics_test.go
rename to collector/drbd/drbd_test.go
index e5da90f..93bca79 100644
--- a/drbd_metrics_test.go
+++ b/collector/drbd/drbd_test.go
@@ -1,4 +1,4 @@
-package main
+package drbd
 
 import (
 	"strings"
@@ -6,6 +6,8 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
+
+	assertcustom "github.com/ClusterLabs/ha_cluster_exporter/internal/assert"
 )
 
 func TestDrbdParsing(t *testing.T) {
@@ -141,32 +143,32 @@ func TestDrbdParsing(t *testing.T) {
 }
 
 func TestNewDrbdCollector(t *testing.T) {
-	_, err := NewDrbdCollector("test/fake_drbdsetup.sh", "splitbrainpath")
+	_, err := NewCollector("../../test/fake_drbdsetup.sh", "splitbrainpath")
 
 	assert.Nil(t, err)
 }
 
 func TestNewDrbdCollectorChecksDrbdsetupExistence(t *testing.T) {
-	_, err := NewDrbdCollector("test/nonexistent", "splitbrainfake")
+	_, err := NewCollector("../../test/nonexistent", "splitbrainfake")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/nonexistent' does not exist")
+	assert.Contains(t, err.Error(), "'../../test/nonexistent' does not exist")
 }
 
 func TestNewDrbdCollectorChecksDrbdsetupExecutableBits(t *testing.T) {
-	_, err := NewDrbdCollector("test/dummy", "splibrainfake")
+	_, err := NewCollector("../../test/dummy", "splibrainfake")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/dummy' is not executable")
+	assert.Contains(t, err.Error(), "'../../test/dummy' is not executable")
 }
 
 func TestDRBDCollector(t *testing.T) {
-	collector, _ := NewDrbdCollector("test/fake_drbdsetup.sh", "fake")
-	expectMetrics(t, collector, "drbd.metrics")
+	collector, _ := NewCollector("../../test/fake_drbdsetup.sh", "fake")
+	assertcustom.Metrics(t, collector, "drbd.metrics")
 }
 
 func TestDRBDSplitbrainCollector(t *testing.T) {
-	collector, _ := NewDrbdCollector("test/fake_drbdsetup.sh", "test/drbd-splitbrain")
+	collector, _ := NewCollector("../../test/fake_drbdsetup.sh", "../../test/drbd-splitbrain")
 
 	expect := `
 	# HELP ha_cluster_drbd_split_brain Whether a split brain has been detected; 1 line per resource, per volume.
diff --git a/pacemaker_metrics_test.go b/collector/pacemaker/pacemaker_test.go
similarity index 91%
rename from pacemaker_metrics_test.go
rename to collector/pacemaker/pacemaker_test.go
index e061d71..40ee226 100644
--- a/pacemaker_metrics_test.go
+++ b/collector/pacemaker/pacemaker_test.go
@@ -1,9 +1,11 @@
-package main
+package pacemaker
 
 import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	assertcustom "github.com/ClusterLabs/ha_cluster_exporter/internal/assert"
 )
 
 func TestParsePacemakerXML(t *testing.T) {
@@ -90,28 +92,28 @@ func TestParsePacemakerXML(t *testing.T) {
 }
 
 func TestNewPacemakerCollector(t *testing.T) {
-	_, err := NewPacemakerCollector("test/fake_crm_mon.sh", "test/fake_cibadmin.sh")
+	_, err := NewCollector("../../test/fake_crm_mon.sh", "../../test/fake_cibadmin.sh")
 
 	assert.Nil(t, err)
 }
 
 func TestNewPacemakerCollectorChecksCrmMonExistence(t *testing.T) {
-	_, err := NewPacemakerCollector("test/nonexistent", "")
+	_, err := NewCollector("../../test/nonexistent", "")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/nonexistent' does not exist")
+	assert.Contains(t, err.Error(), "'../../test/nonexistent' does not exist")
 }
 
 func TestNewPacemakerCollectorChecksCrmMonExecutableBits(t *testing.T) {
-	_, err := NewPacemakerCollector("test/dummy", "")
+	_, err := NewCollector("../../test/dummy", "")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/dummy' is not executable")
+	assert.Contains(t, err.Error(), "'../../test/dummy' is not executable")
 }
 
 func TestPacemakerCollector(t *testing.T) {
-	collector, err := NewPacemakerCollector("test/fake_crm_mon.sh", "test/fake_cibadmin.sh")
+	collector, err := NewCollector("../../test/fake_crm_mon.sh", "../../test/fake_cibadmin.sh")
 
 	assert.Nil(t, err)
-	expectMetrics(t, collector, "pacemaker.metrics")
+	assertcustom.Metrics(t, collector, "pacemaker.metrics")
 }
diff --git a/sbd_metrics_test.go b/collector/sbd/sbd_test.go
similarity index 86%
rename from sbd_metrics_test.go
rename to collector/sbd/sbd_test.go
index ae2841b..958bd7b 100644
--- a/sbd_metrics_test.go
+++ b/collector/sbd/sbd_test.go
@@ -1,13 +1,15 @@
-package main
+package sbd
 
 import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	assertcustom "github.com/ClusterLabs/ha_cluster_exporter/internal/assert"
 )
 
 func TestReadSbdConfFileError(t *testing.T) {
-	sbdConfFile, err := readSdbFile("test/nonexistent")
+	sbdConfFile, err := readSdbFile("../../test/nonexistent")
 
 	assert.Nil(t, sbdConfFile)
 	assert.Error(t, err)
@@ -185,33 +187,33 @@ SBD_DEVICE=/dev/vdc;/dev/vdd`
 }
 
 func TestNewSbdCollector(t *testing.T) {
-	_, err := NewSbdCollector("test/fake_sbd.sh", "test/fake_sbdconfig")
+	_, err := NewCollector("../../test/fake_sbd.sh", "../../test/fake_sbdconfig")
 
 	assert.Nil(t, err)
 }
 
 func TestNewSbdCollectorChecksSbdConfigExistence(t *testing.T) {
-	_, err := NewSbdCollector("test/fake_sbd.sh", "test/nonexistent")
+	_, err := NewCollector("../../test/fake_sbd.sh", "../../test/nonexistent")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/nonexistent' does not exist")
+	assert.Contains(t, err.Error(), "'../../test/nonexistent' does not exist")
 }
 
 func TestNewSbdCollectorChecksSbdExistence(t *testing.T) {
-	_, err := NewSbdCollector("test/nonexistent", "test/fake_sbdconfig")
+	_, err := NewCollector("../../test/nonexistent", "../../test/fake_sbdconfig")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/nonexistent' does not exist")
+	assert.Contains(t, err.Error(), "'../../test/nonexistent' does not exist")
 }
 
 func TestNewSbdCollectorChecksSbdExecutableBits(t *testing.T) {
-	_, err := NewSbdCollector("test/dummy", "test/fake_sbdconfig")
+	_, err := NewCollector("../../test/dummy", "../../test/fake_sbdconfig")
 
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "'test/dummy' is not executable")
+	assert.Contains(t, err.Error(), "'../../test/dummy' is not executable")
 }
 
 func TestSBDCollector(t *testing.T) {
-	collector, _ := NewSbdCollector("test/fake_sbd.sh", "test/fake_sbdconfig")
-	expectMetrics(t, collector, "sbd.metrics")
+	collector, _ := NewCollector("../../test/fake_sbd.sh", "../../test/fake_sbdconfig")
+	assertcustom.Metrics(t, collector, "sbd.metrics")
 }
diff --git a/ha_cluster_exporter_test.go b/ha_cluster_exporter_test.go
deleted file mode 100644
index 3ff1448..0000000
--- a/ha_cluster_exporter_test.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package main
-
-import (
-	"os"
-	"path"
-	"testing"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/testutil"
-	dto "github.com/prometheus/client_model/go"
-	config "github.com/spf13/viper"
-	"github.com/stretchr/testify/assert"
-)
-
-type StoppedClock struct{}
-
-const TEST_TIMESTAMP = 1234
-
-func (StoppedClock) Now() time.Time {
-	ms := TEST_TIMESTAMP * time.Millisecond
-	return time.Date(1970, 1, 1, 0, 0, 0, int(ms.Nanoseconds()), time.UTC)
-	// 1234 millisecond after Unix epoch (1970-01-01 00:00:01.234 +0000 UTC)
-	// this will allow us to use a fixed timestamped when running assertions
-}
-
-// borrowed from haproxy_exporter
-// https://github.com/prometheus/haproxy_exporter/blob/0ddc4bc5cb4074ba95d57257f63ab82ab451a45b/haproxy_exporter_test.go
-func expectMetrics(t *testing.T, c prometheus.Collector, fixture string) {
-	exp, err := os.Open(path.Join("test", fixture))
-	if err != nil {
-		t.Fatalf("Error opening fixture file %q: %v", fixture, err)
-	}
-	if err := testutil.CollectAndCompare(c, exp); err != nil {
-		t.Fatal("Unexpected metrics returned:", err)
-	}
-}
-
-func TestMetricFactory(t *testing.T) {
-	SUT := &DefaultCollector{}
-	SUT.setDescriptor("test_metric", "", nil)
-
-	metric := SUT.makeGaugeMetric("test_metric", 1)
-
-	assert.Equal(t, SUT.getDescriptor("test_metric"), metric.Desc())
-}
-
-func TestMetricFactoryWithTimestamp(t *testing.T) {
-	config.Set("enable-timestamps", true)
-	defer func() {
-		config.Set("enable-timestamps", false)
-		clock = SystemClock{}
-	}()
-
-	clock = StoppedClock{}
-	SUT := &DefaultCollector{}
-	SUT.setDescriptor("test_metric", "", nil)
-
-	metric := SUT.makeGaugeMetric("test_metric", 1)
-	metricDto := &dto.Metric{}
-	err := metric.Write(metricDto)
-
-	assert.Nil(t, err, "Unexpected error")
-
-	assert.Equal(t, int64(TEST_TIMESTAMP), *metricDto.TimestampMs)
-}