Repository: NetApp/harvest
Pull request #264 (2021-06-28 10:07:32)

Grafana global prefix - re-opened PR
Addresses feature request #87

Re-opened a PR #228 and fixed issues in the review.


Patch:
diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go
index 46a9beb7..e3afc51f 100644
--- a/cmd/tools/generate/generate.go
+++ b/cmd/tools/generate/generate.go
@@ -2,6 +2,7 @@ package generate
 
 import (
 	"github.com/spf13/cobra"
+	"goharvest2/cmd/harvest/version"
 	"goharvest2/pkg/color"
 	"goharvest2/pkg/conf"
 	"os"
@@ -14,6 +15,8 @@ type PollerPort struct {
 	Port       int
 	ConfigFile string
 	LogLevel   int
+	Image      string
+	Version    string
 }
 
 type PollerTemplate struct {
@@ -22,10 +25,12 @@ type PollerTemplate struct {
 
 type options struct {
 	loglevel int
+	image    string
 }
 
 var opts = &options{
 	loglevel: 2,
+	image:    "harvest:latest",
 }
 
 var Cmd = &cobra.Command{
@@ -73,7 +78,7 @@ func generateDockerCompose(path string) {
 	conf.ValidatePortInUse = true
 	for _, v := range conf.Config.PollersOrdered {
 		port, _ := conf.GetPrometheusExporterPorts(v)
-		pollerTemplate.Pollers = append(pollerTemplate.Pollers, PollerPort{v, port, absPath, opts.loglevel})
+		pollerTemplate.Pollers = append(pollerTemplate.Pollers, PollerPort{v, port, absPath, opts.loglevel, opts.image, version.VERSION})
 	}
 
 	t, err := template.New("docker-compose.tmpl").ParseFiles("docker/onePollerPerContainer/docker-compose.tmpl")
@@ -124,4 +129,10 @@ func init() {
 		2,
 		"logging level (0=trace, 1=debug, 2=info, 3=warning, 4=error, 5=critical)",
 	)
+	dockerCmd.PersistentFlags().StringVar(
+		&opts.image,
+		"image",
+		"harvest:latest",
+		"Harvest image",
+	)
 }
diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go
index 53759e32..dbe89351 100644
--- a/cmd/tools/grafana/grafana.go
+++ b/cmd/tools/grafana/grafana.go
@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -47,6 +48,7 @@ type options struct {
 	client         *http.Client
 	headers        http.Header
 	config         string
+	prefix         string
 	useHttps       bool
 	useInsecureTLS bool
 }
@@ -231,6 +233,11 @@ func importDashboards(opts *options) error {
 			return err
 		}
 
+		// optionally add prefix to all metric names in the queries
+		if opts.prefix != "" {
+			addGlobalPrefix(dashboard, opts.prefix)
+		}
+
 		request = make(map[string]interface{})
 		request["overwrite"] = true
 		request["folderId"] = opts.folderId
@@ -252,6 +259,130 @@ func importDashboards(opts *options) error {
 	return nil
 }
 
+// addGlobalPrefix adds the given prefix to all metric names in the
+// dashboards. It assumes that metrics are in Prometheus-format.
+//
+// A more reliable implementation of this feature would be, to
+// add a constant prefix to all metrics, before they are pushed
+// to Github, then replace them with a user-defined prefix
+// (or empty string) when the import tool is used.
+func addGlobalPrefix(db map[string]interface{}, prefix string) {
+
+	var (
+		panels, targets, templates                 []interface{}
+		panel, target, templating, template, query map[string]interface{}
+		p, t                                       interface{}
+		queryString, definition, expr              string
+		ok, has                                    bool
+		regex                                      *regexp.Regexp
+	)
+
+	// regex we will use to match metric names
+	// used in addPrefixToMatricNames, but better to initialize it once
+	regex = regexp.MustCompile(`([a-zA-Z_+]+){.+?}`)
+
+	// make sure prefix ends with _
+	if !strings.HasSuffix(prefix, "_") {
+		prefix += "_"
+	}
+
+	// apply to queries in panels
+	if panels, ok = db["panels"].([]interface{}); !ok {
+		return
+	}
+
+	for _, p = range panels {
+		if panel, ok = p.(map[string]interface{}); !ok {
+			continue
+		}
+
+		if _, has = panel["targets"]; !has {
+			continue
+		}
+
+		if targets, ok = panel["targets"].([]interface{}); !ok {
+			continue
+		}
+
+		for _, t = range targets {
+
+			if target, ok = t.(map[string]interface{}); !ok {
+				continue
+			}
+
+			if _, has = target["expr"]; has {
+				if expr, ok = target["expr"].(string); ok {
+					target["expr"] = addPrefixToMetricNames(expr, prefix, regex)
+				}
+			}
+		}
+	}
+
+	// apply to variables
+	if templating, ok = db["templating"].(map[string]interface{}); !ok {
+		return
+	}
+
+	if templates, ok = templating["list"].([]interface{}); !ok {
+		return
+	}
+
+	for _, t = range templates {
+		if template, ok = t.(map[string]interface{}); ok {
+			if definition, ok = template["definition"].(string); ok {
+				template["definition"] = addPrefixToMetricNames(definition, prefix, regex)
+			}
+			if query, ok = template["query"].(map[string]interface{}); ok {
+				if queryString, ok = query["query"].(string); ok {
+					query["query"] = addPrefixToMetricNames(queryString, prefix, regex)
+				}
+			}
+		}
+	}
+}
+
+// addPrefixToMetricNames adds prefix to metric names in expr. Note that
+// this function will only work with the Prometheus-dashboards of Harvest.
+// It will recognize a number of ways in which metrics are used as queries.
+// (E.g. a single metric, multiple metrics used in addition, etc -- for examples
+// see the test). If we change queries of our dashboards, we have to review
+// this function as well (or come up with a better solution).
+func addPrefixToMetricNames(expr, prefix string, regex *regexp.Regexp) string {
+	var (
+		match    [][]string
+		submatch []string
+	)
+
+	// variable queries
+	if strings.HasPrefix(expr, "label_values(") {
+		if strings.Contains(expr, ", ") {
+			return strings.Replace(expr, "label_values(", "label_values("+prefix, 1)
+		} else {
+			// no metric name
+			return expr
+		}
+	}
+
+	// everything else is for graph queries
+	match = regex.FindAllStringSubmatch(expr, -1)
+
+	for _, m := range match {
+		// multiple metrics used to summarize
+		if strings.Contains(m[1], "+") {
+			submatch = strings.Split(m[1], "+")
+			for i := range submatch {
+				submatch[i] = prefix + submatch[i]
+			}
+			expr = strings.Replace(expr, m[1], strings.Join(submatch, "+"), 1)
+			// single metric
+		} else {
+			expr = strings.Replace(expr, m[1], prefix+m[1], 1)
+		}
+	}
+
+	return expr
+}
+
 func checkToken(opts *options, ignoreConfig bool) error {
 
 	// @TODO check and handle expired API token
@@ -531,6 +662,7 @@ func init() {
 	GrafanaCmd.PersistentFlags().StringVarP(&opts.token, "token", "t", "", "API token issued by Grafana server for authentication")
 	GrafanaCmd.PersistentFlags().StringVarP(&opts.dir, "directory", "d", "grafana/prometheus/", "when importing, directory that contains dashboards.\nWhen exporting, directory to write dashboards to")
 	GrafanaCmd.PersistentFlags().StringVarP(&opts.folder, "folder", "f", grafanaFolderTitle, "Grafana folder name for the dashboards")
+	GrafanaCmd.PersistentFlags().StringVarP(&opts.prefix, "prefix", "p", "", "Use global metric prefix in queries")
 	GrafanaCmd.PersistentFlags().StringVarP(&opts.datasource, "datasource", "s", grafanaDataSource, "Grafana datasource for the dashboards")
 	GrafanaCmd.PersistentFlags().BoolVarP(&opts.variable, "variable", "v", false, "use datasource as variable, overrides: --datasource")
 	GrafanaCmd.PersistentFlags().BoolVarP(&opts.useHttps, "https", "S", false, "use HTTPS")
diff --git a/docker/allPollerInOneContainer/README.md b/docker/allPollerInOneContainer/README.md
index d6d1a339..cff66f51 100644
--- a/docker/allPollerInOneContainer/README.md
+++ b/docker/allPollerInOneContainer/README.md
@@ -7,6 +7,13 @@ docker build -f docker/allPollerInOneContainer/Dockerfile -t harvest:latest  .
 ```
 
 Modify docker-compose (docker/allPollerInOneContainer/docker-compose.yml) file for port mappings and start docker containers
+
 ```
 docker-compose -f docker/allPollerInOneContainer/docker-compose.yml up -d
 ```
+
+Stop docker containers
+
+```
+docker-compose -f docker/allPollerInOneContainer/docker-compose.yml down
+```
\ No newline at end of file
diff --git a/docker/onePollerPerContainer/README.md b/docker/onePollerPerContainer/README.md
index 72bbee22..b358d88c 100644
--- a/docker/onePollerPerContainer/README.md
+++ b/docker/onePollerPerContainer/README.md
@@ -17,3 +17,9 @@ Start docker containers
 ```
 docker-compose -f docker-compose.yml up -d --remove-orphans
 ```
+
+Stop docker containers
+
+```
+docker-compose -f docker-compose.yml down
+```
\ No newline at end of file
diff --git a/docker/onePollerPerContainer/docker-compose.tmpl b/docker/onePollerPerContainer/docker-compose.tmpl
index 30df96f3..69ba704c 100644
--- a/docker/onePollerPerContainer/docker-compose.tmpl
+++ b/docker/onePollerPerContainer/docker-compose.tmpl
@@ -2,8 +2,8 @@ version: "3.5"
 services:
 {{range .Pollers}}
   {{ .PollerName }}:
-    image: harvest:latest
-    container_name: harvest_{{ .PollerName }}
+    image: {{ .Image }}
+    container_name: harvest_{{ .Version }}_{{ .PollerName }}
     restart: unless-stopped
     {{if .Port }}
     ports:


Test patch:
diff --git a/cmd/exporters/prometheus/prometheus_test.go b/cmd/exporters/prometheus/prometheus_test.go
new file mode 100644
index 00000000..192702ab
--- /dev/null
+++ b/cmd/exporters/prometheus/prometheus_test.go
@@ -0,0 +1,50 @@
+/*
+ * Copyright NetApp Inc, 2021 All rights reserved
+ */
+
+package prometheus
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestFilterMetaTags(t *testing.T) {
+
+	example := [][]byte{
+		[]byte(`# HELP some_metric help text`),
+		[]byte(`# TYPE some_metric type`),
+		[]byte(`some_metric{node="node_1"} 0.0`),
+		[]byte(`# HELP some_other_metric help text`),
+		[]byte(`# TYPE some_other_metric type`),
+		[]byte(`some_other_metric{node="node_2"} 0.0`),
+		[]byte(`# HELP some_other_metric DUPLICATE help text`),
+		[]byte(`# TYPE some_other_metric type`),
+		[]byte(`some_other_metric{node="node_3"} 0.0`),
+	}
+
+	expected := [][]byte{
+		[]byte(`# HELP some_metric help text`),
+		[]byte(`# TYPE some_metric type`),
+		[]byte(`some_metric{node="node_1"} 0.0`),
+		[]byte(`# HELP some_other_metric help text`),
+		[]byte(`# TYPE some_other_metric type`),
+		[]byte(`some_other_metric{node="node_2"} 0.0`),
+		[]byte(`some_other_metric{node="node_3"} 0.0`),
+	}
+
+	output := filterMetaTags(example)
+
+	if len(output) != len(expected) {
+		t.Fatalf("filtered data should have %d, but got %d lines", len(expected), len(output))
+	}
+
+	// output should have exact same lines in same order
+	for i := range output {
+		if !bytes.Equal(output[i], expected[i]) {
+			t.Fatalf("line:%d - output = [%s], expected = [%s]", i, string(output[i]), string(expected[i]))
+		}
+	}
+
+	t.Log("OK - output is exactly what is expected")
+}
diff --git a/cmd/tools/grafana/grafana_test.go b/cmd/tools/grafana/grafana_test.go
index 83e4e35d..6e239d8e 100644
--- a/cmd/tools/grafana/grafana_test.go
+++ b/cmd/tools/grafana/grafana_test.go
@@ -1,6 +1,7 @@
 package grafana
 
 import (
+	"regexp"
 	"testing"
 )
 
@@ -46,3 +47,48 @@ func TestHttpsAddr(t *testing.T) {
 		t.Errorf("Expected opts.addr to be %s but got %s", "https://1.1.1.1:3000", opts.addr)
 	}
 }
+
+// Since there can be many exceptions in how metrics are using in queries,
+// the best we can do is check all queries in dashboards (especially after
+// we have changed them), e.g. by running:
+// $ grep \"expr\": grafana/prometheus*
+func TestAddPrefixToMetricNames(t *testing.T) {
+
+	var (
+		examples, expected [7]string
+		prefix, result     string
+		regex              *regexp.Regexp
+		i                  int
+	)
+
+	regex = regexp.MustCompile(`([a-zA-Z_+]+){.+?}`)
+
+	prefix = "xx_"
+
+	examples = [7]string{
+		`sum(volume_read_data{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"}) by (cluster) + sum(volume_write_data{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"}) by(cluster)`,
+		`sum(topk($TopResources, volume_total_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}))`,
+		`volume_size_used_percent{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}`,
+		`avg by(iscsi_lif) (iscsi_lif_iscsi_read_ops+iscsi_lif_iscsi_write_ops+iscsi_lif_iscsi_other_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",node=~\"$Node\"})`,
+		`label_values(metadata_component_status{type="collector",poller=~"$Poller"}, name)`,
+		`label_values(poller_status, datacenter)`,
+		`label_values(datacenter)`,
+	}
+
+	expected = [7]string{
+		`sum(xx_volume_read_data{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"}) by (cluster) + sum(xx_volume_write_data{datacenter=\"$Datacenter\",cluster=~\"$Cluster\"}) by(cluster)`,
+		`sum(topk($TopResources, xx_volume_total_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}))`,
+		`xx_volume_size_used_percent{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}`,
+		`avg by(iscsi_lif) (xx_iscsi_lif_iscsi_read_ops+xx_iscsi_lif_iscsi_write_ops+xx_iscsi_lif_iscsi_other_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",node=~\"$Node\"})`,
+		`label_values(xx_metadata_component_status{type="collector",poller=~"$Poller"}, name)`,
+		`label_values(xx_poller_status, datacenter)`,
+		`label_values(datacenter)`, // no metric name
+	}
+
+	for i = range examples {
+		result = addPrefixToMetricNames(examples[i], prefix, regex)
+		if result != expected[i] {
+			t.Errorf("\nExpected: [%s]\n     Got: [%s]", expected[i], result)
+		}
+	}
+}