Repository: NetApp/harvest
Pull request #537 (2021-10-01 12:43:33)

feat: Harvest should monitor ONTAP sensors

cgrinds: ```
sensor_threshold_status{datacenter="DC_002",cluster="umeng-aff300-05-06",discrete_state="normal",discrete_value="OK",node="umeng-aff300-05",sensor="PSU1 Fan2 Fault",type="discrete",threshold_state="normal"} 1
sensor_discrete_status{datacenter="DC_002",cluster="umeng-aff300-05-06",discrete_state="normal",discrete_value="OK",node="umeng-aff300-05",sensor="PSU1 Fan2 Fault",type="discrete",threshold_state="normal"} 1
sensor_threshold_status{datacenter="DC_002",cluster="umeng-aff300-05-06",discrete_value="OK",node="umeng-aff300-05",sensor="PSU1 Over Volt",type="discrete",threshold_state="normal",discrete_state="normal"} 1
sensor_discrete_status{datacenter="DC_002",cluster="umeng-aff300-05-06",discrete_value="OK",node="umeng-aff300-05",sensor="PSU1 Over Volt",type="discrete",threshold_state="normal",discrete_state="normal"} 1
sensor_threshold_status{datacenter="DC_002",cluster="umeng-aff300-05-06",node="umeng-aff300-06",sensor="Ambient Temp 1",type="thermal",threshold_state="normal",unit="C",warning_high="47",warning_low="5"} 1
sensor_discrete_status{datacenter="DC_002",cluster="umeng-aff300-05-06",node="umeng-aff300-06",sensor="Ambient Temp 1",type="thermal",threshold_state="normal",unit="C",warning_high="47",warning_low="5"} 0
```

Patch:
diff --git a/README.md b/README.md
index 7d5cfcf4..257ae06a 100644
--- a/README.md
+++ b/README.md
@@ -211,6 +211,7 @@ All pollers are defined in `harvest.yml`, the main configuration file of Harvest
 | `username`, `password` | required if `auth_style` is `basic_auth` |  |              |
 | `ssl_cert`, `ssl_key`  | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.<br /><br />If not provided, the poller will look for `<hostname>.key` and `<hostname>.pem` in `$HARVEST_HOME/cert/`.<br/><br/>To create certificates for ONTAP systems, see [using certificate authentication](docs/AuthAndPermissions.md#using-certificate-authentication)                        |              |
 | `use_insecure_tls`     | optional, bool |  If true, disable TLS verification when connecting to ONTAP cluster  | false         |
+| `labels`               | optional, list of key-value pairs              | each of the key-value pairs will be added to a poller's metrics. Details [below](#labels)                                                                                                                                                                                                                                                                                 |                    |
 | `log_max_bytes`        |  | Maximum size of the log file before it will be rotated | `10000000` (10 mb) |
 | `log_max_files`        |  | Number of rotated log files to keep | `10` |
 | `log`                  | optional, list of collector names              | matching collectors log their ZAPI request/response                                                                                                                                                                                                                                                                                                                         |                    |
@@ -276,3 +277,23 @@ For a list of collector-specific parameters, see the documentation of each colle
 
 ### [Unix](cmd/collectors/unix/README.md)
 
+### Labels
+
+Labels offer a way to add additional key-value pairs to a poller's metrics. These allow you to tag a cluster's metrics in a cross-cutting fashion. Here's an example:
+
+```
+  cluster-03:
+    datacenter: DC-01
+    addr: 10.0.1.1
+    labels:
+      - org: meg       # add an org label with the value "meg"
+      - site: rtp      # add a site label with the value "rtp"
+```
+
+These settings add two key-value pairs to each metric collected from `cluster-03` like this:
+
+```
+node_vol_cifs_write_data{org="meg",site="rtp",datacenter="DC-01",cluster="cluster-03",node="umeng-aff300-05"} 10
+```
+
+Keep in mind that each unique combination of key-value pairs increases the amount of stored data. Use them sparingly. See [PrometheusNaming](https://prometheus.io/docs/practices/naming/#labels) for details.
diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go
index 9152aa52..c63149a8 100644
--- a/cmd/collectors/zapi/collector/zapi.go
+++ b/cmd/collectors/zapi/collector/zapi.go
@@ -198,6 +198,11 @@ func (me *Zapi) InitMatrix() error {
 
 	// Add system (cluster) name
 	me.Matrix.SetGlobalLabel("cluster", me.Client.Name())
+	if me.Params.HasChildS("labels") {
+		for _, l := range me.Params.GetChildS("labels").GetChildren() {
+			me.Matrix.SetGlobalLabel(l.GetNameS(), l.GetContentS())
+		}
+	}
 	// For 7mode cluster is same as node
 	if !me.Client.IsClustered() {
 		me.Matrix.SetGlobalLabel("node", me.Client.Name())
diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go
index c14ccda6..317eed6d 100644
--- a/cmd/poller/poller.go
+++ b/cmd/poller/poller.go
@@ -708,7 +708,13 @@ func Union2(hNode *node.Node, poller *conf.Poller) {
 					case "!!seq":
 						// the poller node that's missing is a sequence so add all the children of the sequence
 						for _, seqNode := range valNode.Content {
-							newNode.NewChildS(seqNode.Value, seqNode.Value)
+							if seqNode.Tag == "!!str" {
+								newNode.NewChildS(seqNode.Value, seqNode.Value)
+							} else if seqNode.Tag == "!!map" {
+								for ci := 0; ci < len(seqNode.Content); ci += 2 {
+									newNode.NewChildS(seqNode.Content[ci].Value, seqNode.Content[ci+1].Value)
+								}
+							}
 						}
 					}
 					hNode.AddChild(newNode)
diff --git a/conf/zapi/cdot/9.8.0/sensor.yaml b/conf/zapi/cdot/9.8.0/sensor.yaml
new file mode 100644
index 00000000..77fd5c56
--- /dev/null
+++ b/conf/zapi/cdot/9.8.0/sensor.yaml
@@ -0,0 +1,28 @@
+name:             Sensor
+query:            environment-sensors-get-iter
+object:           environment_sensor
+
+metric_type:      int64
+
+counters:
+  environment-sensors-info:
+    - critical-high-threshold           => critical_high
+    - critical-low-threshold            => critical_low
+    - ^discrete-sensor-state            => discrete_state
+    - ^discrete-sensor-value            => discrete_value
+    - ^^node-name                       => node
+    - ^^sensor-name                     => sensor
+    - ^sensor-type                      => type
+    - ^threshold-sensor-state           => threshold_state
+    - threshold-sensor-value            => threshold_value
+    - ^value-units                      => unit
+    - ^warning-high-threshold           => warning_high
+    - ^warning-low-threshold            => warning_low
+
+plugins:
+  - LabelAgent:
+    value_to_num: # metric label zapi_value rest_value `default_value`
+      - status threshold_state normal normal `0`
+
+export_options:
+  include_all_labels: true
\ No newline at end of file
diff --git a/conf/zapi/default.yaml b/conf/zapi/default.yaml
index 1b2d76b1..a5985257 100644
--- a/conf/zapi/default.yaml
+++ b/conf/zapi/default.yaml
@@ -17,3 +17,4 @@ objects:
   Subsystem:        subsystem.yaml
   Lun:              lun.yaml
   Status_7mode:     status_7.yaml
+  Sensor:           sensor.yaml
diff --git a/harvest.cue b/harvest.cue
index ca65a912..389b5f9b 100644
--- a/harvest.cue
+++ b/harvest.cue
@@ -2,6 +2,8 @@ package harvest
 
 Exporters: [Name=_]: #Prom | #Influx
 
+label: [string]: string
+
 #Prom: {
 	local_http_addr?: "0.0.0.0" | "localhost" | "127.0.0.1"
 	addr?: string // deprecated
@@ -45,4 +47,5 @@ Pollers: [Name=_]: #Poller
 	collectors?: [...#CollectorDef] | [...string]
 	exporters: [...string]
 	log: [...string]
+	labels?: [...label]
 }
\ No newline at end of file
diff --git a/integration/go.mod b/integration/go.mod
index 7683082a..01cba7bd 100644
--- a/integration/go.mod
+++ b/integration/go.mod
@@ -5,7 +5,7 @@ go 1.15
 replace goharvest2 => ../
 
 require (
-	github.com/containerd/containerd v1.5.5 // indirect
+	github.com/containerd/containerd v1.5.7 // indirect
 	github.com/docker/docker v20.10.8+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go
index dffc170a..3070a9f7 100644
--- a/pkg/conf/conf.go
+++ b/pkg/conf/conf.go
@@ -60,8 +60,8 @@ var Config = HarvestConfig{}
 var configRead = false
 var ValidatePortInUse = false
 
-// Used by testing code to reload a new config
-func loadHarvestConfig(configPath string) error {
+// TestLoadHarvestConfig is used by testing code to reload a new config
+func TestLoadHarvestConfig(configPath string) error {
 	configRead = false
 	return LoadHarvestConfig(configPath)
 }
@@ -416,22 +416,23 @@ type Collector struct {
 }
 
 type Poller struct {
-	Datacenter     *string      `yaml:"datacenter,omitempty"`
-	Addr           *string      `yaml:"addr,omitempty"`
-	AuthStyle      *string      `yaml:"auth_style,omitempty"`
-	Username       *string      `yaml:"username,omitempty"`
-	Password       string       `yaml:"password,omitempty"`
-	UseInsecureTls *bool        `yaml:"use_insecure_tls,omitempty"`
-	SslCert        *string      `yaml:"ssl_cert,omitempty"`
-	SslKey         *string      `yaml:"ssl_key,omitempty"`
-	LogMaxBytes    *int64       `yaml:"log_max_bytes,omitempty"`
-	LogMaxFiles    *int         `yaml:"log_max_files,omitempty"`
-	Exporters      *[]string    `yaml:"exporters,omitempty"`
-	Collectors     *[]Collector `yaml:"collectors,omitempty"`
-	IsKfs          *bool        `yaml:"is_kfs,omitempty"`
-	PollerSchedule *string      `yaml:"poller_schedule,omitempty"`
-	ClientTimeout  *string      `yaml:"client_timeout,omitempty"`
-	LogSet         *[]string    `yaml:"log,omitempty"`
+	Datacenter     *string               `yaml:"datacenter,omitempty"`
+	Addr           *string               `yaml:"addr,omitempty"`
+	AuthStyle      *string               `yaml:"auth_style,omitempty"`
+	Username       *string               `yaml:"username,omitempty"`
+	Password       string                `yaml:"password,omitempty"`
+	UseInsecureTls *bool                 `yaml:"use_insecure_tls,omitempty"`
+	SslCert        *string               `yaml:"ssl_cert,omitempty"`
+	SslKey         *string               `yaml:"ssl_key,omitempty"`
+	LogMaxBytes    *int64                `yaml:"log_max_bytes,omitempty"`
+	LogMaxFiles    *int                  `yaml:"log_max_files,omitempty"`
+	Exporters      *[]string             `yaml:"exporters,omitempty"`
+	Collectors     *[]Collector          `yaml:"collectors,omitempty"`
+	IsKfs          *bool                 `yaml:"is_kfs,omitempty"`
+	PollerSchedule *string               `yaml:"poller_schedule,omitempty"`
+	ClientTimeout  *string               `yaml:"client_timeout,omitempty"`
+	LogSet         *[]string             `yaml:"log,omitempty"`
+	Labels         *[]*map[string]string `yaml:"labels,omitempty"`
 }
 
 func (p *Poller) Union(defaults *Poller) {


Test patch:
diff --git a/cmd/poller/poller_test.go b/cmd/poller/poller_test.go
index 7de0c30a..edb1ebb9 100644
--- a/cmd/poller/poller_test.go
+++ b/cmd/poller/poller_test.go
@@ -1,6 +1,10 @@
 package main
 
-import "testing"
+import (
+	"goharvest2/pkg/conf"
+	"goharvest2/pkg/tree/node"
+	"testing"
+)
 
 func TestPingParsing(t *testing.T) {
 	poller := Poller{}
@@ -52,3 +56,36 @@ round-trip min/avg/max = 0.088/0.088/0.088 ms`,
 		})
 	}
 }
+
+func TestUnion2(t *testing.T) {
+	configPath := "../../cmd/tools/doctor/testdata/testConfig.yml"
+	n := node.NewS("foople")
+	err := conf.TestLoadHarvestConfig(configPath)
+	if err != nil {
+		panic(err)
+	}
+	p, err := conf.GetPoller2(configPath, "infinity2")
+	Union2(n, p)
+	labels := n.GetChildS("labels")
+	if labels == nil {
+		t.Errorf("got nil, want labels")
+	}
+	type label struct {
+		key string
+		val string
+	}
+	wants := []label{
+		{key: "org", val: "abc"},
+		{key: "site", val: "RTP"},
+		{key: "floor", val: "3"},
+	}
+	for i, c := range labels.Children {
+		want := wants[i]
+		if want.key != c.GetNameS() {
+			t.Errorf("got key=%s, want=%s", c.GetNameS(), want.key)
+		}
+		if want.val != c.GetContentS() {
+			t.Errorf("got key=%s, want=%s", c.GetContentS(), want.val)
+		}
+	}
+}
diff --git a/cmd/tools/doctor/testdata/testConfig.yml b/cmd/tools/doctor/testdata/testConfig.yml
index 5beb35b6..54af274f 100644
--- a/cmd/tools/doctor/testdata/testConfig.yml
+++ b/cmd/tools/doctor/testdata/testConfig.yml
@@ -112,6 +112,10 @@ Pollers:
     datacenter: DC-01
     addr: 10.0.0.2
     collectors: [ Zapi ]
+    labels:
+      - org: abc
+      - site: RTP
+      - floor: 3
 
   unix-01:
     datacenter: local
diff --git a/pkg/conf/conf_test.go b/pkg/conf/conf_test.go
index 2d12d4ce..e4ab692b 100644
--- a/pkg/conf/conf_test.go
+++ b/pkg/conf/conf_test.go
@@ -52,7 +52,7 @@ func TestGetPrometheusExporterPortsIssue284(t *testing.T) {
 }
 
 func loadTestData(yml string) {
-	err := loadHarvestConfig(yml)
+	err := TestLoadHarvestConfig(yml)
 	if err != nil {
 		panic(err)
 	}
@@ -225,7 +225,7 @@ func TestCollectorConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := loadHarvestConfig(tt.path)
+			err := TestLoadHarvestConfig(tt.path)
 			if err != nil {
 				panic(err)
 			}