Repository: NetApp/harvest
Pull request #2208 (2023-07-17 10:34:06)

feat: Supporting topk in S3 dashboard
Prior the changes:

<img width="1724" alt="image" src="https://github.com/NetApp/harvest/assets/83282894/5e012460-bb80-45c0-b70a-011b159132c5">


Now with changes:

<img width="1733" alt="image" src="https://github.com/NetApp/harvest/assets/83282894/551865e5-9c5d-40a6-8779-74cbe0de64ac">



Patch:
diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go
index 4d3edf69..2919f4ea 100644
--- a/cmd/collectors/storagegrid/rest/client.go
+++ b/cmd/collectors/storagegrid/rest/client.go
@@ -128,7 +128,7 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie
 				InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
 		}
 	} else {
-		username := poller.Username
+		username := pollerAuth.Username
 		password := pollerAuth.Password
 		client.username = username
 		if username == "" {
diff --git a/cmd/tools/grafana/metrics.go b/cmd/tools/grafana/metrics.go
index 53ad7d93..eb06afae 100644
--- a/cmd/tools/grafana/metrics.go
+++ b/cmd/tools/grafana/metrics.go
@@ -101,6 +101,7 @@ type variable struct {
 	query   string
 	refresh string
 	path    string
+	options []gjson.Result
 }
 
 func allVariables(data []byte) []variable {
@@ -116,6 +117,7 @@ func allVariables(data []byte) []variable {
 			kind:    value.Get("type").String(),
 			query:   value.Get("query.query").String(),
 			refresh: value.Get("refresh").String(),
+			options: value.Get("options").Array(),
 			path:    key.String(),
 		})
 		return true
diff --git a/conf/restperf/9.12.0/wafl_hya_per_aggr.yaml b/conf/restperf/9.12.0/wafl_hya_per_aggr.yaml
index 650afe14..099539da 100644
--- a/conf/restperf/9.12.0/wafl_hya_per_aggr.yaml
+++ b/conf/restperf/9.12.0/wafl_hya_per_aggr.yaml
@@ -4,7 +4,7 @@ object:                   flashpool
 
 counters:
   - ^^id
-  - ^hya_aggr_name                                 => aggr
+  - ^hya_aggregate_name                            => aggr
   - ^node.name                                     => node
   - evict_destage_rate
   - evict_remove_rate
diff --git a/docs/install/overview.md b/docs/install/overview.md
index 49d317e4..b40d44dc 100644
--- a/docs/install/overview.md
+++ b/docs/install/overview.md
@@ -13,18 +13,13 @@ Instructions on how to install Harvest via [NAbox](https://nabox.org/documentati
 
 ## Source
 
-To build Harvest from source code, first make sure you have a working Go environment 
-with [version 1.19 or greater installed](https://golang.org/doc/install).
-
-Clone the repo and build everything.
-
-```
-git clone https://github.com/NetApp/harvest.git
-cd harvest
-make build
-bin/harvest version
-```
-
-If you're building on a Mac use `GOOS=darwin make build`
+To build Harvest from source code follow these steps.
+
+1. `git clone https://github.com/NetApp/harvest.git`
+2. cd `harvest`
+3. check the version of go required in the `go.mod` file
+4. ensure you have a working Go environment at that version or newer. Go installs found [here](https://golang.org/doc/install). 
+5. `make build` (if you want to run Harvest from a Mac use `GOOS=darwin make build`) 
+6. `bin/harvest version`
 
 Checkout the `Makefile` for other targets of interest.
diff --git a/grafana/dashboards/cmode/network.json b/grafana/dashboards/cmode/network.json
index 989e23ae..d7846b63 100644
--- a/grafana/dashboards/cmode/network.json
+++ b/grafana/dashboards/cmode/network.json
@@ -3317,8 +3317,8 @@
         "allValue": null,
         "current": {
           "selected": true,
-          "text": "20",
-          "value": "20"
+          "text": "5",
+          "value": "5"
         },
         "description": null,
         "error": null,
@@ -3349,7 +3349,7 @@
             "value": "4"
           },
           {
-            "selected": false,
+            "selected": true,
             "text": "5",
             "value": "5"
           },
@@ -3374,7 +3374,7 @@
             "value": "15"
           },
           {
-            "selected": true,
+            "selected": false,
             "text": "20",
             "value": "20"
           },
diff --git a/grafana/dashboards/cmode/node.json b/grafana/dashboards/cmode/node.json
index 8ef39919..d00c4fd6 100644
--- a/grafana/dashboards/cmode/node.json
+++ b/grafana/dashboards/cmode/node.json
@@ -4815,9 +4815,9 @@
       {
         "allValue": null,
         "current": {
-          "selected": false,
-          "text": "8",
-          "value": "8"
+          "selected": true,
+          "text": "5",
+          "value": "5"
         },
         "description": null,
         "error": null,
@@ -4848,7 +4848,7 @@
             "value": "4"
           },
           {
-            "selected": false,
+            "selected": true,
             "text": "5",
             "value": "5"
           },
@@ -4858,7 +4858,7 @@
             "value": "6"
           },
           {
-            "selected": true,
+            "selected": false,
             "text": "8",
             "value": "8"
           },
diff --git a/grafana/dashboards/cmode/qtree.json b/grafana/dashboards/cmode/qtree.json
index c347d514..f4e7dbd0 100644
--- a/grafana/dashboards/cmode/qtree.json
+++ b/grafana/dashboards/cmode/qtree.json
@@ -883,9 +883,9 @@
       {
         "allValue": null,
         "current": {
-          "selected": false,
-          "text": "3",
-          "value": "3"
+          "selected": true,
+          "text": "5",
+          "value": "5"
         },
         "description": null,
         "error": null,
@@ -906,7 +906,7 @@
             "value": "2"
           },
           {
-            "selected": true,
+            "selected": false,
             "text": "3",
             "value": "3"
           },
@@ -916,7 +916,7 @@
             "value": "4"
           },
           {
-            "selected": false,
+            "selected": true,
             "text": "5",
             "value": "5"
           },
diff --git a/grafana/dashboards/cmode/s3ObjectStorage.json b/grafana/dashboards/cmode/s3ObjectStorage.json
index c0d86d4f..e226f629 100644
--- a/grafana/dashboards/cmode/s3ObjectStorage.json
+++ b/grafana/dashboards/cmode/s3ObjectStorage.json
@@ -488,14 +488,14 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "ontaps3_logical_used_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$Bucket\"}",
+          "expr": "topk($TopResources, ontaps3_logical_used_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$TopS3LogicalUsedSize\"})",
           "hide": false,
           "interval": "",
           "legendFormat": "{{svm}} - {{bucket}}",
           "refId": "A"
         }
       ],
-      "title": "Bucket Used Size",
+      "title": "Top $TopResources Buckets by Used Size",
       "transformations": [],
       "type": "timeseries"
     },
@@ -606,14 +606,14 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "exemplar": false,
-          "expr": "ontaps3_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$Bucket\"}",
+          "expr": "topk($TopResources, ontaps3_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$TopS3UsedPercent\"})",
           "hide": false,
           "interval": "",
           "legendFormat": "{{svm}} - {{bucket}}",
           "refId": "A"
         }
       ],
-      "title": "Bucket Used Size Percent",
+      "title": "Top $TopResources Buckets by Used Size Percent",
       "transformations": [],
       "type": "timeseries"
     },
@@ -842,16 +842,8 @@
         "type": "query"
       },
       {
-        "allValue": "(.*)",
-        "current": {
-          "selected": true,
-          "text": [
-            "All"
-          ],
-          "value": [
-            "$__all"
-          ]
-        },
+        "allValue": null,
+        "current": {},
         "datasource": "${DS_PROMETHEUS}",
         "definition": "label_values(ontaps3_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}, bucket)",
         "hide": 0,
@@ -870,6 +862,143 @@
         "skipUrlSync": false,
         "sort": 1,
         "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {
+          "selected": true,
+          "text": "5",
+          "value": "5"
+        },
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "TopResources",
+        "options": [
+          {
+            "selected": false,
+            "text": "1",
+            "value": "1"
+          },
+          {
+            "selected": false,
+            "text": "2",
+            "value": "2"
+          },
+          {
+            "selected": false,
+            "text": "3",
+            "value": "3"
+          },
+          {
+            "selected": false,
+            "text": "4",
+            "value": "4"
+          },
+          {
+            "selected": true,
+            "text": "5",
+            "value": "5"
+          },
+          {
+            "selected": false,
+            "text": "6",
+            "value": "6"
+          },
+          {
+            "selected": false,
+            "text": "8",
+            "value": "8"
+          },
+          {
+            "selected": false,
+            "text": "10",
+            "value": "10"
+          },
+          {
+            "selected": false,
+            "text": "15",
+            "value": "15"
+          },
+          {
+            "selected": false,
+            "text": "25",
+            "value": "25"
+          },
+          {
+            "selected": false,
+            "text": "50",
+            "value": "50"
+          },
+          {
+            "selected": false,
+            "text": "100",
+            "value": "100"
+          },
+          {
+            "selected": false,
+            "text": "250",
+            "value": "250"
+          },
+          {
+            "selected": false,
+            "text": "500",
+            "value": "500"
+          }
+        ],
+        "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": null,
+        "definition": "query_result(topk($TopResources, avg_over_time(ontaps3_logical_used_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$Bucket\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopS3LogicalUsedSize",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(ontaps3_logical_used_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$Bucket\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": ".*bucket=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": null,
+        "definition": "query_result(topk($TopResources, avg_over_time(ontaps3_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$Bucket\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopS3UsedPercent",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(ontaps3_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",bucket=~\"$Bucket\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": ".*bucket=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
       }
     ]
   },
diff --git a/grafana/dashboards/cmode/svm.json b/grafana/dashboards/cmode/svm.json
index b6a29a35..4fbe367f 100644
--- a/grafana/dashboards/cmode/svm.json
+++ b/grafana/dashboards/cmode/svm.json
@@ -13210,8 +13210,8 @@
         "allValue": null,
         "current": {
           "selected": true,
-          "text": "3",
-          "value": "3"
+          "text": "5",
+          "value": "5"
         },
         "description": null,
         "error": null,
@@ -13232,7 +13232,7 @@
             "value": "2"
           },
           {
-            "selected": true,
+            "selected": false,
             "text": "3",
             "value": "3"
           },
@@ -13242,7 +13242,7 @@
             "value": "4"
           },
           {
-            "selected": false,
+            "selected": true,
             "text": "5",
             "value": "5"
           },
diff --git a/grafana/dashboards/storagegrid/overview.json b/grafana/dashboards/storagegrid/overview.json
index 351584eb..fd84cb4f 100644
--- a/grafana/dashboards/storagegrid/overview.json
+++ b/grafana/dashboards/storagegrid/overview.json
@@ -2241,9 +2241,9 @@
       {
         "allValue": null,
         "current": {
-          "selected": false,
-          "text": "3",
-          "value": "3"
+          "selected": true,
+          "text": "5",
+          "value": "5"
         },
         "description": null,
         "error": null,
@@ -2264,7 +2264,7 @@
             "value": "2"
           },
           {
-            "selected": true,
+            "selected": false,
             "text": "3",
             "value": "3"
           },
@@ -2274,7 +2274,7 @@
             "value": "4"
           },
           {
-            "selected": false,
+            "selected": true,
             "text": "5",
             "value": "5"
           },
diff --git a/pkg/api/ontapi/zapi/client.go b/pkg/api/ontapi/zapi/client.go
index 369d750a..28d269d1 100644
--- a/pkg/api/ontapi/zapi/client.go
+++ b/pkg/api/ontapi/zapi/client.go
@@ -147,13 +147,13 @@ func New(poller *conf.Poller, c *auth.Credentials) (*Client, error) {
 		}
 	} else {
 		password := pollerAuth.Password
-		if poller.Username == "" {
+		if pollerAuth.Username == "" {
 			return nil, errs.New(errs.ErrMissingParam, "username")
 		} else if password == "" {
 			return nil, errs.New(errs.ErrMissingParam, "password")
 		}
 
-		request.SetBasicAuth(poller.Username, password)
+		request.SetBasicAuth(pollerAuth.Username, password)
 		transport = &http.Transport{
 			Proxy:           http.ProxyFromEnvironment,
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go
index c5e4bc5e..020f29e5 100644
--- a/pkg/auth/auth.go
+++ b/pkg/auth/auth.go
@@ -3,6 +3,7 @@ package auth
 import (
 	"bytes"
 	"context"
+	"dario.cat/mergo"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/logging"
 	"os/exec"
@@ -162,7 +163,7 @@ func (c *Credentials) GetPollerAuth() (PollerAuth, error) {
 	if auth.IsCert {
 		return auth, nil
 	}
-	if auth.Password != "" {
+	if auth.Username != "" && auth.Password != "" {
 		return auth, nil
 	}
 
@@ -186,7 +187,8 @@ func (c *Credentials) GetPollerAuth() (PollerAuth, error) {
 	if auth.Username != "" {
 		defaultAuth.Username = auth.Username
 	}
-	return defaultAuth, nil
+	_ = mergo.Merge(&auth, defaultAuth)
+	return auth, nil
 }
 
 func getPollerAuth(c *Credentials, poller *conf.Poller) (PollerAuth, error) {
diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go
index 2121e842..0f6d4d5f 100644
--- a/pkg/conf/conf.go
+++ b/pkg/conf/conf.go
@@ -136,19 +136,25 @@ func ReadCredentialFile(credPath string, p *Poller) error {
 			return errs.New(errs.ErrInvalidParam, "poller not found in credentials file")
 		}
 	}
-	if credPoller.SslKey != "" {
+
+	// Merge the poller and defaults from the credential file
+	if credConfig.Defaults != nil {
+		_ = mergo.Merge(credPoller, credConfig.Defaults)
+	}
+
+	if p.SslKey == "" {
 		p.SslKey = credPoller.SslKey
 	}
-	if credPoller.SslCert != "" {
+	if p.SslCert == "" {
 		p.SslCert = credPoller.SslCert
 	}
-	if credPoller.CaCertPath != "" {
+	if p.CaCertPath == "" {
 		p.CaCertPath = credPoller.CaCertPath
 	}
-	if credPoller.Username != "" {
+	if p.Username == "" {
 		p.Username = credPoller.Username
 	}
-	if credPoller.Password != "" {
+	if p.Password == "" {
 		p.Password = credPoller.Password
 	}
 	return nil


Test patch:
diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go
index b48c2511..82230b9b 100644
--- a/cmd/tools/grafana/dashboard_test.go
+++ b/cmd/tools/grafana/dashboard_test.go
@@ -760,6 +760,25 @@ func checkTopKRange(t *testing.T, path string, data []byte) {
 	}
 
 	for _, v := range variables {
+		if v.name == "TopResources" {
+			for _, optionVal := range v.options {
+				selected := optionVal.Get("selected").Bool()
+				text := optionVal.Get("text").String()
+				value := optionVal.Get("value").String()
+
+				// Test if text and value match, except for the special case with "All" and "$__all"
+				if text != value && !(text == "All" && value == "$__all") {
+					t.Errorf("In dashboard %s, variable %s uses topk, but text '%s' does not match value '%s'",
+						shortPath(path), v.name, text, value)
+				}
+
+				// Test if the selected value matches the expected text "5"
+				if (text == "5") != selected {
+					t.Errorf("In dashboard %s, variable %s uses topk, but text '%s' has incorrect selected state: %t",
+						shortPath(path), v.name, text, selected)
+				}
+			}
+		}
 		if !strings.Contains(v.query, "topk") || !strings.Contains(v.query, "__range") {
 			continue
 		}
diff --git a/pkg/auth/auth_test.go b/pkg/auth/auth_test.go
index a304b061..ff88bc09 100644
--- a/pkg/auth/auth_test.go
+++ b/pkg/auth/auth_test.go
@@ -51,6 +51,50 @@ Pollers:
 		credentials_file: testdata/secrets.yaml`,
 		},
 
+		{
+			name:           "poller username from credentials_file defaults",
+			pollerName:     "test",
+			want:           PollerAuth{Username: "default-user", Password: "from-secrets-file"},
+			defaultDefined: true,
+			yaml: `
+Defaults:
+	auth_style: certificate_auth
+	credentials_file: secrets/openlab
+Pollers:
+	test:
+		addr: a.b.c
+		credentials_file: testdata/secrets.yaml`,
+		},
+
+		{
+			name:           "poller username from credentials_file, password from poller",
+			pollerName:     "test",
+			want:           PollerAuth{Username: "default-user", Password: "moon"},
+			defaultDefined: true,
+			yaml: `
+Defaults:
+	credentials_file: testdata/secrets.yaml
+Pollers:
+	test:
+		addr: a.b.c
+		password: moon`,
+		},
+
+		{
+			name:           "poller username from credentials_file",
+			pollerName:     "test2",
+			want:           PollerAuth{Username: "test2-user", Password: "from-secrets-file"},
+			defaultDefined: true,
+			yaml: `
+Defaults:
+	auth_style: certificate_auth
+	credentials_file: secrets/openlab
+Pollers:
+	test2:
+		addr: a.b.c
+		credentials_file: testdata/secrets.yaml`,
+		},
+
 		{
 			name:           "default cert_auth",
 			pollerName:     "test",
@@ -252,6 +296,20 @@ Pollers:
 		username: flo
 		credentials_script:
 			path: testdata/get_pass
+`,
+		},
+
+		{
+			name:         "password with space",
+			pollerName:   "test",
+			want:         PollerAuth{Username: "flo", Password: "abc def"},
+			wantSchedule: "42m",
+			yaml: `
+Pollers:
+	test:
+		addr: a.b.c
+		username: flo
+		password: abc def
 `,
 		},
 	}
@@ -284,9 +342,6 @@ Pollers:
 			if tt.want.Password != got.Password {
 				t.Errorf("got password=[%s], want password=[%s]", got.Password, tt.want.Password)
 			}
-			if tt.want.Username != poller.Username {
-				t.Errorf("poller got username=[%s], want username=[%s]", poller.Username, tt.want.Username)
-			}
 			if tt.want.IsCert != got.IsCert {
 				t.Errorf("got IsCert=[%t], want IsCert=[%t]", got.IsCert, tt.want.IsCert)
 			}
diff --git a/pkg/auth/testdata/secrets.yaml b/pkg/auth/testdata/secrets.yaml
index a827c363..f6966529 100644
--- a/pkg/auth/testdata/secrets.yaml
+++ b/pkg/auth/testdata/secrets.yaml
@@ -6,4 +6,7 @@ Defaults:
 
 Pollers:
   test:
+    password: from-secrets-file
+  test2:
+    username: test2-user
     password: from-secrets-file
\ No newline at end of file