Repository: NetApp/harvest
Pull request #478 (2021-08-30 11:51:00)

feat: add asup flag to jenkins


Patch:
diff --git a/cmd/collectors/zapi/plugins/shelf/shelf.go b/cmd/collectors/zapi/plugins/shelf/shelf.go
index b91c7cae..acd2fc16 100644
--- a/cmd/collectors/zapi/plugins/shelf/shelf.go
+++ b/cmd/collectors/zapi/plugins/shelf/shelf.go
@@ -73,7 +73,7 @@ func (my *Shelf) Init() error {
 
 		my.instanceLabels[attribute] = dict.New()
 
-		my.data[attribute] = matrix.New(my.Parent+".Shelf", "shelf_"+objectName)
+		my.data[attribute] = matrix.New(my.Parent+".Shelf", "shelf_"+objectName, "shelf_"+objectName)
 		my.data[attribute].SetGlobalLabel("datacenter", my.ParentParams.GetChildContentS("datacenter"))
 		my.data[attribute].SetGlobalLabel("cluster", my.client.Name())
 
diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go
index 877bef77..220e6d4f 100644
--- a/cmd/exporters/prometheus/prometheus.go
+++ b/cmd/exporters/prometheus/prometheus.go
@@ -229,7 +229,7 @@ func (me *Prometheus) Export(data *matrix.Matrix) error {
 	}
 
 	// store metrics in cache
-	key := data.UUID + "." + data.Object
+	key := data.UUID + "." + data.Object + "." + data.Identifier
 
 	// lock cache, to prevent HTTPd reading while we are mutating it
 	me.cache.Lock()
diff --git a/cmd/poller/collector/collector.go b/cmd/poller/collector/collector.go
index 1fdcc1d9..33e88c2d 100644
--- a/cmd/poller/collector/collector.go
+++ b/cmd/poller/collector/collector.go
@@ -170,7 +170,7 @@ func Init(c Collector) error {
 	c.SetSchedule(s)
 
 	// Initialize Matrix, the container of collected data
-	mx := matrix.New(name, object)
+	mx := matrix.New(name, object, object)
 	if exportOptions := params.GetChildS("export_options"); exportOptions != nil {
 		mx.SetExportOptions(exportOptions)
 	} else {
@@ -201,7 +201,7 @@ func Init(c Collector) error {
 	}
 
 	// Initialize metadata
-	md := matrix.New(name, "metadata_collector")
+	md := matrix.New(name, "metadata_collector", "metadata_collector")
 
 	md.SetGlobalLabel("hostname", opts.Hostname)
 	md.SetGlobalLabel("version", opts.Version)
diff --git a/cmd/poller/exporter/exporter.go b/cmd/poller/exporter/exporter.go
index 83906cdd..0a1f1d1f 100644
--- a/cmd/poller/exporter/exporter.go
+++ b/cmd/poller/exporter/exporter.go
@@ -73,7 +73,7 @@ func New(c, n string, o *options.Options, p conf.Exporter) *AbstractExporter {
 
 // InitAbc() initializes AbstractExporter
 func (me *AbstractExporter) InitAbc() error {
-	me.Metadata = matrix.New(me.Name, "metadata_exporter")
+	me.Metadata = matrix.New(me.Name, "metadata_exporter", "metadata_exporter")
 	me.Metadata.SetGlobalLabel("hostname", me.Options.Hostname)
 	me.Metadata.SetGlobalLabel("version", me.Options.Version)
 	me.Metadata.SetGlobalLabel("poller", me.Options.Poller)
diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go
index 2161b2fd..4057d2e1 100644
--- a/cmd/poller/poller.go
+++ b/cmd/poller/poller.go
@@ -795,7 +795,7 @@ func (p *Poller) getExporter(name string) exporter.Exporter {
 // initialize matrices to be used as metadata
 func (p *Poller) loadMetadata() {
 
-	p.metadata = matrix.New("poller", "metadata_component")
+	p.metadata = matrix.New("poller", "metadata_component", "metadata_component")
 	p.metadata.NewMetricUint8("status")
 	p.metadata.NewMetricUint64("count")
 	p.metadata.SetGlobalLabel("poller", p.name)
@@ -804,7 +804,7 @@ func (p *Poller) loadMetadata() {
 	p.metadata.SetExportOptions(matrix.DefaultExportOptions())
 
 	// metadata for target system
-	p.status = matrix.New("poller", "metadata_target")
+	p.status = matrix.New("poller", "metadata_target", "metadata_component")
 	p.status.NewMetricUint8("status")
 	p.status.NewMetricFloat32("ping")
 	p.status.NewMetricUint32("goroutines")
diff --git a/grafana/dashboards/harvest_dashboard_volume_details.json b/grafana/dashboards/harvest_dashboard_volume_details.json
index 88769583..4c5282aa 100644
--- a/grafana/dashboards/harvest_dashboard_volume_details.json
+++ b/grafana/dashboards/harvest_dashboard_volume_details.json
@@ -4048,200 +4048,6 @@
       "title": "Per Volume Statistics",
       "type": "row"
     },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 24,
-        "w": 8,
-        "x": 0,
-        "y": 19
-      },
-      "hiddenSeries": false,
-      "id": 82,
-      "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": true,
-        "max": true,
-        "min": false,
-        "show": true,
-        "sort": "avg",
-        "sortDesc": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "7.5.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [
-        {
-          "$$hashKey": "object:327",
-          "alias": "/ - SPACE USED/",
-          "dashes": true,
-          "yaxis": 2
-        }
-      ],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": false,
-          "expr": "volume_total_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}",
-          "interval": "",
-          "legendFormat": "{{svm}} - {{volume}} - TOTAL IOPS",
-          "refId": "A"
-        },
-        {
-          "exemplar": false,
-          "expr": "volume_space_physical_used{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "{{svm}} - {{volume}} - SPACE USED",
-          "refId": "B"
-        },
-        {
-          "exemplar": false,
-          "expr": "volume_read_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "{{svm}} - {{volume}} - READ IOPS",
-          "refId": "C"
-        },
-        {
-          "exemplar": false,
-          "expr": "volume_write_ops{datacenter=\"$Datacenter\",cluster=\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "{{svm}} - {{volume}} - WRITE IOPS",
-          "refId": "D"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Per Volume IO Density",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "transformations": [
-        {
-          "id": "calculateField",
-          "options": {
-            "alias": "Total",
-            "binary": {
-              "left": "volume_total_ops{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", svm=\"cpoc_vmware\", type=\"flexvol\", volume=\"cpoc_vmware_poc\"}",
-              "operator": "/",
-              "reducer": "sum",
-              "right": "{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", style=\"flexvol\", svm=\"cpoc_vmware\", volume=\"cpoc_vmware_poc\"}"
-            },
-            "mode": "binary",
-            "reduce": {
-              "reducer": "sum"
-            },
-            "replaceFields": false
-          }
-        },
-        {
-          "id": "calculateField",
-          "options": {
-            "alias": "-Read-",
-            "binary": {
-              "left": "volume_read_ops{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", svm=\"cpoc_vmware\", type=\"flexvol\", volume=\"cpoc_vmware_poc\"}",
-              "operator": "/",
-              "reducer": "sum",
-              "right": "{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", style=\"flexvol\", svm=\"cpoc_vmware\", volume=\"cpoc_vmware_poc\"}"
-            },
-            "mode": "binary",
-            "reduce": {
-              "reducer": "sum"
-            }
-          }
-        },
-        {
-          "id": "calculateField",
-          "options": {
-            "alias": "-Write-",
-            "binary": {
-              "left": "volume_write_ops{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", svm=\"cpoc_vmware\", type=\"flexvol\", volume=\"cpoc_vmware_poc\"}",
-              "operator": "/",
-              "reducer": "sum",
-              "right": "{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", style=\"flexvol\", svm=\"cpoc_vmware\", volume=\"cpoc_vmware_poc\"}"
-            },
-            "mode": "binary",
-            "reduce": {
-              "reducer": "sum"
-            }
-          }
-        },
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "volume_read_ops{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", svm=\"cpoc_vmware\", type=\"flexvol\", volume=\"cpoc_vmware_poc\"}": true,
-              "volume_total_ops{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", svm=\"cpoc_vmware\", type=\"flexvol\", volume=\"cpoc_vmware_poc\"}": true,
-              "volume_write_ops{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", svm=\"cpoc_vmware\", type=\"flexvol\", volume=\"cpoc_vmware_poc\"}": true,
-              "{aggr=\"n1_data\", cluster=\"bteam\", datacenter=\"CPOC\", instance=\"10.61.101.69:12994\", job=\"harvest\", node=\"n1\", style=\"flexvol\", svm=\"cpoc_vmware\", volume=\"cpoc_vmware_poc\"}": true
-            },
-            "indexByName": {},
-            "renameByName": {}
-          }
-        }
-      ],
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:123",
-          "format": "iops",
-          "label": "IOPs/TB",
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:124",
-          "format": "decbytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
     {
       "aliasColors": {},
       "bars": false,
@@ -4257,8 +4063,8 @@
       "fillGradient": 1,
       "gridPos": {
         "h": 12,
-        "w": 8,
-        "x": 8,
+        "w": 12,
+        "x": 0,
         "y": 19
       },
       "hiddenSeries": false,
@@ -4392,8 +4198,8 @@
       "fillGradient": 0,
       "gridPos": {
         "h": 12,
-        "w": 8,
-        "x": 16,
+        "w": 12,
+        "x": 12,
         "y": 19
       },
       "hiddenSeries": false,
@@ -4510,8 +4316,8 @@
       "fillGradient": 0,
       "gridPos": {
         "h": 12,
-        "w": 8,
-        "x": 8,
+        "w": 12,
+        "x": 0,
         "y": 31
       },
       "hiddenSeries": false,
@@ -4636,8 +4442,8 @@
       "fillGradient": 0,
       "gridPos": {
         "h": 12,
-        "w": 8,
-        "x": 16,
+        "w": 12,
+        "x": 12,
         "y": 31
       },
       "hiddenSeries": false,
diff --git a/jenkins/artifacts/jenkinsfile b/jenkins/artifacts/jenkinsfile
index 272e458b..7f1ef310 100644
--- a/jenkins/artifacts/jenkinsfile
+++ b/jenkins/artifacts/jenkinsfile
@@ -9,8 +9,9 @@ pipeline {
         string(name: 'VERSION', defaultValue: '', description: '[Optional] RPM Version should match Harvest version with rc or nightly stripped. If not filled default is YY.mm.ddHH. See https://github.com/NetApp/harvest/wiki/Release-Checklist#version-names for details')
         string(name: 'RELEASE', defaultValue: 'nightly', description: '[Optional] Example: nightly (default) See https://github.com/NetApp/harvest-private/wiki/Release-Checklist#rpm-and-debian-names for details ')
         string(name: 'BRANCH', defaultValue: 'main', description: '[Optional] Branch name to clone. Default (main) ')
-		string(name: 'IS_PRODUCTION', defaultValue: 'false', description: 'true to publish to dockerhub and "production" AutoSupport binary. Default (no dockerhub publish "build" AutoSupport binary)')
-		string(name: 'OVERWRITE_DOCKER_LATEST_TAG', defaultValue: 'false', description: 'true to overwrite latest tag at dockerhub. Works only if IS_PRODUCTION is true')
+		string(name: 'ASUP_MAKE_TARGET', defaultValue: 'build', description: 'make target for AutoSupport binary build|production')
+		string(name: 'DOCKER_PUBLISH', defaultValue: 'false', description: 'true to publish to dockerhub')
+		string(name: 'OVERWRITE_DOCKER_LATEST_TAG', defaultValue: 'false', description: 'true to overwrite latest tag at dockerhub. Works only if DOCKER_PUBLISH is true')
     }
 
     environment {
@@ -24,13 +25,15 @@ pipeline {
         BRANCH =sh (returnStdout: true, script: """
         echo \"${params.BRANCH}\"
         """).trim()
-		IS_PRODUCTION =sh (returnStdout: true, script: """
-        echo \"${params.IS_PRODUCTION}\"
+		ASUP_MAKE_TARGET =sh (returnStdout: true, script: """
+        echo \"${params.ASUP_MAKE_TARGET}\"
         """).trim()
-		OVERWRITE_DOCKER_LATEST_TAG =sh (returnStdout: true, script: """
+		DOCKER_PUBLISH =sh (returnStdout: true, script: """
+        echo \"${params.DOCKER_PUBLISH}\"
+        """).trim()
+        OVERWRITE_DOCKER_LATEST_TAG =sh (returnStdout: true, script: """
         echo \"${params.OVERWRITE_DOCKER_LATEST_TAG}\"
         """).trim()
-        ASUP_MAKE_TARGET = "${params.IS_PRODUCTION == "true" ? "production" : "build"}"
         targetParentLocation = "/opt/home/nightly/"
         harvestPath = "harvest"
     }
@@ -110,7 +113,7 @@ pipeline {
        stage('Build Docker Image and Publish') {
                 when {
                   expression {
-                    return env.IS_PRODUCTION == 'true';
+                    return env.DOCKER_PUBLISH == 'true';
                   }
                 }
                 steps {
@@ -139,5 +142,4 @@ pipeline {
             }
         }
     }
-
 }
diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go
index c070dcec..9c8d8693 100644
--- a/pkg/matrix/matrix.go
+++ b/pkg/matrix/matrix.go
@@ -20,6 +20,7 @@ import (
 type Matrix struct {
 	UUID          string
 	Object        string
+	Identifier    string
 	globalLabels  *dict.Dict
 	instances     map[string]*Instance
 	metrics       map[string]Metric
@@ -27,8 +28,8 @@ type Matrix struct {
 	exportable    bool
 }
 
-func New(uuid, object string) *Matrix {
-	me := Matrix{UUID: uuid, Object: object}
+func New(uuid, object string, identifier string) *Matrix {
+	me := Matrix{UUID: uuid, Object: object, Identifier: identifier}
 	me.globalLabels = dict.New()
 	me.instances = make(map[string]*Instance, 0)
 	me.metrics = make(map[string]Metric, 0)
@@ -62,7 +63,7 @@ func (me *Matrix) SetExportable(b bool) {
 }
 
 func (me *Matrix) Clone(with_data, with_metrics, with_instances bool) *Matrix {
-	clone := New(me.UUID, me.Object)
+	clone := New(me.UUID, me.Object, me.Identifier)
 	clone.globalLabels = me.globalLabels
 	clone.exportOptions = me.exportOptions
 	clone.exportable = me.exportable


Test patch:
diff --git a/cmd/exporters/influxdb/influxdb_test.go b/cmd/exporters/influxdb/influxdb_test.go
index 41c12000..eef2091e 100644
--- a/cmd/exporters/influxdb/influxdb_test.go
+++ b/cmd/exporters/influxdb/influxdb_test.go
@@ -83,7 +83,7 @@ func TestExportDebug(t *testing.T) {
 	influx := setupInfluxDB(exporterName, t)
 
 	// matrix with fake data
-	data := matrix.New("test_exporter", "influxd_test_data")
+	data := matrix.New("test_exporter", "influxd_test_data", "influxd_test_data")
 	data.SetExportOptions(matrix.DefaultExportOptions())
 
 	// add metric
diff --git a/cmd/poller/plugin/aggregator/aggregator_test.go b/cmd/poller/plugin/aggregator/aggregator_test.go
index 0363ecfc..e2152d15 100644
--- a/cmd/poller/plugin/aggregator/aggregator_test.go
+++ b/cmd/poller/plugin/aggregator/aggregator_test.go
@@ -30,7 +30,7 @@ func TestInitPlugin(t *testing.T) {
 func TestRuleSimpleAggregation(t *testing.T) {
 
 	// create artifical data
-	m = matrix.New("", "")
+	m = matrix.New("", "", "")
 	var n *matrix.Matrix
 
 	metricA, err := m.NewMetricUint8("metricA")
diff --git a/cmd/poller/plugin/label_agent/label_agent_test.go b/cmd/poller/plugin/label_agent/label_agent_test.go
index 5abf373b..0618a613 100644
--- a/cmd/poller/plugin/label_agent/label_agent_test.go
+++ b/cmd/poller/plugin/label_agent/label_agent_test.go
@@ -225,7 +225,7 @@ func TestValueMappingRule(t *testing.T) {
 		err                  error
 	)
 	// should match
-	m := matrix.New("TestLabelAgent", "test")
+	m := matrix.New("TestLabelAgent", "test", "test")
 
 	if instanceA, err = m.NewInstance("A"); err != nil {
 		t.Fatal(err)
@@ -299,7 +299,7 @@ func TestValueToNumRule(t *testing.T) {
 		err                   error
 	)
 	// should match
-	m := matrix.New("TestLabelAgent", "test")
+	m := matrix.New("TestLabelAgent", "test", "test")
 
 	if instanceA, err = m.NewInstance("A"); err != nil {
 		t.Fatal(err)
diff --git a/pkg/matrix/matrix_test.go b/pkg/matrix/matrix_test.go
index 1a709406..57b9f5fb 100644
--- a/pkg/matrix/matrix_test.go
+++ b/pkg/matrix/matrix_test.go
@@ -5,7 +5,7 @@ import (
 )
 
 func setUpMatrix() *Matrix {
-	m := New("TestRemoveInstance", "test")
+	m := New("TestRemoveInstance", "test", "test")
 	speed, _ := m.NewMetricUint32("max_speed")
 	length, _ := m.NewMetricFloat32("length_in_mm")
 	instanceNames := []string{"A", "B", "C", "D"}