Repository: NetApp/harvest
Pull request #780 (2022-01-13 10:32:18)

feat: rest gaps aggr, qtree, shelf
covered the changes from 775 PR. 

6 aggr fields:
aggr_snapshot_maxfiles_possible rejected --> Done
aggr_hybrid_cache_size_total -> Done
aggr_raid_plex_count -> Done
aggr_snapshot_inode_used_percent rejected
aggr_space_used_percent rejected 
aggr_space_reserved -> rejected

Qtree and Shelf, basic template changes are ready, plugin will be next.


Patch:
diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
index 1a0b6254..4ff25cb2 100644
--- a/cmd/collectors/rest/rest.go
+++ b/cmd/collectors/rest/rest.go
@@ -30,16 +30,22 @@ type endPoint struct {
 }
 
 type prop struct {
+	object         string
 	query          string
 	instanceKeys   []string
 	instanceLabels map[string]string
-	metrics        []string
+	metrics        []metric
 	counters       map[string]string
 	returnTimeOut  string
 	fields         []string
 	apiType        string // public, private
 }
 
+type metric struct {
+	label string
+	name  string
+}
+
 func init() {
 	plugin.RegisterModule(Rest{})
 }
@@ -79,18 +85,32 @@ func (r *Rest) Init(a *collector.AbstractCollector) error {
 		return err
 	}
 
-	r.Logger.Info().Str("cluster", r.client.Cluster().Name).Msgf("connected to %s", r.client.Cluster().Info)
-
-	r.Matrix.SetGlobalLabel("cluster", r.client.Cluster().Name)
-
 	if err = r.initCache(); err != nil {
 		return err
 	}
+
+	if err = r.InitMatrix(); err != nil {
+		return err
+	}
 	r.Logger.Info().Msgf("initialized cache with %d metrics", len(r.Matrix.GetMetrics()))
 
 	return nil
 }
 
+func (r *Rest) InitMatrix() error {
+	// overwrite from abstract collector
+	r.Matrix.Object = r.prop.object
+	// Add system (cluster) name
+	r.Matrix.SetGlobalLabel("cluster", r.client.Cluster().Name)
+	if r.Params.HasChildS("labels") {
+		for _, l := range r.Params.GetChildS("labels").GetChildren() {
+			r.Matrix.SetGlobalLabel(l.GetNameS(), l.GetContentS())
+		}
+	}
+
+	return nil
+}
+
 func (r *Rest) getClient(a *collector.AbstractCollector, config *node.Node) (*rest.Client, error) {
 	var (
 		poller *conf.Poller
@@ -142,7 +162,7 @@ func (r *Rest) initEndPoints() error {
 			prop.instanceKeys = make([]string, 0)
 			prop.instanceLabels = make(map[string]string)
 			prop.counters = make(map[string]string)
-			prop.metrics = make([]string, 0)
+			prop.metrics = make([]metric, 0)
 			prop.apiType = "public"
 
 			for _, line1 := range line.GetChildren() {
@@ -161,9 +181,9 @@ func (r *Rest) initEndPoints() error {
 							prop.instanceKeys = append(prop.instanceKeys, name)
 						case "label":
 							prop.instanceLabels[name] = display
-						default:
-							prop.instanceLabels[name] = display
-							prop.metrics = append(prop.metrics, name)
+						case "float":
+							m := metric{label: display, name: name}
+							prop.metrics = append(prop.metrics, m)
 						}
 					}
 					if prop.apiType == "private" {
@@ -337,11 +357,20 @@ func (r *Rest) PollData() (*matrix.Matrix, error) {
 			}
 		}
 
-		for key, metric := range r.Matrix.GetMetrics() {
-			f := instanceData.Get(key)
+		for _, metric := range r.prop.metrics {
+			metr, ok := r.Matrix.GetMetrics()[metric.name]
+			if !ok {
+				if metr, err = r.Matrix.NewMetricFloat64(metric.name); err != nil {
+					r.Logger.Error().Err(err).
+						Str("name", metric.name).
+						Msg("NewMetricFloat64")
+				}
+			}
+			f := instanceData.Get(metric.name)
 			if f.Exists() {
-				if err = metric.SetValueFloat64(instance, f.Float()); err != nil {
-					r.Logger.Error().Err(err).Str("key", key).Str("metric", metric.GetName()).
+				metr.SetName(metric.label)
+				if err = metr.SetValueFloat64(instance, f.Float()); err != nil {
+					r.Logger.Error().Err(err).Str("key", metric.name).Str("metric", metric.label).
 						Msg("Unable to set float key on metric")
 				}
 				count++
@@ -464,25 +493,20 @@ func (r *Rest) processEndPoints(data *matrix.Matrix) error {
 				}
 
 				for _, metric := range endpoint.prop.metrics {
-					f := instanceData.Get(metric)
+					metr, ok := data.GetMetrics()[metric.name]
+					if !ok {
+						if metr, err = data.NewMetricFloat64(metric.name); err != nil {
+							r.Logger.Error().Err(err).
+								Str("name", metric.name).
+								Msg("NewMetricFloat64")
+						}
+					}
+					f := instanceData.Get(metric.name)
 					if f.Exists() {
-						if metr, ok := data.GetMetrics()[metric]; !ok {
-							if metr, _ = data.NewMetricFloat64(metric); err != nil {
-								r.Logger.Error().Err(err).
-									Str("name", metric).
-									Msg("NewMetricFloat64")
-							}
-							metr.SetName(endpoint.prop.instanceLabels[metric])
-							if err = metr.SetValueFloat64(instance, f.Float()); err != nil {
-								r.Logger.Error().Err(err).Str("key", metric).Str("metric", endpoint.prop.instanceLabels[metric]).
-									Msg("Unable to set float key on metric")
-							}
-						} else {
-							metr.SetName(endpoint.prop.instanceLabels[metric])
-							if err = metr.SetValueFloat64(instance, f.Float()); err != nil {
-								r.Logger.Error().Err(err).Str("key", metric).Str("metric", endpoint.prop.instanceLabels[metric]).
-									Msg("Unable to set float key on metric")
-							}
+						metr.SetName(metric.label)
+						if err = metr.SetValueFloat64(instance, f.Float()); err != nil {
+							r.Logger.Error().Err(err).Str("key", metric.name).Str("metric", metric.label).
+								Msg("Unable to set float key on metric")
 						}
 					}
 				}
diff --git a/cmd/collectors/rest/templating.go b/cmd/collectors/rest/templating.go
index f05dc250..ce82f0c7 100644
--- a/cmd/collectors/rest/templating.go
+++ b/cmd/collectors/rest/templating.go
@@ -2,7 +2,6 @@ package rest
 
 import (
 	"goharvest2/pkg/errors"
-	"goharvest2/pkg/matrix"
 	"goharvest2/pkg/tree/node"
 	"strings"
 )
@@ -28,16 +27,13 @@ func (r *Rest) initCache() error {
 	var (
 		counters            *node.Node
 		display, name, kind string
-		metr                matrix.Metric
-		err                 error
 	)
 
 	if x := r.Params.GetChildContentS("object"); x != "" {
-		r.Object = x
+		r.prop.object = x
 	} else {
-		r.Object = strings.ToLower(r.Object)
+		r.prop.object = strings.ToLower(r.Object)
 	}
-	r.Matrix.Object = r.Object
 
 	if e := r.Params.GetChildS("export_options"); e != nil {
 		r.Matrix.SetExportOptions(e)
@@ -78,14 +74,8 @@ func (r *Rest) initCache() error {
 			case "label":
 				r.prop.instanceLabels[name] = display
 			case "float":
-				if metr, err = r.Matrix.NewMetricFloat64(name); err != nil {
-					r.Logger.Error().Err(err).
-						Str("name", name).
-						Msg("NewMetricFloat64")
-					return err
-				}
-				metr.SetName(display)
-				metr.SetProperty("etl.float")
+				m := metric{label: display, name: name}
+				r.prop.metrics = append(r.prop.metrics, m)
 			}
 		}
 	}
@@ -117,11 +107,8 @@ func (r *Rest) initCache() error {
 	}
 
 	r.Logger.Info().Strs("extracted Instance Keys", r.prop.instanceKeys).Msg("")
-	r.Logger.Info().Int("count metrics", len(r.Matrix.GetMetrics())).Int("count labels", len(r.prop.instanceLabels)).Msg("initialized metric cache")
+	r.Logger.Info().Int("count metrics", len(r.prop.metrics)).Int("count labels", len(r.prop.instanceLabels)).Msg("initialized metric cache")
 
-	if len(r.Matrix.GetMetrics()) == 0 && r.Params.GetChildContentS("collect_only_labels") != "true" {
-		return errors.New(errors.ERR_NO_METRIC, "failed to parse numeric metrics")
-	}
 	return nil
 }
 
diff --git a/cmd/collectors/zapi/plugins/quota/quota.go b/cmd/collectors/zapi/plugins/quota/quota.go
index 32a00e97..c0b6ccee 100644
--- a/cmd/collectors/zapi/plugins/quota/quota.go
+++ b/cmd/collectors/zapi/plugins/quota/quota.go
@@ -183,6 +183,14 @@ func (my *Quota) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 
 				if attrValue := quota.GetChildContentS(attribute); attrValue != "" {
 					qtreeInstance := data.GetInstance(tree + "." + volume + "." + vserver)
+					if qtreeInstance == nil {
+						my.Logger.Warn().
+							Str("tree", tree).
+							Str("volume", volume).
+							Str("vserver", vserver).
+							Msg("No instance matching tree.volume.vserver")
+						continue
+					}
 					if !qtreeInstance.IsExportable() {
 						continue
 					}
diff --git a/cmd/exporters/prometheus/README.md b/cmd/exporters/prometheus/README.md
index 3987f348..0150c60c 100644
--- a/cmd/exporters/prometheus/README.md
+++ b/cmd/exporters/prometheus/README.md
@@ -24,7 +24,7 @@ All parameters of the exporter are defined in the `Exporters` section of `harves
 An overview of all parameters:
 
 | parameter           | type                                           | description                                                                                                                                                                                                     | default   |
-| ------------------- | ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+|---------------------|------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
 | `port_range`        | int-int (range), overrides `port` if specified | lower port to upper port (inclusive) of the HTTP end-point to create when a poller specifies this exporter. Starting at lower port, each free port will be tried sequentially up to the upper port.             |           |
 | `port`              | int, required if port_range is not specified   | port of the HTTP end-point                                                                                                                                                                                      |           |
 | `local_http_addr`   | string, optional                               | address of the HTTP server Harvest starts for Prometheus to scrape:<br />use `localhost` to serve only on the local machine<br />use `0.0.0.0` (default) if Prometheus is scrapping from another machine        | `0.0.0.0` |
@@ -33,6 +33,7 @@ An overview of all parameters:
 | `allow_addrs_regex` | list of strings, optional                      | allow access only if host address matches at least one of the regular expressions                                                                                                                               |           |
 | `cache_max_keep`    | string (Go duration format), optional          | maximum amount of time metrics are cached (in case Prometheus does not timely collect the metrics)                                                                                                              | `300s`    |
 | `add_meta_tags`     | bool, optional                                 | add `HELP` and `TYPE` [metatags](https://prometheus.io/docs/instrumenting/exposition_formats/#comments-help-text-and-type-information) to metrics (currently no useful information, but required by some tools) | `false`   |
+| `sort_labels`       | bool, optional                                 | sort metric labels before exporting. Some [open-metrics scrapers report](https://github.com/NetApp/harvest/issues/756) stale metrics when labels are not sorted.                                                | `false`   |
 
 A few examples:
 
diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go
index b9d3ffdf..3ff7f026 100644
--- a/cmd/exporters/prometheus/prometheus.go
+++ b/cmd/exporters/prometheus/prometheus.go
@@ -29,6 +29,7 @@ import (
 	"goharvest2/pkg/matrix"
 	"goharvest2/pkg/set"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -361,18 +362,32 @@ func (me *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 
 			// @TODO, check at least one label is found?
 			if len(instance_labels) != 0 {
-				label_data := fmt.Sprintf("%s_labels{%s,%s} 1.0", prefix, strings.Join(instance_keys, ","), strings.Join(instance_labels, ","))
+				var labelData string
+				if me.Params.SortLabels {
+					allLabels := make([]string, len(instance_labels))
+					copy(allLabels, instance_labels)
+					for _, instanceKey := range instance_keys {
+						allLabels = append(allLabels, instanceKey)
+					}
+					sort.Strings(allLabels)
+					labelData = fmt.Sprintf("%s_labels{%s} 1.0", prefix, strings.Join(allLabels, ","))
+				} else {
+					labelData = fmt.Sprintf("%s_labels{%s,%s} 1.0", prefix, strings.Join(instance_keys, ","), strings.Join(instance_labels, ","))
+				}
 				if me.addMetaTags && !tagged.Has(prefix+"_labels") {
 					tagged.Add(prefix + "_labels")
 					rendered = append(rendered, []byte("# HELP "+prefix+"_labels Pseudo-metric for "+data.Object+" labels"))
 					rendered = append(rendered, []byte("# TYPE "+prefix+"_labels gauge"))
 				}
-				rendered = append(rendered, []byte(label_data))
+				rendered = append(rendered, []byte(labelData))
 			} else {
 				me.Logger.Trace().Msgf("skip instance labels, no labels parsed (%v) (%v)", instance_keys, instance_labels)
 			}
 		}
 
+		if me.Params.SortLabels {
+			sort.Strings(instance_keys)
+		}
 		for mkey, metric := range data.GetMetrics() {
 
 			if !metric.IsExportable() {
diff --git a/cmd/poller/plugin/README.md b/cmd/poller/plugin/README.md
index 2509c03f..c6f84d3f 100644
--- a/cmd/poller/plugin/README.md
+++ b/cmd/poller/plugin/README.md
@@ -125,13 +125,15 @@ Notice that the rules are executed in the same order as you've added them. List
   - [include_regex](#include_regex)
   - [value_mapping](#value_mapping)
   - [value_to_num](#value_to_num)
+  - [compute_metric](#compute_metric)
 
 ## split
 
 Rule syntax:
 
 ```yaml
-split: LABEL `SEP` LABEL1,LABEL2,LABEL3
+split: 
+  - LABEL `SEP` LABEL1,LABEL2,LABEL3
 # source label - separator - comma-seperated target labels
 ```
 
@@ -140,7 +142,8 @@ Splits the value of a given label by separator `SEP` and creates new labels if t
 Example:
 
 ```yaml
-split: node `/` ,aggr,plex,disk
+split: 
+  - node `/` ,aggr,plex,disk
 # will split the value of "node" using separator "/"
 # will expect 4 values: first will be discarded, remaining
 # three will be stored as labels "aggr", "plex" and "disk"
@@ -153,13 +156,15 @@ Does the same as `split_regex` but uses a regular expression instead of a separa
 Rule syntax:
 
 ```yaml
-split_simple: LABEL `REGEX` LABEL1,LABEL2,LABEL3
+split_simple: 
+  - LABEL `REGEX` LABEL1,LABEL2,LABEL3
 ```
 
 Example:
 
 ```yaml
-node `.*_(ag\d+)_(p\d+)_(d\d+)` aggr,plex,disk
+split_simple: 
+  - node `.*_(ag\d+)_(p\d+)_(d\d+)` aggr,plex,disk
 # will look for "_ag", "_p", "_d", each followed by one
 # or more numbers, if there is a match, the submatches
 # will be stored as "aggr", "plex" and "disk"
@@ -171,7 +176,8 @@ node `.*_(ag\d+)_(p\d+)_(d\d+)` aggr,plex,disk
 Rule syntax:
 
 ```yaml
-split_pairs: LABEL `SEP1` `SEP2`
+split_pairs: 
+  - LABEL `SEP1` `SEP2`
 # source label - pair separator - key-value separator
 ```
 
@@ -180,7 +186,8 @@ Extracts key-value pairs from the value of source label `LABEL`. Note that you n
 Example:
 
 ```yaml
-split_pairs: comment ` ` `:`
+split_pairs: 
+  - comment ` ` `:`
 # will split pairs using a single space and split key-values using colon
 # e.g. if comment="owner:jack contact:some@email", the result wll be
 # two new labels: owner="jack" and contact="some@email"
@@ -193,14 +200,16 @@ Join multiple label values using separator `SEP` and create a new label.
 Rule syntax:
 
 ```yaml
-join: LABEL `SEP` LABEL1,LABEL2,LABEL3
+join: 
+  - LABEL `SEP` LABEL1,LABEL2,LABEL3
 # target label - separator - comma-seperated source labels
 ```
 
 Example:
 
 ```yaml
-join: plex_long `_` aggr,plex
+join: 
+  - plex_long `_` aggr,plex
 # will look for the values of labels "aggr" and "plex",
 # if they are set, a new "plex_long" label will be added
 # by joining their values with "_"
@@ -213,14 +222,16 @@ Substitute substring `OLD` with `NEW` in label `SOURCE` and store in `TARGET`. N
 Rule syntax:
 
 ```yaml
-replace: SOURCE TARGET `OLD` `NEW`
+replace: 
+  - SOURCE TARGET `OLD` `NEW`
 # source label - target label - substring to replace - replace with
 ```
 
 Example:
 
 ```yaml
-replace: node node_short `node_` ``
+replace: 
+  - node node_short `node_` ``
 # this rule will just remove "node_" from all values of label
 # "node". E.g. if label is "node_jamaica1", it will rewrite it 
 # as "jamaica1"
@@ -232,14 +243,16 @@ Same as `replace_simple`, but will use a regular expression instead of `OLD`. No
 Rule syntax:
 
 ```yaml
-replace_regex: SOURCE TARGET `REGEX` `NEW`
+replace_regex: 
+  - SOURCE TARGET `REGEX` `NEW`
 # source label - target label - substring to replace - replace with
 ```
 
 Example:
 
 ```yaml
-replace_regex: node node `^(node)_(\d+)_.*$` `Node-$2`
+replace_regex: 
+  - node node `^(node)_(\d+)_.*$` `Node-$2`
 # if there is a match, will capitalize "Node" and remove suffixes.
 # E.g. if label is "node_10_dc2", it will rewrite it as
 # will rewrite it as "Node-10"
@@ -252,14 +265,16 @@ Exclude each instance, if the value of `LABEL` is exactly `VALUE`. Exclude means
 Rule syntax:
 
 ```yaml
-exclude_equals: LABEL `VALUE`
+exclude_equals: 
+  - LABEL `VALUE`
 # label name - label value
 ```
 
 Example:
 
 ```yaml
-exclude_equals: vol_type `flexgroup_constituent`
+exclude_equals: 
+  - vol_type `flexgroup_constituent`
 # all instances, which have label "vol_type" with value
 # "flexgroup_constituent" will not be exported
 ```
@@ -271,14 +286,16 @@ Same as `exclude_equals`, but all labels that *contain* `VALUE` will be excluded
 Rule syntax:
 
 ```yaml
-exclude_contains: LABEL `VALUE`
+exclude_contains: 
+  - LABEL `VALUE`
 # label name - label value
 ```
 
 Example:
 
 ```yaml
-exclude_contains: vol_type `flexgroup_`
+exclude_contains: 
+  - vol_type `flexgroup_`
 # all instances, which have label "vol_type" which contain
 # "flexgroup_" will not be exported
 ```
@@ -290,14 +307,16 @@ Same as `exclude_equals`, but will use a regular expression and all matching ins
 Rule syntax:
 
 ```yaml
-exclude_regex: LABEL `REGEX`
+exclude_regex: 
+  - LABEL `REGEX`
 # label name - regular expression
 ```
 
 Example:
 
 ```yaml
-exclude_regex: vol_type `^flex`
+exclude_regex: 
+  - vol_type `^flex`
 # all instances, which have label "vol_type" which starts with
 # "flex" will not be exported
 ```
@@ -309,14 +328,16 @@ Include each instance, if the value of `LABEL` is exactly `VALUE`. Include means
 Rule syntax:
 
 ```yaml
-include_equals: LABEL `VALUE`
+include_equals: 
+  - LABEL `VALUE`
 # label name - label value
 ```
 
 Example:
 
 ```yaml
-include_equals: vol_type `flexgroup_constituent`
+include_equals: 
+  - vol_type `flexgroup_constituent`
 # all instances, which have label "vol_type" with value
 # "flexgroup_constituent" will be exported
 ```
@@ -328,14 +349,16 @@ Same as `include_equals`, but all labels that *contain* `VALUE` will be included
 Rule syntax:
 
 ```yaml
-include_contains: LABEL `VALUE`
+include_contains: 
+  - LABEL `VALUE`
 # label name - label value
 ```
 
 Example:
 
 ```yaml
-include_contains: vol_type `flexgroup_`
+include_contains: 
+  - vol_type `flexgroup_`
 # all instances, which have label "vol_type" which contain
 # "flexgroup_" will be exported
 ```
@@ -347,14 +370,16 @@ Same as `include_equals`, but a regular expression will be used for inclusion. S
 Rule syntax:
 
 ```yaml
-include_regex: LABEL `REGEX`
+include_regex: 
+  - LABEL `REGEX`
 # label name - regular expression
 ```
 
 Example:
 
 ```yaml
-include_regex: vol_type `^flex`
+include_regex: 
+  - vol_type `^flex`
 # all instances, which have label "vol_type" which starts with
 # "flex" will be exported
 ```
@@ -375,7 +400,8 @@ Note that you don't define the numeric values yourself, instead, you only provid
 Rule syntax:
 
 ```yaml
-value_to_num: METRIC LABEL ZAPI_VALUE REST_VALUE `N`
+value_to_num: 
+  - METRIC LABEL ZAPI_VALUE REST_VALUE `N`
 # map values of LABEL to 1 if it is ZAPI_VALUE or REST_VALUE
 # otherwise, value of METRIC is set to N
 ```
@@ -384,7 +410,8 @@ The default value `N` is optional, if no default value is given and the label va
 Examples:
 
 ```yaml
-value_to_num: status state up online `0`
+value_to_num: 
+  - status state up online `0`
 # a new metric will be created with the name "status"
 # if an instance has label "state" with value "up", the metric value will be 1,
 # if it's "online", the value will be set to 1,
@@ -392,13 +419,69 @@ value_to_num: status state up online `0`
 ```
 
 ```yaml
-value_to_num: status state up online `4`
+value_to_num: 
+  - status state up online `4`
 # metric value will be set to 1 if "state" is "up", otherwise to **4**
 ```
 
 ```yaml
-value_to_num: status outage - - `0` #ok_value is empty value. 
+value_to_num: 
+  - status outage - - `0` #ok_value is empty value. 
 # metric value will be set to 1 if "outage" is empty, if it's any other value, it will be set to the default, 0
 # '-' is a special symbol in this mapping, and it will be converted to blank while processing.
 ```
+## compute_metric
 
+This rule creates the new metric (of type `float64`) using the existing metric values.
+
+Note that you don't define the numeric values yourself, instead, you only provide the metric names with operation, 
+the plugin will fetch the value of each given metric and store the result of mathematical operation in new given custom metric.
+
+Currently, we support these operations: ADD SUBTRACT MULTIPLY DIVIDE
+
+Rule syntax:
+
+```yaml
+compute_metric: 
+  - METRIC OPERATION METRIC1 METRIC2 METRIC3 
+# target new metric - mathematical operation - input metric names 
+# apply OPERATION on metric values of METRIC1, METRIC2 and METRIC3 and set result in METRIC
+```
+
+Examples:
+
+```yaml
+compute_metric:
+  - space_total ADD space_available space_used
+# a new metric will be created with the name "space_total"
+# if an instance has metric "space_available" with value "1000", and "space_used" with value "400",
+# the result value will be "1400" and set to metric "space_total".
+```
+
+```yaml
+compute_metric:
+  - disk_count ADD primary.disk_count secondary.disk_count hybrid.disk_count
+# value of metric "disk_count" would be addition of all the given disk_counts metric values.
+# disk_count = primary.disk_count + secondary.disk_count + hybrid.disk_count
+```
+
+```yaml
+compute_metric:
+  - files_available SUBTRACT files files_used
+# value of metric "files_available" would be subtraction of the metric value of files_used from metric value of files.
+# files_available = files - files_used
+```
+
+```yaml
+compute_metric:
+  - total_bytes MULTIPLY bytes_per_sector sector_count
+# value of metric "total_bytes" would be multiplication of metric value of bytes_per_sector and metric value of sector_count.
+# total_bytes = bytes_per_sector * sector_count
+```
+
+```yaml
+compute_metric:
+  - transmission_rate DIVIDE transfer.bytes_transferred transfer.total_duration
+# value of metric "transmission_rate" would be division of metric value of transfer.bytes_transferred by metric value of transfer.total_duration.
+# transmission_rate = transfer.bytes_transferred / transfer.total_duration
+```
\ No newline at end of file
diff --git a/cmd/poller/plugin/label_agent/label_agent.go b/cmd/poller/plugin/label_agent/label_agent.go
index 71f2e745..b18646ae 100644
--- a/cmd/poller/plugin/label_agent/label_agent.go
+++ b/cmd/poller/plugin/label_agent/label_agent.go
@@ -28,6 +28,7 @@ type LabelAgent struct {
 	includeContainsRules []includeContainsRule
 	includeRegexRules    []includeRegexRule
 	valueToNumRules      []valueToNumRule
+	computeMetricRules   []computeMetricRule
 }
 
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
@@ -321,3 +322,72 @@ func (me *LabelAgent) mapValues(m *matrix.Matrix) error {
 
 	return nil
 }
+
+func (me *LabelAgent) computeMetrics(m *matrix.Matrix) error {
+
+	var (
+		metric                    matrix.Metric
+		metricVal, firstMetricVal matrix.Metric
+		err                       error
+	)
+
+	// map values for compute_metric mapping rules
+	for _, r := range me.computeMetricRules {
+
+		if metric = m.GetMetric(r.metric); metric == nil {
+			if metric, err = m.NewMetricFloat64(r.metric); err != nil {
+				me.Logger.Error().Stack().Err(err).Str("new metric", r.metric).Msg("computeMetrics: failed to create metric")
+				return err
+			} else {
+				metric.SetProperty("compute_metric mapping")
+			}
+		}
+
+		for _, instance := range m.GetInstances() {
+			var result float64
+
+			// Parse first operand and store in result for further processing
+			if firstMetricVal = m.GetMetric(r.metricNames[0]); firstMetricVal != nil {
+				if val, ok := firstMetricVal.GetValueFloat64(instance); ok {
+					result = val
+				}
+			} else {
+				me.Logger.Warn().Stack().Err(err).Str("metricName", r.metricNames[0]).Msg("computeMetrics: metric not found")
+			}
+
+			// Parse other operands and process them
+			for i := 1; i < len(r.metricNames); i++ {
+				if metricVal = m.GetMetric(r.metricNames[i]); metricVal != nil {
+					v, _ := metricVal.GetValueFloat64(instance)
+
+					switch r.operation {
+					case "ADD":
+						result += v
+					case "SUBTRACT":
+						result -= v
+					case "MULTIPLY":
+						result *= v
+					case "DIVIDE":
+						if v != 0 {
+							result /= v
+						} else {
+							me.Logger.Error().
+								Str("operation", r.operation).
+								Msg("Division by zero operation")
+						}
+					default:
+						me.Logger.Warn().
+							Str("operation", r.operation).
+							Msg("Unknown operation")
+					}
+				} else {
+					me.Logger.Warn().Stack().Err(err).Str("metricName", r.metricNames[i]).Msg("computeMetrics: metric not found")
+				}
+			}
+
+			_ = metric.SetValueFloat64(instance, result)
+			me.Logger.Trace().Str("metricName", r.metric).Float64("metricValue", result).Msg("computeMetrics: new metric created")
+		}
+	}
+	return nil
+}
diff --git a/cmd/poller/plugin/label_agent/parse_rules.go b/cmd/poller/plugin/label_agent/parse_rules.go
index 8521699e..c24dda2f 100644
--- a/cmd/poller/plugin/label_agent/parse_rules.go
+++ b/cmd/poller/plugin/label_agent/parse_rules.go
@@ -24,6 +24,7 @@ func (me *LabelAgent) parseRules() int {
 	me.excludeRegexRules = make([]excludeRegexRule, 0)
 	me.splitPairsRules = make([]splitPairsRule, 0)
 	me.valueToNumRules = make([]valueToNumRule, 0)
+	me.computeMetricRules = make([]computeMetricRule, 0)
 
 	for _, c := range me.Params.GetChildren() {
 		name := c.GetNameS()
@@ -60,6 +61,8 @@ func (me *LabelAgent) parseRules() int {
 				me.parseIncludeRegexRule(rule)
 			case "value_to_num":
 				me.parseValueToNumRule(rule)
+			case "compute_metric":
+				me.parseComputeMetricRule(rule)
 			default:
 				me.Logger.Warn().
 					Str("object", me.ParentParams.GetChildContentS("object")).
@@ -137,9 +140,13 @@ func (me *LabelAgent) parseRules() int {
 		case "value_to_num":
 			if len(me.valueToNumRules) != 0 {
 				me.actions = append(me.actions, me.mapValues)
-				count += len(me.includeRegexRules)
+				count += len(me.valueToNumRules)
+			}
+		case "compute_metric":
+			if len(me.computeMetricRules) != 0 {
+				me.actions = append(me.actions, me.computeMetrics)
+				count += len(me.computeMetricRules)
 			}
-			count += len(me.valueToNumRules)
 		default:
 			me.Logger.Warn().
 				Str("object", me.ParentParams.GetChildContentS("object")).
@@ -514,3 +521,24 @@ func (me *LabelAgent) parseValueToNumRule(rule string) {
 	}
 	me.Logger.Warn().Msgf("(value_to_num) rule has invalid format [%s]", rule)
 }
+
+type computeMetricRule struct {
+	metric      string
+	operation   string
+	metricNames []string
+}
+
+func (me *LabelAgent) parseComputeMetricRule(rule string) {
+	if fields := strings.Fields(rule); len(fields) >= 4 {
+		r := computeMetricRule{metric: fields[0], operation: fields[1], metricNames: make([]string, 0)}
+
+		for i := 2; i < len(fields); i++ {
+			r.metricNames = append(r.metricNames, fields[i])
+		}
+
+		me.computeMetricRules = append(me.computeMetricRules, r)
+		me.Logger.Debug().Msgf("(compute_metric) parsed rule [%v]", r)
+		return
+	}
+	me.Logger.Warn().Msgf("(compute_metric) rule has invalid format [%s]", rule)
+}
diff --git a/conf/rest/9.13.0/aggr.yaml b/conf/rest/9.13.0/aggr.yaml
index 94398aab..bc136525 100644
--- a/conf/rest/9.13.0/aggr.yaml
+++ b/conf/rest/9.13.0/aggr.yaml
@@ -8,24 +8,14 @@ counters:
   - ^^uuid  => uuid
   - ^home_node.name  => node
   #- ^aggr-get-iter.aggr-attributes.aggr-raid-attributes.aggregate-type  => raid_type  #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-raid-attributes.disk-count   => raid_disk_count #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-raid-attributes.plex-count  => raid_plex_count #mapping missing
   - block_storage.primary.raid_size => raid_size
   - ^state  => state
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.files-private-used  => inode_files_private_used #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.files-total => inode_files_total #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.files-used => inode_files_used #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.inodefile-private-capacity => inodefile_private_capacity #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.inodefile-public-capacity => inodefile_public_capacity #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.maxfiles-available => inode_maxfiles_available #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.maxfiles-possible => inode_maxfiles_possible #mapping missing
-  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.maxfiles-used => inode_maxfiles_used #mapping missing
   #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.percent-inode-used-capacity => percent_inode_used_capacity #mapping missing
   - space.cloud_storage.used => space_capacity_tier_used
   - space.block_storage.data_compacted_count => space_data_compacted_count
   - space.block_storage.data_compaction_space_saved => space_data_compaction_saved
   - space.block_storage.data_compaction_space_saved_percent  => space_data_compaction_saved_percent
-  - space.block_storage.hybrid_cache.size  => hybrid_cache_size_total  #mapping seems wrong in doc
+  - block_storage.hybrid_cache.size  => hybrid_cache_size_total
   - snapshot.percent_used_capacity => space_used_percent #mapping seems wrong in doc
   - space.block_storage.inactive_user_data => space_performance_tier_inactive_user_data
   - space.block_storage.inactive_user_data_percent => space_performance_tier_inactive_user_data_percent
@@ -38,25 +28,40 @@ counters:
   - space.block_storage.size => space_total
   - space.block_storage.used => space_used
   #- aggr-get-iter.aggr-attributes.aggr-space-attributes.total-reserved-space => space_reserved #Missing mapping
-  #- aggr-get-iter.aggr-attributes.aggr-volume-count-attributes.flexvol-count => volume_count_flexvol #Missing mapping
   - snapshot.files_total => snapshot_files_total
   - snapshot.files_used => snapshot_files_used
   - snapshot.max_files_available => snapshot_maxfiles_available
   - snapshot.max_files_used => snapshot_maxfiles_used
+  - inode_attributes.used_percent => inode_used_percent
+  - inode_attributes.file_public_capacity => inode_inodefile_public_capacity
+  - inode_attributes.file_private_capacity => inode_inodefile_private_capacity
+  - inode_attributes.files_used => inode_files_used
+  - inode_attributes.files_total => inode_files_total
+  - inode_attributes.files_private_used => inode_files_private_used
+  - inode_attributes.max_files_available => inode_maxfiles_available
+  - inode_attributes.max_files_possible => inode_maxfiles_possible
+  - inode_attributes.max_files_used => inode_maxfiles_used
   #- aggr-get-iter.aggr-attributes.aggr-snapshot-attributes.percent-inode-used-capacity => snapshot_inode_used_percent #missing mapping
-  #- aggr-get-iter.aggr-attributes.aggr-snapshot-attributes.percent-used-capacity => snapshot_used_percent #missing mapping
-  - snapshot.size_available => snapshot_size_available  #mapping seems wrong in doc
-  - snapshot.size_total => snapshot_size_total  #mapping seems wrong in doc
-  - snapshot.size_used => snapshot_size_used  #mapping seems wrong in doc
-  - snapshot.percent_snapshot_space => snapshot_reserve_percent #mapping seems wrong in doc
+  - space.snapshot.available => snapshot_size_available
+  - space.snapshot.used => snapshot_size_used
+  - space.snapshot.used_percent => snapshot_used_percent
+  - space.snapshot.reserve_percent => snapshot_reserve_percent
+  - space.snapshot.total => snapshot_size_total
+  - volume_count => volume_count_flexvol
+  - block_storage.primary.disk_count => primary_disk_count
+  - block_storage.hybrid_cache.disk_count => hybrid_disk_count
+  - block_storage.plexes.# => raid_plex_count
   - hidden_fields:
-      - space.block_storage.inactive_user_data
-      - space.block_storage.inactive_user_data_percent
+      - space.block_storage
+      - inode_attributes
 
 plugins:
   - LabelAgent:
       value_to_num:
         - new_status state online online `0`
+      compute_metric:
+        - snapshot_maxfiles_possible ADD snapshot.max_files_available snapshot.max_files_used
+        - raid_disk_count ADD block_storage.primary.disk_count block_storage.hybrid_cache.disk_count
 
 export_options:
   instance_keys:
diff --git a/conf/rest/9.13.0/qtree.yaml b/conf/rest/9.13.0/qtree.yaml
new file mode 100644
index 00000000..9ecbac20
--- /dev/null
+++ b/conf/rest/9.13.0/qtree.yaml
@@ -0,0 +1,32 @@
+
+name:                       Qtree
+query:                      api/storage/qtrees
+object:                     qtree
+
+counters:
+  - ^^svm.name => svm
+  - ^^volume.name => volume
+  - ^^name => qtree
+  - ^export_policy.name => export_policy
+  - ^security_style => security_style
+  - id => id
+
+
+endpoints:
+  - query: api/private/cli/qtree
+    counters:
+      - ^^vserver => svm
+      - ^^volume => volume
+      - ^^qtree => qtree
+      - ^status => status
+
+
+export_options:
+  instance_keys:
+    - qtree
+    - volume
+    - svm
+  instance_labels:
+    - export_policy
+    - security_style
+    - status
diff --git a/conf/rest/9.13.0/quota.yaml b/conf/rest/9.13.0/quota.yaml
new file mode 100644
index 00000000..432fde6a
--- /dev/null
+++ b/conf/rest/9.13.0/quota.yaml
@@ -0,0 +1,33 @@
+
+name:                       Quota
+query:                      api/storage/quota/reports
+object:                     quota
+
+counters:
+  - ^^svm.name => svm
+  - ^^volume.name => volume
+  - ^^index => quota_id
+  - ^qtree.name => qtree
+  - ^type => quota_type
+  - space.hard_limit => disk_limit
+  - space.soft_limit => soft_disk_limit
+  - space.used.total => disk_used
+  - space.used.hard_limit_percent => disk_used_pct_disk_limit
+  - space.used.soft_limit_percent => disk_used_pct_soft_disk_limit
+#  - space.used.soft_limit_percent => disk_used_pct_threshold  # wrong mapping ??
+  - files.hard_limit => file_limit
+  - files.soft_limit => soft_file_limit
+  - files.used.total => files_used
+  - files.used.hard_limit_percent => files_used_pct_file_limit
+  - files.used.soft_limit_percent => files_used_pct_soft_file_limit
+  #- threshold //deprecated
+
+
+export_options:
+  instance_keys:
+    - svm
+    - volume
+    - quota_id
+  instance_labels:
+    - qtree
+    - quota_type
diff --git a/conf/rest/9.13.0/shelf.yaml b/conf/rest/9.13.0/shelf.yaml
new file mode 100644
index 00000000..49dbec92
--- /dev/null
+++ b/conf/rest/9.13.0/shelf.yaml
@@ -0,0 +1,33 @@
+
+name:               StorageShelf
+query:              api/storage/shelves
+object:             shelf
+
+counters:
+  - ^^uid => shelf_uid
+  - ^module_type => module_type
+  - ^serial_number => serial_number
+  - ^name => shelf
+  - ^model => model
+  - ^state => state
+  - ^manufacturer.name => vendor_name
+  - disk_count => disk_count
+  #- ^state => op_status
+
+plugins:
+  LabelAgent:
+    # metric label zapi_value rest_value `default_value`
+    value_to_num:
+      - new_status state online ok `0`
+
+export_options:
+  instance_keys:
+    - shelf_uid
+    - shelf
+  instance_labels:
+    - module_type
+    - serial_number
+    - model
+    - state
+    - op_status
+    - vendor_name
diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml
index 30efaa4d..931e1bc1 100644
--- a/conf/rest/default.yaml
+++ b/conf/rest/default.yaml
@@ -15,4 +15,6 @@ objects:
   Subsystem:        subsystem.yaml
   Snapshot:         snapshot.yaml
   Port:             port.yaml
-
+  Qtree:            qtree.yaml
+  Quota:            quota.yaml
+  Shelf:            shelf.yaml
diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go
index f77eb6e6..cab02966 100644
--- a/pkg/conf/conf.go
+++ b/pkg/conf/conf.go
@@ -407,6 +407,7 @@ type Exporter struct {
 
 	// Prometheus specific
 	HeartBeatUrl string `yaml:"heart_beat_url,omitempty"`
+	SortLabels   bool   `yaml:"sort_labels,omitempty"`
 
 	// InfluxDB specific
 	Bucket        *string `yaml:"bucket,omitempty"`


Test patch:
diff --git a/cmd/poller/plugin/label_agent/label_agent_test.go b/cmd/poller/plugin/label_agent/label_agent_test.go
index 8d7ba2fd..81001a5d 100644
--- a/cmd/poller/plugin/label_agent/label_agent_test.go
+++ b/cmd/poller/plugin/label_agent/label_agent_test.go
@@ -53,6 +53,16 @@ func TestInitPlugin(t *testing.T) {
 	params.NewChildS("value_to_num", "").NewChildS("", "new_stage stage init start `4`")
 	// create metric "new_outage", if empty value is expected and non empty means wrong, use default value "0"
 	params.NewChildS("value_to_num", "").NewChildS("", "new_outage outage - - `0`")
+	// create metric "space_total", which is addition of the metric value of space_available and space_used
+	params.NewChildS("compute_metric", "").NewChildS("", "space_total ADD space_available space_used")
+	// create metric "disk_count", which is addition of the metric value of primary.disk_count, secondary.disk_count and hybrid.disk_count
+	params.NewChildS("compute_metric", "").NewChildS("", "disk_count ADD primary.disk_count secondary.disk_count hybrid.disk_count")
+	// create metric "files_available", which is subtraction of the metric value of files_used from metric value of files
+	params.NewChildS("compute_metric", "").NewChildS("", "files_available SUBTRACT files files_used")
+	// create metric "total_bytes", which is multiplication of the metric value of bytes_per_sector and sector_count
+	params.NewChildS("compute_metric", "").NewChildS("", "total_bytes MULTIPLY bytes_per_sector sector_count")
+	// create metric "transmission_rate", which is division of the metric value of transfer.bytes_transferred by transfer.total_duration
+	params.NewChildS("compute_metric", "").NewChildS("", "transmission_rate DIVIDE transfer.bytes_transferred transfer.total_duration")
 
 	abc := plugin.New("Test", nil, params, nil)
 	p = &LabelAgent{AbstractPlugin: abc}
@@ -386,3 +396,164 @@ func TestValueToNumRule(t *testing.T) {
 		t.Logf("OK - metric [outage]: value for instanceB set to %d", v)
 	}
 }
+
+func TestComputeMetricsRule(t *testing.T) {
+
+	var (
+		instanceA, instanceB                                                *matrix.Instance
+		metricAvail, metricUsed, metricTotal                                matrix.Metric
+		metricDiskP, metricDiskS, metricDiskH, metricDiskTotal              matrix.Metric
+		metricFiles, metricFilesUsed, metricFilesAvailable                  matrix.Metric
+		metricBytesPSector, metricSectorCount, metricTotalBytes             matrix.Metric
+		metricBytesTransferred, metricTotalDuration, metricTransmissionRate matrix.Metric
+		expected                                                            float64
+		err                                                                 error
+	)
+
+	m := matrix.New("TestLabelAgent", "test", "test")
+
+	if instanceA, err = m.NewInstance("A"); err != nil {
+		t.Fatal(err)
+	}
+
+	// space metric for addition
+	if metricAvail, err = m.NewMetricFloat64("space_available"); err == nil {
+		metricAvail.SetValueFloat64(instanceA, 1010101010)
+	} else {
+		t.Error("metric [space_available]  not created for InstanceA")
+	}
+	if metricUsed, err = m.NewMetricFloat64("space_used"); err == nil {
+		metricUsed.SetValueFloat64(instanceA, 5050505050)
+	} else {
+		t.Error("metric [space_used]  not created for InstanceA")
+	}
+	// files metric for subtraction
+	if metricFiles, err = m.NewMetricFloat64("files"); err == nil {
+		metricFiles.SetValueFloat64(instanceA, 1024)
+	} else {
+		t.Error("metric [files]  not created for InstanceA")
+	}
+	if metricFilesUsed, err = m.NewMetricFloat64("files_used"); err == nil {
+		metricFilesUsed.SetValueFloat64(instanceA, 216)
+	} else {
+		t.Error("metric [files_used]  not created for InstanceA")
+	}
+
+	if instanceB, err = m.NewInstance("B"); err != nil {
+		t.Fatal(err)
+	}
+
+	// disk metric for addition
+	if metricDiskP, err = m.NewMetricFloat64("primary.disk_count"); err == nil {
+		metricDiskP.SetValueFloat64(instanceB, 8)
+	} else {
+		t.Error("metric [primary.disk_count]  not created for InstanceB")
+	}
+	if metricDiskS, err = m.NewMetricFloat64("secondary.disk_count"); err == nil {
+		metricDiskS.SetValueFloat64(instanceB, 10)
+	} else {
+		t.Error("metric [secondary.disk_count]  not created for InstanceB")
+	}
+	if metricDiskH, err = m.NewMetricFloat64("hybrid.disk_count"); err == nil {
+		metricDiskH.SetValueFloat64(instanceB, 4)
+	} else {
+		t.Error("metric [hybrid.disk_count]  not created for InstanceB")
+	}
+	// bytes metric for multiplication
+	if metricBytesPSector, err = m.NewMetricFloat64("bytes_per_sector"); err == nil {
+		metricBytesPSector.SetValueFloat64(instanceB, 10000)
+	} else {
+		t.Error("metric [bytes_per_sector]  not created for InstanceB")
+	}
+	if metricSectorCount, err = m.NewMetricFloat64("sector_count"); err == nil {
+		metricSectorCount.SetValueFloat64(instanceB, 12)
+	} else {
+		t.Error("metric [sector_count]  not created for InstanceB")
+	}
+	// transmission metric for division
+	if metricBytesTransferred, err = m.NewMetricFloat64("transfer.bytes_transferred"); err == nil {
+		metricBytesTransferred.SetValueFloat64(instanceB, 9000000)
+	} else {
+		t.Error("metric [transfer.bytes_transferred]  not created for InstanceB")
+	}
+	if metricTotalDuration, err = m.NewMetricFloat64("transfer.total_duration"); err == nil {
+		metricTotalDuration.SetValueFloat64(instanceB, 3600)
+	} else {
+		t.Error("metric [transfer.total_duration]  not created for InstanceB")
+	}
+
+	if err = p.computeMetrics(m); err != nil {
+		t.Fatal(err)
+	}
+
+	// check "space_total" for instanceA
+	expected = 6060606060
+	if metricTotal = m.GetMetric("space_total"); metricTotal != nil {
+		if metricTotalVal, ok := metricTotal.GetValueFloat64(instanceA); !ok {
+			t.Error("metric [space_total]: value for InstanceA not set")
+		} else if metricTotalVal != expected {
+			t.Errorf("metric [space_total]: value for InstanceA is %f, expected %f", metricTotalVal, expected)
+		} else {
+			t.Logf("OK - metric [space_total]: value for instanceA set to %f", metricTotalVal)
+		}
+	} else {
+		t.Error("metric [space_total] missing")
+	}
+
+	// check "disk_count" for instanceB
+	expected = 22
+	if metricDiskTotal = m.GetMetric("disk_count"); metricDiskTotal != nil {
+		if metricDiskTotalVal, ok := metricDiskTotal.GetValueFloat64(instanceB); !ok {
+			t.Error("metric [disk_count]: value for InstanceB not set")
+		} else if metricDiskTotalVal != expected {
+			t.Errorf("metric [disk_count]: value for InstanceB is %f, expected %f", metricDiskTotalVal, expected)
+		} else {
+			t.Logf("OK - metric [disk_count]: value for instanceB set to %f", metricDiskTotalVal)
+		}
+	} else {
+		t.Error("metric [disk_count] missing")
+	}
+
+	// check "files_available" for instanceA
+	expected = 808
+	if metricFilesAvailable = m.GetMetric("files_available"); metricFilesAvailable != nil {
+		if metricFilesAvailableVal, ok := metricFilesAvailable.GetValueFloat64(instanceA); !ok {
+			t.Error("metric [files_available]: value for InstanceA not set")
+		} else if metricFilesAvailableVal != expected {
+			t.Errorf("metric [files_available]: value for InstanceA is %f, expected %f", metricFilesAvailableVal, expected)
+		} else {
+			t.Logf("OK - metric [files_available]: value for instanceA set to %f", metricFilesAvailableVal)
+		}
+	} else {
+		t.Error("metric [files_available] missing")
+	}
+
+	// check "total_bytes" for instanceB
+	expected = 120000
+	if metricTotalBytes = m.GetMetric("total_bytes"); metricTotalBytes != nil {
+		if metricTotalBytesVal, ok := metricTotalBytes.GetValueFloat64(instanceB); !ok {
+			t.Error("metric [total_bytes]: value for InstanceB not set")
+		} else if metricTotalBytesVal != expected {
+			t.Errorf("metric [total_bytes]: value for InstanceB is %f, expected %f", metricTotalBytesVal, expected)
+		} else {
+			t.Logf("OK - metric [total_bytes]: value for instanceB set to %f", metricTotalBytesVal)
+		}
+	} else {
+		t.Error("metric [total_bytes] missing")
+	}
+
+	// check "transmission_rate" for instanceB
+	expected = 2500
+	if metricTransmissionRate = m.GetMetric("transmission_rate"); metricTransmissionRate != nil {
+		if metricTransmissionRateVal, ok := metricTransmissionRate.GetValueFloat64(instanceB); !ok {
+			t.Error("metric [transmission_rate]: value for InstanceB not set")
+		} else if metricTransmissionRateVal != expected {
+			t.Errorf("metric [transmission_rate]: value for InstanceB is %f, expected %f", metricTransmissionRateVal, expected)
+		} else {
+			t.Logf("OK - metric [transmission_rate]: value for instanceB set to %f", metricTransmissionRateVal)
+		}
+	} else {
+		t.Error("metric [transmission_rate] missing")
+	}
+
+}