Repository: NetApp/harvest (go)

Issue:
Collecting Prometheus Histogram Stats with Harvest
**Is your feature request related to a problem? Please describe.**
We have a requirement to monitor some stats that are of type histogram.  However, after enabling them in harvest, we have noticed that the stat is not formatted in a Prometheus histogram format.  This means the Prometheus function call "histogram_quantile" does not work.  At the moment, histogram stats seem unusable in their current format (as far as we can tell).

**Describe the solution you'd like**
Format the histograms collected from Harvest in a Prometheus histogram format.
There are 3 stats generated in a Prometheus histogram from one Netapp histogram collected.

1. <stat_name>_bucket{le="<bucket_name>"}
2. <stat_name>_sum
3. <stat_name>_count

Details about each can be found here:  [Prometheus Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram)

Currently, histograms get formatted like this:
```
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<90s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<10s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<200us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<600ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<14ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<100ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<8s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<1s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<20s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<4ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<100us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<60us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<2ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<1ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<6s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<30s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<6ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<60ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<600us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<4s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<80us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric=">=120s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<200ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<800us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<2s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<800ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<20ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<400ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<18ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<120s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<8ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<60s"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<40us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<16ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<12ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<80ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<400us"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<10ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<40ms"} 0
svm_cifs_cifs_latency_hist{datacenter="DC1",cluster="cluster1",svm="svm1",metric="<20us"} 0
```
This same stat would get formatted like this to match the Prometheus scheme:

```
# HELP netapp_perf_cifs_vserver_cifs_latency_hist netapp_perf_cifs_vserver_cifs_latency_hist cifs_latency_hist
# TYPE netapp_perf_cifs_vserver_cifs_latency_hist histogram
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.02"} 1439
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.04"} 2579
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.06"} 2607
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.08"} 2608
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.1"} 2611
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.2"} 2759
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.4"} 2908
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.6"} 3108
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="0.8"} 3119
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="1"} 3126
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="2"} 3143
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="4"} 3172
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="6"} 3179
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="8"} 3215
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="10"} 3220
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="12"} 3228
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="14"} 3232
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="16"} 3235
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="18"} 3238
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="20"} 3242
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="40"} 3253
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="60"} 3256
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="80"} 3256
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="100"} 3257
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="200"} 3262
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="400"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="600"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="800"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="1000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="2000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="4000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="6000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="8000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="10000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="20000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="30000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="60000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="90000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="120000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="240000"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_bucket{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1",le="+Inf"} 3265
netapp_perf_cifs_vserver_cifs_latency_hist_sum{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1"} 4085.44
netapp_perf_cifs_vserver_cifs_latency_hist_count{cluster_name="cluster1",instance="collector:25035",instance_name="svm1",vserver_name="svm1"} 3265
```

To make this work, the label "metric" harvest is collecting would need to be converted to a number value and made into a bucket label "le".

Example: metric="<20us" => le="0.02" (standarizing all buckets to ms)

**Describe alternatives you've considered**
We currently can collect these histograms in an internal perf collection tool, but we would like to transfer all collections to harvest since it is way better in so many ways.



Pull request:
feat: export ONTAP histograms as Prometheus histograms
Fixes #1309

Converting ONTAP histograms to Prometheus histograms requires making sense of ONTAP's counter labels since those labels need to be normalized into a bucket interval for Prometheus. It is not always possible to generically convert ONTAP labels, since differnt counter subsystems use different free-form text. There is no standard or schema for these label names.

Harvest currently handles histograms that use this form of labels:

- `<20us   6`
- `<6ms    1`
- `<10s    0`
- `>=120s  4`

In this example, if these metrics are from the `svm` object and the `cifs_latency_hist` counter they will be exported as shown below. Since microseconds are the smallest unit, all latencies will be normalized to microseconds.

```
svm_cifs_cifs_latency_hist_bucket{cluster="cluster1",datacenter="dc",svm="vs4",le="20"} 6
svm_cifs_cifs_latency_hist_bucket{cluster="cluster1",datacenter="dc",svm="vs4",le="6000"} 1
svm_cifs_cifs_latency_hist_bucket{cluster="cluster1",datacenter="dc",svm="vs4",le="10000000"} 0
svm_cifs_cifs_latency_hist_bucket{cluster="cluster1",datacenter="dc",svm="vs4",le="+Inf"} 4
```

Currently, Harvest only understands how to parse units named: `us`, `ms|msec`, `s|sec`. If all labels are parsable, the histogram will be exported as a Prometheus histogram, otherwise Harvest will export the ONTAP histogram as a flattened set of metrics where the `metric` label equals the ONTAP metric.

Patch:
diff --git a/cmd/collectors/rest/plugins/certificate/certificate.go b/cmd/collectors/rest/plugins/certificate/certificate.go
index 9afcf08e..1e5ee474 100644
--- a/cmd/collectors/rest/plugins/certificate/certificate.go
+++ b/cmd/collectors/rest/plugins/certificate/certificate.go
@@ -40,7 +40,7 @@ func (my *Certificate) Init() error {
 		return err
 	}
 
-	timeout := rest.DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go
index ecf54e06..b2652456 100644
--- a/cmd/collectors/rest/plugins/qtree/qtree.go
+++ b/cmd/collectors/rest/plugins/qtree/qtree.go
@@ -57,7 +57,7 @@ func (my *Qtree) Init() error {
 	}
 
 	clientTimeout := my.ParentParams.GetChildContentS("client_timeout")
-	timeout := rest.DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	duration, err := time.ParseDuration(clientTimeout)
 	if err == nil {
 		timeout = duration
diff --git a/cmd/collectors/rest/plugins/shelf/shelf.go b/cmd/collectors/rest/plugins/shelf/shelf.go
index cf146554..a14661fc 100644
--- a/cmd/collectors/rest/plugins/shelf/shelf.go
+++ b/cmd/collectors/rest/plugins/shelf/shelf.go
@@ -97,7 +97,7 @@ func (my *Shelf) Init() error {
 		return err
 	}
 
-	timeout := rest.DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
diff --git a/cmd/collectors/rest/plugins/snapmirror/snapmirror.go b/cmd/collectors/rest/plugins/snapmirror/snapmirror.go
index b7ae58cc..c9d8fff8 100644
--- a/cmd/collectors/rest/plugins/snapmirror/snapmirror.go
+++ b/cmd/collectors/rest/plugins/snapmirror/snapmirror.go
@@ -42,7 +42,7 @@ func (my *SnapMirror) Init() error {
 		return err
 	}
 
-	timeout := rest.DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
diff --git a/cmd/collectors/rest/plugins/volume/volume.go b/cmd/collectors/rest/plugins/volume/volume.go
index f69b5ca3..d27a2845 100644
--- a/cmd/collectors/rest/plugins/volume/volume.go
+++ b/cmd/collectors/rest/plugins/volume/volume.go
@@ -40,7 +40,7 @@ func (my *Volume) Init() error {
 		return err
 	}
 
-	timeout := rest.DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
index 29b3b4b0..fb0f3718 100644
--- a/cmd/collectors/rest/rest.go
+++ b/cmd/collectors/rest/rest.go
@@ -98,6 +98,8 @@ func (r *Rest) Init(a *collector.AbstractCollector) error {
 		return err
 	}
 
+	r.InitVars(a.Params)
+
 	if err = r.initEndPoints(); err != nil {
 		return err
 	}
@@ -121,11 +123,24 @@ func (r *Rest) Init(a *collector.AbstractCollector) error {
 	return nil
 }
 
+func (r *Rest) InitVars(config *node.Node) {
+
+	var err error
+
+	clientTimeout := config.GetChildContentS("client_timeout")
+	duration, err := time.ParseDuration(clientTimeout)
+	if err == nil {
+		r.Client.Timeout = duration
+	} else {
+		r.Logger.Info().Str("timeout", rest.DefaultTimeout).Msg("Using default timeout")
+	}
+}
+
 func (r *Rest) InitClient() error {
 
 	var err error
 	a := r.AbstractCollector
-	if r.Client, err = r.getClient(a, a.Params); err != nil {
+	if r.Client, err = r.getClient(a); err != nil {
 		return err
 	}
 
@@ -153,7 +168,7 @@ func (r *Rest) InitMatrix() error {
 	return nil
 }
 
-func (r *Rest) getClient(a *collector.AbstractCollector, config *node.Node) (*rest.Client, error) {
+func (r *Rest) getClient(a *collector.AbstractCollector) (*rest.Client, error) {
 	var (
 		poller *conf.Poller
 		err    error
@@ -169,15 +184,7 @@ func (r *Rest) getClient(a *collector.AbstractCollector, config *node.Node) (*re
 		r.Logger.Error().Str("poller", opt.Poller).Msg("Address is empty")
 		return nil, errs.New(errs.ErrMissingParam, "addr")
 	}
-
-	clientTimeout := config.GetChildContentS("client_timeout")
-	timeout := rest.DefaultTimeout * time.Second
-	duration, err := time.ParseDuration(clientTimeout)
-	if err == nil {
-		timeout = duration
-	} else {
-		r.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout")
-	}
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	if client, err = rest.New(*poller, timeout); err != nil {
 		r.Logger.Error().Err(err).Str("poller", opt.Poller).Msg("error creating new client")
 		os.Exit(1)
diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go
index c48383d8..b98e650b 100644
--- a/cmd/collectors/restperf/restperf.go
+++ b/cmd/collectors/restperf/restperf.go
@@ -98,6 +98,8 @@ func (r *RestPerf) Init(a *collector.AbstractCollector) error {
 		return err
 	}
 
+	r.InitVars(a.Params)
+
 	if err = collector.Init(r); err != nil {
 		return err
 	}
@@ -900,7 +902,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	_ = r.Metadata.LazySetValueInt64("calc_time", "data", calcD.Microseconds())
 
 	r.Logger.Info().
-		Int("instances", len(instanceKeys)).
+		Int("instances", len(newData.GetInstances())).
 		Uint64("metrics", count).
 		Str("apiD", apiD.Round(time.Millisecond).String()).
 		Str("parseD", parseD.Round(time.Millisecond).String()).
@@ -1108,8 +1110,9 @@ func (r *RestPerf) PollInstance() (map[string]*matrix.Matrix, error) {
 			}
 		}
 
-		if oldInstances.Delete(instanceKey) {
+		if oldInstances.Has(instanceKey) {
 			// instance already in cache
+			oldInstances.Remove(instanceKey)
 			r.Logger.Debug().Msgf("updated instance [%s%s%s%s]", color.Bold, color.Yellow, instanceKey, color.End)
 		} else if instance, err := mat.NewInstance(instanceKey); err != nil {
 			r.Logger.Error().Err(err).Str("Instance key", instanceKey).Msg("add instance")
diff --git a/cmd/collectors/unix/main.go b/cmd/collectors/unix/main.go
index 46269410..0adba1f7 100644
--- a/cmd/collectors/unix/main.go
+++ b/cmd/collectors/unix/main.go
@@ -247,7 +247,7 @@ func (u *Unix) loadMetrics(counters *node.Node) error {
 
 				if !labels.Has(label) {
 					u.Logger.Warn().Msgf("invalid histogram metric [%s]", label)
-					wanted.Delete(w)
+					wanted.Remove(w)
 					continue
 				}
 
@@ -311,7 +311,7 @@ func (u *Unix) PollInstance() (map[string]*matrix.Matrix, error) {
 			instance.SetLabel("pid", strconv.Itoa(pid))
 			u.Logger.Debug().Str("name", name).Int("pid", pid).Msg("Add instance")
 		} else {
-			currInstances.Delete(name)
+			currInstances.Remove(name)
 			instance.SetLabel("pid", strconv.Itoa(pid))
 			u.Logger.Debug().Str("name", name).Int("pid", pid).Msg("Update instance")
 		}
diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go
index 163e1a08..fc56e44e 100644
--- a/cmd/collectors/zapi/collector/zapi.go
+++ b/cmd/collectors/zapi/collector/zapi.go
@@ -410,7 +410,7 @@ func (z *Zapi) CollectAutoSupport(p *collector.Payload) {
 		}
 	}
 
-	clientTimeout := strconv.Itoa(client.DefaultTimeout)
+	clientTimeout := client.DefaultTimeout
 	newTimeout := z.Params.GetChildContentS("client_timeout")
 	if newTimeout != "" {
 		clientTimeout = newTimeout
diff --git a/cmd/collectors/zapiperf/README.md b/cmd/collectors/zapiperf/README.md
index c52db136..39245517 100644
--- a/cmd/collectors/zapiperf/README.md
+++ b/cmd/collectors/zapiperf/README.md
@@ -54,16 +54,16 @@ This configuration file (the "template") contains a list of objects that should
 
 Additionally, this file contains the parameters that are applied as defaults to all objects. (As mentioned before, any of these parameters can be defined in the Harvest or object configuration files as well).
 
-| parameter              | type         | description                                      | default                |
-|------------------------|--------------|--------------------------------------------------|------------------------|
-| `use_insecure_tls` | bool, optional | skip verifying TLS certificate of the target system | `false`               |
-| `client_timeout`   | duration (Go-syntax)  | how long to wait for server responses             | 10s                  |
-| `batch_size`       | int, optional  | max instances per API request                       | `500`                 |
-| `latency_io_reqd`  | int, optional  | threshold of IOPs for calculating latency metrics (latencies based on very few IOPs are unreliable) | `100`  |
-| `schedule`         | list, required | the poll frequencies of the collector/object, should include exactly these three elements in the exact same other: | |
-|    - `counter`         | duration (Go-syntax) | poll frequency of updating the counter metadata cache (example value: `1200s` = `20m`) | |
-|    - `instance`         | duration (Go-syntax) | poll frequency of updating the instance cache (example value: `600s` = `10m`) | |
-|    - `data`         | duration (Go-syntax) | poll frequency of updating the data cache (example value: `60s` = `1m`)<br /><br />**Note** Harvest allows defining poll intervals on sub-second level (e.g. `1ms`), however keep in mind the following:<br /><ul><li>API response of an ONTAP system can take several seconds, so the collector is likely to enter failed state if the poll interval is less than `client_timeout`.</li><li>Small poll intervals will create significant workload on the ONTAP system, as many counters are aggregated on-demand.</li><li>Some metric values become less significant if they are calculated for very short intervals (e.g. latencies)</li></ul> | |
+| parameter              | type         | description                                      | default |
+|------------------------|--------------|--------------------------------------------------|---------|
+| `use_insecure_tls` | bool, optional | skip verifying TLS certificate of the target system | `false` |
+| `client_timeout`   | duration (Go-syntax)  | how long to wait for server responses             | 30s     |
+| `batch_size`       | int, optional  | max instances per API request                       | `500`   |
+| `latency_io_reqd`  | int, optional  | threshold of IOPs for calculating latency metrics (latencies based on very few IOPs are unreliable) | `100`   |
+| `schedule`         | list, required | the poll frequencies of the collector/object, should include exactly these three elements in the exact same other: |         |
+|    - `counter`         | duration (Go-syntax) | poll frequency of updating the counter metadata cache (example value: `1200s` = `20m`) |         |
+|    - `instance`         | duration (Go-syntax) | poll frequency of updating the instance cache (example value: `600s` = `10m`) |         |
+|    - `data`         | duration (Go-syntax) | poll frequency of updating the data cache (example value: `60s` = `1m`)<br /><br />**Note** Harvest allows defining poll intervals on sub-second level (e.g. `1ms`), however keep in mind the following:<br /><ul><li>API response of an ONTAP system can take several seconds, so the collector is likely to enter failed state if the poll interval is less than `client_timeout`.</li><li>Small poll intervals will create significant workload on the ONTAP system, as many counters are aggregated on-demand.</li><li>Some metric values become less significant if they are calculated for very short intervals (e.g. latencies)</li></ul> |         |
 
 The template should define objects in the `objects` section. Example:
 
diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go
index 883bf893..ee8baf21 100644
--- a/cmd/collectors/zapiperf/zapiperf.go
+++ b/cmd/collectors/zapiperf/zapiperf.go
@@ -924,12 +924,12 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 
 		// string metric, add as instance label
 		if strings.Contains(counter.GetChildContentS("properties"), "string") {
-			oldLabels.Delete(key)
+			oldLabels.Remove(key)
 			z.instanceLabels[key] = display
 			z.Logger.Trace().Msgf("%s+[%s] added as label name (%s)%s", color.Yellow, key, display, color.End)
 		} else {
 			// add counter as numeric metric
-			oldMetrics.Delete(key)
+			oldMetrics.Remove(key)
 			if r := z.addCounter(counter, key, display, true, counters); r != "" && !wanted.Has(r) {
 				missing.Add(r) // required base counter, missing in template
 				z.Logger.Trace().Msgf("%smarking [%s] as required base counter for [%s]%s", color.Red, r, key, color.End)
@@ -942,7 +942,7 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 		z.Logger.Debug().Msgf("attempting to retrieve metadata of %d replaced counters", replaced.Size())
 		for name, counter := range counters {
 			if replaced.Has(name) {
-				oldMetrics.Delete(name)
+				oldMetrics.Remove(name)
 				z.Logger.Debug().Msgf("adding [%s] (replacement for deprecated counter)", name)
 				if r := z.addCounter(counter, name, name, true, counters); r != "" && !wanted.Has(r) {
 					missing.Add(r) // required base counter, missing in template
@@ -959,7 +959,7 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 			Msg("Attempting to retrieve metadata of missing base counters")
 		for name, counter := range counters {
 			if missing.Has(name) {
-				oldMetrics.Delete(name)
+				oldMetrics.Remove(name)
 				z.Logger.Debug().Str("name", name).Msg("Adding missing base counter")
 				z.addCounter(counter, name, "", false, counters)
 			}
@@ -987,10 +987,10 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 		if z.Query == objWorkloadDetail || z.Query == objWorkloadDetailVolume {
 
 			var service, wait, visits, ops matrix.Metric
-			oldMetrics.Delete("service_time")
-			oldMetrics.Delete("wait_time")
-			oldMetrics.Delete("visits")
-			oldMetrics.Delete("ops")
+			oldMetrics.Remove("service_time")
+			oldMetrics.Remove("wait_time")
+			oldMetrics.Remove("visits")
+			oldMetrics.Remove("ops")
 
 			if service = mat.GetMetric("service_time"); service == nil {
 				z.Logger.Error().Err(nil).Msg("metric [service_time] required to calculate workload missing")
@@ -1028,7 +1028,7 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 					resource := x.GetContentS()
 
 					if m := mat.GetMetric(name); m != nil {
-						oldMetrics.Delete(name)
+						oldMetrics.Remove(name)
 						continue
 					}
 					if m, err := mat.NewMetricFloat64(name); err != nil {
@@ -1040,7 +1040,7 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 						// base counter is the ops of the same resource
 						m.SetComment("ops")
 
-						oldMetrics.Delete(name)
+						oldMetrics.Remove(name)
 						z.Logger.Debug().Msgf("+ [%s] (=> %s) added workload latency metric", name, resource)
 					}
 				}
@@ -1060,7 +1060,7 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) {
 				}
 				z.qosLabels[label] = display
 				//me.instanceLabels[label] = display
-				//oldLabels.Delete(label)
+				//oldLabels.Remove(label)
 			}
 		}
 	}
@@ -1130,8 +1130,15 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 	// counter type is array, each element will be converted to a metric instance
 	if counter.GetChildContentS("type") == "array" {
 
-		var labels, baseLabels []string
-		var e string
+		var (
+			labels, baseLabels []string
+			e                  string
+			description        string
+			isHistogram        bool
+			histogramMetric    matrix.Metric
+		)
+
+		description = strings.ToLower(counter.GetChildContentS("desc"))
 
 		if labels, e = parseHistogramLabels(counter); e != "" {
 			z.Logger.Warn().Msgf("skipping [%s] of type array: %s", name, e)
@@ -1156,15 +1163,39 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 			}
 		}
 
-		for _, label := range labels {
+		baseKey := baseCounter
+		if baseCounter != "" && len(baseLabels) != 0 {
+			baseKey += "." + baseLabels[0]
+		}
+
+		// ONTAP does not have a `type` for histogram. Harvest tests the `desc` field to determine
+		// if a counter is a histogram
+		isHistogram = false
+		if len(labels) > 0 && strings.Contains(description, "histogram") {
+			key := name + "." + "bucket"
+			histogramMetric = mat.GetMetric(key)
+			if histogramMetric != nil {
+				z.Logger.Trace().Str("metric", key).Msg("Updating array metric attributes")
+			} else {
+				histogramMetric, err = mat.NewMetricFloat64(key)
+				if err != nil {
+					z.Logger.Error().Err(err).Str("key", key).Msg("unable to create histogram metric")
+					return ""
+				}
+			}
+			histogramMetric.SetName(display)
+			histogramMetric.SetProperty(property)
+			histogramMetric.SetComment(baseKey)
+			histogramMetric.SetExportable(enabled)
+			histogramMetric.SetBuckets(&labels)
+			isHistogram = true
+		}
+
+		for i, label := range labels {
 
 			var m matrix.Metric
 
 			key := name + "." + label
-			baseKey := baseCounter
-			if baseCounter != "" && len(baseLabels) != 0 {
-				baseKey += "." + baseLabels[0]
-			}
 
 			if m = mat.GetMetric(key); m != nil {
 				z.Logger.Trace().Msgf("updating array metric [%s] attributes", key)
@@ -1185,6 +1216,11 @@ func (z *ZapiPerf) addCounter(counter *node.Node, name, display string, enabled
 				m.SetLabel("submetric", x[1])
 			} else {
 				m.SetLabel("metric", label)
+				if isHistogram {
+					// Save the index of this label so the labels can be exported in order
+					m.SetLabel("comment", strconv.Itoa(i))
+					m.SetHistogram(true)
+				}
 			}
 		}
 		// cache labels only when parsing counter was success
@@ -1342,8 +1378,9 @@ func (z *ZapiPerf) PollInstance() (map[string]*matrix.Matrix, error) {
 				name := i.GetChildContentS(nameAttr)
 				uuid := i.GetChildContentS(uuidAttr)
 				z.Logger.Debug().Msgf("skip instance, missing key [%s] (name=%s, uuid=%s)", z.instanceKey, name, uuid)
-			} else if oldInstances.Delete(key) {
+			} else if oldInstances.Has(key) {
 				// instance already in cache
+				oldInstances.Remove(key)
 				z.Logger.Trace().Msgf("updated instance [%s%s%s%s]", color.Bold, color.Yellow, key, color.End)
 				continue
 			} else if instance, err := mat.NewInstance(key); err != nil {
diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go
index fb657633..78081a81 100644
--- a/cmd/exporters/prometheus/prometheus.go
+++ b/cmd/exporters/prometheus/prometheus.go
@@ -183,6 +183,7 @@ func (p *Prometheus) Init() error {
 	// @TODO: implement error checking to enter failed state if HTTPd failed
 	// (like we did in Alpha)
 
+	//goland:noinspection HttpUrlsUsage
 	p.Logger.Debug().Msgf("initialized, HTTP daemon started at [http://%s:%d]", addr, port)
 
 	return nil
@@ -267,16 +268,21 @@ func (p *Prometheus) Export(data *matrix.Matrix) error {
 
 func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 	var (
-		rendered                                     [][]byte
-		tagged                                       *set.Set
-		labelsToInclude, keysToInclude, globalLabels []string
-		prefix                                       string
-		err                                          error
-		replacer                                     *strings.Replacer
+		rendered         [][]byte
+		tagged           *set.Set
+		labelsToInclude  []string
+		keysToInclude    []string
+		globalLabels     []string
+		prefix           string
+		err              error
+		replacer         *strings.Replacer
+		histograms       map[string]*histogram
+		normalizedLabels map[string][]string // cache of histogram normalized labels
 	)
 
 	rendered = make([][]byte, 0)
 	globalLabels = make([]string, 0)
+	normalizedLabels = make(map[string][]string)
 	replacer = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "\\n")
 
 	if p.addMetaTags {
@@ -388,6 +394,7 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 		if p.Params.SortLabels {
 			sort.Strings(instanceKeys)
 		}
+		histograms = make(map[string]*histogram)
 		for mkey, metric := range data.GetMetrics() {
 
 			if !metric.IsExportable() {
@@ -399,13 +406,42 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 
 			if value, ok, pass := metric.GetValueString(instance); ok && pass {
 
-				// metric is histogram
+				// metric is array, determine if this is a plain array or histogram
 				if metric.HasLabels() {
-					metricLabels := make([]string, 0)
+					if metric.IsHistogram() {
+						// metric is histogram. Create a new metric to accumulate
+						// the flattened metrics and export them in order
+						bucketMetric := data.GetMetric(metric.GetName() + ".bucket")
+						if bucketMetric == nil {
+							p.Logger.Debug().
+								Str("metric", metric.GetName()).
+								Msg("Unable to find bucket for metric, skip")
+							continue
+						}
+						metricIndex := metric.GetLabel("comment")
+						index, err := strconv.Atoi(metricIndex)
+						if err != nil {
+							p.Logger.Error().Err(err).
+								Str("metric", metric.GetName()).
+								Str("index", metricIndex).
+								Msg("Unable to find index of metric, skip")
+						}
+						histogram := histogramFromBucket(histograms, bucketMetric)
+						histogram.values[index] = value
+						continue
+					}
+					metricLabels := make([]string, 0, metric.GetLabels().Size())
 					for k, v := range metric.GetLabels().Map() {
 						metricLabels = append(metricLabels, escape(replacer, k, v))
 					}
-					x := fmt.Sprintf("%s_%s{%s,%s} %s", prefix, metric.GetName(), strings.Join(instanceKeys, ","), strings.Join(metricLabels, ","), value)
+					x := fmt.Sprintf(
+						"%s_%s{%s,%s} %s",
+						prefix,
+						metric.GetName(),
+						strings.Join(instanceKeys, ","),
+						strings.Join(metricLabels, ","),
+						value,
+					)
 
 					if p.addMetaTags && !tagged.Has(prefix+"_"+metric.GetName()) {
 						tagged.Add(prefix + "_" + metric.GetName())
@@ -430,6 +466,62 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 				p.Logger.Trace().Str("mkey", mkey).Msg("skipped: no data value")
 			}
 		}
+		// all metrics have been processed and flattened metrics accumulated. Determine which histograms can be
+		// normalized and export
+		for _, h := range histograms {
+			metric := h.metric
+			bucketNames := metric.Buckets()
+			objectMetric := data.Object + "_" + metric.GetName()
+			_, ok := normalizedLabels[objectMetric]
+			if !ok {
+				canNormalize := true
+				normalizedNames := make([]string, 0, len(*bucketNames))
+				// check if the buckets can be normalized and collect normalized names
+				for _, bucketName := range *bucketNames {
+					normalized := p.normalizeHistogram(metric, bucketName, data.Object)
+					if normalized == "" {
+						canNormalize = false
+						break
+					}
+					normalizedNames = append(normalizedNames, normalized)
+				}
+				if canNormalize {
+					normalizedLabels[objectMetric] = normalizedNames
+				}
+			}
+
+			if p.addMetaTags && !tagged.Has(prefix+"_"+metric.GetName()) {
+				tagged.Add(prefix + "_" + metric.GetName())
+				rendered = append(rendered, []byte("# HELP "+prefix+"_"+metric.GetName()+" Metric for "+data.Object))
+				rendered = append(rendered, []byte("# TYPE "+prefix+"_"+metric.GetName()+" histogram"))
+			}
+
+			normalizedNames, canNormalize := normalizedLabels[objectMetric]
+			for i, value := range h.values {
+				bucketName := (*bucketNames)[i]
+				var x string
+				if canNormalize {
+					x = fmt.Sprintf(
+						"%s_%s{%s,%s} %s",
+						prefix,
+						metric.GetName()+"_bucket",
+						strings.Join(instanceKeys, ","),
+						`le="`+normalizedNames[i]+`"`,
+						value,
+					)
+				} else {
+					x = fmt.Sprintf(
+						"%s_%s{%s,%s} %s",
+						prefix,
+						metric.GetName(),
+						strings.Join(instanceKeys, ","),
+						escape(replacer, "metric", bucketName),
+						value,
+					)
+				}
+				rendered = append(rendered, []byte(x))
+			}
+		}
 	}
 	p.Logger.Debug().
 		Str("object", data.Object).
@@ -439,6 +531,66 @@ func (p *Prometheus) render(data *matrix.Matrix) ([][]byte, error) {
 	return rendered, nil
 }
 
+var numAndUnitRe = regexp.MustCompile(`(\d+)\s*(\w+)`)
+
+// normalizeHistogram tries to normalize ONTAP values by converting units to multiples of the smallest unit.
+// When the unit can not be determined, return an empty string
+func (p *Prometheus) normalizeHistogram(metric matrix.Metric, ontap string, object string) string {
+	numAndUnit := ontap
+	if strings.HasPrefix(ontap, "<") {
+		numAndUnit = ontap[1:]
+	} else if strings.HasPrefix(ontap, ">") {
+		return "+Inf"
+	}
+	submatch := numAndUnitRe.FindStringSubmatch(numAndUnit)
+	if len(submatch) != 3 {
+		p.Logger.Trace().
+			Str("object", object).
+			Str("metric", metric.GetName()).
+			Str("numAndUnit", numAndUnit).
+			Msg("No units found")
+		return ""
+	}
+	num := submatch[1]
+	unit := submatch[2]
+	float, err := strconv.ParseFloat(num, 64)
+	if err != nil {
+		p.Logger.Trace().Str("num", num).Msg("Unable to convert to float64")
+		return ""
+	}
+	normal := 0.0
+	switch unit {
+	case "us":
+		return num
+	case "ms", "msec":
+		normal = 1_000 * float
+	case "s", "sec":
+		normal = 1_000_000 * float
+	default:
+		p.Logger.Trace().Str("unit", unit).Msg("Unknown unit")
+		return ""
+	}
+	return strconv.FormatFloat(normal, 'f', -1, 64)
+}
+
+func histogramFromBucket(histograms map[string]*histogram, metric matrix.Metric) *histogram {
+	h, ok := histograms[metric.GetName()]
+	if ok {
+		return h
+	}
+	buckets := metric.Buckets()
+	var capacity int
+	if buckets != nil {
+		capacity = len(*buckets)
+	}
+	h = &histogram{
+		metric: metric,
+		values: make([]string, capacity),
+	}
+	histograms[metric.GetName()] = h
+	return h
+}
+
 func escape(replacer *strings.Replacer, key string, value string) string {
 	// See https://prometheus.io/docs/instrumenting/exposition_formats/#comments-help-text-and-type-information
 	// label_value can be any sequence of UTF-8 characters, but the backslash (\), double-quote ("),
@@ -446,3 +598,8 @@ func escape(replacer *strings.Replacer, key string, value string) string {
 
 	return fmt.Sprintf("%s=\"%s\"", key, replacer.Replace(value))
 }
+
+type histogram struct {
+	metric matrix.Metric
+	values []string
+}
diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go
index 78aefde8..c4d88526 100644
--- a/cmd/poller/poller.go
+++ b/cmd/poller/poller.go
@@ -1100,7 +1100,7 @@ func (p *Poller) upgradeCollector(c conf.Collector) (conf.Collector, error) {
 func (p *Poller) newRestClient() (*rest.Rest, error) {
 	params := node.NewS("")
 	// Set client_timeout to suppress logging a msg about the default client_timeout during Rest client creation
-	params.NewChildS("client_timeout", fmt.Sprintf("%ds", rest2.DefaultTimeout))
+	params.NewChildS("client_timeout", rest2.DefaultTimeout)
 	Union2(params, p.params)
 	delegate := collector.New("Rest", "", p.options, params)
 	r := &rest.Rest{
diff --git a/cmd/tools/rest/client.go b/cmd/tools/rest/client.go
index 4100f29d..fa0e0974 100644
--- a/cmd/tools/rest/client.go
+++ b/cmd/tools/rest/client.go
@@ -21,7 +21,7 @@ import (
 
 const (
 	// DefaultTimeout should be > than ONTAP's default REST timeout, which is 15 seconds for GET requests
-	DefaultTimeout = 30
+	DefaultTimeout = "30s"
 )
 
 type Client struct {
@@ -246,7 +246,7 @@ func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool)
 		return 0, err
 	}
 
-	timeout := DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(DefaultTimeout)
 	if restClient, err = New(*poller, timeout); err != nil {
 		return 0, fmt.Errorf("error creating new client %w", err)
 	}
diff --git a/cmd/tools/rest/rest.go b/cmd/tools/rest/rest.go
index 964c2063..8d71a24a 100644
--- a/cmd/tools/rest/rest.go
+++ b/cmd/tools/rest/rest.go
@@ -174,7 +174,7 @@ func doData() {
 		return
 	}
 
-	timeout := DefaultTimeout * time.Second
+	timeout, _ := time.ParseDuration(DefaultTimeout)
 	if client, err = New(*poller, timeout); err != nil {
 		fmt.Printf("error creating new client %+v\n", err)
 		os.Exit(1)
diff --git a/conf/zapiperf/default.yaml b/conf/zapiperf/default.yaml
index e4ca4371..7301a596 100644
--- a/conf/zapiperf/default.yaml
+++ b/conf/zapiperf/default.yaml
@@ -42,7 +42,7 @@ objects:
   FcpLif:                 fcp_lif.yaml
   ISCSI:                  iscsi_lif.yaml
   LIF:                    lif.yaml
-  LUN:                    lun.yaml
+  Lun:                    lun.yaml
   NFSv3:                  nfsv3.yaml
   NFSv41:                 nfsv4_1.yaml
   NFSv42:                 nfsv4_2.yaml
diff --git a/pkg/api/ontapi/zapi/client.go b/pkg/api/ontapi/zapi/client.go
index f390261e..98b23153 100644
--- a/pkg/api/ontapi/zapi/client.go
+++ b/pkg/api/ontapi/zapi/client.go
@@ -25,7 +25,7 @@ import (
 
 const (
 	DefaultAPIVersion = "1.3"
-	DefaultTimeout    = 10
+	DefaultTimeout    = "30s"
 )
 
 type Client struct {
@@ -185,14 +185,16 @@ func parseClientTimeout(clientTimeout string) (time.Duration, error) {
 	if charIndex != -1 {
 		duration, err := time.ParseDuration(clientTimeout)
 		if err != nil {
-			return time.Duration(DefaultTimeout) * time.Second, err
+			timeout, _ := time.ParseDuration(DefaultTimeout)
+			return timeout, err
 		}
 		return duration, nil
 	}
 	t, err := strconv.Atoi(clientTimeout)
 	if err != nil {
 		// when there is an error return the default timeout
-		return time.Duration(DefaultTimeout) * time.Second, nil //nolint:nilerr
+		timeout, _ := time.ParseDuration(DefaultTimeout)
+		return timeout, nil //nolint:nilerr
 	}
 	return time.Duration(t) * time.Second, nil
 }
diff --git a/pkg/matrix/metric.go b/pkg/matrix/metric.go
index 5a6fee48..441517d1 100644
--- a/pkg/matrix/metric.go
+++ b/pkg/matrix/metric.go
@@ -36,7 +36,11 @@ type Metric interface {
 	SetComment(string)
 	IsArray() bool
 	SetArray(bool)
+	IsHistogram() bool
+	SetHistogram(bool)
 	Clone(bool) Metric
+	SetBuckets(*[]string)
+	Buckets() *[]string
 
 	// methods for resizing metric storage
 
@@ -91,8 +95,10 @@ type AbstractMetric struct {
 	property   string
 	comment    string
 	array      bool
+	histogram  bool
 	exportable bool
 	labels     *dict.Dict
+	buckets    *[]string
 	record     []bool
 	pass       []bool
 }
@@ -105,6 +111,8 @@ func (m *AbstractMetric) Clone(deep bool) *AbstractMetric {
 		comment:    m.comment,
 		exportable: m.exportable,
 		array:      m.array,
+		histogram:  m.histogram,
+		buckets:    m.buckets,
 	}
 	if m.labels != nil {
 		clone.labels = m.labels.Copy()
@@ -173,6 +181,22 @@ func (m *AbstractMetric) SetLabel(key, value string) {
 	m.labels.Set(key, value)
 }
 
+func (m *AbstractMetric) SetHistogram(b bool) {
+	m.histogram = b
+}
+
+func (m *AbstractMetric) IsHistogram() bool {
+	return m.histogram
+}
+
+func (m *AbstractMetric) Buckets() *[]string {
+	return m.buckets
+}
+
+func (m *AbstractMetric) SetBuckets(buckets *[]string) {
+	m.buckets = buckets
+}
+
 func (m *AbstractMetric) SetLabels(labels *dict.Dict) {
 	m.labels = labels
 }
diff --git a/pkg/set/set.go b/pkg/set/set.go
index 5e7cfce3..48295cd9 100644
--- a/pkg/set/set.go
+++ b/pkg/set/set.go
@@ -31,21 +31,10 @@ func (s *Set) AddValues(values []string) {
 }
 
 // Remove removes an element from the set
-// See also Delete
 func (s *Set) Remove(value string) {
 	delete(s.set, value)
 }
 
-// Delete removes an element from the set
-// return true if the element was in the set, false otherwise
-func (s *Set) Delete(value string) bool {
-	if s.Has(value) {
-		delete(s.set, value)
-		return true
-	}
-	return false
-}
-
 func (s *Set) Has(value string) bool {
 	_, has := s.set[value]
 	return has


Test patch:
diff --git a/pkg/api/ontapi/zapi/client_test.go b/pkg/api/ontapi/zapi/client_test.go
index 7784a2f9..6971c62f 100644
--- a/pkg/api/ontapi/zapi/client_test.go
+++ b/pkg/api/ontapi/zapi/client_test.go
@@ -69,13 +69,14 @@ func TestClientTimeout(t *testing.T) {
 		hasErr       bool
 	}
 
+	timeout, _ := time.ParseDuration(DefaultTimeout)
 	tests := []test{
 		{"no units", "180", 180 * time.Second, false},
 		{"no units", "123", 123 * time.Second, false},
-		{"empty", "", DefaultTimeout * time.Second, true},
+		{"empty", "", timeout, true},
 		{"zero", "0", 0 * time.Second, false},
 		{"with units", "5m4s", 5*time.Minute + 4*time.Second, false},
-		{"invalid", "bob", DefaultTimeout * time.Second, true},
+		{"invalid", "bob", timeout, true},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {