Repository: NetApp/harvest
Pull request #1831 (2023-03-20 12:46:37)

feat: Harvest should collect QOS policy groups


Patch:
diff --git a/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go b/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go
new file mode 100644
index 00000000..9bd973e5
--- /dev/null
+++ b/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go
@@ -0,0 +1,52 @@
+package qospolicyfixed
+
+import (
+	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed"
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+	"strings"
+)
+
+type QosPolicyFixed struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &QosPolicyFixed{AbstractPlugin: p}
+}
+
+func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+	data := dataMap[p.Object]
+
+	for _, instance := range data.GetInstances() {
+		p.setFixed(instance)
+	}
+
+	return nil, nil
+}
+
+func (p *QosPolicyFixed) setFixed(instance *matrix.Instance) {
+	label := instance.GetLabel("throughput_policy")
+	if label == "" {
+		return
+	}
+	before, after, found := strings.Cut(label, "-")
+	if !found {
+		p.Logger.Warn().Str("label", label).Msg("Unable to parse fixed xput label")
+		return
+	}
+	min, err := qospolicyfixed.ZapiXputToRest(before)
+	if err != nil {
+		p.Logger.Error().Err(err).Str("label", before).Msg("Failed to parse fixed xput label")
+		return
+	}
+	max, err := qospolicyfixed.ZapiXputToRest(after)
+	if err != nil {
+		p.Logger.Error().Err(err).Str("label", after).Msg("Failed to parse fixed xput label")
+		return
+	}
+	instance.SetLabel("min_throughput_iops", min.IOPS)
+	instance.SetLabel("max_throughput_iops", max.IOPS)
+	instance.SetLabel("min_throughput_mbps", min.Mbps)
+	instance.SetLabel("max_throughput_mbps", max.Mbps)
+}
diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
index 57cc3b7e..d5640862 100644
--- a/cmd/collectors/rest/rest.go
+++ b/cmd/collectors/rest/rest.go
@@ -5,6 +5,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/certificate"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/disk"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/netroute"
+	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qospolicyfixed"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/qtree"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/securityaccount"
 	"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/sensor"
@@ -384,6 +385,8 @@ func (r *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin
 		return sensor.New(abc)
 	case "SecurityAccount":
 		return securityaccount.New(abc)
+	case "QosPolicyFixed":
+		return qospolicyfixed.New(abc)
 	default:
 		r.Logger.Warn().Str("kind", kind).Msg("no rest plugin found ")
 	}
diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go
index 7d11d65d..da583932 100644
--- a/cmd/collectors/zapi/collector/zapi.go
+++ b/cmd/collectors/zapi/collector/zapi.go
@@ -7,6 +7,7 @@ package zapi
 import (
 	"fmt"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/certificate"
+	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qtree"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/security"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/sensor"
@@ -17,6 +18,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/set"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"sort"
 	"strconv"
 	"strings"
@@ -149,6 +151,9 @@ func (z *Zapi) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin
 		return svm.New(abc)
 	case "Security":
 		return security.New(abc)
+	case "QosPolicyFixed":
+		return qospolicyfixed.New(abc)
+
 	default:
 		z.Logger.Info().Msgf("no zapi plugin found for %s", kind)
 	}
@@ -234,7 +239,7 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) {
 		tag               string
 		err               error
 		ad, pd            time.Duration // Request/API time, Parse time, Fetch time
-		fetch             func(*matrix.Instance, *node.Node, []string)
+		fetch             func(*matrix.Instance, *node.Node, []string, bool)
 		instances         []*node.Node
 	)
 
@@ -251,15 +256,23 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) {
 		oldInstances.Add(key)
 	}
 
-	fetch = func(instance *matrix.Instance, node *node.Node, path []string) {
+	fetch = func(instance *matrix.Instance, node *node.Node, path []string, isAppend bool) {
 
 		newpath := append(path, node.GetNameS())
 		key := strings.Join(newpath, ".")
 		z.Logger.Trace().Msgf(" > %s(%s)%s <%s%d%s> name=[%s%s%s%s] value=[%s%s%s]", color.Grey, newpath, color.End, color.Red, len(node.GetChildren()), color.End, color.Bold, color.Cyan, node.GetNameS(), color.End, color.Yellow, node.GetContentS(), color.End)
+
 		if value := node.GetContentS(); value != "" {
 			if label, has := z.instanceLabelPaths[key]; has {
-				instance.SetLabel(label, value)
-				z.Logger.Trace().Msgf(" > %slabel (%s) [%s] set value (%s)%s", color.Yellow, key, label, value, color.End)
+				// Handling array with comma separated values
+				previousValue := instance.GetLabel(label)
+				if isAppend && previousValue != "" {
+					instance.SetLabel(label, previousValue+","+value)
+					z.Logger.Trace().Msgf(" > %slabel (%s) [%s] set value (%s)%s", color.Yellow, key, label, instance.GetLabel(label)+","+value, color.End)
+				} else {
+					instance.SetLabel(label, value)
+					z.Logger.Trace().Msgf(" > %slabel (%s) [%s] set value (%s)%s", color.Yellow, key, label, value, color.End)
+				}
 				count++
 			} else if metric := mat.GetMetric(key); metric != nil {
 				if err := metric.SetValueString(instance, value); err != nil {
@@ -279,7 +292,12 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) {
 		}
 
 		for _, child := range node.GetChildren() {
-			fetch(instance, child, newpath)
+			if util.HasDuplicates(child.GetAllChildNamesS()) {
+				z.Logger.Debug().Msgf("Array detected for %s", child.GetNameS())
+				fetch(instance, child, newpath, true)
+			} else {
+				fetch(instance, child, newpath, isAppend)
+			}
 		}
 	}
 
@@ -333,7 +351,7 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) {
 					return nil, err
 				}
 			}
-			fetch(instance, instances[0], make([]string, 0))
+			fetch(instance, instances[0], make([]string, 0), false)
 			oldInstances.Remove("cluster")
 			break
 		}
@@ -360,7 +378,7 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) {
 			oldInstances.Remove(key)
 			// clear all instance labels as there are some fields which may be missing between polls
 			instance.ClearLabels()
-			fetch(instance, instanceElem, make([]string, 0))
+			fetch(instance, instanceElem, make([]string, 0), false)
 		}
 	}
 
diff --git a/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go b/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go
new file mode 100644
index 00000000..66dd0d9a
--- /dev/null
+++ b/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go
@@ -0,0 +1,111 @@
+package qospolicyfixed
+
+import (
+	"errors"
+	"fmt"
+	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/matrix"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type QosPolicyFixed struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &QosPolicyFixed{AbstractPlugin: p}
+}
+
+func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+	// Change ZAPI max-throughput/min-throughput to match what REST publishes
+
+	data := dataMap[p.Object]
+	for _, instance := range data.GetInstances() {
+		policyClass := instance.GetLabel("class")
+		if policyClass != "user_defined" {
+			// Only export user_defined policy classes - ignore all others
+			// REST only returns user_defined while ZAPI returns all
+			instance.SetExportable(false)
+			continue
+		}
+		p.setThroughput(instance, "max_xput", "max_throughput_iops", "max_throughput_mbps")
+		p.setThroughput(instance, "min_xput", "min_throughput_iops", "min_throughput_mbps")
+	}
+
+	return nil, nil
+}
+
+func (p *QosPolicyFixed) setThroughput(instance *matrix.Instance, labelName string, iopLabel string, mbpsLabel string) {
+	val := instance.GetLabel(labelName)
+	xput, err := ZapiXputToRest(val)
+	if err != nil {
+		p.Logger.Warn().Str(labelName, val).Msg("Unable to convert label, skipping")
+		return
+	}
+	instance.SetLabel(iopLabel, xput.IOPS)
+	instance.SetLabel(mbpsLabel, xput.Mbps)
+}
+
+var iopsRe = regexp.MustCompile(`(\d+)iops`)
+var bpsRe = regexp.MustCompile(`(\d+(\.\d+)?)(\w+)/s`)
+
+var unitToMb = map[string]float32{
+	"b":  1 / float32(1024*1024),
+	"kb": 1 / float32(1024),
+	"mb": 1,
+	"gb": 1024,
+	"tb": 1024 * 1024,
+}
+
+type MaxXput struct {
+	IOPS string
+	Mbps string
+}
+
+func ZapiXputToRest(zapi string) (MaxXput, error) {
+	lower := strings.ToLower(zapi)
+	empty := MaxXput{IOPS: "0", Mbps: "0"}
+	if lower == "inf" || lower == "0" {
+		return empty, nil
+	}
+
+	// check for a combination
+	before, after, found := strings.Cut(lower, ",")
+	if found {
+		l, err1 := ZapiXputToRest(before)
+		r, err2 := ZapiXputToRest(after)
+		if err1 != nil || err2 != nil {
+			return empty, errors.Join(err1, err2)
+		}
+		return MaxXput{
+			IOPS: l.IOPS,
+			Mbps: r.Mbps,
+		}, nil
+	}
+
+	// check for iops
+	matches := iopsRe.FindStringSubmatch(lower)
+	if len(matches) == 2 {
+		return MaxXput{IOPS: matches[1], Mbps: "0"}, nil
+	}
+
+	// check for bps and normalize units to Mbps
+	matches = bpsRe.FindStringSubmatch(lower)
+	if len(matches) != 4 {
+		return empty, fmt.Errorf("unknown qos-policy format [%s]", zapi)
+	}
+	numStr := matches[1]
+	unit := matches[3]
+	multiple, ok := unitToMb[unit]
+	if !ok {
+		return empty, fmt.Errorf("unknown qos-policy unit [%s] of [%s]", unit, zapi)
+	}
+	num, err := strconv.ParseFloat(numStr, 32)
+	if err != nil {
+		return empty, fmt.Errorf("failed to convert qos-policy unit [%s] of [%s]", numStr, zapi)
+	}
+	mbps := float32(num) * multiple
+	return MaxXput{Mbps: strconv.Itoa(int(mbps)), IOPS: "0"}, nil
+}
diff --git a/cmd/poller/collector/collector.go b/cmd/poller/collector/collector.go
index 3459299f..634855db 100644
--- a/cmd/poller/collector/collector.go
+++ b/cmd/poller/collector/collector.go
@@ -448,9 +448,9 @@ func (c *AbstractCollector) Start(wg *sync.WaitGroup) {
 		if nd := c.Schedule.NextDue(); nd > 0 {
 			c.Logger.Debug().Msgf("sleeping %s until next poll", nd.String()) //DEBUG
 			c.Schedule.Sleep()
-			// log if lagging by more than 50 ms
+			// log if lagging by more than 500 ms
 			// < is used since larger durations are more negative
-		} else if nd.Milliseconds() <= -50 && !c.Schedule.IsStandBy() {
+		} else if nd.Milliseconds() <= -500 && !c.Schedule.IsStandBy() {
 			c.Logger.Warn().
 				Str("lag", (-nd).String()).
 				Msg("lagging behind schedule")
diff --git a/cmd/tools/grafana/units.yaml b/cmd/tools/grafana/units.yaml
index 645e1f73..882950b0 100644
--- a/cmd/tools/grafana/units.yaml
+++ b/cmd/tools/grafana/units.yaml
@@ -3,6 +3,7 @@
 # short  turns 12990 into 13.0 K
 # locale turns 12990 into 12,990
 # string turns 12990 into 12990
+# https://github.com/grafana/grafana/blob/661e44ae6d78dbec91e62161769072f38311ab9f/packages/grafana-data/src/valueFormats/categories.ts#L144
 
 - metric: aggr_space_total
   ontapUnit: bytes
diff --git a/conf/rest/9.12.0/qos_policy_fixed.yaml b/conf/rest/9.12.0/qos_policy_fixed.yaml
new file mode 100644
index 00000000..c15b9893
--- /dev/null
+++ b/conf/rest/9.12.0/qos_policy_fixed.yaml
@@ -0,0 +1,31 @@
+
+name:               QosPolicyFixed
+query:              api/private/cli/qos/policy-group
+object:             qos_policy_fixed
+
+counters:
+  - ^^uuid                => uuid
+  - ^policy_group         => name
+  - ^vserver              => svm
+  - ^class                => class
+  - ^num_workloads        => object_count
+  - ^is_shared            => capacity_shared
+  - ^throughput_policy    => throughput_policy
+  - filter:
+      - class=user_defined
+
+plugins:
+  - QosPolicyFixed
+
+export_options:
+  instance_keys:
+    - svm
+  instance_labels:
+    - name
+    - capacity_shared
+    - max_throughput_iops
+    - max_throughput_mbps
+    - min_throughput_iops
+    - min_throughput_mbps
+    - object_count
+    - class
\ No newline at end of file
diff --git a/conf/rest/9.12.0/volume.yaml b/conf/rest/9.12.0/volume.yaml
index c91e9763..d8be3182 100644
--- a/conf/rest/9.12.0/volume.yaml
+++ b/conf/rest/9.12.0/volume.yaml
@@ -9,6 +9,7 @@ counters:
   - ^name                                         => volume
   - ^svm.name                                     => svm
   - ^aggregates.#.uuid                            => aggrUuid        # handled in plugin for flexgroup
+  - ^aggregates.#.name                            => aggr
   - ^style                                        => style
   - ^type                                         => type
   - ^snapshot_policy.name                         => snapshot_policy
@@ -54,8 +55,7 @@ endpoints:
   - query: api/private/cli/volume
     counters:
       - ^^instance_uuid                           => instance_uuid
-      - ^node                                     => node
-      - ^aggregate                                => aggr
+      - ^nodes                                    => node
       - ^is_sis_volume                            => is_sis_volume
       - compression_space_saved_percent           => sis_compress_saved_percent
       - dedupe_space_saved_percent                => sis_dedup_saved_percent
diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml
index 6ecbb172..e7ed794e 100644
--- a/conf/rest/default.yaml
+++ b/conf/rest/default.yaml
@@ -20,6 +20,7 @@ objects:
   NtpServer:                   ntpserver.yaml
   OntapS3:                     ontap_s3.yaml
   Port:                        port.yaml
+  QosPolicyFixed:              qos_policy_fixed.yaml
   Qtree:                       qtree.yaml
   Security:                    security.yaml
   SecurityAccount:             security_account.yaml
diff --git a/conf/restperf/9.12.0/workload.yaml b/conf/restperf/9.12.0/workload.yaml
index 2128f277..2655a004 100644
--- a/conf/restperf/9.12.0/workload.yaml
+++ b/conf/restperf/9.12.0/workload.yaml
@@ -32,12 +32,13 @@ override:
   - read_io_type_base: delta
 
 qos_labels:
-  - svm.name                  => svm
+  - name         => workload
+  - svm.name     => svm
   - volume
   - qtree
   - lun
   - file
-  - policy.name               => policy_group
+  - policy.name  => policy_group
   - wid
 
 export_options:
@@ -49,3 +50,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/conf/restperf/9.12.0/workload_detail.yaml b/conf/restperf/9.12.0/workload_detail.yaml
index 3e5f84a8..a3e59c1a 100644
--- a/conf/restperf/9.12.0/workload_detail.yaml
+++ b/conf/restperf/9.12.0/workload_detail.yaml
@@ -32,6 +32,11 @@ resource_map:
   DELAY_CENTER_WAFL_SUSP_OTHER: suspend
   DELAY_CENTER_QOS_MIN_THROUGHPUT: qos_min
   DELAY_CENTER_WAFL_SUSP_CP: cp
+  DELAY_CENTER_FLEXCACHE_RAL: flexcache_ral
+  DELAY_CENTER_FLEXCACHE_SPINHI: flexcache_spinhi
+  DELAY_CENTER_SYNC_REPL: sync_repl
+  DELAY_CENTER_COP: cop
+  DELAY_CENTER_WAFL_ADM_CTRL: admission
   # CPU_dblade_background :
   # CPU_exempt :
   # CPU_ha :
@@ -41,13 +46,10 @@ resource_map:
   # CPU_protocol :
   # CPU_raid :
   # CPU_wafl_exempt :
-  # DELAY_CENTER_FLEXCACHE_RAL :
-  # DELAY_CENTER_FLEXCACHE_SPINHI :
-  # DELAY_CENTER_SYNC_REPL :
   # DISK_SSD_OTHER :
 
 qos_labels:
-  - name
+  - name         => workload
   - svm.name     => svm
   - volume
   - qtree
@@ -65,4 +67,5 @@ export_options:
     - qtree
     - lun
     - file
-    - policy_group
\ No newline at end of file
+    - policy_group
+    - workload
\ No newline at end of file
diff --git a/conf/restperf/9.12.0/workload_detail_volume.yaml b/conf/restperf/9.12.0/workload_detail_volume.yaml
index faaec9de..b6143b7f 100644
--- a/conf/restperf/9.12.0/workload_detail_volume.yaml
+++ b/conf/restperf/9.12.0/workload_detail_volume.yaml
@@ -30,6 +30,11 @@ resource_map:
   DELAY_CENTER_WAFL_SUSP_OTHER : suspend
   DELAY_CENTER_QOS_MIN_THROUGHPUT: qos_min
   DELAY_CENTER_WAFL_SUSP_CP: cp
+  DELAY_CENTER_FLEXCACHE_RAL: flexcache_ral
+  DELAY_CENTER_FLEXCACHE_SPINHI: flexcache_spinhi
+  DELAY_CENTER_SYNC_REPL: sync_repl
+  DELAY_CENTER_COP: cop
+  DELAY_CENTER_WAFL_ADM_CTRL: admission
   # CPU_dblade_background : 
   # CPU_exempt : 
   # CPU_ha : 
@@ -38,14 +43,11 @@ resource_map:
   # CPU_network : 
   # CPU_protocol : 
   # CPU_raid : 
-  # CPU_wafl_exempt : 
-  # DELAY_CENTER_FLEXCACHE_RAL : 
-  # DELAY_CENTER_FLEXCACHE_SPINHI : 
-  # DELAY_CENTER_SYNC_REPL : 
+  # CPU_wafl_exempt :
   # DISK_SSD_OTHER :
 
 qos_labels:
-  - name
+  - name          => workload
   - svm.name      => svm
   - volume
   - qtree
@@ -63,3 +65,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/conf/restperf/9.12.0/workload_volume.yaml b/conf/restperf/9.12.0/workload_volume.yaml
index 5d9dcdc8..8565796f 100644
--- a/conf/restperf/9.12.0/workload_volume.yaml
+++ b/conf/restperf/9.12.0/workload_volume.yaml
@@ -33,12 +33,13 @@ override:
   - read_io_type_base: delta
 
 qos_labels:
-  - svm.name                   => svm
+  - name         => workload
+  - svm.name     => svm
   - volume
   - qtree
   - lun
   - file
-  - policy.name                => policy_group
+  - policy.name  => policy_group
   - wid
 
 export_options:
@@ -50,3 +51,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/conf/zapi/cdot/9.8.0/qos_policy_fixed.yaml b/conf/zapi/cdot/9.8.0/qos_policy_fixed.yaml
new file mode 100644
index 00000000..33f099b1
--- /dev/null
+++ b/conf/zapi/cdot/9.8.0/qos_policy_fixed.yaml
@@ -0,0 +1,33 @@
+
+name:             QosPolicyFixed
+query:            qos-policy-group-get-iter
+object:           qos_policy_fixed
+
+counters:
+  qos-policy-group-info:
+    - ^^uuid               => uuid
+    - ^policy-group        => name
+    - ^vserver             => svm
+    - ^is-shared           => capacity_shared
+    - ^max-throughput      => max_xput
+    - ^min-throughput      => min_xput
+    - ^num-workloads       => object_count
+    - ^policy-group-class  => class
+
+collect_only_labels: true
+
+plugins:
+  - QosPolicyFixed
+
+export_options:
+  instance_keys:
+    - svm
+  instance_labels:
+    - name
+    - capacity_shared
+    - max_throughput_iops
+    - max_throughput_mbps
+    - min_throughput_iops
+    - min_throughput_mbps
+    - object_count
+    - class
\ No newline at end of file
diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml
index e1f7506f..f4418c70 100644
--- a/conf/zapi/cdot/9.8.0/volume.yaml
+++ b/conf/zapi/cdot/9.8.0/volume.yaml
@@ -14,12 +14,14 @@ counters:
     - volume-id-attributes:
       - ^^instance-uuid             => instance_uuid
       - ^name                       => volume
-      - ^node                       => node
       - ^owning-vserver-name        => svm
-      - ^containing-aggregate-name  => aggr
+      - aggr-list:
+          - ^aggr-name              => aggr
       - ^containing-aggregate-uuid  => aggrUuid
       - ^style-extended             => style
       - ^type                       => type
+      - nodes:
+          - ^node-name              => node
 
     - volume-inode-attributes:
       - files-used
diff --git a/conf/zapi/default.yaml b/conf/zapi/default.yaml
index e80ea609..a0e283e6 100644
--- a/conf/zapi/default.yaml
+++ b/conf/zapi/default.yaml
@@ -14,6 +14,7 @@ objects:
 #  NetPort:                     netPort.yaml
   Node:                        node.yaml
   NtpServer:                   ntpserver.yaml
+  QosPolicy:                   qos_policy_fixed.yaml
   Qtree:                       qtree.yaml
   Security:                    security.yaml
   SecurityAccount:             security_account.yaml
diff --git a/conf/zapiperf/cdot/9.8.0/workload.yaml b/conf/zapiperf/cdot/9.8.0/workload.yaml
index df2b12a0..65d0e629 100644
--- a/conf/zapiperf/cdot/9.8.0/workload.yaml
+++ b/conf/zapiperf/cdot/9.8.0/workload.yaml
@@ -35,13 +35,14 @@ override:
   - read_io_type_base: delta
 
 qos_labels:
-  - vserver => svm
+  - vserver       => svm
   - volume
   - qtree
   - lun
   - file
   - policy-group
   - wid
+  - workload-name => workload
 
 export_options:
   instance_keys:
@@ -52,3 +53,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/conf/zapiperf/cdot/9.8.0/workload_detail.yaml b/conf/zapiperf/cdot/9.8.0/workload_detail.yaml
index 2aeeda8d..14acae93 100644
--- a/conf/zapiperf/cdot/9.8.0/workload_detail.yaml
+++ b/conf/zapiperf/cdot/9.8.0/workload_detail.yaml
@@ -33,6 +33,11 @@ resource_map:
   DELAY_CENTER_WAFL_SUSP_OTHER: suspend
   DELAY_CENTER_QOS_MIN_THROUGHPUT: qos_min
   DELAY_CENTER_WAFL_SUSP_CP: cp
+  DELAY_CENTER_FLEXCACHE_RAL: flexcache_ral
+  DELAY_CENTER_FLEXCACHE_SPINHI: flexcache_spinhi
+  DELAY_CENTER_SYNC_REPL: sync_repl
+  DELAY_CENTER_COP: cop
+  DELAY_CENTER_WAFL_ADM_CTRL: admission
   # CPU_dblade_background : 
   # CPU_exempt : 
   # CPU_ha : 
@@ -41,21 +46,19 @@ resource_map:
   # CPU_network : 
   # CPU_protocol : 
   # CPU_raid : 
-  # CPU_wafl_exempt : 
-  # DELAY_CENTER_FLEXCACHE_RAL : 
-  # DELAY_CENTER_FLEXCACHE_SPINHI : 
-  # DELAY_CENTER_SYNC_REPL : 
+  # CPU_wafl_exempt :
   # DISK_SSD_OTHER :
 
 qos_labels:
   - name
-  - vserver => svm
+  - vserver       => svm
   - volume
   - qtree
   - lun
   - file
   - policy-group
   - wid
+  - workload-name => workload
 
 export_options:
   instance_keys:
@@ -66,3 +69,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml b/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml
index 5d5c7a99..1b21b064 100644
--- a/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml
+++ b/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml
@@ -32,28 +32,31 @@ resource_map:
   DELAY_CENTER_WAFL_SUSP_OTHER : suspend
   DELAY_CENTER_QOS_MIN_THROUGHPUT: qos_min
   DELAY_CENTER_WAFL_SUSP_CP: cp
-  # CPU_dblade_background : 
-  # CPU_exempt : 
-  # CPU_ha : 
-  # CPU_idle : 
-  # CPU_kahuna : 
-  # CPU_network : 
-  # CPU_protocol : 
-  # CPU_raid : 
-  # CPU_wafl_exempt : 
-  # DELAY_CENTER_FLEXCACHE_RAL : 
-  # DELAY_CENTER_FLEXCACHE_SPINHI : 
-  # DELAY_CENTER_SYNC_REPL : 
-  # DISK_SSD_OTHER :
+  DELAY_CENTER_FLEXCACHE_RAL : flexcache_ral
+  DELAY_CENTER_FLEXCACHE_SPINHI : flexcache_spinhi
+  DELAY_CENTER_SYNC_REPL : sync_repl
+  DELAY_CENTER_COP: cop
+  DELAY_CENTER_WAFL_ADM_CTRL: admission
+#  CPU_dblade_background :
+#  CPU_exempt :
+#  CPU_ha :
+#  CPU_idle :
+#  CPU_kahuna :
+#  CPU_network :
+#  CPU_protocol :
+#  CPU_raid :
+#  CPU_wafl_exempt :
+#  DISK_SSD_OTHER :
 
 qos_labels:
-  - vserver => svm
+  - vserver       => svm
   - volume
   - qtree
   - lun
   - file
   - policy-group
   - wid
+  - workload-name => workload
 
 export_options:
   instance_keys:
@@ -64,3 +67,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/conf/zapiperf/cdot/9.8.0/workload_volume.yaml b/conf/zapiperf/cdot/9.8.0/workload_volume.yaml
index ec33c319..7bb5e8bc 100644
--- a/conf/zapiperf/cdot/9.8.0/workload_volume.yaml
+++ b/conf/zapiperf/cdot/9.8.0/workload_volume.yaml
@@ -36,13 +36,14 @@ override:
   - read_io_type_base: delta
 
 qos_labels:
-  - vserver => svm
+  - vserver       => svm
   - volume
   - qtree
   - lun
   - file
   - policy-group
   - wid
+  - workload-name => workload
 
 export_options:
   instance_keys:
@@ -53,3 +54,4 @@ export_options:
     - lun
     - file
     - policy_group
+    - workload
diff --git a/go.mod b/go.mod
index d1d4d3df..4ecf405a 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/bbrks/wrap/v2 v2.5.0
 	github.com/go-openapi/spec v0.20.8
 	github.com/hashicorp/go-version v1.6.0
-	github.com/imdario/mergo v0.3.13
+	github.com/imdario/mergo v0.3.14
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/rs/zerolog v1.29.0
 	github.com/shirou/gopsutil/v3 v3.23.2
diff --git a/go.sum b/go.sum
index b56d2046..1ff6e354 100644
--- a/go.sum
+++ b/go.sum
@@ -27,8 +27,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
-github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/imdario/mergo v0.3.14 h1:fOqeC1+nCuuk6PKQdg9YmosXX7Y7mHX6R/0ZldI9iHo=
+github.com/imdario/mergo v0.3.14/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -137,6 +137,5 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/grafana/dashboards/cmode/delay_center.json b/grafana/dashboards/cmode/delay_center.json
new file mode 100644
index 00000000..2f6a5001
--- /dev/null
+++ b/grafana/dashboards/cmode/delay_center.json
@@ -0,0 +1,2488 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "8.1.8"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "text",
+      "name": "Text",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "",
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "id": null,
+  "iteration": 1678947394838,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "cdot"
+      ],
+      "targetBlank": false,
+      "title": "Related Dashboards",
+      "tooltip": "",
+      "type": "dashboards",
+      "url": ""
+    }
+  ],
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 125,
+      "panels": [],
+      "title": "Important Information",
+      "type": "row"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 4,
+        "w": 24,
+        "x": 0,
+        "y": 1
+      },
+      "id": 107,
+      "options": {
+        "content": "This dashboard requires Workload/QOS counter templates. Two actions are required to use this dashboard: <br>\n1. If you are using the ZapiPerf collector, then you must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml. <br>\n2. If you are using the RestPerf Collector, then you must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml. <br>\n\nMore information about [ONTAP Performance](https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide).",
+        "mode": "markdown"
+      },
+      "pluginVersion": "8.1.8",
+      "type": "text"
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 123,
+      "panels": [],
+      "title": "Delay Center",
+      "type": "row"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 2,
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 103,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "(avg(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\"}) by (resource))",
+          "interval": "",
+          "legendFormat": "{{resource}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Latency by Delay center",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 17
+      },
+      "id": 105,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFrontend\",resource=\"frontend\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFrontend\",resource=\"frontend\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for CPU N-blade (frontend)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "id": 104,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"}) by (workload)/ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"}))",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for CPU D-blade (backend)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "id": 108,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Network",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 27
+      },
+      "id": 110,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Cluster Interconnect",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 37
+      },
+      "id": 109,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Qos Limit (throttle)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 37
+      },
+      "id": 113,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSQosMin\",resource=\"qos_min\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSQosMin\",resource=\"qos_min\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Qos Min Throughput (qos_min)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 47
+      },
+      "id": 111,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for WAFL Susp CP",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 47
+      },
+      "id": 112,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for WAFL Susp Other",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 57
+      },
+      "id": 114,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Disk IO",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 57
+      },
+      "id": 115,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Cloud IO",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 67
+      },
+      "id": 116,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNVLog\",resource=\"nvlog\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNVLog\",resource=\"nvlog\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for NVLog Transfer",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 67
+      },
+      "id": 117,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheRAL\",resource=\"flexcache_ral\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheRAL\",resource=\"flexcache_ral\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for FlexCache RAL",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 77
+      },
+      "id": 118,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheSpinhi\",resource=\"flexcache_spinhi\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheSpinhi\",resource=\"flexcache_spinhi\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for FlexCache Spinhi",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 77
+      },
+      "id": 119,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSyncRepl\",resource=\"sync_repl\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSyncRepl\",resource=\"sync_repl\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for Sync Repl",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 87
+      },
+      "id": 120,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCOP\",resource=\"cop\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCOP\",resource=\"cop\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for COP",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "decimals": 1,
+          "mappings": [
+            {
+              "options": {
+                "match": "null+nan",
+                "result": {
+                  "index": 0,
+                  "text": "0%"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 87
+      },
+      "id": 121,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.1.8",
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSWaflAdmission\",resource=\"admission\"}) by (workload)) / ignoring(workload) group_left  sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSWaflAdmission\",resource=\"admission\"})",
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{workload}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Top $TopResources Workloads for WAFL Admission",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 30,
+  "style": "dark",
+  "tags": [
+    "harvest",
+    "ontap",
+    "cdot"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": false,
+        "label": "Data Source",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "label_values(volume_labels{system_type!=\"7mode\"}, datacenter)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": true,
+        "name": "Datacenter",
+        "options": [],
+        "query": {
+          "query": "label_values(volume_labels{system_type!=\"7mode\"}, datacenter)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": true,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, workload)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Workload",
+        "options": [],
+        "query": {
+          "query": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, workload)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}, volume)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Volume",
+        "options": [],
+        "query": {
+          "query": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}, volume)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "selected": false,
+          "text": "5",
+          "value": "5"
+        },
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "TopResources",
+        "options": [
+          {
+            "selected": false,
+            "text": "1",
+            "value": "1"
+          },
+          {
+            "selected": false,
+            "text": "2",
+            "value": "2"
+          },
+          {
+            "selected": false,
+            "text": "3",
+            "value": "3"
+          },
+          {
+            "selected": false,
+            "text": "4",
+            "value": "4"
+          },
+          {
+            "selected": true,
+            "text": "5",
+            "value": "5"
+          },
+          {
+            "selected": false,
+            "text": "6",
+            "value": "6"
+          },
+          {
+            "selected": false,
+            "text": "8",
+            "value": "8"
+          },
+          {
+            "selected": false,
+            "text": "10",
+            "value": "10"
+          },
+          {
+            "selected": false,
+            "text": "15",
+            "value": "15"
+          },
+          {
+            "selected": false,
+            "text": "25",
+            "value": "25"
+          },
+          {
+            "selected": false,
+            "text": "50",
+            "value": "50"
+          },
+          {
+            "selected": false,
+            "text": "100",
+            "value": "100"
+          },
+          {
+            "selected": false,
+            "text": "250",
+            "value": "250"
+          },
+          {
+            "selected": false,
+            "text": "500",
+            "value": "500"
+          }
+        ],
+        "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSBackend",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSFrontend",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSThrottle",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSNetwork",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSCluster",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSCP",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSQosMin",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSSuspend",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSDisk",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSCloud",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSNVLog",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_ral\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSFlexcacheRAL",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_ral\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_spinhi\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSFlexcacheSpinhi",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_spinhi\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"sync_repl\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSSyncRepl",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"sync_repl\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cop\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSCOP",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cop\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"admission\"}[${__range}])))",
+        "description": null,
+        "error": null,
+        "hide": 2,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "TopVolumeQOSWaflAdmission",
+        "options": [],
+        "query": {
+          "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"admission\"}[${__range}])))",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": ".*volume=\\\"(.*?)\\\".*",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "ONTAP: Delay Center",
+  "uid": "",
+  "version": 1
+}
\ No newline at end of file
diff --git a/grafana/dashboards/cmode/disk.json b/grafana/dashboards/cmode/disk.json
index f01b4a5a..059ba34c 100644
--- a/grafana/dashboards/cmode/disk.json
+++ b/grafana/dashboards/cmode/disk.json
@@ -1189,7 +1189,7 @@
               }
             ]
           },
-          "unit": "short"
+          "unit": "kbytes"
         },
         "overrides": []
       },
@@ -1284,7 +1284,7 @@
               }
             ]
           },
-          "unit": "deckbytes"
+          "unit": "kbytes"
         },
         "overrides": []
       },
diff --git a/grafana/dashboards/cmode/lun.json b/grafana/dashboards/cmode/lun.json
index 48b044bf..55e92eb5 100644
--- a/grafana/dashboards/cmode/lun.json
+++ b/grafana/dashboards/cmode/lun.json
@@ -71,7 +71,7 @@
   "gnetId": null,
   "graphTooltip": 1,
   "id": null,
-  "iteration": 1664349128957,
+  "iteration": 1678897124924,
   "links": [
     {
       "asDropdown": true,
@@ -1369,7 +1369,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 33
+        "y": 7
       },
       "id": 28,
       "panels": [
@@ -1932,7 +1932,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 34
+        "y": 8
       },
       "id": 55,
       "panels": [
@@ -2253,21 +2253,21 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 35
+        "y": 9
       },
       "id": 61,
       "panels": [
         {
           "datasource": "${DS_PROMETHEUS}",
           "gridPos": {
-            "h": 3,
+            "h": 4,
             "w": 24,
             "x": 0,
-            "y": 36
+            "y": 10
           },
           "id": 73,
           "options": {
-            "content": "You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.\n",
+            "content": "This dashboard requires Workload/QOS counters templates. Two actions are required to use this dashboard: <br>\n1. If you are using ZapiPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml. <br>\n2. If you are using RestPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml. <br>\n\nRefer ONTAP Performance - Resolution Guide https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide",
             "mode": "markdown"
           },
           "pluginVersion": "8.1.8",
@@ -2324,7 +2324,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2332,7 +2332,7 @@
             "h": 9,
             "w": 8,
             "x": 0,
-            "y": 39
+            "y": 15
           },
           "id": 63,
           "options": {
@@ -2353,9 +2353,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNetwork\",resource=\"network\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNetwork\",resource=\"network\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2415,7 +2415,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2423,7 +2423,7 @@
             "h": 9,
             "w": 8,
             "x": 8,
-            "y": 39
+            "y": 15
           },
           "id": 64,
           "options": {
@@ -2444,9 +2444,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyThrottle\",resource=\"throttle\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyThrottle\",resource=\"throttle\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2506,7 +2506,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2514,7 +2514,7 @@
             "h": 9,
             "w": 8,
             "x": 16,
-            "y": 39
+            "y": 15
           },
           "id": 95,
           "options": {
@@ -2535,9 +2535,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyMin\",resource=\"qos_min\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyMin\",resource=\"qos_min\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2596,7 +2596,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2604,7 +2604,7 @@
             "h": 9,
             "w": 8,
             "x": 0,
-            "y": 48
+            "y": 24
           },
           "id": 66,
           "options": {
@@ -2625,9 +2625,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCluster\",resource=\"cluster\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCluster\",resource=\"cluster\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2686,7 +2686,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2694,7 +2694,7 @@
             "h": 9,
             "w": 8,
             "x": 8,
-            "y": 48
+            "y": 24
           },
           "id": 67,
           "options": {
@@ -2715,9 +2715,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyBackend\",resource=\"backend\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyBackend\",resource=\"backend\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2777,7 +2777,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2785,7 +2785,7 @@
             "h": 9,
             "w": 8,
             "x": 16,
-            "y": 48
+            "y": 24
           },
           "id": 97,
           "options": {
@@ -2806,9 +2806,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCP\",resource=\"cp\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCP\",resource=\"cp\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2867,7 +2867,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2875,7 +2875,7 @@
             "h": 9,
             "w": 8,
             "x": 0,
-            "y": 57
+            "y": 33
           },
           "id": 69,
           "options": {
@@ -2897,9 +2897,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencySuspend\",resource=\"suspend\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencySuspend\",resource=\"suspend\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}} ",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}} ",
               "refId": "A"
             }
           ],
@@ -2959,7 +2959,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -2967,7 +2967,7 @@
             "h": 9,
             "w": 8,
             "x": 8,
-            "y": 57
+            "y": 33
           },
           "id": 70,
           "options": {
@@ -2988,9 +2988,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCloud\",resource=\"cloud\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCloud\",resource=\"cloud\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}} ",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}} ",
               "refId": "A"
             }
           ],
@@ -3049,7 +3049,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3057,7 +3057,7 @@
             "h": 9,
             "w": 8,
             "x": 16,
-            "y": 57
+            "y": 33
           },
           "id": 65,
           "options": {
@@ -3078,9 +3078,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyFrontend\",resource=\"frontend\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyFrontend\",resource=\"frontend\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3139,7 +3139,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3147,7 +3147,7 @@
             "h": 9,
             "w": 8,
             "x": 0,
-            "y": 66
+            "y": 42
           },
           "id": 71,
           "options": {
@@ -3168,9 +3168,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNvlog\",resource=\"nvlog\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNvlog\",resource=\"nvlog\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}} ",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}} ",
               "refId": "A"
             }
           ],
@@ -3229,7 +3229,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3237,7 +3237,7 @@
             "h": 9,
             "w": 8,
             "x": 8,
-            "y": 66
+            "y": 42
           },
           "id": 68,
           "options": {
@@ -3258,9 +3258,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyDisk\",resource=\"disk\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyDisk\",resource=\"disk\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3280,7 +3280,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 36
+        "y": 10
       },
       "id": 75,
       "panels": [
@@ -3845,7 +3845,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 37
+        "y": 11
       },
       "id": 85,
       "panels": [
@@ -5321,5 +5321,5 @@
   "timezone": "",
   "title": "ONTAP: LUN",
   "uid": "",
-  "version": 5
-}
+  "version": 6
+}
\ No newline at end of file
diff --git a/grafana/dashboards/cmode/smb2.json b/grafana/dashboards/cmode/smb2.json
index 84162574..30e184f4 100644
--- a/grafana/dashboards/cmode/smb2.json
+++ b/grafana/dashboards/cmode/smb2.json
@@ -59,7 +59,7 @@
   "gnetId": null,
   "graphTooltip": 1,
   "id": null,
-  "iteration": 1677735866675,
+  "iteration": 1679387873656,
   "links": [
     {
       "asDropdown": true,
@@ -1131,7 +1131,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "sum(increase(smb2_session_setup_latency_histogram_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[$__interval])) by (le)",
+              "expr": "sum(increase(smb2_oplock_break_latency_histogram_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[$__interval])) by (le)",
               "format": "heatmap",
               "instant": false,
               "interval": "",
@@ -1141,7 +1141,7 @@
             }
           ],
           "timeFrom": null,
-          "title": "Session Setup Latency Heatmap",
+          "title": "Oplock Break Latency Heatmap",
           "tooltip": {
             "show": true,
             "showHistogram": false
@@ -1301,5 +1301,5 @@
   "timezone": "",
   "title": "ONTAP: SMB2",
   "uid": "",
-  "version": 1
+  "version": 2
 }
\ No newline at end of file
diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json
index 8b919492..6bdfe319 100644
--- a/grafana/dashboards/cmode/volume.json
+++ b/grafana/dashboards/cmode/volume.json
@@ -71,7 +71,7 @@
   "gnetId": null,
   "graphTooltip": 1,
   "id": null,
-  "iteration": 1670268827108,
+  "iteration": 1678952925478,
   "links": [
     {
       "asDropdown": true,
@@ -2193,6 +2193,23 @@
       },
       "id": 66,
       "panels": [
+        {
+          "datasource": "${DS_PROMETHEUS}",
+          "gridPos": {
+            "h": 4,
+            "w": 24,
+            "x": 0,
+            "y": 16
+          },
+          "id": 91,
+          "options": {
+            "content": "This dashboard requires Workload/QOS counters templates. Two actions are required to use this dashboard: <br>\n1. If you are using ZapiPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml. <br>\n2. If you are using RestPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml. <br>\n\nRefer ONTAP Performance - Resolution Guide https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide",
+            "mode": "markdown"
+          },
+          "pluginVersion": "8.1.8",
+          "title": "Important Information on Top Volume QoS Resource Latency Graphs",
+          "type": "text"
+        },
         {
           "datasource": "${DS_PROMETHEUS}",
           "fieldConfig": {
@@ -2243,15 +2260,15 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
           "gridPos": {
             "h": 10,
-            "w": 8,
+            "w": 12,
             "x": 0,
-            "y": 4
+            "y": 21
           },
           "id": 72,
           "options": {
@@ -2272,9 +2289,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadLatency\"})/1000",
+              "expr": "topk($TopResources, qos_volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadLatency\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2333,17 +2350,17 @@
                   }
                 ]
               },
-              "unit": "Bps"
+              "unit": "µs"
             },
             "overrides": []
           },
           "gridPos": {
             "h": 10,
-            "w": 8,
-            "x": 8,
-            "y": 4
+            "w": 12,
+            "x": 12,
+            "y": 21
           },
-          "id": 74,
+          "id": 73,
           "options": {
             "legend": {
               "calcs": [
@@ -2362,15 +2379,15 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadData\"})",
+              "expr": "topk($TopResources, qos_volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteLatency\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Top $TopResources Volumes by QoS Volume Read Throughput",
+          "title": "Top $TopResources Volumes by QoS Volume Write Latency",
           "type": "timeseries"
         },
         {
@@ -2429,9 +2446,9 @@
           },
           "gridPos": {
             "h": 10,
-            "w": 8,
-            "x": 16,
-            "y": 4
+            "w": 12,
+            "x": 0,
+            "y": 31
           },
           "id": 76,
           "options": {
@@ -2454,7 +2471,7 @@
               "exemplar": false,
               "expr": "topk($TopResources, qos_volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadOps\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2513,17 +2530,17 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "iops"
             },
             "overrides": []
           },
           "gridPos": {
             "h": 10,
-            "w": 8,
-            "x": 0,
-            "y": 14
+            "w": 12,
+            "x": 12,
+            "y": 31
           },
-          "id": 73,
+          "id": 77,
           "options": {
             "legend": {
               "calcs": [
@@ -2542,15 +2559,15 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteLatency\"})/1000",
+              "expr": "topk($TopResources, qos_volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteOps\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Top $TopResources Volumes by QoS Volume Write Latency",
+          "title": "Top $TopResources Volumes by QoS Volume Write IOPS",
           "type": "timeseries"
         },
         {
@@ -2609,11 +2626,11 @@
           },
           "gridPos": {
             "h": 10,
-            "w": 8,
-            "x": 8,
-            "y": 14
+            "w": 12,
+            "x": 0,
+            "y": 41
           },
-          "id": 75,
+          "id": 74,
           "options": {
             "legend": {
               "calcs": [
@@ -2632,15 +2649,15 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteData\"})",
+              "expr": "topk($TopResources, qos_volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadData\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Top $TopResources Volumes by QoS Volume Write Throughput",
+          "title": "Top $TopResources Volumes by QoS Volume Read Throughput",
           "type": "timeseries"
         },
         {
@@ -2693,17 +2710,17 @@
                   }
                 ]
               },
-              "unit": "iops"
+              "unit": "Bps"
             },
             "overrides": []
           },
           "gridPos": {
             "h": 10,
-            "w": 8,
-            "x": 16,
-            "y": 14
+            "w": 12,
+            "x": 12,
+            "y": 41
           },
-          "id": 77,
+          "id": 75,
           "options": {
             "legend": {
               "calcs": [
@@ -2722,15 +2739,15 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteOps\"})",
+              "expr": "topk($TopResources, qos_volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteData\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Top $TopResources Volumes by QoS Volume Write IOPS",
+          "title": "Top $TopResources Volumes by QoS Volume Write Throughput",
           "type": "timeseries"
         },
         {
@@ -2769,6 +2786,8 @@
                 }
               },
               "mappings": [],
+              "max": 1,
+              "min": 0,
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
@@ -2782,9 +2801,7 @@
                   }
                 ]
               },
-              "unit": "percentunit",
-              "min": 0,
-              "max": 1
+              "unit": "percentunit"
             },
             "overrides": []
           },
@@ -2792,7 +2809,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 24
+            "y": 51
           },
           "id": 79,
           "options": {
@@ -2815,7 +2832,7 @@
               "exemplar": false,
               "expr": "topk($TopResources, qos_volume_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSequentialReads\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2883,7 +2900,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 24
+            "y": 51
           },
           "id": 80,
           "options": {
@@ -2906,7 +2923,7 @@
               "exemplar": false,
               "expr": "topk($TopResources, qos_volume_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSequentialWrites\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -2933,14 +2950,14 @@
         {
           "datasource": "${DS_PROMETHEUS}",
           "gridPos": {
-            "h": 3,
+            "h": 5,
             "w": 24,
             "x": 0,
-            "y": 5
+            "y": 17
           },
-          "id": 91,
+          "id": 103,
           "options": {
-            "content": "You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.\n",
+            "content": "This dashboard requires Workload/QOS counters templates. Two actions are required to use this dashboard: <br>\n1. If you are using ZapiPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml. <br>\n2. If you are using RestPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml. <br>\n\nRefer ONTAP Performance - Resolution Guide https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide",
             "mode": "markdown"
           },
           "pluginVersion": "8.1.8",
@@ -2997,7 +3014,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3005,7 +3022,7 @@
             "h": 11,
             "w": 8,
             "x": 0,
-            "y": 8
+            "y": 22
           },
           "id": 48,
           "options": {
@@ -3026,9 +3043,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3088,7 +3105,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3096,7 +3113,7 @@
             "h": 11,
             "w": 8,
             "x": 8,
-            "y": 8
+            "y": 22
           },
           "id": 50,
           "options": {
@@ -3117,9 +3134,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3179,7 +3196,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3187,7 +3204,7 @@
             "h": 11,
             "w": 8,
             "x": 16,
-            "y": 8
+            "y": 22
           },
           "id": 88,
           "options": {
@@ -3208,9 +3225,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSMin\",resource=\"qos_min\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSMin\",resource=\"qos_min\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3269,7 +3286,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3277,7 +3294,7 @@
             "h": 11,
             "w": 8,
             "x": 0,
-            "y": 19
+            "y": 33
           },
           "id": 54,
           "options": {
@@ -3298,9 +3315,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3359,7 +3376,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3367,7 +3384,7 @@
             "h": 11,
             "w": 8,
             "x": 8,
-            "y": 19
+            "y": 33
           },
           "id": 56,
           "options": {
@@ -3388,9 +3405,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3450,7 +3467,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3458,7 +3475,7 @@
             "h": 11,
             "w": 8,
             "x": 16,
-            "y": 19
+            "y": 33
           },
           "id": 89,
           "options": {
@@ -3479,9 +3496,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3540,7 +3557,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3548,7 +3565,7 @@
             "h": 11,
             "w": 8,
             "x": 0,
-            "y": 30
+            "y": 44
           },
           "id": 60,
           "options": {
@@ -3570,9 +3587,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}} ",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}} ",
               "refId": "A"
             }
           ],
@@ -3632,7 +3649,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3640,7 +3657,7 @@
             "h": 11,
             "w": 8,
             "x": 8,
-            "y": 30
+            "y": 44
           },
           "id": 62,
           "options": {
@@ -3661,9 +3678,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}} ",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}} ",
               "refId": "A"
             }
           ],
@@ -3722,7 +3739,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3730,7 +3747,7 @@
             "h": 11,
             "w": 8,
             "x": 16,
-            "y": 30
+            "y": 44
           },
           "id": 52,
           "options": {
@@ -3751,9 +3768,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSfrontend\",resource=\"frontend\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSfrontend\",resource=\"frontend\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -3812,7 +3829,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3820,7 +3837,7 @@
             "h": 11,
             "w": 8,
             "x": 0,
-            "y": 41
+            "y": 55
           },
           "id": 64,
           "options": {
@@ -3841,9 +3858,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNvlog\",resource=\"nvlog\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNvlog\",resource=\"nvlog\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}} ",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}} ",
               "refId": "A"
             }
           ],
@@ -3902,7 +3919,7 @@
                   }
                 ]
               },
-              "unit": "ms"
+              "unit": "µs"
             },
             "overrides": []
           },
@@ -3910,7 +3927,7 @@
             "h": 11,
             "w": 8,
             "x": 8,
-            "y": 41
+            "y": 55
           },
           "id": 58,
           "options": {
@@ -3931,9 +3948,9 @@
           "targets": [
             {
               "exemplar": false,
-              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"})/1000",
+              "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"})",
               "interval": "",
-              "legendFormat": "{{svm}} / {{volume}}",
+              "legendFormat": "{{svm}} / {{workload}} / {{volume}}",
               "refId": "A"
             }
           ],
@@ -6235,5 +6252,5 @@
   "timezone": "",
   "title": "ONTAP: Volume",
   "uid": "",
-  "version": 8
+  "version": 9
 }
\ No newline at end of file
diff --git a/pkg/util/util.go b/pkg/util/util.go
index f96bc6fa..e5297274 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -391,3 +391,17 @@ func EncodeURL(href string) (string, error) {
 	u.RawQuery = u.Query().Encode()
 	return u.RequestURI(), nil
 }
+
+func HasDuplicates(slice []string) bool {
+	encountered := map[string]bool{}
+
+	for _, v := range slice {
+		if encountered[v] {
+			return true
+		} else {
+			encountered[v] = true
+		}
+	}
+
+	return false
+}
diff --git a/vendor/github.com/imdario/mergo/CONTRIBUTING.md b/vendor/github.com/imdario/mergo/CONTRIBUTING.md
new file mode 100644
index 00000000..0a1ff9f9
--- /dev/null
+++ b/vendor/github.com/imdario/mergo/CONTRIBUTING.md
@@ -0,0 +1,112 @@
+<!-- omit in toc -->
+# Contributing to mergo
+
+First off, thanks for taking the time to contribute! ❤️
+
+All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them. Please make sure to read the relevant section before making your contribution. It will make it a lot easier for us maintainers and smooth out the experience for all involved. The community looks forward to your contributions. 🎉
+
+> And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
+> - Star the project
+> - Tweet about it
+> - Refer this project in your project's readme
+> - Mention the project at local meetups and tell your friends/colleagues
+
+<!-- omit in toc -->
+## Table of Contents
+
+- [Code of Conduct](#code-of-conduct)
+- [I Have a Question](#i-have-a-question)
+- [I Want To Contribute](#i-want-to-contribute)
+- [Reporting Bugs](#reporting-bugs)
+- [Suggesting Enhancements](#suggesting-enhancements)
+
+## Code of Conduct
+
+This project and everyone participating in it is governed by the
+[mergo Code of Conduct](https://github.com/imdario/mergoblob/master/CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code. Please report unacceptable behavior
+to <>.
+
+
+## I Have a Question
+
+> If you want to ask a question, we assume that you have read the available [Documentation](https://pkg.go.dev/github.com/imdario/mergo).
+
+Before you ask a question, it is best to search for existing [Issues](https://github.com/imdario/mergo/issues) that might help you. In case you have found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to search the internet for answers first.
+
+If you then still feel the need to ask a question and need clarification, we recommend the following:
+
+- Open an [Issue](https://github.com/imdario/mergo/issues/new).
+- Provide as much context as you can about what you're running into.
+- Provide project and platform versions (nodejs, npm, etc), depending on what seems relevant.
+
+We will then take care of the issue as soon as possible.
+
+## I Want To Contribute
+
+> ### Legal Notice <!-- omit in toc -->
+> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+
+### Reporting Bugs
+
+<!-- omit in toc -->
+#### Before Submitting a Bug Report
+
+A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to investigate carefully, collect information and describe the issue in detail in your report. Please complete the following steps in advance to help us fix any potential bug as fast as possible.
+
+- Make sure that you are using the latest version.
+- Determine if your bug is really a bug and not an error on your side e.g. using incompatible environment components/versions (Make sure that you have read the [documentation](). If you are looking for support, you might want to check [this section](#i-have-a-question)).
+- To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [bug tracker](https://github.com/imdario/mergoissues?q=label%3Abug).
+- Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue.
+- Collect information about the bug:
+- Stack trace (Traceback)
+- OS, Platform and Version (Windows, Linux, macOS, x86, ARM)
+- Version of the interpreter, compiler, SDK, runtime environment, package manager, depending on what seems relevant.
+- Possibly your input and the output
+- Can you reliably reproduce the issue? And can you also reproduce it with older versions?
+
+<!-- omit in toc -->
+#### How Do I Submit a Good Bug Report?
+
+> You must never report security related issues, vulnerabilities or bugs including sensitive information to the issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by email to .
+<!-- You may add a PGP key to allow the messages to be sent encrypted as well. -->
+
+We use GitHub issues to track bugs and errors. If you run into an issue with the project:
+
+- Open an [Issue](https://github.com/imdario/mergo/issues/new). (Since we can't be sure at this point whether it is a bug or not, we ask you not to talk about a bug yet and not to label the issue.)
+- Explain the behavior you would expect and the actual behavior.
+- Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This usually includes your code. For good bug reports you should isolate the problem and create a reduced test case.
+- Provide the information you collected in the previous section.
+
+Once it's filed:
+
+- The project team will label the issue accordingly.
+- A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced.
+- If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be implemented by someone.
+
+### Suggesting Enhancements
+
+This section guides you through submitting an enhancement suggestion for mergo, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions.
+
+<!-- omit in toc -->
+#### Before Submitting an Enhancement
+
+- Make sure that you are using the latest version.
+- Read the [documentation]() carefully and find out if the functionality is already covered, maybe by an individual configuration.
+- Perform a [search](https://github.com/imdario/mergo/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
+- Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library.
+
+<!-- omit in toc -->
+#### How Do I Submit a Good Enhancement Suggestion?
+
+Enhancement suggestions are tracked as [GitHub issues](https://github.com/imdario/mergo/issues).
+
+- Use a **clear and descriptive title** for the issue to identify the suggestion.
+- Provide a **step-by-step description of the suggested enhancement** in as many details as possible.
+- **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you.
+- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. <!-- this should only be included if the project has a GUI -->
+- **Explain why this enhancement would be useful** to most mergo users. You may also want to point out the other projects that solved it better and which could serve as inspiration.
+
+<!-- omit in toc -->
+## Attribution
+This guide is based on the **contributing-gen**. [Make your own](https://github.com/bttger/contributing-gen)!
diff --git a/vendor/github.com/imdario/mergo/README.md b/vendor/github.com/imdario/mergo/README.md
index 7e6f7aee..4f028749 100644
--- a/vendor/github.com/imdario/mergo/README.md
+++ b/vendor/github.com/imdario/mergo/README.md
@@ -1,6 +1,5 @@
 # Mergo
 
-
 [![GoDoc][3]][4]
 [![GitHub release][5]][6]
 [![GoCard][7]][8]
@@ -9,6 +8,7 @@
 [![Sourcegraph][11]][12]
 [![FOSSA Status][13]][14]
 [![Become my sponsor][15]][16]
+[![Tidelift][17]][18]
 
 [1]: https://travis-ci.org/imdario/mergo.png
 [2]: https://travis-ci.org/imdario/mergo
@@ -26,6 +26,8 @@
 [14]: https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield
 [15]: https://img.shields.io/github/sponsors/imdario
 [16]: https://github.com/sponsors/imdario
+[17]: https://tidelift.com/badges/package/go/github.com%2Fimdario%2Fmergo
+[18]: https://tidelift.com/subscription/pkg/go-github.com-imdario-mergo
 
 A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements.
 
@@ -55,7 +57,6 @@ If Mergo is useful to you, consider buying me a coffee, a beer, or making a mont
 
 ### Mergo in the wild
 
-- [cli/cli](https://github.com/cli/cli)
 - [moby/moby](https://github.com/moby/moby)
 - [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes)
 - [vmware/dispatch](https://github.com/vmware/dispatch)
diff --git a/vendor/github.com/imdario/mergo/SECURITY.md b/vendor/github.com/imdario/mergo/SECURITY.md
new file mode 100644
index 00000000..a5de61f7
--- /dev/null
+++ b/vendor/github.com/imdario/mergo/SECURITY.md
@@ -0,0 +1,14 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.3.x   | :white_check_mark: |
+| < 0.3   | :x:                |
+
+## Security contact information
+
+To report a security vulnerability, please use the
+[Tidelift security contact](https://tidelift.com/security).
+Tidelift will coordinate the fix and disclosure.
diff --git a/vendor/github.com/imdario/mergo/map.go b/vendor/github.com/imdario/mergo/map.go
index a13a7ee4..f739255a 100644
--- a/vendor/github.com/imdario/mergo/map.go
+++ b/vendor/github.com/imdario/mergo/map.go
@@ -44,7 +44,7 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf
 			}
 		}
 		// Remember, remember...
-		visited[h] = &visit{addr, typ, seen}
+		visited[h] = &visit{typ, seen, addr}
 	}
 	zeroValue := reflect.Value{}
 	switch dst.Kind() {
diff --git a/vendor/github.com/imdario/mergo/merge.go b/vendor/github.com/imdario/mergo/merge.go
index 8b4e2f47..1bf26cfe 100644
--- a/vendor/github.com/imdario/mergo/merge.go
+++ b/vendor/github.com/imdario/mergo/merge.go
@@ -38,10 +38,10 @@ func isExportedComponent(field *reflect.StructField) bool {
 }
 
 type Config struct {
+	Transformers                 Transformers
 	Overwrite                    bool
 	AppendSlice                  bool
 	TypeCheck                    bool
-	Transformers                 Transformers
 	overwriteWithEmptyValue      bool
 	overwriteSliceWithEmptyValue bool
 	sliceDeepCopy                bool
@@ -76,7 +76,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co
 			}
 		}
 		// Remember, remember...
-		visited[h] = &visit{addr, typ, seen}
+		visited[h] = &visit{typ, seen, addr}
 	}
 
 	if config.Transformers != nil && !isReflectNil(dst) && dst.IsValid() {
@@ -205,6 +205,16 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co
 				dst.SetMapIndex(key, srcElement)
 			}
 		}
+
+		// Ensure that all keys in dst are deleted if they are not in src.
+		if overwriteWithEmptySrc {
+			for _, key := range dst.MapKeys() {
+				srcElement := src.MapIndex(key)
+				if !srcElement.IsValid() {
+					dst.SetMapIndex(key, reflect.Value{})
+				}
+			}
+		}
 	case reflect.Slice:
 		if !dst.CanSet() {
 			break
diff --git a/vendor/github.com/imdario/mergo/mergo.go b/vendor/github.com/imdario/mergo/mergo.go
index 9fe362d4..d2af2a95 100644
--- a/vendor/github.com/imdario/mergo/mergo.go
+++ b/vendor/github.com/imdario/mergo/mergo.go
@@ -28,9 +28,9 @@ var (
 // checks in progress are true when it reencounters them.
 // Visited are stored in a map indexed by 17 * a1 + a2;
 type visit struct {
-	ptr  uintptr
 	typ  reflect.Type
 	next *visit
+	ptr  uintptr
 }
 
 // From src/pkg/encoding/json/encode.go.
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 0eb4df9e..ff8563a1 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -21,7 +21,7 @@ github.com/go-openapi/swag
 # github.com/hashicorp/go-version v1.6.0
 ## explicit
 github.com/hashicorp/go-version
-# github.com/imdario/mergo v0.3.13
+# github.com/imdario/mergo v0.3.14
 ## explicit; go 1.13
 github.com/imdario/mergo
 # github.com/inconshreveable/mousetrap v1.1.0


Test patch:
diff --git a/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed_test.go b/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed_test.go
new file mode 100644
index 00000000..a4dcfd06
--- /dev/null
+++ b/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed_test.go
@@ -0,0 +1,43 @@
+package qospolicyfixed
+
+import (
+	"strconv"
+	"testing"
+)
+
+func Test_zapiXputToRest(t *testing.T) {
+	tests := []struct {
+		zapi  string
+		want  MaxXput
+		isErr bool
+	}{
+		{zapi: "100IOPS", want: MaxXput{IOPS: "100", Mbps: "0"}},
+		{zapi: "100iops", want: MaxXput{IOPS: "100", Mbps: "0"}},
+		{zapi: "111111IOPS", want: MaxXput{IOPS: "111111", Mbps: "0"}},
+		{zapi: "0", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "INF", want: MaxXput{IOPS: "0", Mbps: "0"}},
+
+		{zapi: "1GB/s", want: MaxXput{IOPS: "0", Mbps: "1024"}},
+		{zapi: "100B/s", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "10KB/s", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "1mb/s", want: MaxXput{IOPS: "0", Mbps: "1"}},
+		{zapi: "1tb/s", want: MaxXput{IOPS: "0", Mbps: "1048576"}},
+		{zapi: "1000KB/s", want: MaxXput{IOPS: "0", Mbps: "0"}},
+		{zapi: "15000IOPS,468.8MB/s", want: MaxXput{IOPS: "15000", Mbps: "468"}},
+		{zapi: "50000IOPS,1.53GB/s", want: MaxXput{IOPS: "50000", Mbps: "1566"}},
+
+		{zapi: "1 foople/s", want: MaxXput{IOPS: "0", Mbps: "0"}, isErr: true},
+	}
+	for i, tt := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			got, err := ZapiXputToRest(tt.zapi)
+			if tt.isErr && err == nil {
+				t.Errorf("ZapiXputToRest(%s) got=%+v, want err but got err=%s", tt.zapi, got, err)
+				return
+			}
+			if got.IOPS != tt.want.IOPS || got.Mbps != tt.want.Mbps {
+				t.Errorf("ZapiXputToRest(%s) got=%+v, want=%+v", tt.zapi, got, tt.want)
+			}
+		})
+	}
+}
diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go
index 048bdd25..81073826 100644
--- a/cmd/tools/grafana/dashboard_test.go
+++ b/cmd/tools/grafana/dashboard_test.go
@@ -582,9 +582,10 @@ func checkTopKRange(t *testing.T, path string, data []byte) {
 
 func TestOnlyHighlightsExpanded(t *testing.T) {
 	exceptions := map[string]int{
-		"cmode/shelf.json":    2,
-		"cmode/security.json": 3,
-		"cmode/fsa.json":      2,
+		"cmode/shelf.json":        2,
+		"cmode/security.json":     3,
+		"cmode/fsa.json":          2,
+		"cmode/delay_center.json": 2,
 	}
 	// count number of expanded sections in dashboard and ensure num expanded = 1
 	visitDashboards(
diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go
index 05ff60be..a3ba1ca1 100644
--- a/pkg/util/util_test.go
+++ b/pkg/util/util_test.go
@@ -85,3 +85,41 @@ func TestParseMetricType(t *testing.T) {
 		})
 	}
 }
+
+func TestHasDuplicates(t *testing.T) {
+	type test struct {
+		testCase      string
+		childNames    []string
+		expectedArray bool
+	}
+
+	tests := []test{
+		{
+			testCase:      "testcasenochild",
+			childNames:    []string{},
+			expectedArray: false,
+		},
+		{
+			testCase:      "testcaseonechild",
+			childNames:    []string{"type"},
+			expectedArray: false,
+		},
+		{
+			testCase:      "testcasemultipleChild",
+			childNames:    []string{"type", "style", "aggr"},
+			expectedArray: false,
+		},
+		{
+			testCase:      "testcasearray",
+			childNames:    []string{"aggr-name", "aggr-name", "aggr-name", "aggr-name"},
+			expectedArray: true,
+		},
+	}
+
+	for _, testcase := range tests {
+		isArray := HasDuplicates(testcase.childNames)
+		if isArray != testcase.expectedArray {
+			t.Errorf("array hasn't been detected properly for %s, isArray is = %v, isArray should = %v", testcase.testCase, isArray, testcase.expectedArray)
+		}
+	}
+}