Repository: NetApp/harvest
Pull request #565 (2021-10-13 14:26:14)

feat: rest infrastructure


Patch:
diff --git a/.github/fetch-asup b/.github/fetch-asup
index c580aef4..daff3552 100755
--- a/.github/fetch-asup
+++ b/.github/fetch-asup
@@ -10,9 +10,9 @@ ASUP_BIN_VERSION="$2"
 BIN_PLATFORM="$3"
 mkdir -p "$ASUP_BIN_DIR"
 
-COUNT_REPO_FOLDERS=($(find $ASUP_BIN_DIR/github.com/!net!app -mindepth 1 -maxdepth 1 -type d  | wc -l))
-if [ $COUNT_REPO_FOLDERS -gt 5 ]; then
-  rm -rf $ASUP_BIN_DIR/*
+COUNT_REPO_FOLDERS=( "$(find "$ASUP_BIN_DIR"/github.com/!net!app -mindepth 1 -maxdepth 1 -type d  | wc -l)" )
+if [ "${COUNT_REPO_FOLDERS[*]}" -gt 5 ]; then
+  rm -rf "${ASUP_BIN_DIR:?}"/*
 fi
 GOMODCACHE=$ASUP_BIN_DIR go mod download github.com/NetApp/harvest-metrics@"$ASUP_BIN_VERSION"
 mod_bin=$(find "$ASUP_BIN_DIR" -name "*asup*" -name "*$BIN_PLATFORM*" -exec ls -t {} + | head -n 1)
diff --git a/cmd/collectors/rest/plugins/disk/disk.go b/cmd/collectors/rest/plugins/disk/disk.go
new file mode 100644
index 00000000..13a26f18
--- /dev/null
+++ b/cmd/collectors/rest/plugins/disk/disk.go
@@ -0,0 +1,40 @@
+/*
+ * Copyright NetApp Inc, 2021 All rights reserved
+ */
+package disk
+
+import (
+	"goharvest2/cmd/poller/plugin"
+	"goharvest2/pkg/matrix"
+)
+
+type Disk struct {
+	*plugin.AbstractPlugin
+}
+
+func New(p *plugin.AbstractPlugin) plugin.Plugin {
+	return &Disk{AbstractPlugin: p}
+}
+
+func (me *Disk) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
+
+	for _, instance := range data.GetInstances() {
+
+		containerType := instance.GetLabel("container_type")
+
+		if containerType == "shared" {
+			instance.SetLabel("shared", "true")
+		} else {
+			instance.SetLabel("shared", "false")
+		}
+
+		if containerType == "broken" {
+			instance.SetLabel("failed", "true")
+		} else {
+			instance.SetLabel("failed", "false")
+		}
+
+	}
+
+	return nil, nil
+}
diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
new file mode 100644
index 00000000..7f5a55f4
--- /dev/null
+++ b/cmd/collectors/rest/rest.go
@@ -0,0 +1,392 @@
+package rest
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/tidwall/gjson"
+	"goharvest2/cmd/collectors/rest/plugins/disk"
+	"goharvest2/cmd/poller/collector"
+	"goharvest2/cmd/poller/plugin"
+	"goharvest2/cmd/tools/rest"
+	"goharvest2/pkg/conf"
+	"goharvest2/pkg/errors"
+	"goharvest2/pkg/matrix"
+	"goharvest2/pkg/tree/node"
+	"goharvest2/pkg/util"
+	"os"
+	"strings"
+	"time"
+)
+
+type Rest struct {
+	*collector.AbstractCollector
+	client         *rest.Client
+	apiPath        string
+	instanceKeys   []string
+	instanceLabels map[string]string
+	counters       map[string]string
+	fields         []string
+	queryFields    []string //TODO remove later. Used for finding missing fields in templates
+	misses         []string
+	returnTimeOut  string
+}
+
+var allFieldQuery = true // Used to query fields=* query during rest call
+
+func init() {
+	plugin.RegisterModule(Rest{})
+}
+
+func (Rest) HarvestModule() plugin.ModuleInfo {
+	return plugin.ModuleInfo{
+		ID:  "harvest.collector.rest",
+		New: func() plugin.Module { return new(Rest) },
+	}
+}
+
+func (r *Rest) Init(a *collector.AbstractCollector) error {
+
+	var err error
+
+	r.AbstractCollector = a
+
+	if r.client, err = r.getClient(a, a.Params); err != nil {
+		return err
+	}
+
+	if err = r.client.Init(5); err != nil {
+		return err
+	}
+
+	if err = r.InitVars(); err != nil {
+		return err
+	}
+
+	if err = collector.Init(r); err != nil {
+		return err
+	}
+
+	r.Logger.Info().Str("cluster", r.client.Cluster().Name).Msgf("connected to %s", r.client.Cluster().Info)
+
+	r.Matrix.SetGlobalLabel("cluster", r.client.Cluster().Name)
+
+	if err = r.initCache(); err != nil {
+		return err
+	}
+	r.Logger.Info().Msgf("initialized cache with %d metrics", len(r.Matrix.GetMetrics()))
+
+	return nil
+}
+
+func (r *Rest) InitVars() error {
+
+	var (
+		template *node.Node
+		err      error
+	)
+
+	// import template
+	if template, err = r.ImportSubTemplate("", r.getTemplateFn(), r.client.Cluster().Version); err != nil {
+		return err
+	}
+
+	r.Params.Union(template)
+	r.fields = []string{"*"}
+	if x := r.Params.GetChildS("fields"); x != nil {
+		r.fields = append(r.fields, x.GetAllChildContentS()...)
+	}
+	return nil
+}
+
+func (r *Rest) getClient(a *collector.AbstractCollector, config *node.Node) (*rest.Client, error) {
+	var (
+		poller *conf.Poller
+		err    error
+		client *rest.Client
+	)
+
+	opt := a.GetOptions()
+	if poller, err = conf.PollerNamed(opt.Poller); err != nil {
+		r.Logger.Error().Stack().Err(err).Str("poller", opt.Poller).Msgf("")
+		return nil, err
+	}
+	if poller.Addr == "" {
+		r.Logger.Error().Stack().Str("poller", opt.Poller).Msg("Address is empty")
+		return nil, errors.New(errors.MISSING_PARAM, "addr")
+	}
+
+	clientTimeout := config.GetChildContentS("client_timeout")
+	timeout := rest.DefaultTimeout * time.Second
+	duration, err := time.ParseDuration(clientTimeout)
+	if err == nil {
+		timeout = duration
+		r.Logger.Info().Str("timeout", timeout.String()).Msg("Using timeout")
+	} else {
+		r.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout")
+	}
+	if client, err = rest.New(poller, timeout); err != nil {
+		r.Logger.Error().Stack().Err(err).Str("poller", opt.Poller).Msg("error creating new client")
+		os.Exit(1)
+	}
+
+	return client, err
+}
+
+func (r *Rest) getTemplateFn() string {
+	var fn string
+	objects := r.Params.GetChildS("objects")
+	if objects != nil {
+		fn = objects.GetChildContentS(r.Object)
+	}
+	return fn
+}
+
+// Returns a slice of keys in dot notation from json
+func getFieldName(source string, parent string) []string {
+	res := make([]string, 0)
+	var arr map[string]gjson.Result
+	r := gjson.Parse(source)
+	if r.IsArray() {
+		newR := r.Get("0")
+		arr = newR.Map()
+	} else if r.IsObject() {
+		arr = r.Map()
+	} else {
+		return []string{parent}
+	}
+	if len(arr) == 0 {
+		return []string{parent}
+	}
+	for key, val := range arr {
+		var temp []string
+		if parent == "" {
+			temp = getFieldName(val.Raw, key)
+		} else {
+			temp = getFieldName(val.Raw, parent+"."+key)
+		}
+		res = append(res, temp...)
+	}
+	return res
+}
+
+func (r *Rest) PollData() (*matrix.Matrix, error) {
+
+	var (
+		content      []byte
+		count        uint64
+		apiD, parseD time.Duration
+		startTime    time.Time
+		err          error
+		records      []interface{}
+	)
+
+	r.Logger.Debug().Msg("starting data poll")
+	r.Matrix.Reset()
+
+	startTime = time.Now()
+
+	// check if always run fields=* query
+	if allFieldQuery {
+		href := rest.BuildHref(r.apiPath, strings.Join(r.fields, ","), nil, "", "", "", r.returnTimeOut)
+		r.Logger.Debug().Str("href", href).Msg("")
+		err = rest.FetchData(r.client, href, &records)
+		if err != nil {
+			r.Logger.Error().Stack().Err(err).Str("href", href).Msg("Failed to fetch data")
+			return nil, err
+		}
+		err = r.matchFields(records)
+		if err != nil {
+			r.Logger.Warn().Err(err).Msg("Error while matching fields")
+			return nil, err
+		}
+	} else {
+		// Check if fields are set for rest call
+		if len(r.fields) > 0 {
+			href := rest.BuildHref(r.apiPath, strings.Join(r.fields, ","), nil, "", "", "", r.returnTimeOut)
+			r.Logger.Debug().Str("href", href).Msg("")
+			err = rest.FetchData(r.client, href, &records)
+			if err != nil {
+				r.Logger.Error().Stack().Err(err).Str("href", href).Msg("Failed to fetch data")
+				return nil, err
+			}
+		} else {
+			href := rest.BuildHref(r.apiPath, "*", nil, "", "", "", r.returnTimeOut)
+			r.Logger.Debug().Str("href", href).Msg("")
+			err = rest.FetchData(r.client, href, &records)
+			if err != nil {
+				r.Logger.Error().Stack().Err(err).Str("href", href).Msg("Failed to fetch data")
+				return nil, err
+			}
+			err = r.matchFields(records)
+			if err != nil {
+				r.Logger.Warn().Err(err).Msg("Error while matching fields")
+				return nil, err
+			}
+		}
+	}
+
+	if len(r.misses) > 0 {
+		r.Logger.Warn().
+			Str("mis-configured counters", strings.Join(r.misses, ",")).
+			Str("ApiPath", r.apiPath).
+			Msg("")
+	}
+
+	all := rest.Pagination{
+		Records:    records,
+		NumRecords: len(records),
+	}
+	apiD = time.Since(startTime)
+
+	content, err = json.Marshal(all)
+	if err != nil {
+		r.Logger.Error().Err(err).Str("ApiPath", r.apiPath).Msg("Unable to marshal rest pagination")
+	}
+
+	startTime = time.Now()
+	if !gjson.ValidBytes(content) {
+		return nil, fmt.Errorf("json is not valid for: %s", r.apiPath)
+	}
+	parseD = time.Since(startTime)
+
+	results := gjson.GetManyBytes(content, "num_records", "records")
+	numRecords := results[0]
+	if numRecords.Int() == 0 {
+		return nil, errors.New(errors.ERR_NO_INSTANCE, "no "+r.Object+" instances on cluster")
+	}
+
+	r.Logger.Debug().Str("object", r.Object).Str("number of records extracted", numRecords.String()).Msg("")
+
+	results[1].ForEach(func(key, instanceData gjson.Result) bool {
+		var (
+			instanceKey string
+			instance    *matrix.Instance
+		)
+
+		if !instanceData.IsObject() {
+			r.Logger.Warn().Str("type", instanceData.Type.String()).Msg("Instance data is not object, skipping")
+			return true
+		}
+
+		// extract instance key(s)
+		for _, k := range r.instanceKeys {
+			value := instanceData.Get(k)
+			if value.Exists() {
+				instanceKey += value.String()
+			} else {
+				r.Logger.Warn().Str("key", k).Msg("skip instance, missing key")
+				break
+			}
+		}
+
+		if r.Params.GetChildContentS("only_cluster_instance") != "true" {
+			if instanceKey == "" {
+				return true
+			}
+		}
+
+		if instance = r.Matrix.GetInstance(instanceKey); instance == nil {
+			if instance, err = r.Matrix.NewInstance(instanceKey); err != nil {
+				r.Logger.Error().Err(err).Str("Instance key", instanceKey).Msg("")
+				return true
+			}
+		}
+
+		for label, display := range r.instanceLabels {
+			value := instanceData.Get(label)
+			if value.Exists() {
+				instance.SetLabel(display, value.String())
+				count++
+			} else {
+				// spams a lot currently due to missing label mappings. Moved to debug for now till rest gaps are filled
+				r.Logger.Debug().Str("Instance key", instanceKey).Str("label", label).Msg("Missing label value")
+			}
+		}
+
+		for key, metric := range r.Matrix.GetMetrics() {
+
+			if metric.GetProperty() == "etl.bool" {
+				b := instanceData.Get(key)
+				if b.Exists() {
+					if err = metric.SetValueBool(instance, b.Bool()); err != nil {
+						r.Logger.Error().Err(err).Str("key", key).Str("metric", metric.GetName()).Msg("Unable to set bool key on metric")
+					}
+					count++
+				}
+			} else if metric.GetProperty() == "etl.float" {
+				f := instanceData.Get(key)
+				if f.Exists() {
+					if err = metric.SetValueFloat64(instance, f.Float()); err != nil {
+						r.Logger.Error().Err(err).Str("key", key).Str("metric", metric.GetName()).
+							Msg("Unable to set float key on metric")
+					}
+					count++
+				}
+			}
+		}
+		return true
+	})
+
+	r.Logger.Info().
+		Uint64("dataPoints", count).
+		Str("apiTime", apiD.String()).
+		Str("parseTime", parseD.String()).
+		Msg("Collected")
+
+	_ = r.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
+	_ = r.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
+	_ = r.Metadata.LazySetValueUint64("count", "data", count)
+	r.AddCollectCount(count)
+
+	return r.Matrix, nil
+}
+
+func (r *Rest) matchFields(records []interface{}) error {
+	all := rest.Pagination{
+		Records:    records,
+		NumRecords: len(records),
+	}
+	c, err := json.Marshal(all)
+	if err != nil {
+		r.Logger.Error().Err(err).Str("ApiPath", r.apiPath).Msg("Unable to marshal rest pagination")
+		return err
+	}
+
+	results := gjson.GetBytes(c, "records")
+	if len(results.String()) > 0 {
+		// fetch first record from json
+		firstValue := results.Get("0")
+		res := getFieldName(firstValue.String(), "")
+		var searchKeys []string
+		for k := range r.counters {
+			searchKeys = append(searchKeys, k)
+		}
+		// find keys from rest json response which matches counters defined in templates
+		matches, misses := util.Intersection(res, searchKeys)
+		r.queryFields = []string{}
+		r.misses = []string{}
+		for _, param := range matches {
+			r.queryFields = append(r.queryFields, param)
+		}
+		for _, param := range misses {
+			r.misses = append(r.misses, param)
+		}
+
+	}
+	return nil
+}
+
+func (me *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin {
+	switch kind {
+	case "Disk":
+		return disk.New(abc)
+	default:
+		me.Logger.Warn().Str("kind", kind).Msg("no rest plugin found ")
+	}
+	return nil
+}
+
+// Interface guards
+var (
+	_ collector.Collector = (*Rest)(nil)
+)
diff --git a/cmd/collectors/rest/templating.go b/cmd/collectors/rest/templating.go
new file mode 100644
index 00000000..b5698e28
--- /dev/null
+++ b/cmd/collectors/rest/templating.go
@@ -0,0 +1,120 @@
+package rest
+
+import (
+	"goharvest2/pkg/errors"
+	"goharvest2/pkg/matrix"
+	"goharvest2/pkg/tree/node"
+	"strings"
+)
+
+func (r *Rest) initCache() error {
+
+	var (
+		counters            *node.Node
+		display, name, kind string
+		metr                matrix.Metric
+		err                 error
+	)
+
+	if x := r.Params.GetChildContentS("object"); x != "" {
+		r.Object = x
+	} else {
+		r.Object = strings.ToLower(r.Object)
+	}
+	r.Matrix.Object = r.Object
+
+	if e := r.Params.GetChildS("export_options"); e != nil {
+		r.Matrix.SetExportOptions(e)
+	}
+
+	if r.apiPath = r.Params.GetChildContentS("query"); r.apiPath == "" {
+		return errors.New(errors.MISSING_PARAM, "query")
+	}
+
+	// create metric cache
+	if counters = r.Params.GetChildS("counters"); counters == nil {
+		return errors.New(errors.MISSING_PARAM, "counters")
+	}
+
+	// default value for ONTAP is 15 sec
+	if returnTimeout := r.Params.GetChildContentS("return_timeout"); returnTimeout != "" {
+		r.returnTimeOut = returnTimeout
+	}
+
+	r.instanceKeys = make([]string, 0)
+	r.instanceLabels = make(map[string]string)
+	r.counters = make(map[string]string)
+
+	for _, c := range counters.GetAllChildContentS() {
+		name, display, kind = parseMetric(c)
+		r.Logger.Debug().
+			Str("kind", kind).
+			Str("name", name).
+			Str("display", display).
+			Msg("Collected")
+
+		r.counters[name] = display
+		switch kind {
+		case "key":
+			r.instanceLabels[name] = display
+			r.instanceKeys = append(r.instanceKeys, name)
+		case "label":
+			r.instanceLabels[name] = display
+		case "bool":
+			if metr, err = r.Matrix.NewMetricUint8(name); err != nil {
+				r.Logger.Error().Err(err).
+					Str("name", name).
+					Msg("NewMetricUint8")
+				return err
+			}
+			metr.SetName(display)
+			metr.SetProperty("etl.bool") // to distinct from internally generated metrics, e.g. from plugins
+		case "float":
+			if metr, err = r.Matrix.NewMetricFloat64(name); err != nil {
+				r.Logger.Error().Err(err).
+					Str("name", name).
+					Msg("NewMetricFloat64")
+				return err
+			}
+			metr.SetName(display)
+			metr.SetProperty("etl.float")
+		}
+	}
+
+	r.Logger.Info().Strs("extracted Instance Keys", r.instanceKeys).Msg("")
+	r.Logger.Info().Int("count metrics", len(r.Matrix.GetMetrics())).Int("count labels", len(r.instanceLabels)).Msg("initialized metric cache")
+
+	if len(r.Matrix.GetMetrics()) == 0 && r.Params.GetChildContentS("collect_only_labels") != "true" {
+		return errors.New(errors.ERR_NO_METRIC, "failed to parse numeric metrics")
+	}
+	return nil
+
+}
+
+func parseMetric(rawName string) (string, string, string) {
+	var (
+		name, display string
+		values        []string
+	)
+	if values = strings.SplitN(rawName, "=>", 2); len(values) == 2 {
+		name = strings.TrimSpace(values[0])
+		display = strings.TrimSpace(values[1])
+	} else {
+		name = rawName
+		display = strings.ReplaceAll(rawName, ".", "_")
+	}
+
+	if strings.HasPrefix(name, "^^") {
+		return strings.TrimPrefix(name, "^^"), strings.TrimPrefix(display, "^^"), "key"
+	}
+
+	if strings.HasPrefix(name, "^") {
+		return strings.TrimPrefix(name, "^"), strings.TrimPrefix(display, "^"), "label"
+	}
+
+	if strings.HasPrefix(name, "?") {
+		return strings.TrimPrefix(name, "?"), strings.TrimPrefix(display, "?"), "bool"
+	}
+
+	return name, display, "float"
+}
diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go
index 1e52b13b..91185ae8 100644
--- a/cmd/collectors/zapi/collector/zapi.go
+++ b/cmd/collectors/zapi/collector/zapi.go
@@ -14,6 +14,7 @@ import (
 	"goharvest2/cmd/collectors/zapiperf/plugins/nic"
 	"goharvest2/cmd/collectors/zapiperf/plugins/volume"
 	"goharvest2/cmd/poller/plugin"
+	"goharvest2/pkg/conf"
 	"sort"
 	"strconv"
 	"strings"
@@ -82,7 +83,7 @@ func (me *Zapi) InitVars() error {
 
 	var err error
 
-	if me.Client, err = client.New(me.Params); err != nil { // convert to connection error, so poller aborts
+	if me.Client, err = client.New(conf.ZapiPoller(me.Params)); err != nil { // convert to connection error, so poller aborts
 		return errors.New(errors.ERR_CONNECTION, err.Error())
 	}
 	me.Client.TraceLogSet(me.Name, me.Params)
diff --git a/cmd/collectors/zapi/plugins/quota/quota.go b/cmd/collectors/zapi/plugins/quota/quota.go
index 0802c4f0..3c20db30 100644
--- a/cmd/collectors/zapi/plugins/quota/quota.go
+++ b/cmd/collectors/zapi/plugins/quota/quota.go
@@ -1,12 +1,12 @@
-/*
- * Copyright NetApp Inc, 2021 All rights reserved
- */
+// Copyright NetApp Inc, 2021 All rights reserved
+
 package quota
 
 import (
 	"goharvest2/cmd/poller/collector"
 	"goharvest2/cmd/poller/plugin"
 	"goharvest2/pkg/api/ontapi/zapi"
+	"goharvest2/pkg/conf"
 	"goharvest2/pkg/dict"
 	"goharvest2/pkg/errors"
 	"goharvest2/pkg/matrix"
@@ -15,7 +15,7 @@ import (
 	"strings"
 )
 
-const QuotaBatchSize = "500"
+const BatchSize = "500"
 
 // Quota plugin is needed to match qtrees with quotas.
 type Quota struct {
@@ -40,7 +40,7 @@ func (my *Quota) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(my.ParentParams); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}
@@ -98,8 +98,8 @@ func (my *Quota) Init() error {
 				my.Logger.Info().Msgf("using batch-size [%s]", my.batchSize)
 			}
 		} else {
-			my.batchSize = QuotaBatchSize
-			my.Logger.Trace().Msgf("using default batch-size [%s]", QuotaBatchSize)
+			my.batchSize = BatchSize
+			my.Logger.Trace().Str("BatchSize", BatchSize).Msg("Using default batch-size")
 		}
 	}
 
diff --git a/cmd/collectors/zapi/plugins/shelf/shelf.go b/cmd/collectors/zapi/plugins/shelf/shelf.go
index 5fc673c2..c5932c44 100644
--- a/cmd/collectors/zapi/plugins/shelf/shelf.go
+++ b/cmd/collectors/zapi/plugins/shelf/shelf.go
@@ -7,6 +7,7 @@ import (
 	"goharvest2/cmd/poller/collector"
 	"goharvest2/cmd/poller/plugin"
 	"goharvest2/pkg/api/ontapi/zapi"
+	"goharvest2/pkg/conf"
 	"goharvest2/pkg/dict"
 	"goharvest2/pkg/errors"
 	"goharvest2/pkg/matrix"
@@ -35,7 +36,7 @@ func (my *Shelf) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(my.ParentParams); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}
@@ -141,7 +142,7 @@ func (my *Shelf) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) {
 	}
 
 	// Set all global labels from zapi.go if already not exist
-	for a, _ := range my.instanceLabels {
+	for a := range my.instanceLabels {
 		my.data[a].SetGlobalLabels(data.GetGlobalLabels())
 	}
 
diff --git a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go
index 1ef5208d..4c790016 100644
--- a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go
+++ b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go
@@ -6,6 +6,7 @@ package snapmirror
 import (
 	"goharvest2/cmd/poller/plugin"
 	"goharvest2/pkg/api/ontapi/zapi"
+	"goharvest2/pkg/conf"
 	"goharvest2/pkg/dict"
 	"goharvest2/pkg/matrix"
 	"goharvest2/pkg/tree/node"
@@ -34,7 +35,7 @@ func (my *SnapMirror) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(my.ParentParams); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}
diff --git a/cmd/exporters/prometheus/README.md b/cmd/exporters/prometheus/README.md
index 6375d283..ed4fa4f6 100644
--- a/cmd/exporters/prometheus/README.md
+++ b/cmd/exporters/prometheus/README.md
@@ -1,5 +1,4 @@
 
-
 # Prometheus Exporter
 
 ## Overview
@@ -125,7 +124,7 @@ To use HTTP service discovery you need to:
 
 Add the following to your `harvest.yml`
 
-```
+```yaml
 Admin:
   httpsd:
     listen: :8887
@@ -172,7 +171,7 @@ Edit your `prometheus.yml` and add the following section
 
 `$ vim /etc/prometheus/prometheus.yml`
 
-```
+```yaml
 scrape_configs:
   - job_name: harvest
     http_sd_configs:
@@ -181,7 +180,7 @@ scrape_configs:
  
 Harvest and Prometheus both support basic authentication for HTTP SD end-points. To enable basic auth, add the following to your Harvest config.
 
-```
+```yaml
 Admin:
   httpsd:
     listen: :8887
@@ -201,7 +200,7 @@ Notice that none of the pollers specify an exporter. Instead, all the pollers sh
 
 If you add or remove more clusters in the `Pollers` section, you do not have to change Prometheus since it dynamically pulls the targets from the Harvest admin node.
 
-```
+```yaml
 Admin:
   httpsd:
     listen: :8887
diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go
index 02ce7fde..2ee8a9c5 100644
--- a/cmd/poller/poller.go
+++ b/cmd/poller/poller.go
@@ -31,6 +31,7 @@ import (
 	errors2 "errors"
 	"fmt"
 	"github.com/spf13/cobra"
+	_ "goharvest2/cmd/collectors/rest"
 	_ "goharvest2/cmd/collectors/simple"
 	_ "goharvest2/cmd/collectors/unix"
 	_ "goharvest2/cmd/collectors/zapi/collector"
diff --git a/cmd/tools/rest/client.go b/cmd/tools/rest/client.go
index d9e5e1be..504822c7 100644
--- a/cmd/tools/rest/client.go
+++ b/cmd/tools/rest/client.go
@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"fmt"
+	"github.com/tidwall/gjson"
 	"goharvest2/pkg/conf"
 	"goharvest2/pkg/errors"
 	"goharvest2/pkg/logging"
@@ -19,7 +20,7 @@ import (
 
 const (
 	// DefaultTimeout should be > than ONTAP's default REST timeout, which is 15 seconds for GET requests
-	DefaultTimeout = 30 * time.Second
+	DefaultTimeout = 30
 )
 
 type Client struct {
@@ -28,17 +29,24 @@ type Client struct {
 	buffer   *bytes.Buffer
 	Logger   *logging.Logger
 	baseURL  string
+	cluster  Cluster
 	password string
 	username string
 }
 
-func New(poller *conf.Poller) (*Client, error) {
+type Cluster struct {
+	Name    string
+	Info    string
+	Uuid    string
+	Version [3]int
+}
+
+func New(poller *conf.Poller, timeout time.Duration) (*Client, error) {
 	var (
 		client         Client
 		httpclient     *http.Client
 		transport      *http.Transport
 		cert           tls.Certificate
-		timeout        time.Duration
 		addr           string
 		url            string
 		useInsecureTLS bool
@@ -101,15 +109,6 @@ func New(poller *conf.Poller) (*Client, error) {
 		}
 	}
 
-	timeout = DefaultTimeout
-	if poller.ClientTimeout != "" {
-		timeout, err = time.ParseDuration(poller.ClientTimeout)
-		if err != nil {
-			client.Logger.Error().Msgf("err paring client timeout of=[%s] err=%+v\n", timeout, err)
-		}
-	}
-	client.Logger.Debug().Msgf("using timeout [%d]", timeout)
-
 	httpclient = &http.Client{Transport: transport, Timeout: timeout}
 	client.client = httpclient
 
@@ -193,7 +192,8 @@ func downloadSwagger(poller *conf.Poller, path string, url string) (int64, error
 		return 0, err
 	}
 
-	if restClient, err = New(poller); err != nil {
+	timeout := DefaultTimeout * time.Second
+	if restClient, err = New(poller, timeout); err != nil {
 		return 0, fmt.Errorf("error creating new client %w\n", err)
 	}
 
@@ -213,3 +213,57 @@ func downloadSwagger(poller *conf.Poller, path string, url string) (int64, error
 	}
 	return n, nil
 }
+
+func (c *Client) Init(retries int) error {
+
+	var (
+		err     error
+		content []byte
+		i       int
+	)
+
+	for i = 0; i < retries; i++ {
+
+		if content, err = c.GetRest(BuildHref("cluster", "*", nil, "", "", "", "")); err != nil {
+			continue
+		}
+
+		results := gjson.GetManyBytes(content, "name", "uuid", "version.full", "version.generation", "version.major", "version.minor")
+		c.cluster.Name = results[0].String()
+		c.cluster.Uuid = results[1].String()
+		c.cluster.Info = results[2].String()
+		c.cluster.Version[0] = int(results[3].Int())
+		c.cluster.Version[1] = int(results[4].Int())
+		c.cluster.Version[2] = int(results[5].Int())
+		return nil
+	}
+	return err
+}
+
+func BuildHref(apiPath string, fields string, field []string, queryFields string, queryValue string, maxRecords string, returnTimeout string) string {
+	href := strings.Builder{}
+	href.WriteString("api/")
+	href.WriteString(apiPath)
+	href.WriteString("?return_records=true")
+	addArg(&href, "&fields=", fields)
+	for _, f := range field {
+		addArg(&href, "&", f)
+	}
+	addArg(&href, "&query_fields=", queryFields)
+	addArg(&href, "&query=", queryValue)
+	addArg(&href, "&max_records=", maxRecords)
+	addArg(&href, "&return_timeout=", returnTimeout)
+	return href.String()
+}
+
+func addArg(href *strings.Builder, field string, value string) {
+	if value == "" {
+		return
+	}
+	href.WriteString(field)
+	href.WriteString(value)
+}
+
+func (c *Client) Cluster() Cluster {
+	return c.cluster
+}
diff --git a/cmd/tools/rest/rest.go b/cmd/tools/rest/rest.go
index f6754ac2..ba94947a 100644
--- a/cmd/tools/rest/rest.go
+++ b/cmd/tools/rest/rest.go
@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/spf13/cobra"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
 	"goharvest2/pkg/conf"
 	"io"
 	"log"
@@ -170,7 +172,8 @@ func doData() {
 		return
 	}
 
-	if client, err = New(poller); err != nil {
+	timeout := DefaultTimeout * time.Second
+	if client, err = New(poller, timeout); err != nil {
 		fmt.Printf("error creating new client %+v\n", err)
 		os.Exit(1)
 	}
@@ -180,8 +183,14 @@ func doData() {
 		args.Api = args.Api[1:]
 	}
 	var records []interface{}
-	fetchData(client, buildHref(), &records)
+	href := BuildHref(args.Api, args.Fields, args.Field, args.QueryField, args.QueryValue, args.MaxRecords, "")
+	stderr("fetching href=[%s]\n", href)
 
+	err = FetchData(client, href, &records)
+	if err != nil {
+		stderr("error %+v\n", err)
+		return
+	}
 	all := Pagination{
 		Records:    records,
 		NumRecords: len(records),
@@ -210,43 +219,36 @@ func getPollerAndAddr() (*conf.Poller, string, error) {
 	return poller, poller.Addr, nil
 }
 
-func buildHref() string {
-	href := strings.Builder{}
-	href.WriteString("api/")
-	href.WriteString(args.Api)
-	href.WriteString("?return_records=true")
-	addArg(&href, "&fields=", args.Fields)
-	for _, field := range args.Field {
-		addArg(&href, "&", field)
+func FetchData(client *Client, href string, records *[]interface{}) error {
+	getRest, err := client.GetRest(href)
+	if err != nil {
+		return fmt.Errorf("error making request api=%s err=%+v\n", href, err)
 	}
-	addArg(&href, "&query_fields=", args.QueryField)
-	addArg(&href, "&query=", args.QueryValue)
-	addArg(&href, "&max_records=", args.MaxRecords)
-
-	return href.String()
-}
 
-func addArg(href *strings.Builder, field string, value string) {
-	if value == "" {
-		return
+	isNonIterRestCall := false
+	value := gjson.GetBytes(getRest, "records")
+	if value.String() == "" {
+		isNonIterRestCall = true
 	}
-	href.WriteString(field)
-	href.WriteString(value)
-}
 
-func fetchData(client *Client, href string, records *[]interface{}) {
-	stderr("fetching href=[%s]\n", href)
-	getRest, err := client.GetRest(href)
-	if err != nil {
-		stderr("error making request api=%s err=%+v\n", href, err)
-		return
+	if isNonIterRestCall {
+		contentJson := `{"records":[]}`
+		response, err := sjson.SetRawBytes([]byte(contentJson), "records.-1", getRest)
+		if err != nil {
+			return fmt.Errorf("error setting record %+v\n", err)
+		}
+		var page Pagination
+		err = json.Unmarshal(response, &page)
+		if err != nil {
+			return fmt.Errorf("error unmarshalling json %+v\n", err)
+		}
+		*records = append(*records, page.Records...)
 	} else {
 		// extract returned records since paginated records need to be merged into a single list
 		var page Pagination
 		err := json.Unmarshal(getRest, &page)
 		if err != nil {
-			stderr("error unmarshalling json %+v\n", err)
-			return
+			return fmt.Errorf("error unmarshalling json %+v\n", err)
 		}
 
 		*records = append(*records, page.Records...)
@@ -257,12 +259,13 @@ func fetchData(client *Client, href string, records *[]interface{}) {
 			if nextLink != "" {
 				if nextLink == href {
 					// nextLink is same as previous link, no progress is being made, exit
-					return
+					return nil
 				}
-				fetchData(client, nextLink, records)
+				FetchData(client, nextLink, records)
 			}
 		}
 	}
+	return nil
 }
 
 func stderr(format string, a ...interface{}) {
diff --git a/cmd/tools/zapi/zapi.go b/cmd/tools/zapi/zapi.go
index 6978e31e..721bbe62 100644
--- a/cmd/tools/zapi/zapi.go
+++ b/cmd/tools/zapi/zapi.go
@@ -145,11 +145,7 @@ func doCmd(cmd string) {
 	if poller, err = conf.PollerNamed(args.Poller); err != nil {
 		log.Fatal(err)
 	}
-	n, err := poller.AsNode(args.Poller)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if connection, err = client.New(n); err != nil {
+	if connection, err = client.New(*poller); err != nil {
 		log.Fatal(err)
 	}
 
diff --git a/conf/rest/9.13.0/aggr.yaml b/conf/rest/9.13.0/aggr.yaml
new file mode 100644
index 00000000..21c4207e
--- /dev/null
+++ b/conf/rest/9.13.0/aggr.yaml
@@ -0,0 +1,66 @@
+
+name:               Aggregate
+query:              storage/aggregates
+object:             aggr
+
+fields:
+  - space.block_storage.inactive_user_data
+  - space.block_storage.inactive_user_data_percent
+
+counters:
+  - ^name  => aggr
+  - ^^uuid  => uuid
+  - ^home_node.name  => node
+  #- ^aggr-get-iter.aggr-attributes.aggr-raid-attributes.aggregate-type  => type  #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-raid-attributes.disk-count   => disk_count #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-raid-attributes.plex-count  => plex_count #mapping missing
+  - block_storage.primary.raid_size => raid_size
+  - ^state  => state
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.files-private-used  => files_private_used #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.files-total => files_total #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.files-used => files_used #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.inodefile-private-capacity => inodefile_private_capacity #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.inodefile-public-capacity => inodefile_public_capacity #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.maxfiles-available => maxfiles_available #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.maxfiles-possible => maxfiles_possible #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.maxfiles-used => maxfiles_used #mapping missing
+  #- aggr-get-iter.aggr-attributes.aggr-inode-attributes.percent-inode-used-capacity => percent_inode_used_capacity #mapping missing
+  - space.cloud_storage.used => capacity_tier_used
+  - space.block_storage.data_compacted_count => data_compacted_count
+  - space.block_storage.data_compaction_space_saved => data_compaction_space_saved
+  - space.block_storage.data_compaction_space_saved_percent  => data_compaction_space_saved_percent
+  - space.block_storage.hybrid_cache.size  => hybrid_cache_size_total  #mapping seems wrong in doc
+  - snapshot.percent_used_capacity => space_used_percent #mapping seems wrong in doc
+  - space.block_storage.inactive_user_data => performance_tier_inactive_user_data
+  - space.block_storage.inactive_user_data_percent => performance_tier_inactive_user_data_percent
+  - space.block_storage.physical_used_percent => physical_used_percent
+  - space.block_storage.volume_deduplication_shared_count => sis_shared_count
+  - space.block_storage.volume_deduplication_space_saved => sis_space_saved
+  - space.block_storage.volume_deduplication_space_saved_percent => sis_space_saved_percent
+  - space.block_storage.available => space_available
+  - space.block_storage.size => space_total
+  - space.block_storage.used => space_used
+  #- aggr-get-iter.aggr-attributes.aggr-space-attributes.total-reserved-space => space_reserved #Missing mapping
+  #- aggr-get-iter.aggr-attributes.aggr-volume-count-attributes.flexvol-count => flexvol_count #Missing mapping
+  - snapshot.files_total => files_total
+  - snapshot.files_used => files_used
+  - snapshot.max_files_available => maxfiles_available
+  #- aggr-get-iter.aggr-attributes.aggr-snapshot-attributes.percent-inode-used-capacity => snapshot_inode_used_percent #missing mapping
+  #- aggr-get-iter.aggr-attributes.aggr-snapshot-attributes.percent-used-capacity => snapshot_used_percent #missing mapping
+  - snapshot.size_available => size_available  #mapping seems wrong in doc
+  - snapshot.size_total => size_total  #mapping seems wrong in doc
+  - snapshot.size_used => size_used  #mapping seems wrong in doc
+  - snapshot.percent_snapshot_space => snapshot_reserve_percent #mapping seems wrong in doc
+
+plugins:
+  LabelAgent:
+    value_to_num:
+      - new_status state online online `0`
+
+export_options:
+  instance_keys:
+    - aggr
+    - node
+  instance_labels:
+    - type
+    - state
diff --git a/conf/rest/9.13.0/disk.yaml b/conf/rest/9.13.0/disk.yaml
new file mode 100644
index 00000000..03a376b4
--- /dev/null
+++ b/conf/rest/9.13.0/disk.yaml
@@ -0,0 +1,48 @@
+
+name:             Disk
+query:            storage/disks
+object:           disk
+
+counters:
+  - ^^uid
+  - ^name                     => disk
+  - bytes_per_sector          => bytes_per_sector
+  - sector_count              => sector_count
+  - ^type
+  - ^container_type
+  - ^model
+  - ^serial_number
+  - ^shelf.uid                => shelf
+  - ^bay                      => shelf_bay
+  - stats.average_latency     => average_latency
+  - stats.throughput          => disk_io_kbps
+  - stats.power_on_hours      => uptime
+  #- sectors-read  #missing mapping
+  #- sectors-written #missing mapping
+  - outage.reason             => outage
+  - usable_size
+  - ^state
+  - ^node.uuid
+  - ^node.name                => node
+  - ^home_node.name           => owner_node
+
+plugins:
+  - Disk
+  - LabelAgent:
+    value_to_num:
+      - new_status outage - - `0` #ok_value is empty value, '-' would be converted to blank while processing.
+
+export_options:
+  instance_keys:
+    - node
+    - disk
+  instance_labels:
+    - type
+    - model
+    - outage
+    - owner_node
+    - shared
+    - shelf
+    - shelf_bay
+    - serial_number
+    - failed
diff --git a/conf/rest/9.13.0/lun.yaml b/conf/rest/9.13.0/lun.yaml
new file mode 100644
index 00000000..163a1d34
--- /dev/null
+++ b/conf/rest/9.13.0/lun.yaml
@@ -0,0 +1,34 @@
+
+name:                       Lun
+query:                      storage/luns
+object:                     lun
+
+counters:
+  - ^location.node.name => node
+  - ^name => path
+  - ^location.qtree.name => qtree
+  - space.size => size
+  - space.used => size_used
+  - ^status.state => state
+  - ^^uuid
+  - ^location.volume.name => volume
+  - ^svm.name => svm
+
+plugins:
+  - LabelAgent:
+    value_to_num:
+      - new_status state online online `0`
+    # path is something like "/vol/vol_georg_fcp401/lun401"
+    # we only want lun name, which is 4th element
+    split:
+      - path `/` ,,,lun
+
+export_options:
+  instance_keys:
+    - node
+    - qtree
+    - lun
+    - volume
+    - svm
+  instance_labels:
+    - state
diff --git a/conf/rest/9.13.0/node.yaml b/conf/rest/9.13.0/node.yaml
new file mode 100644
index 00000000..5e041d60
--- /dev/null
+++ b/conf/rest/9.13.0/node.yaml
@@ -0,0 +1,61 @@
+
+name:                       Node
+query:                      cluster/nodes
+object:                     node
+
+fields:
+  - metric.processor_utilization
+  - is_capacity_optimized
+  - is_all_flash_optimized
+  - vendor_serial_number   #seems hidden but no response in rest
+  - is_capacity_optimized
+  - is_all_flash_select_optimized
+
+counters:
+  - metric.processor_utilization => cpu_busytime   #hidden field
+  - controller.failed_fan.count => failed_fan
+  - ^controller.failed_fan.message.message => failed_fan
+  - controller.failed_power_supply.count => failed_power
+  - ^controller.failed_power_supply.message.message => failed_power_message
+  - ^controller.over_temperature => over_temperature
+  - ^is_all_flash_optimized => is_all_flash_optimized
+  - ^is_all_flash_select_optimized => is_all_flash_select_optimized
+  - ^is_capacity_optimized => is_capacity_optimized
+  #- ^system-node-get-iter.is-cloud-optimized => is_cloud_optimized #missing mapping
+  #- system-node-get-iter.is-node-cluster-eligible => is_node_cluster_eligible #missing mapping
+  - ^state => healthy
+  - ^is_performance_optimized => is_perf_optimized  #not found in infinity rest call
+  #- ^system-node-get-iter.maximum-aggregate-size => max_aggr_size  #missing mapping
+  #- ^system-node-get-iter.maximum-number-of-volumes => max_vol_num #missing mapping
+  #- ^system-node-get-iter.maximum-volume-size => max_vol_size #missing mapping
+  - ^^name => node
+  - ^location
+  - ^model
+  - ^serial_number => serial
+  - uptime
+  - ^vendor_serial_number => vendor
+  - ^version.full => version
+
+
+plugins:
+  - LabelAgent:
+    value_to_num:
+      - new_status healthy true up `0`
+    join:
+      - warnings `; ` failed_fan_message,failed_power_message,over_temperature
+
+export_options:
+  instance_keys:
+    - node
+  instance_labels:
+    - max_aggr_size
+    - max_vol_size
+    - max_vol_num
+    - warnings
+    - healthy
+    - location
+    - model
+    - serial
+    - uptime
+    - vendor
+    - version
diff --git a/conf/rest/9.13.0/snapmirror.yaml b/conf/rest/9.13.0/snapmirror.yaml
new file mode 100644
index 00000000..fdf61e0d
--- /dev/null
+++ b/conf/rest/9.13.0/snapmirror.yaml
@@ -0,0 +1,57 @@
+
+name:                         SnapMirror
+query:                        snapmirror/relationships
+object:                       snapmirror
+
+counters:
+  #- snapmirror-get-iter.break-failed-count   #missing mapping
+  #- snapmirror-get-iter.break-successful-count #missing-mapping
+  - ^destination.path  => destination_volume
+  #- ^destination_volume_node       => destination_node #missing-mapping
+  - ^destination.svm.name => destination_vserver
+  - ^healthy
+  - lag_time
+  - transfer.total_duration => last_transfer_duration
+  - transfer.end_time => last_transfer_end_timestamp
+  - transfer.bytes_transferred  => last_transfer_size
+  - ^transfer.bytes_transferred  => last_transfer_size
+  #- newest-snapshot-timestamp #missing mapping
+  - ^^uuid => relationship_id
+  - ^transfer.state => relationship_status
+  #- ^relationship-type  #missing mapping
+  #- ^relationship-group-type      => group_type #missing mapping
+  #- resync-failed-count #missing mapping
+  #- resync-successful-count #missing mapping
+  #- ^schedule  #maps to snapmirror/policies?fields=transfer_schedule.name . will need plugin change
+  - ^source.path => source_volume
+  - ^source.svm.name => source_vserver
+  #- ^source-node #missing mapping
+  #- total-transfer-time-secs #missing mapping
+  - ^unhealthy_reason.0.message => unhealthy_reason
+  #- update-failed-count #missing mapping
+  #- update-successful-count #missing mapping
+
+
+#Need snapmirror plugin development to support missing field in export
+#plugins:
+#  - Snapmirror
+
+export_options:
+  instance_keys:
+    - relationship_id
+    - destination_volume
+    - destination_node
+    - destination_vserver
+    - source_volume
+    - source_vserver
+    - source_node
+  instance_labels:
+    - healthy
+    - unhealthy_reason
+    - last_transfer_type
+    - relationship_status
+    - relationship_type
+    - schedule
+    - destination_node_limit
+    - source_node_limit
+    - group_type
diff --git a/conf/rest/9.13.0/status.yaml b/conf/rest/9.13.0/status.yaml
new file mode 100644
index 00000000..661a4756
--- /dev/null
+++ b/conf/rest/9.13.0/status.yaml
@@ -0,0 +1,21 @@
+
+name:       Status
+query:      cluster
+object:     cluster
+fields:
+  - health
+
+counters:
+  - ^health => status
+
+collect_only_labels: true
+only_cluster_instance: true
+no_max_records: true
+
+plugins:
+  - LabelAgent:
+    value_to_num:
+      - new_status status ok ok `0`
+
+export_options:
+  include_all_labels: true
diff --git a/conf/rest/9.13.0/volume.yaml b/conf/rest/9.13.0/volume.yaml
new file mode 100644
index 00000000..84d273f4
--- /dev/null
+++ b/conf/rest/9.13.0/volume.yaml
@@ -0,0 +1,72 @@
+name:                     Volume
+query:                    storage/volumes
+object:                   volume
+
+fields:
+  - space
+  - autosize
+  - files
+
+counters:
+  - autosize.maximum => maximum_size
+  - autosize.grow_threshold => grow_threshold_percent
+  - ^^uuid => instance_uuid
+  - ^name                       => volume
+  - ^node                       => node  #missing mapping
+  - ^svm.name        => svm
+  - ^aggregates.0.name  => aggr   #zapi maps to field containing-aggregate-name while rest mapping is an array?
+  - ^style             => style
+  - files.used => files_used
+  - files.maximum => files_maximum
+#  - compression-space-saved               => sis_compress_saved #missing mapping
+#  - deduplication-space-saved             => sis_dedup_saved #missing mapping
+#  - total-space-saved                     => sis_total_saved #missing mapping
+#  - percentage-compression-space-saved    => sis_compress_saved_percent #missing mapping
+#  - percentage-deduplication-space-saved  => sis_dedup_saved_percent #missing mapping
+#  - percentage-total-space-saved          => sis_total_saved_percent #missing mapping
+#  - ^is-sis-volume                        => is-sis-volume #missing mapping
+  - space.expected_available => expected_available
+  - space.filesystem_size                       => filesystem_size
+  - space.logical_space.available  => logical_available  #not found in infincity cluster
+  - space.logical_space.used => logical_used
+  - space.logical_space.used_by_afs => logical_used_by_afs
+  - space.logical_space.used_by_snapshots  => logical_used_by_snapshots
+  - space.logical_space.used_percent => logical_used_percent
+  - space.physical_used => physical_used
+  - space.physical_used_percent => physical_used_percent
+  - space.size                          => size
+  - space.available                     => size_available
+  - space.afs_total                          => size_total
+  - space.used                           => size_used
+  - space.percent_used                => size_used_percent
+  - space.snapshot.used              => snapshots_size_used
+  - space.size_available_for_snapshots        => snapshots_size_available
+  - space.snapshot.reserve_available          => snapshot_reserve_available
+  - space.snapshot.reserve_size               => snapshot_reserve_size
+  - space.snapshot.reserve_percent         => snapshot_reserve_percent
+  - space.snapshot.space_used_percent    => snapshot_reserve_used_percent
+  - ^state   => state
+
+
+plugins:
+  LabelAgent:
+    value_to_num:
+      - new_status state online online `0`
+    exclude_equals:
+      - style `flexgroup_constituent`
+    replace:
+      - style style `flexgroup_constituent` `flexgroup`
+  Aggregator:
+    volume<style=flexgroup>volume node,svm,aggr,style
+
+export_options:
+  instance_keys:
+    - volume
+    - node
+    - svm
+    - aggr
+    - style
+  instance_labels:
+    - state
+    #- is-sis-volume
+
diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml
new file mode 100644
index 00000000..066c39f1
--- /dev/null
+++ b/conf/rest/default.yaml
@@ -0,0 +1,16 @@
+
+collector:          Rest
+
+
+schedule:
+  - data: 180s
+
+objects:
+  Disk:             disk.yaml
+  Lun:              lun.yaml
+  Status:            status.yaml
+  Node:   node.yaml
+  Aggr:  aggr.yaml
+  SnapMirror:       snapmirror.yaml
+  Volume: volume.yaml
+
diff --git a/grafana/dashboards/cmode/harvest_dashboard_mcc_cluster.json b/grafana/dashboards/cmode/harvest_dashboard_mcc_cluster.json
index 06e21e2d..878c3f2a 100644
--- a/grafana/dashboards/cmode/harvest_dashboard_mcc_cluster.json
+++ b/grafana/dashboards/cmode/harvest_dashboard_mcc_cluster.json
@@ -1,4 +1,46 @@
 {
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.5.4"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
   "annotations": {
     "enable": false,
     "list": [
@@ -28,7 +70,7 @@
   "panels": [
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -43,7 +85,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -123,7 +165,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -203,7 +245,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -279,7 +321,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -354,7 +396,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -432,7 +474,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -516,7 +558,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "editable": true,
       "error": false,
@@ -627,7 +669,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 1,
       "editable": true,
       "error": false,
@@ -739,7 +781,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -843,7 +885,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -864,7 +906,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "editable": true,
       "error": false,
@@ -969,7 +1011,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 1,
       "editable": true,
       "error": false,
@@ -1080,7 +1122,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -1184,7 +1226,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1205,7 +1247,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fill": 1,
@@ -1319,7 +1361,7 @@
       "zerofill": true
     },
     {
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1413,7 +1455,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -1520,7 +1562,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -1630,7 +1672,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fill": 1,
@@ -1749,7 +1791,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fill": 1,
@@ -1862,7 +1904,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1883,7 +1925,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "fieldConfig": {
         "defaults": {
@@ -1998,7 +2040,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "fieldConfig": {
         "defaults": {
@@ -2107,7 +2149,7 @@
     },
     {
       "collapsed": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -2128,7 +2170,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -2238,7 +2280,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "${DS_PROMETHEUS}",
       "decimals": 2,
       "editable": true,
       "error": false,
@@ -2375,7 +2417,7 @@
           "text": "mcc",
           "value": "mcc"
         },
-        "datasource": null,
+        "datasource": "${DS_PROMETHEUS}",
         "definition": "label_values(node_labels{system_type!=\"7mode\"}, datacenter)",
         "description": null,
         "error": null,
@@ -2408,7 +2450,7 @@
           "text": "sti8300mcc261262_siteA",
           "value": "sti8300mcc261262_siteA"
         },
-        "datasource": null,
+        "datasource": "${DS_PROMETHEUS}",
         "definition": "label_values(aggr_new_status{system_type!=\"7mode\",datacenter=\"$Datacenter\"}, cluster)",
         "description": null,
         "error": null,
@@ -2440,7 +2482,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "datasource": null,
+        "datasource": "${DS_PROMETHEUS}",
         "definition": "label_values(aggr_new_status{system_type!=\"7mode\",datacenter=\"$Datacenter\",cluster=\"$Cluster\"}, node)",
         "description": null,
         "error": null,
@@ -2468,7 +2510,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "datasource": null,
+        "datasource": "${DS_PROMETHEUS}",
         "definition": "label_values(aggr_new_status{datacenter=\"$Datacenter\",cluster=\"$Cluster\",node=~\"$Node\"}, aggr)",
         "description": null,
         "error": null,
@@ -2501,7 +2543,7 @@
           "text": "1",
           "value": "1"
         },
-        "datasource": null,
+        "datasource": "${DS_PROMETHEUS}",
         "description": null,
         "error": null,
         "hide": 0,
diff --git a/integration/Jenkinsfile b/integration/Jenkinsfile
index e5ec2c0a..3e03c76f 100644
--- a/integration/Jenkinsfile
+++ b/integration/Jenkinsfile
@@ -180,9 +180,8 @@ pipeline {
 
 def void stopAndRemoveDockers() {
     sh '''
-        for container_id in $(docker ps -a -q);do docker stop $container_id && docker rm $container_id;done
-        for image_id in $(docker images -q);do docker rmi $image_id;done
-        for volume_id in $(docker volume ls -q); do docker volume rm -f $volume_id;done
+        docker system prune --all --force --volumes
+
     '''
 }
 
diff --git a/jenkins/artifacts/jenkinsfile b/jenkins/artifacts/jenkinsfile
index 028d16f6..ea8ecddf 100644
--- a/jenkins/artifacts/jenkinsfile
+++ b/jenkins/artifacts/jenkinsfile
@@ -51,7 +51,7 @@ pipeline {
         stage('clean workspace') {
             steps {
                 cleanWs()
-                sh 'rm -rf $targetParentLocation$VERSION$RELEASE'
+                stopAndRemoveDockers()
             }
         }
         stage('Setup GO') {
@@ -198,6 +198,15 @@ pipeline {
     }
 }
 
+def void stopAndRemoveDockers() {
+    sh '''
+        rm -rf /opt/home/nightly/
+        mkdir -p /opt/home/nightly/
+        docker system prune --all --force --volumes
+    '''
+}
+
+
 def getBranchName(gitBranchName, paramBranchName) {
     gitBranchName = gitBranchName.replace('origin/', '')
     if (gitBranchName?.trim() && gitBranchName != "main") {
diff --git a/pkg/api/ontapi/zapi/client.go b/pkg/api/ontapi/zapi/client.go
index 0595970a..4bd2f9cb 100644
--- a/pkg/api/ontapi/zapi/client.go
+++ b/pkg/api/ontapi/zapi/client.go
@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"fmt"
+	"goharvest2/pkg/conf"
 	"goharvest2/pkg/errors"
 	"goharvest2/pkg/logging"
 	"goharvest2/pkg/tree"
@@ -37,7 +38,7 @@ type Client struct {
 	logZapi    bool            // used to log ZAPI request/response
 }
 
-func New(config *node.Node) (*Client, error) {
+func New(poller conf.Poller) (*Client, error) {
 	var (
 		client         Client
 		httpclient     *http.Client
@@ -54,20 +55,20 @@ func New(config *node.Node) (*Client, error) {
 	client.Logger = logging.SubLogger("Zapi", "Client")
 
 	// check required & optional parameters
-	if client.apiVersion = config.GetChildContentS("api_version"); client.apiVersion == "" {
+	if client.apiVersion = poller.ApiVersion; client.apiVersion == "" {
 		client.apiVersion = DefaultApiVersion
 		client.Logger.Debug().Msgf("using default API version [%s]", DefaultApiVersion)
 	}
 
-	if client.vfiler = config.GetChildContentS("api_vfiler"); client.vfiler != "" {
+	if client.vfiler = poller.ApiVfiler; client.vfiler != "" {
 		client.Logger.Debug().Msgf("using vfiler tunneling [%s]", client.vfiler)
 	}
 
-	if addr = config.GetChildContentS("addr"); addr == "" {
+	if addr = poller.Addr; addr == "" {
 		return nil, errors.New(errors.MISSING_PARAM, "addr")
 	}
 
-	if config.GetChildContentS("is_kfs") == "true" {
+	if poller.IsKfs {
 		url = "https://" + addr + ":8443/servlets/netapp.servlets.admin.XMLrequest_filer"
 	} else {
 		url = "https://" + addr + ":443/servlets/netapp.servlets.admin.XMLrequest_filer"
@@ -81,19 +82,16 @@ func New(config *node.Node) (*Client, error) {
 	request.Header.Set("Charset", "utf-8")
 
 	// by default, enforce secure TLS, if not requested otherwise by user
-	if x := config.GetChildContentS("use_insecure_tls"); x != "" {
-		if useInsecureTLS, err = strconv.ParseBool(x); err != nil {
-			client.Logger.Error().Stack().Err(err).Msg("use_insecure_tls")
-		}
-	} else {
-		useInsecureTLS = false
+	useInsecureTLS = false
+	if poller.UseInsecureTls != nil {
+		useInsecureTLS = *poller.UseInsecureTls
 	}
 
 	// set authentication method
-	if config.GetChildContentS("auth_style") == "certificate_auth" {
+	if poller.AuthStyle == "certificate_auth" {
 
-		certPath := config.GetChildContentS("ssl_cert")
-		keyPath := config.GetChildContentS("ssl_key")
+		certPath := poller.SslCert
+		keyPath := poller.SslKey
 
 		if certPath == "" {
 			return nil, errors.New(errors.MISSING_PARAM, "ssl_cert")
@@ -111,16 +109,13 @@ func New(config *node.Node) (*Client, error) {
 		}
 	} else {
 
-		username := config.GetChildContentS("username")
-		password := config.GetChildContentS("password")
-
-		if username == "" {
+		if poller.Username == "" {
 			return nil, errors.New(errors.MISSING_PARAM, "username")
-		} else if password == "" {
+		} else if poller.Password == "" {
 			return nil, errors.New(errors.MISSING_PARAM, "password")
 		}
 
-		request.SetBasicAuth(username, password)
+		request.SetBasicAuth(poller.Username, poller.Password)
 		transport = &http.Transport{
 			Proxy:           http.ProxyFromEnvironment,
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: useInsecureTLS},
@@ -132,7 +127,7 @@ func New(config *node.Node) (*Client, error) {
 	httpclient = &http.Client{Transport: transport, Timeout: timeout}
 
 	client.client = httpclient
-	client.SetTimeout(config.GetChildContentS("client_timeout"))
+	client.SetTimeout(poller.ClientTimeout)
 	// ensure that we can change body dynamically
 	request.GetBody = func() (io.ReadCloser, error) {
 		r := bytes.NewReader(client.buffer.Bytes())
diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go
index 55879fef..824d4626 100644
--- a/pkg/conf/conf.go
+++ b/pkg/conf/conf.go
@@ -9,7 +9,6 @@ import (
 	"github.com/imdario/mergo"
 	"goharvest2/pkg/constant"
 	"goharvest2/pkg/errors"
-	"goharvest2/pkg/tree"
 	"goharvest2/pkg/tree/node"
 	"goharvest2/pkg/util"
 	"gopkg.in/yaml.v3"
@@ -23,10 +22,14 @@ import (
 )
 
 var Config = HarvestConfig{}
-var cfp = ""
 var configRead = false
 var ValidatePortInUse = false
 
+const (
+	DefaultApiVersion = "1.3"
+	DefaultTimeout    = "10s"
+)
+
 // TestLoadHarvestConfig is used by testing code to reload a new config
 func TestLoadHarvestConfig(configPath string) {
 	configRead = false
@@ -41,7 +44,7 @@ func LoadHarvestConfig(configPath string) error {
 		return nil
 	}
 	contents, err := ioutil.ReadFile(configPath)
-	cfp = configPath
+
 	if err != nil {
 		fmt.Printf("error reading config file=[%s] %+v\n", configPath, err)
 		return err
@@ -302,66 +305,65 @@ func (p *Poller) Union(defaults *Poller) {
 	_ = mergo.Merge(p, defaults)
 }
 
-// TODO remove node code after changing bin/zapi to use Poller
-func loadConfig(configPath string) (*node.Node, error) {
-	configNode, err := tree.Import("yaml", configPath)
-	if configNode != nil {
-		// Load HarvestConfig to rewrite passwords - eventually all the code will be refactored to use HarvestConfig.
-		// This is needed because the current yaml parser does not handle password with special characters.
-		// E.g abc#123, that's because the # is interpreted as the beginning of a comment. The code below overwrites
-		// the incorrect password with the correct one by using a better yaml parser for each Poller and Default section
-		err := LoadHarvestConfig(configPath)
-		if err != nil {
-			return nil, err
-		}
-		pollers := configNode.GetChildS("Pollers")
-		if pollers != nil {
-			for _, poller := range pollers.GetChildren() {
-				password := poller.GetChildContentS("password")
-				pollerStruct := (Config.Pollers)[poller.GetNameS()]
-				if pollerStruct.Password != "" && pollerStruct.Password != password {
-					poller.SetChildContentS("password", pollerStruct.Password)
-				}
-			}
-		}
-		// Check Defaults also
-		defaultNode := configNode.GetChildS("Defaults")
-		if defaultNode != nil {
-			password := defaultNode.GetChildContentS("password")
-			defaultStruct := *Config.Defaults
-			if defaultStruct.Password != "" && defaultStruct.Password != password {
-				defaultNode.SetChildContentS("password", defaultStruct.Password)
-			}
-		}
+// ZapiPoller creates a poller out of a node, this is a bridge between the node and struct-based code
+// Used by ZAPI based code
+func ZapiPoller(n *node.Node) Poller {
+	var p Poller
+
+	if Config.Defaults != nil {
+		p = *Config.Defaults
+	} else {
+		p = Poller{}
 	}
-	return configNode, err
-}
 
-// AsNode converts a poller into a node based structure to bridge the struct and node-based code
-func (p *Poller) AsNode(name string) (*node.Node, error) {
-	if p == nil {
-		return nil, nil
+	if apiVersion := n.GetChildContentS("api_version"); apiVersion != "" {
+		p.ApiVersion = apiVersion
+	} else {
+		if p.ApiVersion == "" {
+			p.ApiVersion = DefaultApiVersion
+		}
 	}
-	cfg, err := loadConfig(cfp)
-	if err != nil {
-		return nil, err
+	if vfiler := n.GetChildContentS("api_vfiler"); vfiler != "" {
+		p.ApiVfiler = vfiler
 	}
-	var pollers, poller, defaults *node.Node
-
-	pollers = cfg.GetChildS("Pollers")
-	defaults = cfg.GetChildS("Defaults")
-
-	if pollers == nil {
-		err = errors.New(errors.ERR_CONFIG, "[Pollers] section not found")
-	} else if defaults != nil { // optional
-		for _, p := range pollers.GetChildren() {
-			p.Union(defaults)
+	if addr := n.GetChildContentS("addr"); addr != "" {
+		p.Addr = addr
+	}
+	isKfs := n.GetChildContentS("is_kfs")
+	if isKfs == "true" {
+		p.IsKfs = true
+	} else if isKfs == "false" {
+		p.IsKfs = false
+	}
+	if x := n.GetChildContentS("use_insecure_tls"); x != "" {
+		if insecureTLS, err := strconv.ParseBool(x); err == nil {
+			// err can be ignored since conf was already validated
+			p.UseInsecureTls = &insecureTLS
 		}
 	}
-	if poller = pollers.GetChildS(name); poller == nil {
-		err = errors.New(errors.ERR_CONFIG, "poller ["+name+"] not found")
+	if authStyle := n.GetChildContentS("auth_style"); authStyle != "" {
+		p.AuthStyle = authStyle
+	}
+	if sslCert := n.GetChildContentS("ssl_cert"); sslCert != "" {
+		p.SslCert = sslCert
+	}
+	if sslKey := n.GetChildContentS("ssl_key"); sslKey != "" {
+		p.SslKey = sslKey
+	}
+	if username := n.GetChildContentS("username"); username != "" {
+		p.Username = username
+	}
+	if password := n.GetChildContentS("password"); password != "" {
+		p.Password = password
+	}
+	if clientTimeout := n.GetChildContentS("client_timeout"); clientTimeout != "" {
+		p.ClientTimeout = clientTimeout
+	} else {
+		if p.ClientTimeout == "" {
+			p.ClientTimeout = DefaultTimeout
+		}
 	}
-	return poller, err
+	return p
 }
 
 type Exporter struct {
diff --git a/pkg/matrix/metric.go b/pkg/matrix/metric.go
index 4015fd8e..d0667b8a 100644
--- a/pkg/matrix/metric.go
+++ b/pkg/matrix/metric.go
@@ -51,6 +51,7 @@ type Metric interface {
 	SetValueFloat64(*Instance, float64) error
 	SetValueString(*Instance, string) error
 	SetValueBytes(*Instance, []byte) error
+	SetValueBool(*Instance, bool) error
 
 	AddValueInt(*Instance, int) error
 	AddValueInt32(*Instance, int32) error
@@ -208,3 +209,7 @@ func (me *AbstractMetric) MultiplyByScalar(s int) error {
 func (me *AbstractMetric) AddValueString(i *Instance, s string) error {
 	return errors.New(errors.ERR_IMPLEMENT, me.dtype)
 }
+
+func (me *AbstractMetric) SetValueBool(i *Instance, b bool) error {
+	return errors.New(errors.ERR_IMPLEMENT, me.dtype)
+}
diff --git a/pkg/matrix/metric_uint8.go b/pkg/matrix/metric_uint8.go
index 93cd686f..1842fbf2 100644
--- a/pkg/matrix/metric_uint8.go
+++ b/pkg/matrix/metric_uint8.go
@@ -75,6 +75,16 @@ func (me *MetricUint8) SetValueInt64(i *Instance, v int64) error {
 	return errors.New(OVERFLOW_ERROR, fmt.Sprintf("convert int64 (%d) to uint32", v))
 }
 
+func (me *MetricUint8) SetValueBool(i *Instance, v bool) error {
+	me.record[i.index] = true
+	if v {
+		me.values[i.index] = 1
+	} else {
+		me.values[i.index] = 0
+	}
+	return nil
+}
+
 func (me *MetricUint8) SetValueUint8(i *Instance, v uint8) error {
 	me.record[i.index] = true
 	me.values[i.index] = v
diff --git a/pkg/util/util.go b/pkg/util/util.go
index 1d6c1be8..25a4f9bd 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -306,3 +306,24 @@ type PollerStatus struct {
 	ProfilingPort string
 	PromPort      string
 }
+
+// Intersection returns things from b that are common and missing with a
+func Intersection(a []string, b []string) ([]string, []string) {
+	matches := make([]string, 0)
+	misses := make([]string, 0)
+	hash := make(map[string]bool)
+
+	for _, aa := range a {
+		hash[aa] = true
+	}
+
+	for _, bb := range b {
+		if _, found := hash[bb]; found {
+			matches = append(matches, bb)
+		} else {
+			misses = append(misses, bb)
+		}
+	}
+
+	return matches, misses
+}


Test patch:
diff --git a/cmd/collectors/rest/rest_test.go b/cmd/collectors/rest/rest_test.go
new file mode 100644
index 00000000..16ad36ee
--- /dev/null
+++ b/cmd/collectors/rest/rest_test.go
@@ -0,0 +1,49 @@
+package rest
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func Test_getFieldName(t *testing.T) {
+
+	type test struct {
+		name   string
+		source string
+		parent string
+		want   int
+	}
+
+	var tests = []test{
+		{
+			name:   "Test1",
+			source: readFile("testdata/cluster.json"),
+			parent: "",
+			want:   52,
+		},
+		{
+			name:   "Test2",
+			source: readFile("testdata/sample.json"),
+			parent: "",
+			want:   3,
+		},
+		{
+			name:   "Test3",
+			source: readFile("testdata/test.json"),
+			parent: "",
+			want:   9,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getFieldName(tt.source, tt.parent); len(got) != tt.want {
+				t.Errorf("length of output slice = %v, want %v", len(got), tt.want)
+			}
+		})
+	}
+}
+
+func readFile(path string) string {
+	b, _ := ioutil.ReadFile(path)
+	return string(b)
+}
diff --git a/cmd/collectors/rest/testdata/cluster.json b/cmd/collectors/rest/testdata/cluster.json
new file mode 100644
index 00000000..8eccbb7e
--- /dev/null
+++ b/cmd/collectors/rest/testdata/cluster.json
@@ -0,0 +1,103 @@
+{
+  "name": "ocum-1",
+  "uuid": "1-1-1-1-1",
+  "location": "RTP",
+  "version": {
+    "full": "NetApp Release 9.10.1X12: Wed Oct 13 10:38:21 UTC 2021",
+    "generation": 9,
+    "major": 10,
+    "minor": 1
+  },
+  "dns_domains": [
+    "1.1.1"
+  ],
+  "name_servers": [
+    "1.1.1.1",
+    "1.1.1.1"
+  ],
+  "ntp_servers": [
+    "1.1.1.1.1"
+  ],
+  "management_interfaces": [
+    {
+      "uuid": "1-1-1-1-1",
+      "name": "cluster_mgmt",
+      "ip": {
+        "address": "1.1.1.1"
+      },
+      "_links": {
+        "self": {
+          "href": "/api/network/ip/interfaces/1-1-1-1-1"
+        }
+      }
+    }
+  ],
+  "metric": {
+    "timestamp": "2021-10-29T09:52:45Z",
+    "duration": "PT15S",
+    "status": "ok",
+    "latency": {
+      "other": 87,
+      "total": 149,
+      "read": 0,
+      "write": 155
+    },
+    "iops": {
+      "read": 0,
+      "write": 0,
+      "other": 0,
+      "total": 0
+    },
+    "throughput": {
+      "read": 0,
+      "write": 56,
+      "other": 0,
+      "total": 56
+    }
+  },
+  "statistics": {
+    "timestamp": "2021-10-29T09:52:59Z",
+    "status": "ok",
+    "latency_raw": {
+      "other": 58864044,
+      "total": 227552254,
+      "read": 856638,
+      "write": 167831572
+    },
+    "iops_raw": {
+      "read": 11238,
+      "write": 936570,
+      "other": 801325,
+      "total": 1749133
+    },
+    "throughput_raw": {
+      "read": 4874634,
+      "write": 78671880,
+      "other": 0,
+      "total": 83546514
+    }
+  },
+  "timezone": {
+    "name": "America/New_York"
+  },
+  "certificate": {
+    "uuid": "1-1-1-1-1",
+    "_links": {
+      "self": {
+        "href": "/api/security/certificates/1-1-1-1-1"
+      }
+    }
+  },
+  "san_optimized": false,
+  "health": "ok",
+  "peering_policy": {
+    "minimum_passphrase_length": 8,
+    "authentication_required": true,
+    "encryption_required": false
+  },
+  "_links": {
+    "self": {
+      "href": "/api/cluster"
+    }
+  }
+}
\ No newline at end of file
diff --git a/cmd/collectors/rest/testdata/sample.json b/cmd/collectors/rest/testdata/sample.json
new file mode 100644
index 00000000..e84dab40
--- /dev/null
+++ b/cmd/collectors/rest/testdata/sample.json
@@ -0,0 +1,1 @@
+{"name":{"first":"Janet","last":"Prichard"},"age":47}
\ No newline at end of file
diff --git a/cmd/collectors/rest/testdata/test.json b/cmd/collectors/rest/testdata/test.json
new file mode 100644
index 00000000..ddfd3698
--- /dev/null
+++ b/cmd/collectors/rest/testdata/test.json
@@ -0,0 +1,11 @@
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
+    {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
+    {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
+  ]
+}
\ No newline at end of file
diff --git a/integration/test/dashboard/data_validation.go b/integration/test/dashboard/data_validation.go
index 34ac2e96..1931cce4 100644
--- a/integration/test/dashboard/data_validation.go
+++ b/integration/test/dashboard/data_validation.go
@@ -29,7 +29,7 @@ func HasMinRecord(query string, limit int) bool {
 }
 
 func AssertIfNotPresent(query string) {
-	maxCount := 20
+	maxCount := 10
 	startCount := 1
 	query = fmt.Sprintf("count(%s)", query)
 	log.Info().Msg("Checking whether data is present or not for counter " + query)
@@ -45,8 +45,8 @@ func AssertIfNotPresent(query string) {
 				if len(metricArray) > 1 {
 					totalRecord := metricArray[1].Int()
 					log.Info().Int64("Total Record", totalRecord).Msg("")
-					if totalRecord > 5 {
-						time.Sleep(2 * time.Minute)
+					if totalRecord >= 5 {
+						time.Sleep(3 * time.Minute)
 						return
 					}
 				}
@@ -55,5 +55,5 @@ func AssertIfNotPresent(query string) {
 		startCount++
 		time.Sleep(30 * time.Second)
 	}
-	panic("Data for counter " + query + " not found after 10 min. Check Workload counters are uncommented from conf/zapiperf/default.yml")
+	panic("Data for counter " + query + " not found after 8 min. Check Workload counters are uncommented from conf/zapiperf/default.yml")
 }
diff --git a/integration/test/dashboard_test.go b/integration/test/dashboard_test.go
index 5d087671..720738e2 100644
--- a/integration/test/dashboard_test.go
+++ b/integration/test/dashboard_test.go
@@ -36,6 +36,9 @@ type Dashboard struct {
 	FolderId    int64  `json:"folderId"`
 }
 
+const cDotFolder = "Harvest 21.11 - cDOT"
+const sevenModeFolder = "Harvest 21.11 - 7-mode"
+
 func (suite *DashboardImportTestSuite) SetupSuite() {
 	log.Println("Verify Grafana and Prometheus are configured")
 	if !utils.IsUrlReachable(utils.GetGrafanaHttpUrl()) {
@@ -59,7 +62,7 @@ func (suite *DashboardImportTestSuite) TestImport() {
 	err = json.Unmarshal(data, &dataFolder)
 	utils.PanicIfNotNil(err)
 	for _, values := range dataFolder {
-		if values.Title == "Harvest 2.0 - cDOT" {
+		if values.Title == cDotFolder {
 			return
 		}
 	}
@@ -68,19 +71,19 @@ func (suite *DashboardImportTestSuite) TestImport() {
 }
 
 func (suite *DashboardImportTestSuite) TestCModeDashboardCount() {
-	folderId := GetFolderId("Harvest 21.11 - cDOT", suite.T())
+	folderId := GetFolderId(cDotFolder, suite.T())
 	expectedName := []string{"Harvest Metadata", "NetApp Detail: Aggregate", "NetApp Detail: Cluster",
 		"NetApp Detail: Disk", "NetApp Detail: LUN", "NetApp Detail: Network", "NetApp Detail: Network  - Details",
 		"NetApp Detail: Network with NVMe/FC", "NetApp Detail: Node", "NetApp Detail: Node - Details",
 		"NetApp Detail: Shelf", "NetApp Detail: SnapMirror", "NetApp Detail: SVM", "NetApp Detail: SVM - Details",
-		"NetApp Detail: Volume", "NetApp Detail: Volume - Details"}
+		"NetApp Detail: Volume", "NetApp Detail: Volume - Details", "NetApp Detail: MetroCluster"}
 
 	VerifyDashboards(folderId, expectedName, suite.T())
 }
 
 func (suite *DashboardImportTestSuite) TestSevenModeDashboardCount() {
 
-	folderId := GetFolderId("Harvest 21.11 - 7-mode", suite.T())
+	folderId := GetFolderId(sevenModeFolder, suite.T())
 	expectedName := []string{"NetApp Detail: Aggregate 7 mode", "NetApp Detail: Cluster 7 mode",
 		"NetApp Detail: Disk 7 mode", "NetApp Detail: LUN 7 mode", "NetApp Detail: Network 7 mode", "NetApp Detail: Network with NVMe/FC 7 mode",
 		"NetApp Detail: Node 7 mode", "NetApp Detail: Shelf 7 mode", "NetApp Detail: Volume 7 mode"}
diff --git a/integration/test/test.sh b/integration/test/test.sh
index 81e3224d..ae5a70ae 100644
--- a/integration/test/test.sh
+++ b/integration/test/test.sh
@@ -5,4 +5,4 @@ cd $SCRIPT_DIR/test
 tag=${1?Specify valid test tag}
 export PATH=$PATH:/usr/local/go/bin
 go mod tidy
-go test -tags=$tag
\ No newline at end of file
+go test -timeout 30m -tags=$tag
\ No newline at end of file
diff --git a/pkg/api/ontapi/zapi/client_test.go b/pkg/api/ontapi/zapi/client_test.go
index 863cf79b..245c2ff2 100644
--- a/pkg/api/ontapi/zapi/client_test.go
+++ b/pkg/api/ontapi/zapi/client_test.go
@@ -1,6 +1,7 @@
 package zapi
 
 import (
+	"goharvest2/pkg/conf"
 	"goharvest2/pkg/tree/node"
 	"testing"
 	"time"
@@ -50,7 +51,7 @@ func TestNew(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := New(tt.config)
+			_, err := New(conf.ZapiPoller(tt.config))
 			if (err != nil) != tt.wantErr {
 				t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
 				return
diff --git a/pkg/conf/conf_test.go b/pkg/conf/conf_test.go
index 80843e57..99c8ccdd 100644
--- a/pkg/conf/conf_test.go
+++ b/pkg/conf/conf_test.go
@@ -2,6 +2,7 @@ package conf
 
 import (
 	"fmt"
+	"goharvest2/pkg/tree/node"
 	"reflect"
 	"sort"
 	"strconv"
@@ -249,3 +250,27 @@ func TestCollectorConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestNodeToPoller(t *testing.T) {
+	testArg := func(t *testing.T, want, got string) {
+		if got != want {
+			t.Errorf("want=[%s] got=[%s]", want, got)
+		}
+	}
+
+	Config.Defaults = &Poller{
+		Username: "bob",
+		Password: "bob",
+	}
+
+	defaultNode := node.NewS("root")
+	defaultNode.NewChildS("password", "pass")
+	defaultNode.NewChildS("use_insecure_tls", "true")
+	poller := ZapiPoller(defaultNode)
+
+	testArg(t, DefaultApiVersion, poller.ApiVersion)
+	testArg(t, "bob", poller.Username)
+	testArg(t, "pass", poller.Password)
+	testArg(t, "10s", poller.ClientTimeout)
+	testArg(t, "true", strconv.FormatBool(*poller.UseInsecureTls))
+}
diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go
new file mode 100644
index 00000000..ba91914d
--- /dev/null
+++ b/pkg/util/util_test.go
@@ -0,0 +1,44 @@
+package util
+
+import (
+	"testing"
+)
+
+func TestIntersection(t *testing.T) {
+
+	type test struct {
+		name        string
+		a           []string
+		b           []string
+		matchLength int
+		missLength  int
+	}
+
+	tests := []test{
+		{
+			name:        "test1",
+			a:           []string{"a", "b", "c"},
+			b:           []string{"a", "c"},
+			matchLength: 2,
+			missLength:  0,
+		},
+		{
+			name:        "test2",
+			a:           []string{"a", "b", "c"},
+			b:           []string{"a", "e", "d"},
+			matchLength: 1,
+			missLength:  2,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			match, miss := Intersection(tt.a, tt.b)
+			if len(match) != tt.matchLength {
+				t.Errorf("Intersection() match length = %v, want %v", len(match), tt.matchLength)
+			}
+			if len(miss) != tt.missLength {
+				t.Errorf("Intersection() miss length = %v, want %v", len(miss), tt.missLength)
+			}
+		})
+	}
+}