Repository: NetApp/harvest
Pull request #319 (2021-07-19 13:38:38)

feat: adding value_to_num mapping rules
my prometheus url:
http://10.249.29.132:9090/graph?g0.expr=volume_new_status&g0.tab=1&g0.stacked=0&g0.range_input=1h

vgratian: I got a bit confused seeing examples like: 
```yaml
value_to_num: status state online online `0`
```

does this mean that "ok value" of `status` is `online`, and for `state` - `online` as well?
Hardikl: This is the syntax: metric label zapi_value rest_value `default_value`
status state online online `0`

So, first online is possible zapi value in OK state and second online is possible REST value in OK state. To minimize the changes while REST implementation.

Will add comment in all yamls as well to better readability.

Patch:
diff --git a/README.md b/README.md
index 04155407..3b68e311 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,6 @@ Harvest is compatible with:
 - Prometheus: `2.24` or higher
 - InfluxDB: `v2`
 - Grafana: `7.4.2` or higher
-- Docker: `20.10.0` or higher
 
 # Installation / Upgrade
 
@@ -120,7 +119,7 @@ After editing `/opt/harvest/harvest.yml`, manage Harvest with `systemctl start|s
 
 ## Docker
 
-* See [Harvest for Docker](docker/onePollerPerContainer/README.md)
+* Refer [One Poller One Container](docker/onePollerPerContainer/README.md)
 
 ## Building from source
 
@@ -203,7 +202,7 @@ All pollers are defined in `harvest.yml`, the main configuration file of Harvest
 | `exporters`            | **required** | list of exporter names from the `Exporters` section. Note: this should be the name of the exporter (e.g. `prometheus1`), not the value of the `exporter` key (e.g. `Prometheus`)   |                   |
 | `auth_style`           | required by Zapi* collectors |  either `basic_auth` or `certificate_auth`  | `basic_auth` |
 | `username`, `password` | required if `auth_style` is `basic_auth` |  |              |
-| `ssl_cert`, `ssl_key`  | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.<br /><br />If not provided, the poller will look for `<hostname>.key` and `<hostname>.pem` in `$HARVEST_HOME/cert/`.<br/><br/>To create certificates for ONTAP systems, see [using certificate authentication](docs/AuthAndPermissions.md#using-certificate-authentication)                        |              |
+| `ssl_cert`, `ssl_key`  | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.<br /><br />If not provided, the poller will look for `<hostname>.key` and `<hostname>.pem` in `$HARVEST_HOME/cert/`.<br/><br/>To create certificates for ONTAP systems, see the [Zapi documentation](cmd/collectors/zapi/README.md#authentication)                        |              |
 | `use_insecure_tls`     | optional, bool |  If true, disable TLS verification when connecting to ONTAP cluster  | false         |
 | `log_max_bytes`        |  | Maximum size of the log file before it will be rotated | `10000000` (10 mb) |
 | `log_max_files`        |  | Number of rotated log files to keep | `10` |
diff --git a/cmd/harvest/harvest.go b/cmd/harvest/harvest.go
index bd602daf..af075336 100644
--- a/cmd/harvest/harvest.go
+++ b/cmd/harvest/harvest.go
@@ -24,7 +24,6 @@ import (
 	"goharvest2/cmd/tools/doctor"
 	"goharvest2/cmd/tools/generate"
 	"goharvest2/cmd/tools/grafana"
-	"goharvest2/cmd/tools/rest"
 	"goharvest2/cmd/tools/zapi"
 	"goharvest2/pkg/conf"
 	"goharvest2/pkg/set"
@@ -557,7 +556,7 @@ func init() {
 	rootCmd.AddCommand(manageCmd("stop", true))
 	rootCmd.AddCommand(manageCmd("restart", true))
 	rootCmd.AddCommand(manageCmd("kill", true))
-	rootCmd.AddCommand(config.ConfigCmd, zapi.Cmd, rest.Cmd, grafana.GrafanaCmd, stub.NewCmd)
+	rootCmd.AddCommand(config.ConfigCmd, zapi.Cmd, grafana.GrafanaCmd, stub.NewCmd)
 	rootCmd.AddCommand(generate.Cmd)
 	rootCmd.AddCommand(doctor.Cmd)
 
diff --git a/cmd/poller/plugin/README.md b/cmd/poller/plugin/README.md
index ccd0eae8..bc057dd9 100644
--- a/cmd/poller/plugin/README.md
+++ b/cmd/poller/plugin/README.md
@@ -117,6 +117,7 @@ Notice that the rules are executed in the same order as you've added them. List
 - [exclude_contains](#exclude_contains)
 - [exclude_regex](#exclude_regex)
 - [value_mapping](#value_mapping)
+- [value_to_num](#value_to_num)
 
 ## split
 
@@ -297,6 +298,8 @@ exclude_equals: vol_type `flexgroup_`
 
 ## value_mapping
 
+value_mapping is deprecated and you should always use value_to_num mapping.
+
 Maps values of a given label to a numeric metric (of type `uint8`). This is sometimes handy to manipulate the data in the DB or Grafana (e.g. change color based on status or create alert).
 
 Note that you don't define the numeric values yourself, instead, you only provide the possible (expected) values, the plugin will map each value to its index in the rule.
@@ -323,3 +326,45 @@ value_mapping: status state up,sleeping,unknown,down
 value_mapping: status state up `1`
 # metric value will be set to 0 if "state" is "up", otherwise to **1**
 ```
+
+
+## value_to_num
+
+This rule is similar to value_mapping but in this mapping, healthy is mapped to 1 and all non healthy values are mapped to 0.
+
+Going further, We should always use value_to_num mapping as value_mapping is deprecated.
+
+Maps values of a given label to a numeric metric (of type `uint8`). This is sometimes handy to manipulate the data in the DB or Grafana (e.g. change color based on status or create alert).
+
+Note that you don't define the numeric values yourself, instead, you only provide the possible (expected) values, the plugin will map each value to its index in the rule.
+
+Rule syntax:
+
+```yaml
+value_to_num: METRIC LABEL ZAPI_VALUE REST_VALUE `N`
+# maps values of LABEL to 1 if it is ZAPI_VALUE or REST_VALUE
+# otherwise value of METRIC is set to N
+```
+The default value `N` is optional, if no default value is given and the label value does not match any of the given values, the metric value will not be set.
+
+Examples:
+
+```yaml
+value_to_num: status state up online `0`
+# a new metric will be created with the name "status"
+# if an instance has label "state" with value "up", the metric value will be 1,
+# if it's "online", the value will be set to 1,
+# if it's any other value, it will be set to the specified default, 0
+```
+
+```yaml
+value_to_num: status state up online `4`
+# metric value will be set to 1 if "state" is "up", otherwise to **4**
+```
+
+```yaml
+value_to_num: status outage - - `0` #ok_value is empty value. 
+# metric value will be set to 1 if "outage" is empty, if it's any other value, it will be set to the default, 0
+# '-' is a special symbol in this mapping and it will be converted to blank while processing.
+```
+
diff --git a/cmd/poller/plugin/label_agent/label_agent.go b/cmd/poller/plugin/label_agent/label_agent.go
index c90786ef..8a8370cb 100644
--- a/cmd/poller/plugin/label_agent/label_agent.go
+++ b/cmd/poller/plugin/label_agent/label_agent.go
@@ -25,6 +25,7 @@ type LabelAgent struct {
 	excludeContainsRules []excludeContainsRule
 	excludeRegexRules    []excludeRegexRule
 	valueMappingRules    []valueMappingRule
+	valueToNumRules      []valueToNumRule
 }
 
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
@@ -61,7 +62,8 @@ func (me *LabelAgent) Run(m *matrix.Matrix) ([]*matrix.Matrix, error) {
 		}
 	}
 
-	if len(me.valueMappingRules) != 0 {
+	// if any of the value mapping available, then map values with appropriate rules
+	if len(me.valueMappingRules) != 0 || len(me.valueToNumRules) != 0 {
 		err = me.mapValues(m)
 	}
 
@@ -204,6 +206,7 @@ func (me *LabelAgent) mapValues(m *matrix.Matrix) error {
 		err    error
 	)
 
+	// map values for value mapping rules
 	for _, r := range me.valueMappingRules {
 
 		if metric = m.GetMetric(r.metric); metric == nil {
@@ -226,5 +229,28 @@ func (me *LabelAgent) mapValues(m *matrix.Matrix) error {
 		}
 	}
 
+	// map values for value_to_num mapping rules
+	for _, r := range me.valueToNumRules {
+
+		if metric = m.GetMetric(r.metric); metric == nil {
+			if metric, err = m.NewMetricUint8(r.metric); err != nil {
+				me.Logger.Error().Stack().Err(err).Msgf("valueToNumMapping: new metric [%s]:", r.metric)
+				return err
+			} else {
+				metric.SetProperty("value_to_num mapping")
+			}
+		}
+
+		for key, instance := range m.GetInstances() {
+			if v, ok := r.mapping[instance.GetLabel(r.label)]; ok {
+				metric.SetValueUint8(instance, v)
+				me.Logger.Trace().Msgf("valueToNumMapping: [%s] [%s] mapped (%s) value to %d", r.metric, key, instance.GetLabel(r.label), v)
+			} else if r.hasDefault {
+				metric.SetValueUint8(instance, r.defaultValue)
+				me.Logger.Trace().Msgf("valueToNumMapping: [%s] [%s] mapped (%s) value to default %d", r.metric, key, instance.GetLabel(r.label), r.defaultValue)
+			}
+		}
+	}
+
 	return nil
 }
diff --git a/cmd/poller/plugin/label_agent/parse_rules.go b/cmd/poller/plugin/label_agent/parse_rules.go
index 1cd5fd69..b2b42e45 100644
--- a/cmd/poller/plugin/label_agent/parse_rules.go
+++ b/cmd/poller/plugin/label_agent/parse_rules.go
@@ -24,6 +24,7 @@ func (me *LabelAgent) parseRules() int {
 	me.excludeRegexRules = make([]excludeRegexRule, 0)
 	me.splitPairsRules = make([]splitPairsRule, 0)
 	me.valueMappingRules = make([]valueMappingRule, 0)
+	me.valueToNumRules = make([]valueToNumRule, 0)
 
 	for _, c := range me.Params.GetChildren() {
 		name := c.GetNameS()
@@ -50,6 +51,8 @@ func (me *LabelAgent) parseRules() int {
 			me.parseExcludeRegexRule(rule)
 		case "value_mapping":
 			me.parseValueMappingRule(rule)
+		case "value_to_num":
+			me.parseValueToNumRule(rule)
 		default:
 			me.Logger.Warn().Msgf("unknown rule (%s)", name)
 		}
@@ -104,6 +107,7 @@ func (me *LabelAgent) parseRules() int {
 	}
 
 	count += len(me.valueMappingRules)
+	count += len(me.valueToNumRules)
 
 	return count
 }
@@ -409,3 +413,51 @@ func (me *LabelAgent) parseValueMappingRule(rule string) {
 	}
 	me.Logger.Warn().Msgf("(value_mapping) rule has invalid format [%s]", rule)
 }
+
+type valueToNumRule struct {
+	metric       string
+	label        string
+	defaultValue uint8
+	hasDefault   bool
+	mapping      map[string]uint8
+}
+
+// example rule:
+// metric label zapi_value rest_value `default_value`
+// status state normal ok `0`
+// will create a new metric "status" of type uint8
+// if value of label "state" is normal or ok
+// the metric value will be 1, otherwise it will be 0.
+
+func (me *LabelAgent) parseValueToNumRule(rule string) {
+	if fields := strings.Fields(rule); len(fields) == 4 || len(fields) == 5 {
+		r := valueToNumRule{metric: fields[0], label: fields[1]}
+		r.mapping = make(map[string]uint8)
+
+		// This '-' is used for handling special case in disk.yaml, rest all are handled normally with assigning to 1.
+		for _, v := range strings.Split(fields[2], "-") {
+			r.mapping[v] = uint8(1)
+		}
+		for _, v := range strings.Split(fields[3], "-") {
+			r.mapping[v] = uint8(1)
+		}
+
+		if len(fields) == 5 {
+
+			fields[4] = strings.TrimPrefix(strings.TrimSuffix(fields[4], "`"), "`")
+
+			if v, err := strconv.ParseUint(fields[4], 10, 8); err != nil {
+				me.Logger.Error().Stack().Err(err).Msgf("(value_to_num) parse default value (%s): ", fields[4])
+				return
+			} else {
+				r.hasDefault = true
+				r.defaultValue = uint8(v)
+			}
+		}
+
+		me.valueToNumRules = append(me.valueToNumRules, r)
+		me.Logger.Debug().Msgf("(value_to_num) parsed rule [%v]", r)
+		return
+	}
+	me.Logger.Warn().Msgf("(value_to_num) rule has invalid format [%s]", rule)
+}
diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go
index 8a3525dc..baec30be 100644
--- a/cmd/poller/poller.go
+++ b/cmd/poller/poller.go
@@ -193,7 +193,7 @@ func (p *Poller) Init() error {
 	// check optional parameter auth_style
 	// if certificates are missing use default paths
 	if p.params.AuthStyle != nil && *p.params.AuthStyle == "certificate_auth" {
-		if p.params.SslCert == nil {
+		if p.params.SslCert != nil {
 			fp := path.Join(p.options.HomePath, "cert/", p.options.Hostname+".pem")
 			p.params.SslCert = &fp
 			logger.Debug().Msgf("using default [ssl_cert] path: [%s]", fp)
@@ -202,7 +202,7 @@ func (p *Poller) Init() error {
 				return errors.New(errors.MISSING_PARAM, "ssl_cert: "+err.Error())
 			}
 		}
-		if p.params.SslKey == nil {
+		if p.params.SslKey != nil {
 			fp := path.Join(p.options.HomePath, "cert/", p.options.Hostname+".key")
 			p.params.SslKey = &fp
 			logger.Debug().Msgf("using default [ssl_key] path: [%s]", fp)
diff --git a/cmd/tools/rest/rest.go b/cmd/tools/rest/rest.go
index ddda2d44..b0eb4e24 100644
--- a/cmd/tools/rest/rest.go
+++ b/cmd/tools/rest/rest.go
@@ -31,9 +31,6 @@ type Args struct {
 	Config        string
 	SwaggerPath   string
 	Fields        string
-	Field         []string
-	QueryField    string
-	QueryValue    string
 	DownloadAll   bool
 	MaxRecords    string
 	ForceDownload bool
@@ -121,12 +118,6 @@ func validateArgs(strings []string) check {
 	} else {
 		args.Item = strings[0]
 	}
-	qfSet := args.QueryField != ""
-	qvSet := args.QueryValue != ""
-	if args.Item == "data" && (qfSet != qvSet) {
-		fmt.Printf(`Both "query-fields" and "query-value" must be specified if either is specified.` + "\n")
-		return check{isValid: false}
-	}
 	return check{isValid: true}
 }
 
@@ -210,26 +201,15 @@ func buildHref() string {
 	href := strings.Builder{}
 	href.WriteString("api/")
 	href.WriteString(args.Api)
-	href.WriteString("?return_records=true")
-	addArg(&href, "&fields=", args.Fields)
-	for _, field := range args.Field {
-		addArg(&href, "&", field)
+	href.WriteString("?return_records=true&fields=")
+	href.WriteString(args.Fields)
+	if args.MaxRecords != "" {
+		href.WriteString("&max_records=")
+		href.WriteString(args.MaxRecords)
 	}
-	addArg(&href, "&query_fields=", args.QueryField)
-	addArg(&href, "&query=", args.QueryValue)
-	addArg(&href, "&max_records=", args.MaxRecords)
-
 	return href.String()
 }
 
-func addArg(href *strings.Builder, field string, value string) {
-	if value == "" {
-		return
-	}
-	href.WriteString(field)
-	href.WriteString(value)
-}
-
 func fetchData(client *Client, href string, records *[]interface{}) {
 	stderr("fetching href=[%s]\n", href)
 	getRest, err := client.GetRest(href)
@@ -276,38 +256,8 @@ func init() {
 
 	showFlags := showCmd.Flags()
 	showFlags.StringVarP(&args.Api, "api", "a", "", "REST API PATTERN to show")
+	showFlags.StringVarP(&args.Fields, "fields", "f", "*", "REST fields used to modify query")
 	showFlags.BoolVar(&args.DownloadAll, "all", false, "Collect all records by walking pagination links")
 	showFlags.StringVarP(&args.MaxRecords, "max-records", "m", "", "Limit the number of records returned before providing pagination link")
 	showFlags.BoolVar(&args.ForceDownload, "download", false, "Force download Swagger file instead of using local copy")
-	showFlags.StringVarP(&args.Fields, "fields", "f", "*", "Fields to return in the response <field>[,...].")
-	showFlags.StringArrayVar(&args.Field, "field", []string{}, "Query a field by value (can be specified multiple times.)\n"+
-		`If the value contains query characters (*|,!<>..), it must be quoted to avoid their special meaning
-    *         wildcard
-    < > <= >= comparisons
-    3..10     range
-    !water    negation
-    3|5       matching value in a list
-    {} and "" escape special characters`)
-	showFlags.StringVarP(&args.QueryField, "query-field", "q", "", "Search fields named <string>, matching rows where the value of the field selected by <string> matches <query-value>.\n"+
-		"comma-delimited list of fields, or * to search across all fields.")
-	showFlags.StringVarP(&args.QueryValue, "query-value", "u", "", "Pattern to search for in all fields specified by <query-fields>\n"+
-		"same query characters as <field> apply (see above)")
-
-	Cmd.SetUsageTemplate(Cmd.UsageTemplate() + `
-Examples:
-  harvest rest -p infinity show apis                                        Query cluster infinity for available APIs
-  harvest rest -p infinity show params --api svm/svms                       Query cluster infinity for svm parameters. These query parameters are used
-                                                                            to filter requests.
-  harvest rest -p infinity show models --api svm/svms                       Query cluster infinity for svm models. These describe the REST response
-                                                                            received when sending the svm/svms GET request.
-  harvest rest -p infinity show data --api svm/svms --field "state=stopped" Query cluster infinity for stopped svms.
-
-  harvest rest -p infinity show data --api storage/volumes \                Query cluster infinity for all volumes where  
-      --field "space.physical_used_percent=>70" \                               physical_used_percent is > 70% and 
-      --field "space.total_footprint=>400G" \                                   total_footprint is > 400G 
-      --fields "name,svm,space"                                             The response should contain name, svm, and space attributes of matching volumes.
-
-  harvest rest -p infinity show data --api storage/volumes \                Query cluster infinity for all volumes where the name of any volume or child
-      --query-field "name" --query-value "io_load|scale"                         resource matches io_load or scale.
-`)
 }
diff --git a/cmd/tools/rest/swag.go b/cmd/tools/rest/swag.go
index 9e703e15..761e8526 100644
--- a/cmd/tools/rest/swag.go
+++ b/cmd/tools/rest/swag.go
@@ -13,7 +13,6 @@ import (
 	"html"
 	"io"
 	"io/ioutil"
-	"net/url"
 	"os"
 	"regexp"
 	"sort"
@@ -57,11 +56,6 @@ const (
 	xNtapIntroduced  = "x-ntap-introduced"
 )
 
-type model struct {
-	schema spec.Schema
-	name   string
-}
-
 // showModels prints the swagger definitions matching the specified api
 // Most models have a reference and response. For example: volume has the following three definitions
 // volume, volume_reference, volume_response
@@ -75,35 +69,40 @@ func showModels(a Args, ontapSwag ontap) {
 		fmt.Printf("Error compiling api regex param=[%s] %v\n", a.Api, err)
 		return
 	}
+	sortedKeys := sortSchema(ontapSwag.swagger.Definitions)
 	seen := map[string]interface{}{}
-	var collected []model
-	for _, name := range sortApis(ontapSwag.collectionsApis) {
-		pathItem := ontapSwag.collectionsApis[name]
-		if compile.MatchString(name) {
-			if pathItem.Get.OperationProps.Responses != nil {
-				schema := (*pathItem.Get.OperationProps.Responses).ResponsesProps.StatusCodeResponses[200].ResponseProps.Schema
-				if schema != nil {
-					responseModel := schemaFromRef(schema.Ref.GetURL())
-					if responseModel != "" {
-						collectModels(responseModel, ontapSwag, &collected)
-					}
-				}
-			}
+	for _, respName := range sortedKeys {
+		if compile.MatchString(respName) {
+			printModelTable(respName, seen, ontapSwag)
 		}
 	}
-	// Sort by api name
-	sort.Slice(collected, func(i, j int) bool {
-		return collected[i].name < collected[j].name
-	})
+}
 
-	for _, m := range collected {
-		fmt.Printf("\n# Model: %s\n", m.name)
+func printModelTable(name string, seen map[string]interface{}, ontapSwag ontap) {
+	_, ok := seen[name]
+	if ok {
+		return
+	}
+	def, ok := ontapSwag.swagger.Definitions[name]
+	if ok {
+		seen[name] = true
+		// determine if this is a collection wrapper or a model
+		// if it's a collection wrapper, unwrap the underlying model
+		props := def.SchemaProps.Properties
+		records, hasRecords := def.SchemaProps.Properties["records"]
+		if hasRecords {
+			props = records.Items.Schema.Properties
+		}
+		if props == nil {
+			return
+		}
+		fmt.Printf("\n# Model: %s\n", name)
 
 		table := newTable("name", "type", "description")
 		table.SetColMinWidth(2, descriptionWidth)
 		table.SetColWidth(descriptionWidth)
 
-		for _, orderedItem := range m.schema.Properties.ToOrderedSchemaItems() {
+		for _, orderedItem := range props.ToOrderedSchemaItems() {
 			args := propArgs{
 				table:     table,
 				name:      orderedItem.Name,
@@ -117,44 +116,6 @@ func showModels(a Args, ontapSwag ontap) {
 	}
 }
 
-func collectModels(name string, ontapSwag ontap, collected *[]model) {
-	def, modelName := walkRefs(ontapSwag, name)
-	if def == nil {
-		return
-	}
-	*collected = append(*collected, model{
-		schema: *def,
-		name:   modelName,
-	})
-}
-
-func walkRefs(ontapSwag ontap, name string) (*spec.Schema, string) {
-	def, ok := ontapSwag.swagger.Definitions[name]
-	if !ok {
-		return nil, ""
-	}
-	records, hasRecords := def.SchemaProps.Properties["records"]
-	if hasRecords {
-		schemaProps := records.Items.Schema.SchemaProps
-		if schemaProps.Type == nil {
-			ref := schemaFromRef(schemaProps.Ref.GetURL())
-			if len(ref) > 0 {
-				schema, ok := ontapSwag.swagger.Definitions[ref]
-				if ok {
-					return &schema, ref
-				}
-			}
-		} else {
-			ref := schemaFromRef(records.Items.Schema.Ref.GetURL())
-			if len(ref) == 0 {
-				ref = name
-			}
-			return records.Items.Schema, ref
-		}
-	}
-	return nil, ""
-}
-
 func newTable(headers ...string) *tw.Table {
 	table := tw.NewWriter(os.Stdout)
 	table.SetBorder(false)
@@ -170,7 +131,7 @@ func printProperty(args propArgs) {
 	}
 	if args.schema.Type == nil {
 		// check if this is a reference to another schema
-		refModel := schemaFromRef(args.schema.Ref.GetURL())
+		refModel := schemaFromRef(args.schema)
 		if len(refModel) > 0 {
 			_, seen := args.seen[refModel]
 			if !seen {
@@ -213,6 +174,15 @@ func sortApis(schema map[string]spec.PathItem) []string {
 	return keys
 }
 
+func sortSchema(schema map[string]spec.Schema) []string {
+	var keys []string
+	for name := range schema {
+		keys = append(keys, name)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
 func showParams(a Args, ontapSwag ontap) {
 	compile, err := regexp.Compile(a.Api)
 	if err != nil {
@@ -232,7 +202,7 @@ func showParams(a Args, ontapSwag ontap) {
 			if pathItem.Get.OperationProps.Responses != nil {
 				schema := (*pathItem.Get.OperationProps.Responses).ResponsesProps.StatusCodeResponses[200].ResponseProps.Schema
 				if schema != nil {
-					responseModel = schemaFromRef(schema.Ref.GetURL())
+					responseModel = schemaFromRef(*schema)
 				}
 			}
 
@@ -259,7 +229,8 @@ func showParams(a Args, ontapSwag ontap) {
 	}
 }
 
-func schemaFromRef(url *url.URL) string {
+func schemaFromRef(schema spec.Schema) string {
+	url := schema.Ref.GetURL()
 	responseModel := ""
 	if url != nil {
 		responseModel = url.Fragment
diff --git a/cmd/tools/zapi/zapi.go b/cmd/tools/zapi/zapi.go
index a8ac0d46..86525903 100644
--- a/cmd/tools/zapi/zapi.go
+++ b/cmd/tools/zapi/zapi.go
@@ -340,7 +340,7 @@ func init() {
 	showCmd.SetUsageTemplate("item to show should be one of: " + strings.Join(validShowArgs, ", "))
 
 	// Append usage examples
-	Cmd.SetUsageTemplate(Cmd.UsageTemplate() + `
+	Cmd.SetUsageTemplate(Cmd.UsageString() + `
 Examples:
   harvest zapi -p infinity show apis                                      Query cluster infinity for available APIs
   harvest zapi -p infinity show attrs --api volume-get-iter               Query cluster infinity for volume-get-iter metrics
diff --git a/conf/zapi/cdot/9.8.0/aggr.yaml b/conf/zapi/cdot/9.8.0/aggr.yaml
index f0d420b8..3caace6c 100644
--- a/conf/zapi/cdot/9.8.0/aggr.yaml
+++ b/conf/zapi/cdot/9.8.0/aggr.yaml
@@ -61,6 +61,8 @@ counters:
 plugins:
   LabelAgent:
     value_mapping: status state online `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status state online online `0`
 
 export_options:
   instance_keys:
diff --git a/conf/zapi/cdot/9.8.0/disk.yaml b/conf/zapi/cdot/9.8.0/disk.yaml
index 81809a0c..bd2b2114 100644
--- a/conf/zapi/cdot/9.8.0/disk.yaml
+++ b/conf/zapi/cdot/9.8.0/disk.yaml
@@ -34,6 +34,8 @@ counters:
 plugins:
   - LabelAgent:
     value_mapping: status outage , `0` # ok_value is empty value
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status outage - - `0` #ok_value is empty value, '-' would be converted to blank while processing.
 
 export_options:
   instance_keys:
diff --git a/conf/zapi/cdot/9.8.0/lun.yaml b/conf/zapi/cdot/9.8.0/lun.yaml
index 4bcf7523..d9631ce0 100644
--- a/conf/zapi/cdot/9.8.0/lun.yaml
+++ b/conf/zapi/cdot/9.8.0/lun.yaml
@@ -18,6 +18,8 @@ counters:
 plugins:
   - LabelAgent:
     value_mapping: status state online `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status state online online `0`
     # path is something like "/vol/vol_georg_fcp401/lun401"
     # we only want lun name, which is 4th element
     split: path `/` ,,,lun
diff --git a/conf/zapi/cdot/9.8.0/node.yaml b/conf/zapi/cdot/9.8.0/node.yaml
index 3d498c31..1b13f9b0 100644
--- a/conf/zapi/cdot/9.8.0/node.yaml
+++ b/conf/zapi/cdot/9.8.0/node.yaml
@@ -42,6 +42,8 @@ counters:
 plugins:
   - LabelAgent:
     value_mapping: status healthy true `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status healthy true up `0`
     join: warnings `; ` failed_fan_message,failed_power_message,over_temperature
 
 export_options:
diff --git a/conf/zapi/cdot/9.8.0/shelf.yaml b/conf/zapi/cdot/9.8.0/shelf.yaml
index 00d7fcc5..8e95d32b 100644
--- a/conf/zapi/cdot/9.8.0/shelf.yaml
+++ b/conf/zapi/cdot/9.8.0/shelf.yaml
@@ -64,6 +64,8 @@ plugins:
 
   LabelAgent:
     value_mapping: status op_status normal `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status state online ok `0`
 
 export_options:
   instance_keys:
diff --git a/conf/zapi/cdot/9.8.0/status.yaml b/conf/zapi/cdot/9.8.0/status.yaml
index 8297286a..82a95334 100644
--- a/conf/zapi/cdot/9.8.0/status.yaml
+++ b/conf/zapi/cdot/9.8.0/status.yaml
@@ -13,6 +13,8 @@ no_max_records: true
 plugins:
   - LabelAgent:
     value_mapping: status status ok `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status status ok ok `0`
 
 export_options:
   include_all_labels: true
diff --git a/conf/zapi/cdot/9.8.0/subsystem.yaml b/conf/zapi/cdot/9.8.0/subsystem.yaml
index 7d8ed461..e94a3bef 100644
--- a/conf/zapi/cdot/9.8.0/subsystem.yaml
+++ b/conf/zapi/cdot/9.8.0/subsystem.yaml
@@ -13,6 +13,8 @@ counters:
 plugins:
   - LabelAgent:
     value_mapping: status health ok `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status health ok todo `0`  # REST gap will be filled up later.
 
 export_options:
   instance_keys:
diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml
index a3c1bee1..eb8e5692 100644
--- a/conf/zapi/cdot/9.8.0/volume.yaml
+++ b/conf/zapi/cdot/9.8.0/volume.yaml
@@ -57,6 +57,8 @@ counters:
 plugins:
   LabelAgent:
     value_mapping: status state online `1`
+    # metric label zapi_value rest_value `default_value`
+    value_to_num: new_status state online online `0`
     exclude_equals: style `flexgroup_constituent`
     replace: style style `flexgroup_constituent` `flexgroup`
   Aggregator:
diff --git a/docker/onePollerPerContainer/README.md b/docker/onePollerPerContainer/README.md
index 2cd7d6bf..80e12516 100644
--- a/docker/onePollerPerContainer/README.md
+++ b/docker/onePollerPerContainer/README.md
@@ -1,58 +1,24 @@
-# Harvest for Docker
+## Docker
 
-Harvest is container-ready and supports several different deployment strategies:
- 
-- A poller-per-container model that offers a simple, flexible way to run Harvest. This deployment also enables a broad range of orchestrators (Nomad, Mesosphere, Swarm, K8, etc.)
-
-- A Harvest subcommand `harvest generate docker` can be use to create a Docker compose file with a container for each poller in your `harvest.yaml` file as well as the necessary volume and network mounts.
-
-- Docker + Ansible workflow to standup Harvest, Grafana, and Prometheus. A nice way to kick the tires and try everything out. See [Monitor all of your ONTAP clusters with Harvest (easy mode)](https://netapp.io/2021/05/21/monitor-all-of-your-ontap-clusters-with-harvest-easy-mode/) for details.
-## Deployment
-
-Make sure Docker is installed and `docker --version` meets the [minimum required version](https://github.com/NetApp/harvest#requirements).
-
-Harvest releases are published on [Dockerhub](https://hub.docker.com/r/rahulguptajss/harvest)
-
-### Docker Compose Poller per Container
-
-If you want to create a separate container for each poller in your `harvest.yaml` file, download the latest version of Harvest and run 
+Build Docker Image
 
 ```
-bin/harvest generate docker --image "rahulguptajss/harvest" > docker-compose.yml
-
-docker-compose up -d --remove-orphans
+docker build -f docker/onePollerPerContainer/Dockerfile -t harvest:latest  . --no-cache
 ```
 
-Stop docker containers
-
+Generate docker-compose file and save contents as docker-compose.yml
 ```
-docker-compose down
+bin/harvest generate docker --config ./harvest.yml
 ```
-### Poller per Container
-
-This option is typically used when you want to use an orchestrator or need more control over the volume, network, or configuration of your container(s).
 
-Adjust example command below as needed:
+Start docker containers
 
-- `PROM_PORT` is set to 12991 and used to bind port 12991 of the container to TCP port 12991 of the host
-- The `harvest.yml` file in the local directory is mapped to `/opt/harvest.yml` in the container. If you want to run with a different Harvest config file, change `$(pwd)/harvest.yml` to point to the desired local file.
-   
 ```
-PROM_PORT=12991 ; docker run --rm -it \
--p $PROM_PORT:$PROM_PORT \
---volume $(pwd)/harvest.yml:/opt/harvest/harvest.yml \
- rahulguptajss/harvest \
- --poller infinity --promPort $PROM_PORT
+docker-compose -f docker-compose.yml up -d --remove-orphans
 ```
 
-You can also generate a [docker-compose file as described above](#docker-compose-poller-per-container) and use it as a guide without using Docker compose.
-
-# Building Harvest Docker Image
-
-You only need to do build a Harvest Docker image if you want to custom image. If you are wanting to run Harvest with Docker see [deployment](#deployment) above.
-
-Build Docker Image
+Stop docker containers
 
 ```
-docker build -f docker/onePollerPerContainer/Dockerfile -t harvest:latest  . --no-cache
-```
+docker-compose -f docker-compose.yml down
+```
\ No newline at end of file


Test patch:
diff --git a/cmd/poller/plugin/label_agent/label_agent_test.go b/cmd/poller/plugin/label_agent/label_agent_test.go
index db3bef23..5abf373b 100644
--- a/cmd/poller/plugin/label_agent/label_agent_test.go
+++ b/cmd/poller/plugin/label_agent/label_agent_test.go
@@ -45,6 +45,12 @@ func TestInitPlugin(t *testing.T) {
 	params.NewChildS("value_mapping", "status state up,sleeping,down")
 	// similar to above, but if none of the values is matching, use default value "4"
 	params.NewChildS("value_mapping", "stage stage init `1`")
+	// create metric "new_status", if label "state" is one of the up/ok[zapi/rest], map metric value to respective index
+	params.NewChildS("value_to_num", "new_status state up ok")
+	// create metric "new_stage", but if none of the values is matching, use default value "4"
+	params.NewChildS("value_to_num", "new_stage stage init start `4`")
+	// create metric "new_outage", if empty value is expected and non empty means wrong, use default value "0"
+	params.NewChildS("value_to_num", "new_outage outage - - `0`")
 
 	abc := plugin.New("Test", nil, params, nil)
 	p = &LabelAgent{AbstractPlugin: abc}
@@ -282,3 +288,103 @@ func TestValueMappingRule(t *testing.T) {
 		t.Logf("OK - metric [stage]: value for instanceB set to %d", v)
 	}
 }
+
+func TestValueToNumRule(t *testing.T) {
+
+	var (
+		instanceA, instanceB  *matrix.Instance
+		status, stage, outage matrix.Metric
+		v, expected           uint8
+		ok                    bool
+		err                   error
+	)
+	// should match
+	m := matrix.New("TestLabelAgent", "test")
+
+	if instanceA, err = m.NewInstance("A"); err != nil {
+		t.Fatal(err)
+	}
+	instanceA.SetLabel("state", "up")   // "status" should be 1
+	instanceA.SetLabel("stage", "init") // "stage" should be 1
+	instanceA.SetLabel("outage", "")    // "outageStatus" should be 1
+
+	if instanceB, err = m.NewInstance("B"); err != nil {
+		t.Fatal(err)
+	}
+	instanceB.SetLabel("state", "unknown") // "status" should not be set
+	instanceB.SetLabel("stage", "unknown") // "stage" should be 4 (default)
+	instanceB.SetLabel("outage", "failed") // "outage" should be 0 (default)
+
+	if err = p.mapValues(m); err != nil {
+		t.Fatal(err)
+	}
+
+	if status = m.GetMetric("new_status"); status == nil {
+		t.Error("metric [status] missing")
+	}
+
+	if stage = m.GetMetric("new_stage"); stage == nil {
+		t.Error("metric [stage] missing")
+	}
+
+	if outage = m.GetMetric("new_outage"); outage == nil {
+		t.Error("metric [outage] missing")
+	}
+
+	// check "status" for instanceA
+	expected = 1
+	if v, ok = status.GetValueUint8(instanceA); !ok {
+		t.Error("metric [status]: value for InstanceA not set")
+	} else if v != expected {
+		t.Errorf("metric [status]: value for InstanceA is %d, expected %d", v, expected)
+	} else {
+		t.Logf("OK - metric [status]: value for instanceA set to %d", v)
+	}
+
+	// check "status" for instanceB
+	if v, ok = status.GetValueUint8(instanceB); !ok {
+		t.Log("OK - metric [status]: value for InstanceB not set")
+	} else {
+		t.Errorf("metric [status]: value for InstanceA is %d, should not be set", v)
+	}
+
+	// check "stage" for instanceA
+	expected = 1
+	if v, ok = stage.GetValueUint8(instanceA); !ok {
+		t.Error("metric [stage]: value for InstanceA not set")
+	} else if v != expected {
+		t.Errorf("metric [stage]: value for InstanceA is %d, expected %d", v, expected)
+	} else {
+		t.Logf("OK - metric [stage]: value for instanceA set to %d", v)
+	}
+
+	// check "stage" for instanceB
+	expected = 4
+	if v, ok = stage.GetValueUint8(instanceB); !ok {
+		t.Error("metric [stage]: value for InstanceB not set")
+	} else if v != expected {
+		t.Errorf("metric [stage]: value for InstanceB is %d, expected %d", v, expected)
+	} else {
+		t.Logf("OK - metric [stage]: value for instanceB set to %d", v)
+	}
+
+	// check "outage" for instanceA
+	expected = 1
+	if v, ok = outage.GetValueUint8(instanceA); !ok {
+		t.Error("metric [outage]: value for InstanceA not set")
+	} else if v != expected {
+		t.Errorf("metric [outage]: value for InstanceA is %d, expected %d", v, expected)
+	} else {
+		t.Logf("OK - metric [outage]: value for instanceA set to %d", v)
+	}
+
+	// check "outage" for instanceB
+	expected = 0
+	if v, ok = outage.GetValueUint8(instanceB); !ok {
+		t.Error("metric [outage]: value for InstanceB not set")
+	} else if v != expected {
+		t.Errorf("metric [outage]: value for InstanceB is %d, expected %d", v, expected)
+	} else {
+		t.Logf("OK - metric [outage]: value for instanceB set to %d", v)
+	}
+}