Repository: iter8-tools/iter8
Pull request #1296 (2022-06-23 19:22:40)

Custom metrics simplification
Fixes #1295 


Patch:
diff --git a/base/custom_metrics.go b/base/custom_metrics.go
index f49288b7..5bfd5c62 100644
--- a/base/custom_metrics.go
+++ b/base/custom_metrics.go
@@ -14,15 +14,15 @@ import (
 
 	"time"
 
+	"github.com/Masterminds/sprig"
 	"github.com/itchyny/gojq"
 	log "github.com/iter8-tools/iter8/base/log"
-
 	"sigs.k8s.io/yaml"
 )
 
-// MetricsSpec specifies the set of metrics that can be obtained from a provider
-type MetricsSpec struct {
-	// Provider is the label/name of the data source
+// ProviderSpec specifies how to get metrics from a provider
+type ProviderSpec struct {
+	// Provider is the name of the metrics provider
 	Provider string `json:"provider" yaml:"provider"`
 
 	// URL is the database endpoint
@@ -38,7 +38,9 @@ type MetricsSpec struct {
 	Metrics []Metric `json:"metrics" yaml:"metrics"`
 }
 
-// Metric defines how to obtain a metric
+// Metric defines how to construct HTTP requests and parse HTTP responses
+// when querying a database for a metric. Metric struct also includes metadata
+// such as the name and description of the metric.
 type Metric struct {
 	// Name is the name of the metric
 	Name string `json:"name" yaml:"name"`
@@ -53,7 +55,7 @@ type Metric struct {
 	Units *string `json:"units,omitempty" yaml:"units,omitempty"`
 
 	// Params is the set of HTTP parameters that need to be sent
-	Params *[]Params `json:"params,omitempty" yaml:"params,omitempty"`
+	Params *[]HTTPParam `json:"params,omitempty" yaml:"params,omitempty"`
 
 	// Body is the HTTP request body that needs to be sent
 	Body *string `json:"body,omitempty" yaml:"body,omitempty"`
@@ -63,8 +65,8 @@ type Metric struct {
 	JqExpression string `json:"jqExpression" yaml:"jqExpression"`
 }
 
-// Params defines an HTTP parameter
-type Params struct {
+// HTTPParam defines an HTTP parameter
+type HTTPParam struct {
 	// Name is the name of the HTTP parameter
 	Name string `json:"name" yaml:"name"`
 
@@ -74,14 +76,16 @@ type Params struct {
 
 // customMetricsInputs is the input to the custommetrics task
 type customMetricsInputs struct {
-	// ProviderURLs a slice of URLs for metric templates
+	// ProviderURLs	A slice of URLs, each of them provides a metrics template file.
 	ProviderURLs []string `json:"providerURLs" yaml:"providerURLs"`
 
-	// Common	values that are common across versions
-	Common map[string]interface{} `json:"common" yaml:"common"`
+	// Values Values used for substituting placeholders in metric templates.
+	Values map[string]interface{} `json:"values" yaml:"values"`
 
-	// VersionInfo values that are specific to each version
-	VersionInfo []map[string]interface{} `json:"versionInfo" yaml:"versionInfo"`
+	// VersionValues Per version values that override values
+	// For each version, its version values are coalesced with values
+	// The length of this slice equals the number of versions
+	VersionValues []map[string]interface{} `json:"versionValues" yaml:"versionValues"`
 }
 
 const (
@@ -104,6 +108,10 @@ type customMetricsTask struct {
 
 // initializeDefaults sets default values for the custom metrics task
 func (t *customMetricsTask) initializeDefaults() {
+	// initialize versionValues if absent
+	if len(t.With.VersionValues) == 0 {
+		t.With.VersionValues = []map[string]interface{}{t.With.Values}
+	}
 }
 
 // validate task inputs
@@ -111,22 +119,18 @@ func (t *customMetricsTask) validateInputs() error {
 	return nil
 }
 
-// for a given version info and Experiment, calculate the elapsed time that
-// should be used for queries
+// getElapsedTimeSeconds using values and experiment
 //
-// elapsed time is based on the StartingTime in the version info or the
-// starting time in the Experiment
-func getElapsedTimeSeconds(versionInfo map[string]interface{}, exp *Experiment) (int64, error) {
-	// elapsedTimeSeconds should not be provided by the user
-	if versionInfo[elapsedTimeSecondsStr] != nil {
-		return 0, errors.New("elapsedTimeSeconds should not be provided by the user in VersionInfo: " + fmt.Sprintf("%v", versionInfo))
-	}
-
+// elapsedTime = currentTime - startingTime
+//
+// First, check for startingTime in values.
+// If unavailable, use startingTime of the experiment.
+func getElapsedTimeSeconds(values map[string]interface{}, exp *Experiment) (int64, error) {
 	startingTime := exp.Result.StartTime.Time
-	if versionInfo[startingTimeStr] != nil {
+	if values[startingTimeStr] != nil {
 		var err error
 		// Calling Parse() method with its parameters
-		startingTime, err = time.Parse(time.RFC3339, fmt.Sprintf("%v", versionInfo[startingTimeStr]))
+		startingTime, err = time.Parse(time.RFC3339, fmt.Sprintf("%v", values[startingTimeStr]))
 
 		if err != nil {
 			return 0, errors.New("cannot parse startingTime")
@@ -142,7 +146,7 @@ func getElapsedTimeSeconds(versionInfo map[string]interface{}, exp *Experiment)
 //
 // bool return value represents whether the pipeline was able to run to
 // completion (prevents double error statement)
-func queryDatabaseAndGetValue(template MetricsSpec, metric Metric) (interface{}, bool) {
+func queryDatabaseAndGetValue(template ProviderSpec, metric Metric) (interface{}, bool) {
 	var requestBody io.Reader
 	if metric.Body != nil {
 		requestBody = strings.NewReader(*metric.Body)
@@ -217,14 +221,13 @@ func queryDatabaseAndGetValue(template MetricsSpec, metric Metric) (interface{},
 }
 
 // get provider template from URL
-func getProviderTemplate(url string, commonValues map[string]interface{}) (*template.Template, error) {
+func getProviderTemplate(providerURL string) (*template.Template, error) {
 	// fetch b from url
-	resp, err := http.Get(url)
+	resp, err := http.Get(providerURL)
 	if err != nil {
 		log.Logger.Error(err)
 		return nil, err
 	}
-
 	// read responseBody
 	// get the doubly templated metrics spec
 	defer resp.Body.Close()
@@ -232,27 +235,13 @@ func getProviderTemplate(url string, commonValues map[string]interface{}) (*temp
 	if err != nil {
 		return nil, err
 	}
-	dt, err := template.New("doubly-templated").Parse(string(responseBody))
+	tpl, err := template.New("provider template").Funcs(sprig.TxtFuncMap()).Parse(string(responseBody))
 	if err != nil {
 		log.Logger.Error(err)
 		return nil, err
 	}
 
-	// execute template with commonValues
-	// get the singly templated metrics spec
-	var buf bytes.Buffer
-	err = dt.Execute(&buf, commonValues)
-	if err != nil {
-		log.Logger.Error(err)
-		return nil, err
-	}
-	st, err := template.New("singly-templated").Parse(string(responseBody))
-	if err != nil {
-		log.Logger.Error(err)
-		return nil, err
-	}
-
-	return st, nil
+	return tpl, nil
 }
 
 // run executes this task
@@ -272,49 +261,58 @@ func (t *customMetricsTask) run(exp *Experiment) error {
 	// initialize defaults
 	t.initializeDefaults()
 
-	// inputs for this task determine the number of versions participating in the
-	// experiment. Initiate insights with num versions.
-	err = exp.Result.initInsightsWithNumVersions(len(t.With.VersionInfo))
+	err = exp.Result.initInsightsWithNumVersions(len(t.With.VersionValues))
 	if err != nil {
 		return err
 	}
 
 	// collect metrics from all providers and for all versions
-	for _, providerURL := range t.With.ProviderURLs {
+	for j, providerURL := range t.With.ProviderURLs {
 		// finalize metrics spec
-		template, err := getProviderTemplate(providerURL, t.With.Common)
+		template, err := getProviderTemplate(providerURL)
 		if err != nil {
 			return err
 		}
-		for i, versionInfo := range t.With.VersionInfo {
-			// add elapsedTimeSeconds
-			elapsedTimeSeconds, err := getElapsedTimeSeconds(versionInfo, exp)
+
+		for i, versionValues := range t.With.VersionValues {
+			// merge values
+			vals, err := mustMergeOverwrite(t.With.Values, versionValues)
 			if err != nil {
 				return err
 			}
-			if versionInfo == nil {
-				versionInfo = make(map[string]interface{})
+			values := vals.(map[string]interface{})
+			if len(values) == 0 {
+				values = make(map[string]interface{})
+			}
+			// add elapsedTimeSeconds
+			elapsedTimeSeconds, err := getElapsedTimeSeconds(values, exp)
+			if err != nil {
+				return err
 			}
-			versionInfo[elapsedTimeSecondsStr] = elapsedTimeSeconds
+			values[elapsedTimeSecondsStr] = elapsedTimeSeconds
 
 			// get the metrics spec
 			var buf bytes.Buffer
-			err = template.Execute(&buf, versionInfo)
+			err = template.Execute(&buf, values)
 			if err != nil {
 				return err
 			}
-			var metrics MetricsSpec
-			err = yaml.Unmarshal(buf.Bytes(), &metrics)
+			bytes, _ := ioutil.ReadAll(&buf)
+			var provider ProviderSpec
+			err = yaml.Unmarshal(bytes, &provider)
 			if err != nil {
 				return err
 			}
+			log.Logger.Debugf("provider spec %v for version %v\n", j, i)
+			log.Logger.Debug("--------------------------------")
+			log.Logger.Debug(string(bytes))
 
 			// get each metric
-			for _, metric := range metrics.Metrics {
+			for _, metric := range provider.Metrics {
 				log.Logger.Debug("query for metric ", metric.Name)
 
 				// perform database query and extract metric value
-				value, ok := queryDatabaseAndGetValue(metrics, metric)
+				val, ok := queryDatabaseAndGetValue(provider, metric)
 
 				// check if there were any issues querying database and extracting value
 				if !ok {
@@ -323,7 +321,7 @@ func (t *customMetricsTask) run(exp *Experiment) error {
 				}
 
 				// do not save value if it has no value
-				if value == nil {
+				if val == nil {
 					log.Logger.Error("could not extract non-nil value for metric ", metric.Name, ": ", err)
 					continue
 				}
@@ -344,14 +342,14 @@ func (t *customMetricsTask) run(exp *Experiment) error {
 				}
 
 				// convert value to float
-				valueString := fmt.Sprint(value)
+				valueString := fmt.Sprint(val)
 				floatValue, err := strconv.ParseFloat(valueString, 64)
 				if err != nil {
 					log.Logger.Error("could not parse string \""+valueString+"\" to float:", err)
 					continue
 				}
 
-				err = exp.Result.Insights.updateMetric(metrics.Provider+"/"+metric.Name, mm, i, floatValue)
+				err = exp.Result.Insights.updateMetric(provider.Provider+"/"+metric.Name, mm, i, floatValue)
 
 				if err != nil {
 					log.Logger.Error("could not add update metric", err)
diff --git a/base/must_merge_overwrite.go b/base/must_merge_overwrite.go
new file mode 100644
index 00000000..2ae73dc6
--- /dev/null
+++ b/base/must_merge_overwrite.go
@@ -0,0 +1,40 @@
+/*
+Copyright (C) 2013-2020 Masterminds
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package base
+
+import (
+	"github.com/imdario/mergo"
+	"github.com/iter8-tools/iter8/base/log"
+)
+
+// mustMergeOverwrite	merge maps giving precedence to the right side
+func mustMergeOverwrite(dst map[string]interface{}, srcs ...map[string]interface{}) (interface{}, error) {
+	for _, src := range srcs {
+		if err := mergo.MergeWithOverwrite(&dst, src); err != nil {
+			// the following log line is the diff between the original sprig func and ours
+			log.Logger.Error(err)
+			return nil, err
+		}
+	}
+	return dst, nil
+}
diff --git a/custommetrics/istio-prom.tpl b/custommetrics/istio-prom.tpl
new file mode 100644
index 00000000..1d643e5e
--- /dev/null
+++ b/custommetrics/istio-prom.tpl
@@ -0,0 +1,114 @@
+# This file provides templated metric specifications that enable
+# Iter8 to retrieve metrics from Istio's Prometheus add-on.
+# 
+# For a list of metrics supported out-of-the-box by the Istio Prom add-on, 
+# please see https://istio.io/latest/docs/reference/config/metrics/
+#
+# Iter8 substitutes the placeholders in this file with values, 
+# and uses the resulting metric specs to query Prometheus.
+# The placeholders are as follows.
+# 
+# reporter                        string  optional
+# destinationWorkload             string  required
+# destinationWorkloadNamespace    string  required
+# elapsedTimeSeconds              int     implicit
+# startingTime                    string  optional
+# latencyPercentiles              []int   optional
+#
+# For descriptions of reporter, destinationWorkload, and destinationWorkloadNamespace, 
+# please see https://istio.io/latest/docs/reference/config/metrics/
+#
+# elapsedTimeSeconds: this should not be specified directly by the user. 
+# It is implicitly computed by Iter8 according to the following formula
+# elapsedTimeSeconds := (time.Now() - startingTime).Seconds()
+# 
+# startingTime: By default, this is the time at which the Iter8 experiment started.
+# The user can explicitly specify the startingTime for each app version
+# (for example, the user can set the startingTime to the creation time of the app version)
+#
+# latencyPercentiles: Each item in this slice will create a new metric spec.
+# For example, if this is set to [50,75,90,95],
+# then, latency-p50, latency-p75, latency-p90, latency-p95 metric specs are created.
+
+{{- define "istio-prom-reporter"}}
+{{- if .reporter }}
+reporter="{{ .reporter }}",
+{{- end }}
+{{- end }}
+
+{{- define "istio-prom-dest"}}
+{{ template "istio-prom-reporter" . }}
+destination_workload="{{ .destinationWorkload }}",
+destination_workload_namespace="{{ .destinationWorkloadNamespace }}"
+{{- end }}
+
+# url is the HTTP endpoint where the Prometheus service installed by Istio's Prom add-on
+# can be queried for metrics
+
+url: {{ default .istioPromURL "http://prometheus.istio-system:9090/api/v1/query" }}
+provider: istio-prom
+method: GET
+metrics:
+- name: request-count
+  type: counter
+  description: |
+    Number of requests
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_requests_total{
+        {{ template "istio-prom-dest" . }}
+      }[{{ .elapsedTimeSeconds }}s])) or on() vector(0)
+  jqExpression: .data.result[0].value[1] | tonumber
+- name: error-count
+  type: counter
+  description: |
+    Number of unsuccessful requests
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_requests_total{
+        response_code=~'5..',
+        {{ template "istio-prom-dest" . }}
+      }[{{ .elapsedTimeSeconds }}s])) or on() vector(0)
+  jqExpression: .data.result[0].value[1] | tonumber
+- name: error-rate
+  type: gauge
+  description: |
+    Fraction of unsuccessful requests
+  params:
+  - name: query
+    value: |
+      (sum(last_over_time(istio_requests_total{
+        response_code=~'5..',
+        {{ template "istio-prom-dest" . }}
+      }[{{ .elapsedTimeSeconds }}s])) or on() vector(0))/(sum(last_over_time(istio_requests_total{
+        {{ template "istio-prom-dest" . }}
+      }[{{ .elapsedTimeSeconds }}s])) or on() vector(0))
+  jqExpression: .data.result.[0].value.[1]
+- name: latency-mean
+  type: gauge
+  description: |
+    Mean latency
+  params:
+  - name: query
+    value: |
+      (sum(last_over_time(istio_request_duration_milliseconds_sum{
+        {{ template "istio-prom-dest" . }}
+      }[{{ .elapsedTimeSeconds }}s])) or on() vector(0))/(sum(last_over_time(istio_requests_total{
+        {{ template "istio-prom-dest" . }}
+      }[{{ .elapsedTimeSeconds }}s])) or on() vector(0))
+  jqExpression: .data.result[0].value[1] | tonumber
+{{- range $i, $p := .latencyPercentiles }}
+- name: latency-p{{ $p }}
+  type: gauge
+  description: |
+    {{ $p }} percentile latency
+  params:
+  - name: query
+    value: |
+      histogram_quantile(0.{{ $p }}, sum(rate(istio_request_duration_milliseconds_bucket{
+        {{ template "istio-prom-dest" $ }}
+      }[{{ .elapsedTimeSeconds }}s])) by (le))
+  jqExpression: .data.result[0].value[1] | tonumber
+{{- end }}
\ No newline at end of file
diff --git a/custommetrictemplates/istio.tpl b/custommetrictemplates/istio.tpl
deleted file mode 100644
index d72aff10..00000000
--- a/custommetrictemplates/istio.tpl
+++ /dev/null
@@ -1,119 +0,0 @@
-url: {{ .providerURL }}
-provider: istio
-method: GET
-# Inputs for the metrics (output of template):
-#   destination_workload                      string
-#   destination_workload_namespace            string
-#   startingTime                              string
-#
-# Note: elapsedTimeSeconds is produced by Iter8
-metrics:
-- name: request-count
-  type: counter
-  description: |
-    Number of requests
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_requests_total{
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1] | tonumber
-- name: error-count
-  type: counter
-  description: |
-    Number of non-successful requests
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_requests_total{
-        response_code=~'5..',
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1] | tonumber
-- name: error-rate
-  type: gauge
-  description: |
-    Percentage of non-successful requests
-  params:
-  - name: query
-    value: |
-      (sum(last_over_time(istio_requests_total{
-        response_code=~'5..',
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0))/(sum(last_over_time(istio_requests_total{
-        {{- if .response_code }}
-        response_code="{{.response_code}}",
-        {{- end }}
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0))
-  jqExpression: .data.result.[0].value.[1]
-- name: le500ms-latency-percentile
-  type: gauge
-  description: |
-    Less than 500 ms latency
-  params:
-  - name: query
-    value: |
-      (sum(last_over_time(istio_request_duration_milliseconds_bucket{
-        le='500',
-        {{- if .response_code }}
-        response_code="{{.response_code}}",
-        {{- end }}
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0))/(sum(last_over_time(istio_request_duration_milliseconds_bucket{
-        le='+Inf',
-        {{- if .response_code }}
-        response_code="{{.response_code}}",
-        {{- end }}
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0))
-  jqExpression: .data.result[0].value[1] | tonumber
-- name: latency-mean
-  type: gauge
-  description: |
-    Mean latency
-  params:
-  - name: query
-    value: |
-      (sum(last_over_time(istio_request_duration_milliseconds_sum{
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0))/(sum(last_over_time(istio_requests_total{
-        {{- if .response_code }}
-        response_code="{{.response_code}}",
-        {{- end }}
-        {{- if .reporter }}
-        reporter="{{.reporter}}",
-        {{- end }}
-        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0))
-  jqExpression: .data.result[0].value[1] | tonumber
diff --git a/go.mod b/go.mod
index 546e2c87..4718cc44 100644
--- a/go.mod
+++ b/go.mod
@@ -23,6 +23,7 @@ require (
 	github.com/antonmedv/expr v1.9.0
 	github.com/bojand/ghz v0.108.0
 	github.com/hashicorp/go-getter v1.6.1
+	github.com/imdario/mergo v0.3.12
 	github.com/itchyny/gojq v0.12.7
 	github.com/jarcoal/httpmock v1.1.0
 	github.com/mattn/go-shellwords v1.0.12


Test patch:
diff --git a/action/gen_test.go b/action/gen_test.go
index cf797071..3c7c0ed3 100644
--- a/action/gen_test.go
+++ b/action/gen_test.go
@@ -67,7 +67,7 @@ func TestGenDB(t *testing.T) {
 	gOpts := NewGenOpts()
 	gOpts.ChartsParentDir = base.CompletePath("../", "")
 	gOpts.ChartName = "iter8"
-	gOpts.Values = []string{"tasks={custommetrics,assess}", "custommetrics.providerURLs[0]=https://raw.githubusercontent.com/iter8-tools/iter8/master/charts/iter8lib/templates/_metrics-istio.tpl", "custommetrics.common.providerURL=http://prometheus.istio-system:9090/api/v1/query", "custommetrics.versionInfo[0].destination_workload=httpbin-v2", "custommetrics.versionInfo[0].destination_workload_namespace=default", `custommetrics.versionInfo[0].startingTime="2020-02-01T09:44:40Z"`, "assess.SLOs.upper.istio/error-rate=0"}
+	gOpts.Values = []string{"tasks={custommetrics,assess}", "custommetrics.providerURLs[0]=https://raw.githubusercontent.com/iter8-tools/iter8/master/charts/iter8lib/templates/_metrics-istio.tpl", "custommetrics.common.providerURL=http://prometheus.istio-system:9090/api/v1/query", "custommetrics.values.destinationWorkload=httpbin-v2", "custommetrics.values.destinationWorkloadNamespace=default", `custommetrics.values.startingTime="2020-02-01T09:44:40Z"`, "assess.SLOs.upper.istio/error-rate=0"}
 
 	err := gOpts.LocalRun()
 	assert.NoError(t, err)
diff --git a/base/custom_metrics_test.go b/base/custom_metrics_test.go
index 7b1404cc..e5ae619d 100644
--- a/base/custom_metrics_test.go
+++ b/base/custom_metrics_test.go
@@ -47,7 +47,7 @@ const (
 // test getElapsedTimeSeconds()
 func TestGetElapsedTimeSeconds(t *testing.T) {
 	os.Chdir(t.TempDir())
-	versionInfo := map[string]interface{}{
+	versionValues := map[string]interface{}{
 		"startingTime": "2020-02-01T09:44:40Z",
 	}
 
@@ -57,45 +57,22 @@ func TestGetElapsedTimeSeconds(t *testing.T) {
 	}
 
 	// this should add a startingTime that will be overwritten by the one in
-	// versionInfo
+	// versionValues
 	exp.initResults(1)
 
-	elapsedTimeSeconds, _ := getElapsedTimeSeconds(versionInfo, exp)
+	elapsedTimeSeconds, _ := getElapsedTimeSeconds(versionValues, exp)
 
 	// elapsedTimeSeconds should be a large number
 	//
 	// if getElapsedTimeSeconds() used the starting time from the experiment instead of
-	// the one from versionInfo, the elapsed time would be 0 or close to 0
+	// the one from versionValues, the elapsed time would be 0 or close to 0
 	assert.Equal(t, elapsedTimeSeconds > 1000000, true)
 }
 
-// test if a user sets elapsedTimeSeconds getElapsedTimeSeconds()
-func TestSetElapsedTimeSecondsError(t *testing.T) {
-	os.Chdir(t.TempDir())
-	versionInfo := map[string]interface{}{
-		"startingTime":       "2020-02-01T09:44:40Z",
-		"elapsedTimeSeconds": "2020-02-01T09:44:40Z",
-	}
-
-	exp := &Experiment{
-		Spec:   []Task{},
-		Result: &ExperimentResult{},
-	}
-
-	// this should add a startingTime that will be overwritten by the one in
-	// versionInfo
-	exp.initResults(1)
-
-	elapsedTimeSeconds, err := getElapsedTimeSeconds(versionInfo, exp)
-
-	assert.Error(t, err)
-	assert.Equal(t, elapsedTimeSeconds, int64(0))
-}
-
 // test if a user sets startingTime incorrectly getElapsedTimeSeconds()
 func TestStartingTimeFormatError(t *testing.T) {
 	os.Chdir(t.TempDir())
-	versionInfo := map[string]interface{}{
+	versionValues := map[string]interface{}{
 		"startingTime": "1652935205",
 	}
 
@@ -105,22 +82,19 @@ func TestStartingTimeFormatError(t *testing.T) {
 	}
 
 	// this should add a startingTime that will be overwritten by the one in
-	// versionInfo
+	// versionValues
 	exp.initResults(1)
-
-	elapsedTimeSeconds, err := getElapsedTimeSeconds(versionInfo, exp)
-
+	_, err := getElapsedTimeSeconds(versionValues, exp)
 	assert.Error(t, err)
-	assert.Equal(t, elapsedTimeSeconds, int64(0))
 }
 
 // some reusable test code
 func headForTests(t *testing.T, providerURL string) *customMetricsTask {
-	common := map[string]interface{}{
-		"providerURL": "http://prometheus.istio-system:9090/api/v1/query",
-		"IAMToken":    "test-token",
-		"GUID":        "test-guid",
-	}
+	// values := map[string]interface{}{
+	// 	"providerURL": "http://prometheus.istio-system:9090/api/v1/query",
+	// 	"IAMToken":    "test-token",
+	// 	"GUID":        "test-guid",
+	// }
 
 	// valid collect database task... should succeed
 	ct := &customMetricsTask{
@@ -129,8 +103,7 @@ func headForTests(t *testing.T, providerURL string) *customMetricsTask {
 		},
 		With: customMetricsInputs{
 			ProviderURLs: []string{providerURL},
-			Common:       common,
-			VersionInfo:  []map[string]interface{}{{}},
+			// Values:       values,
 		},
 	}
 
@@ -220,12 +193,12 @@ func TestCEOneVersion(t *testing.T) {
 
 }
 
-// basic test with versionInfo, mimicking Code Engine
+// basic test with versionValues, mimicking Code Engine
 // one version, three successful metrics
-func TestCEVersionInfo(t *testing.T) {
+func TestCEVersionValues(t *testing.T) {
 	os.Chdir(t.TempDir())
 	ct := headForTests(t, testCEURL)
-	ct.With.VersionInfo = []map[string]interface{}{{
+	ct.With.VersionValues = []map[string]interface{}{{
 		"ibm_codeengine_revision_name": "v1",
 	}}
 
@@ -417,7 +390,7 @@ func TestCEMultipleVersions(t *testing.T) {
 	os.Chdir(t.TempDir())
 	ct := headForTests(t, testCEURL)
 
-	ct.With.VersionInfo = []map[string]interface{}{{}, {}}
+	ct.With.VersionValues = []map[string]interface{}{{}, {}}
 
 	// request-count
 	httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
@@ -493,7 +466,7 @@ func TestCEMultipleVersions(t *testing.T) {
 func TestCEMultipleVersionsAndMetrics(t *testing.T) {
 	os.Chdir(t.TempDir())
 	ct := headForTests(t, testCEURL)
-	ct.With.VersionInfo = []map[string]interface{}{{}, {}}
+	ct.With.VersionValues = []map[string]interface{}{{}, {}}
 
 	// request-count
 	httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
diff --git a/testdata/assertinputs/experiment.yaml b/testdata/assertinputs/experiment.yaml
index e2bd642e..5b9f3874 100644
--- a/testdata/assertinputs/experiment.yaml
+++ b/testdata/assertinputs/experiment.yaml
@@ -6,8 +6,7 @@ spec:
       duration: 2s
       errorRanges:
       - lower: 500
-      versionInfo:
-      - url: https://httpbin.org/get
+      url: https://httpbin.org/get
   # task 2: validate service level objectives for app using
   # the metrics collected in the above task
   - task: assess
diff --git a/testdata/assertinputs/noinsights/experiment.yaml b/testdata/assertinputs/noinsights/experiment.yaml
index 35314ab4..54b0430d 100644
--- a/testdata/assertinputs/noinsights/experiment.yaml
+++ b/testdata/assertinputs/noinsights/experiment.yaml
@@ -6,8 +6,7 @@ spec:
       duration: 2s
       errorRanges:
       - lower: 500
-      versionInfo:
-      - url: https://httpbin.org/get
+      url: https://httpbin.org/get
   # task 2: validate service level objectives for app using
   # the metrics collected in the above task
   - task: assess
diff --git a/testdata/assertinputsfail/experiment.yaml b/testdata/assertinputsfail/experiment.yaml
index c66aef47..180a387c 100644
--- a/testdata/assertinputsfail/experiment.yaml
+++ b/testdata/assertinputsfail/experiment.yaml
@@ -6,8 +6,7 @@ spec:
       duration: 2s
       errorRanges:
       - lower: 500
-      versionInfo:
-      - url: https://httpbin.org/get
+      url: https://httpbin.org/get
   # task 2: validate service level objectives for app using
   # the metrics collected in the above task
   - task: assess
diff --git a/testdata/experiment_db.yaml b/testdata/experiment_db.yaml
index 92ab31d2..89cb93a2 100644
--- a/testdata/experiment_db.yaml
+++ b/testdata/experiment_db.yaml
@@ -4,7 +4,7 @@ spec:
     with:
       providerURLs:
       - https://raw.githubusercontent.com/iter8-tools/iter8/master/testdata/metrics/kfserving.metrics.yaml
-      versionInfo:
+      versionValues:
       - namespace_name: ns-candidate
         startingTime: Jan 2, 2006 at 3:04pm (MST)
   # task 2: validate service level objectives for app using
diff --git a/testdata/experiment_grpc.yaml b/testdata/experiment_grpc.yaml
index 5165688c..49d901ec 100644
--- a/testdata/experiment_grpc.yaml
+++ b/testdata/experiment_grpc.yaml
@@ -10,9 +10,8 @@ spec:
       timeout: 10s
       connect-timeeout: 5s
       protoURL: "https://raw.githubusercontent.com/bojand/ghz/v0.105.0/testdata/greeter.proto"
-      versionInfo:
-      - call: "helloworld.Greeter.SayHello"
-        host: "127.0.0.1"
+      call: "helloworld.Greeter.SayHello"
+      host: "127.0.0.1"
   # task 2: validate service level objectives for app using
   # the metrics collected in the above task
   - task: assess