Repository: iter8-tools/iter8
Pull request #1264 (2022-05-26 15:35:14)

Add slo-validation-istio to iter8lib


Patch:
diff --git a/.gitignore b/.gitignore
index 213ad88d..70addae4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -133,7 +133,7 @@ Chart.lock
 **/result.yaml
 **/report.html
 **/*.metrics.yaml
-!testdata/metrics/test-ce.metrics.yaml
+!testdata/metrics/*.metrics.yaml
 
 bin/
 _dist/
diff --git a/base/collect_database.go b/base/collect_database.go
index a7d8d6ad..f075bd0f 100644
--- a/base/collect_database.go
+++ b/base/collect_database.go
@@ -248,6 +248,10 @@ func (t *collectDatabaseTask) run(exp *Experiment) error {
 			if err != nil {
 				return err
 			}
+
+			if versionInfo == nil {
+				versionInfo = make(map[string]interface{})
+			}
 			versionInfo[elapsedTimeSecondsStr] = elapsedTimeSeconds
 
 			// finalize metrics spec
diff --git a/charts/iter8lib/templates/_k-spec-secret.tpl b/charts/iter8lib/templates/_k-spec-secret.tpl
index a8b55215..add10d2f 100644
--- a/charts/iter8lib/templates/_k-spec-secret.tpl
+++ b/charts/iter8lib/templates/_k-spec-secret.tpl
@@ -7,5 +7,13 @@ metadata:
     iter8.tools/revision: {{ .Release.Revision | quote }}
 stringData:
   experiment.yaml: |
-{{- include "experiment" . | indent 4 }}
+{{ include "experiment" . | indent 4 }}
+{{- if .Values.providers }}
+{{- range .Values.providers }}
+{{- if eq . "istio"}}
+  istio.metrics.yaml: |
+{{ include "metrics.istio" $ | indent 4 }}
 {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/charts/iter8lib/templates/_metrics-istio.tpl b/charts/iter8lib/templates/_metrics-istio.tpl
new file mode 100644
index 00000000..674668c9
--- /dev/null
+++ b/charts/iter8lib/templates/_metrics-istio.tpl
@@ -0,0 +1,124 @@
+{{- define "metrics.istio" -}}
+url: {{ .Values.endpoint }}/api/v1/query
+provider: istio
+method: GET
+# Inputs for the metrics (output of template):
+#   destination_workload                      string
+#   destination_workload_namespace            string
+#   startingTime                              string
+#
+# Note: elapsedTimeSeconds is produced by Iter8
+metrics:
+- name: request-count
+  type: counter
+  description: |
+    Number of requests
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_requests_total{
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
+  jqExpression: .data.result[0].value[1] | tonumber
+- name: error-count
+  type: counter
+  description: |
+    Number of non-successful requests
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_requests_total{
+        response_code=~'5..',
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
+  jqExpression: .data.result[0].value[1] | tonumber
+- name: error-rate
+  type: gauge
+  description: |
+    Percentage of non-successful requests
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_requests_total{
+        response_code=~'5..',
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)/sum(last_over_time(istio_requests_total{
+        {{- if .Values.response_code }}
+        response_code="{{.Values.response_code}}",
+        {{- end }}
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
+  jqExpression: .data.result.[0].value.[1]
+- name: le500ms-latency-percentile
+  type: gauge
+  description: |
+    Less than 500 ms latency
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_request_duration_milliseconds_bucket{
+        le='500',
+        {{- if .Values.response_code }}
+        response_code="{{.Values.response_code}}",
+        {{- end }}
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)/sum(last_over_time(istio_request_duration_milliseconds_bucket{
+        le='+Inf',
+        {{- if .Values.response_code }}
+        response_code="{{.Values.response_code}}",
+        {{- end }}
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
+  jqExpression: .data.result[0].value[1] | tonumber
+- name: latency-mean
+  type: gauge
+  description: |
+    Mean latency
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(istio_request_duration_milliseconds_sum{
+        {{- if .Values.response_code }}
+        response_code="{{.Values.response_code}}",
+        {{- end }}
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)/sum(last_over_time(istio_requests_total{
+        {{- if .Values.response_code }}
+        response_code="{{.Values.response_code}}",
+        {{- end }}
+        {{- if .Values.reporter }}
+        reporter="{{.Values.reporter}}",
+        {{- end }}
+        destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
+        destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
+      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
+  jqExpression: .data.result[0].value[1] | tonumber
+{{- end }}
\ No newline at end of file
diff --git a/charts/iter8lib/templates/_task-database.tpl b/charts/iter8lib/templates/_task-database.tpl
new file mode 100644
index 00000000..0c3eb8aa
--- /dev/null
+++ b/charts/iter8lib/templates/_task-database.tpl
@@ -0,0 +1,12 @@
+{{- define "task.database" -}}
+- task: collect-metrics-database
+  with:
+    {{- if .Values.providers }}
+    providers:
+    {{- range .Values.providers }}
+      - {{ . }}
+    {{- end }}
+    {{- end }}
+    versionInfo:
+    - {{ .Values.versionInfo | toYaml | indent 6 | trim }}
+{{- end }}
\ No newline at end of file


Test patch:
diff --git a/action/gen_test.go b/action/gen_test.go
index bfdfe9c9..6bb5e901 100644
--- a/action/gen_test.go
+++ b/action/gen_test.go
@@ -51,7 +51,7 @@ func TestGenDB(t *testing.T) {
 	// fix gOpts
 	os.Chdir(t.TempDir())
 	gOpts := NewGenOpts()
-	gOpts.ChartsParentDir = base.CompletePath("../testdata/", "")
+	gOpts.ChartsParentDir = base.CompletePath("../", "")
 	gOpts.ChartName = "slo-validation-istio"
 	gOpts.Values = []string{"endpoint=http://127.0.0.1:9090/graph", "destination_workload=httpbin-v2", "destination_workload_namespace=default", `startingTime="Feb 4\, 2014 at 6:05pm (PST)"`, "istio/error-rate=0"}
 
diff --git a/base/collect_database_test.go b/base/collect_database_test.go
index 4a71db08..4839965c 100644
--- a/base/collect_database_test.go
+++ b/base/collect_database_test.go
@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"testing"
@@ -16,7 +18,9 @@ import (
 const (
 	metricsDirectory  = "../testdata/metrics/"
 	testCe            = "test-ce"
-	testPromURL       = `test-database.com/prometheus/api/v1/query?query=`
+	testRequestBody   = "test-request-body"
+	testPromURL       = `test-database.com/prometheus/api/v1/query`
+	queryString       = "?query="
 	requestCountQuery = "sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
 		"}[0s])) or on() vector(0)\n"
 	errorCountQuery = "sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
@@ -26,6 +30,21 @@ const (
 		"  ibm_codeengine_status!=\"200\",\n" +
 		"}[0s])) or on() vector(0)/sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
 		"}[0s])) or on() vector(0)\n"
+	requestCountWithRevisionNameQuery = "sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
+		"  ibm_codeengine_revision_name=\"v1\",\n" +
+		"}[0s])) or on() vector(0)\n"
+	errorCountWithRevisionNameQuery = "sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
+		"  ibm_codeengine_status!=\"200\",\n" +
+		"  ibm_codeengine_revision_name=\"v1\",\n" +
+		"}[0s])) or on() vector(0)\n"
+	errorRateWithRevisionNameQuery = "sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
+		"  ibm_codeengine_status!=\"200\",\n" +
+		"  ibm_codeengine_revision_name=\"v1\",\n" +
+		"}[0s])) or on() vector(0)/sum(last_over_time(ibm_codeengine_application_requests_total{\n" +
+		"  ibm_codeengine_revision_name=\"v1\",\n" +
+		"}[0s])) or on() vector(0)\n"
+	exampleQueryParameter = "example query parameter\n"
+	exampleRequestBody    = "example request body\n"
 )
 
 type collectDatabaseTemplateInput struct {
@@ -41,9 +60,7 @@ type collectDatabaseTemplateInput struct {
 	to run, the metrics files are copied to a temprorary directory and the current
 	working directory is changed to the temporary directory
 */
-func GoToTempDirectoryAndCopyMetricsFile(t *testing.T, test func()) error {
-	metricsFileName := testCe + experimentMetricsPathSuffix
-
+func GoToTempDirectoryAndCopyMetricsFile(t *testing.T, metricsFileName string, test func()) error {
 	originalPath, err := os.Getwd()
 	if err != nil {
 		return err
@@ -86,8 +103,7 @@ func GoToTempDirectoryAndCopyMetricsFile(t *testing.T, test func()) error {
 // test getElapsedTimeSeconds()
 func TestGetElapsedTimeSeconds(t *testing.T) {
 	versionInfo := map[string]interface{}{
-		"ibm_service_instance": "version1",
-		"startingTime":         "Feb 4, 2014 at 6:05pm (PST)",
+		"startingTime": "Feb 4, 2014 at 6:05pm (PST)",
 	}
 
 	exp := &Experiment{
@@ -111,9 +127,8 @@ func TestGetElapsedTimeSeconds(t *testing.T) {
 // test if a user sets elapsedTimeSeconds getElapsedTimeSeconds()
 func TestSetElapsedTimeSecondsError(t *testing.T) {
 	versionInfo := map[string]interface{}{
-		"ibm_service_instance": "version1",
-		"startingTime":         "Feb 4, 2014 at 6:05pm (PST)",
-		"elapsedTimeSeconds":   "Feb 5, 2014 at 6:05pm (PST)",
+		"startingTime":       "Feb 4, 2014 at 6:05pm (PST)",
+		"elapsedTimeSeconds": "Feb 5, 2014 at 6:05pm (PST)",
 	}
 
 	exp := &Experiment{
@@ -134,8 +149,7 @@ func TestSetElapsedTimeSecondsError(t *testing.T) {
 // test if a user sets startingTime incorrectly getElapsedTimeSeconds()
 func TestStartingTimeFormatError(t *testing.T) {
 	versionInfo := map[string]interface{}{
-		"ibm_service_instance": "version1",
-		"startingTime":         "1652935205",
+		"startingTime": "1652935205",
 	}
 
 	exp := &Experiment{
@@ -156,7 +170,123 @@ func TestStartingTimeFormatError(t *testing.T) {
 // basic test with one version, mimicking Code Engine
 // one version, three successful metrics
 func TestCEOneVersion(t *testing.T) {
-	err := GoToTempDirectoryAndCopyMetricsFile(t, func() {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testCe+experimentMetricsPathSuffix, func() {
+		input := &collectDatabaseTemplateInput{
+			Endpoint: "test-database.com",
+			IAMToken: "test-token",
+			GUID:     "test-guid",
+		}
+
+		// convert input to map[string]interface{}
+		var templateInput map[string]interface{}
+		inrec, err := json.Marshal(input)
+		assert.NoError(t, err)
+
+		json.Unmarshal(inrec, &templateInput)
+
+		// valid collect database task... should succeed
+		ct := &collectDatabaseTask{
+			TaskMeta: TaskMeta{
+				Task: StringPointer(CollectDatabaseTaskName),
+			},
+			With: collectDatabaseInputs{
+				Providers:   []string{testCe},
+				VersionInfo: []map[string]interface{}{{}},
+			},
+		}
+
+		httpmock.Activate()
+
+		// request-count
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery),
+			httpmock.NewStringResponder(200, `{
+				"status": "success",
+				"data": {
+					"resultType": "vector",
+					"result": [
+						{
+							"metric": {},
+							"value": [
+								1645602108.839,
+								"43"
+							]
+						}
+					]
+				}
+			}`))
+
+		// error-count
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorCountQuery),
+			httpmock.NewStringResponder(200, `{
+				"status": "success",
+				"data": {
+					"resultType": "vector",
+					"result": [
+						{
+							"metric": {},
+							"value": [
+								1645648760.725,
+								"6"
+							]
+						}
+					]
+				}
+			}`))
+
+		// error-rate
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorRateQuery),
+			httpmock.NewStringResponder(200, `{
+				"status": "success",
+				"data": {
+					"resultType": "vector",
+					"result": [
+						{
+							"metric": {},
+							"value": [
+								1645043851.825,
+								"0.13953488372093023"
+							]
+						}
+					]
+				}
+			}`))
+
+		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
+		exp := &Experiment{
+			Tasks:  []Task{ct},
+			Result: &ExperimentResult{},
+			driver: &md,
+		}
+		exp.initResults()
+		exp.Result.initInsightsWithNumVersions(1)
+
+		err = ct.run(exp)
+
+		// test should not fail
+		assert.NoError(t, err)
+
+		// all three metrics should exist and have values
+		assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/request-count"][0], float64(43))
+		assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-count"][0], float64(6))
+		assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-rate"][0], 0.13953488372093023)
+
+		httpmock.DeactivateAndReset()
+	})
+
+	assert.NoError(t, err)
+}
+
+// basic test with versionInfo, mimicking Code Engine
+// one version, three successful metrics
+func TestCEVersionInfo(t *testing.T) {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testCe+experimentMetricsPathSuffix, func() {
 		input := &collectDatabaseTemplateInput{
 			Endpoint: "test-database.com",
 			IAMToken: "test-token",
@@ -178,7 +308,7 @@ func TestCEOneVersion(t *testing.T) {
 			With: collectDatabaseInputs{
 				Providers: []string{testCe},
 				VersionInfo: []map[string]interface{}{{
-					"ibm_service_instance": "version1",
+					"ibm_codeengine_revision_name": "v1",
 				}},
 			},
 		}
@@ -186,7 +316,7 @@ func TestCEOneVersion(t *testing.T) {
 		httpmock.Activate()
 
 		// request-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(requestCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountWithRevisionNameQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -204,7 +334,7 @@ func TestCEOneVersion(t *testing.T) {
 			}`))
 
 		// error-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorCountWithRevisionNameQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -222,7 +352,7 @@ func TestCEOneVersion(t *testing.T) {
 			}`))
 
 		// error-rate
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorRateQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorRateWithRevisionNameQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -274,7 +404,7 @@ func TestCEOneVersion(t *testing.T) {
 // test with one version and improper authorization, mimicking Code Engine
 // one version, three successful metrics
 func TestCEUnauthorized(t *testing.T) {
-	err := GoToTempDirectoryAndCopyMetricsFile(t, func() {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testCe+experimentMetricsPathSuffix, func() {
 		input := &collectDatabaseTemplateInput{
 			Endpoint: "test-database.com",
 			IAMToken: "test-token",
@@ -293,25 +423,23 @@ func TestCEUnauthorized(t *testing.T) {
 				Task: StringPointer(CollectDatabaseTaskName),
 			},
 			With: collectDatabaseInputs{
-				Providers: []string{testCe},
-				VersionInfo: []map[string]interface{}{{
-					"ibm_service_instance": "version1",
-				}},
+				Providers:   []string{testCe},
+				VersionInfo: []map[string]interface{}{{}},
 			},
 		}
 
 		httpmock.Activate()
 
 		// request-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(requestCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery),
 			httpmock.NewStringResponder(401, `Unauthorized`))
 
 		// error-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorCountQuery),
 			httpmock.NewStringResponder(401, `Unauthorized`))
 
 		// error-rate
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorRateQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorRateQuery),
 			httpmock.NewStringResponder(401, `Unauthorized`))
 
 		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
@@ -347,7 +475,7 @@ func TestCEUnauthorized(t *testing.T) {
 // test with one version with some values, mimicking Code Engine
 // one version, three successful metrics, one without values
 func TestCESomeValues(t *testing.T) {
-	err := GoToTempDirectoryAndCopyMetricsFile(t, func() {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testCe+experimentMetricsPathSuffix, func() {
 		input := &collectDatabaseTemplateInput{
 			Endpoint: "test-database.com",
 			IAMToken: "test-token",
@@ -366,17 +494,15 @@ func TestCESomeValues(t *testing.T) {
 				Task: StringPointer(CollectDatabaseTaskName),
 			},
 			With: collectDatabaseInputs{
-				Providers: []string{testCe},
-				VersionInfo: []map[string]interface{}{{
-					"ibm_service_instance": "version1",
-				}},
+				Providers:   []string{testCe},
+				VersionInfo: []map[string]interface{}{{}},
 			},
 		}
 
 		httpmock.Activate()
 
 		// request-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
 					"resultType": "vector",
@@ -385,7 +511,7 @@ func TestCESomeValues(t *testing.T) {
 			}`))
 
 		// error-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorCountQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -403,7 +529,7 @@ func TestCESomeValues(t *testing.T) {
 			}`))
 
 		// error-rate
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorRateQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorRateQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -458,7 +584,7 @@ func TestCESomeValues(t *testing.T) {
 // test with two version with some values, mimicking Code Engine
 // two versions, four successful metrics, two without values
 func TestCEMultipleVersions(t *testing.T) {
-	err := GoToTempDirectoryAndCopyMetricsFile(t, func() {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testCe+experimentMetricsPathSuffix, func() {
 		input := &collectDatabaseTemplateInput{
 			Endpoint: "test-database.com",
 			IAMToken: "test-token",
@@ -477,19 +603,15 @@ func TestCEMultipleVersions(t *testing.T) {
 				Task: StringPointer(CollectDatabaseTaskName),
 			},
 			With: collectDatabaseInputs{
-				Providers: []string{testCe},
-				VersionInfo: []map[string]interface{}{{
-					"ibm_service_instance": "version1",
-				}, {
-					"ibm_service_instance": "version2",
-				}},
+				Providers:   []string{testCe},
+				VersionInfo: []map[string]interface{}{{}, {}},
 			},
 		}
 
 		httpmock.Activate()
 
 		// request-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
 					"resultType": "vector",
@@ -498,7 +620,7 @@ func TestCEMultipleVersions(t *testing.T) {
 			}`))
 
 		// error-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorCountQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -516,7 +638,7 @@ func TestCEMultipleVersions(t *testing.T) {
 			}`))
 
 		// error-rate
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorRateQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorRateQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -573,7 +695,7 @@ func TestCEMultipleVersions(t *testing.T) {
 // test with two version with some values, mimicking Code Engine
 // two versions, four successful metrics, two without values
 func TestCEMultipleVersionsAndMetrics(t *testing.T) {
-	err := GoToTempDirectoryAndCopyMetricsFile(t, func() {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testCe+experimentMetricsPathSuffix, func() {
 		input := &collectDatabaseTemplateInput{
 			Endpoint: "test-database.com",
 			IAMToken: "test-token",
@@ -592,19 +714,15 @@ func TestCEMultipleVersionsAndMetrics(t *testing.T) {
 				Task: StringPointer(CollectDatabaseTaskName),
 			},
 			With: collectDatabaseInputs{
-				Providers: []string{testCe},
-				VersionInfo: []map[string]interface{}{{
-					"ibm_service_instance": "version1",
-				}, {
-					"ibm_service_instance": "version2",
-				}},
+				Providers:   []string{testCe},
+				VersionInfo: []map[string]interface{}{{}, {}},
 			},
 		}
 
 		httpmock.Activate()
 
 		// request-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(requestCountQuery), httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
 					"resultType": "vector",
@@ -613,7 +731,7 @@ func TestCEMultipleVersionsAndMetrics(t *testing.T) {
 			}`))
 
 		// error-count
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorCountQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorCountQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -631,7 +749,7 @@ func TestCEMultipleVersionsAndMetrics(t *testing.T) {
 			}`))
 
 		// error-rate
-		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorRateQuery),
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(errorRateQuery),
 			httpmock.NewStringResponder(200, `{
 				"status": "success",
 				"data": {
@@ -684,3 +802,90 @@ func TestCEMultipleVersionsAndMetrics(t *testing.T) {
 
 	assert.NoError(t, err)
 }
+
+// basic test with a request body
+func TestRequestBody(t *testing.T) {
+	err := GoToTempDirectoryAndCopyMetricsFile(t, testRequestBody+experimentMetricsPathSuffix, func() {
+		input := &collectDatabaseTemplateInput{
+			Endpoint: "test-database.com",
+		}
+
+		// convert input to map[string]interface{}
+		var templateInput map[string]interface{}
+		inrec, err := json.Marshal(input)
+		assert.NoError(t, err)
+
+		json.Unmarshal(inrec, &templateInput)
+
+		// valid collect database task... should succeed
+		ct := &collectDatabaseTask{
+			TaskMeta: TaskMeta{
+				Task: StringPointer(CollectDatabaseTaskName),
+			},
+			With: collectDatabaseInputs{
+				Providers:   []string{testRequestBody},
+				VersionInfo: []map[string]interface{}{{}},
+			},
+		}
+
+		httpmock.Activate()
+
+		// request-count
+		httpmock.RegisterResponder("GET", testPromURL+queryString+url.QueryEscape(exampleQueryParameter),
+			func(req *http.Request) (*http.Response, error) {
+				if req.Body != nil {
+					b, err := ioutil.ReadAll(req.Body)
+					if err != nil {
+						panic(err)
+					}
+
+					if string(b) == exampleRequestBody {
+						return httpmock.NewStringResponse(200, `{
+							"status": "success",
+							"data": {
+								"resultType": "vector",
+								"result": [
+									{
+										"metric": {},
+										"value": [
+											1645602108.839,
+											"43"
+										]
+									}
+								]
+							}
+						}`), nil
+					}
+				}
+
+				return nil, errors.New("")
+			})
+
+		template, err := template.ParseFiles(testRequestBody + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
+		exp := &Experiment{
+			Tasks:  []Task{ct},
+			Result: &ExperimentResult{},
+			driver: &md,
+		}
+		exp.initResults()
+		exp.Result.initInsightsWithNumVersions(1)
+
+		err = ct.run(exp)
+
+		// test should not fail
+		assert.NoError(t, err)
+
+		assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0][testRequestBody+"/request-count"][0], float64(43))
+
+		httpmock.DeactivateAndReset()
+	})
+
+	assert.NoError(t, err)
+}
diff --git a/testdata/charts/slo-validation-istio/.helmignore b/charts/slo-validation-istio/.helmignore
similarity index 100%
rename from testdata/charts/slo-validation-istio/.helmignore
rename to charts/slo-validation-istio/.helmignore
diff --git a/testdata/charts/slo-validation-istio/Chart.yaml b/charts/slo-validation-istio/Chart.yaml
similarity index 89%
rename from testdata/charts/slo-validation-istio/Chart.yaml
rename to charts/slo-validation-istio/Chart.yaml
index c84aa3d6..f6b88b47 100644
--- a/testdata/charts/slo-validation-istio/Chart.yaml
+++ b/charts/slo-validation-istio/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: slo-validation-istio
 version: 0.10.0
-description: Load test, benchmark, and validate HTTP services
+description: SLO validation for Istio applications
 type: application
 keywords:
 - Iter8
diff --git a/testdata/charts/slo-validation-istio/templates/_experiment.tpl b/charts/slo-validation-istio/templates/_experiment.tpl
similarity index 100%
rename from testdata/charts/slo-validation-istio/templates/_experiment.tpl
rename to charts/slo-validation-istio/templates/_experiment.tpl
diff --git a/testdata/charts/slo-validation-istio/templates/k8s.yaml b/charts/slo-validation-istio/templates/k8s.yaml
similarity index 100%
rename from testdata/charts/slo-validation-istio/templates/k8s.yaml
rename to charts/slo-validation-istio/templates/k8s.yaml
diff --git a/testdata/charts/slo-validation-istio/values.yaml b/charts/slo-validation-istio/values.yaml
similarity index 95%
rename from testdata/charts/slo-validation-istio/values.yaml
rename to charts/slo-validation-istio/values.yaml
index ef393961..514a41fc 100644
--- a/testdata/charts/slo-validation-istio/values.yaml
+++ b/charts/slo-validation-istio/values.yaml
@@ -1,5 +1,5 @@
 # Prometheus URL
-endpoint: http://127.0.0.1:9090
+endpoint: null
 
 ### SLOs    A map of service level objectives (SLOs) that the app needs to satisfy.
 ### Metrics collected during the load test are used to verify if the app satisfies SLOs.
@@ -20,7 +20,7 @@ providers:
   - istio
 
 ##################################
-# Used by the template
+# Used by the metrics template
 
 # PromQL selectors
 reporter: source
@@ -63,7 +63,7 @@ source_workload: null
 source_workload_namespace: null
 
 ##################################
-# Used by metrics
+# Used by metrics spec
 
 destination_workload: null
 destination_workload_namespace: null
diff --git a/testdata/charts/iter8lib/.helmignore b/testdata/charts/iter8lib/.helmignore
deleted file mode 100644
index 0e8a0eb3..00000000
--- a/testdata/charts/iter8lib/.helmignore
+++ /dev/null
@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/testdata/charts/iter8lib/Chart.yaml b/testdata/charts/iter8lib/Chart.yaml
deleted file mode 100644
index abfd4f55..00000000
--- a/testdata/charts/iter8lib/Chart.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-apiVersion: v2
-name: iter8lib
-version: 0.10.1
-description: Iter8 library chart
-type: library
-home: https://iter8.tools
-sources:
-- https://github.com/iter8-tools/iter8
-maintainers:
-- name: Srinivasan Parthasarathy
-  email: spartha@us.ibm.com
-  url: https://researcher.watson.ibm.com/researcher/view.php?person=us-spartha
-icon: https://github.com/iter8-tools/iter8/raw/master/mkdocs/docs/images/favicon.png
diff --git a/testdata/charts/iter8lib/README.md b/testdata/charts/iter8lib/README.md
deleted file mode 100644
index b2a3fdfa..00000000
--- a/testdata/charts/iter8lib/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Iter8 library chart
-
-> This chart provides reusable templates that are used by Iter8 experiment charts. Experiments can be composed by using this chart as a dependency, and by including templates available in this library chart.
diff --git a/testdata/charts/iter8lib/templates/_k-job.tpl b/testdata/charts/iter8lib/templates/_k-job.tpl
deleted file mode 100644
index 88c9965f..00000000
--- a/testdata/charts/iter8lib/templates/_k-job.tpl
+++ /dev/null
@@ -1,22 +0,0 @@
-{{- define "k.job" -}}
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: {{ .Release.Name }}-{{ .Release.Revision }}-job
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-spec:
-  template:
-    spec:
-      containers:
-      - name: iter8
-        image: {{  .Values.iter8lib.iter8Image }}
-        imagePullPolicy: Always
-        command:
-        - "/bin/sh"
-        - "-c"
-        - |
-          iter8 k run --namespace {{ .Release.Namespace }} --group {{ .Release.Name }}
-      restartPolicy: Never
-  backoffLimit: 0
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-manifest.tpl b/testdata/charts/iter8lib/templates/_k-manifest.tpl
deleted file mode 100644
index 903c6119..00000000
--- a/testdata/charts/iter8lib/templates/_k-manifest.tpl
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- define "k.manifest" -}}
-{{ include "k.spec.secret" . }}
----
-{{ include "k.spec.role" . }}
----
-{{ include "k.spec.rolebinding" . }}
----
-{{ include "k.result.secret" . }}
----
-{{ include "k.result.role" . }}
----
-{{ include "k.result.rolebinding" . }}
-{{- if not .Values.iter8lib.disable.job }}
----
-{{ include "k.job" . }}
-{{- end }}
-{{ include "task.ready.rbac" . }}
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-result-role.tpl b/testdata/charts/iter8lib/templates/_k-result-role.tpl
deleted file mode 100644
index 2e7580db..00000000
--- a/testdata/charts/iter8lib/templates/_k-result-role.tpl
+++ /dev/null
@@ -1,13 +0,0 @@
-{{- define "k.result.role" -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: {{ .Release.Name }}-result-role
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-rules:
-- apiGroups: [""]
-  resourceNames: ["{{ .Release.Name }}-result"]
-  resources: ["secrets"]
-  verbs: ["get", "update"]
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-result-rolebinding.tpl b/testdata/charts/iter8lib/templates/_k-result-rolebinding.tpl
deleted file mode 100644
index 92f96013..00000000
--- a/testdata/charts/iter8lib/templates/_k-result-rolebinding.tpl
+++ /dev/null
@@ -1,16 +0,0 @@
-{{- define "k.result.rolebinding" -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ .Release.Name }}-result-rolebinding
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-subjects:
-- kind: ServiceAccount
-  name: default
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  kind: Role
-  name: {{ .Release.Name }}-result-role
-  apiGroup: rbac.authorization.k8s.io
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-result-secret.tpl b/testdata/charts/iter8lib/templates/_k-result-secret.tpl
deleted file mode 100644
index c6728534..00000000
--- a/testdata/charts/iter8lib/templates/_k-result-secret.tpl
+++ /dev/null
@@ -1,14 +0,0 @@
-{{- define "k.result.secret" -}}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ .Release.Name }}-result
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-stringData:
-  result.yaml: |
-    startTime:         {{ now | toJson }}
-    numCompletedTasks: 0
-    failure:           false
-    iter8Version:      {{ .Values.iter8lib.majorMinor }}
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-spec-role.tpl b/testdata/charts/iter8lib/templates/_k-spec-role.tpl
deleted file mode 100644
index 38d5b1b9..00000000
--- a/testdata/charts/iter8lib/templates/_k-spec-role.tpl
+++ /dev/null
@@ -1,13 +0,0 @@
-{{- define "k.spec.role" -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: {{ .Release.Name }}-spec-role
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-rules:
-- apiGroups: [""]
-  resourceNames: ["{{ .Release.Name }}-spec"]
-  resources: ["secrets"]
-  verbs: ["get"]
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-spec-rolebinding.tpl b/testdata/charts/iter8lib/templates/_k-spec-rolebinding.tpl
deleted file mode 100644
index d253d941..00000000
--- a/testdata/charts/iter8lib/templates/_k-spec-rolebinding.tpl
+++ /dev/null
@@ -1,16 +0,0 @@
-{{- define "k.spec.rolebinding" -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ .Release.Name }}-spec-rolebinding
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-subjects:
-- kind: ServiceAccount
-  name: default
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  kind: Role
-  name: {{ .Release.Name }}-spec-role
-  apiGroup: rbac.authorization.k8s.io
-{{- end }}
diff --git a/testdata/charts/iter8lib/templates/_k-spec-secret.tpl b/testdata/charts/iter8lib/templates/_k-spec-secret.tpl
deleted file mode 100644
index d3288f28..00000000
--- a/testdata/charts/iter8lib/templates/_k-spec-secret.tpl
+++ /dev/null
@@ -1,20 +0,0 @@
-{{- define "k.spec.secret" -}}
-{{ $globalContext := . }}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ .Release.Name }}-spec
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-stringData:
-  experiment.yaml: |
-{{ include "experiment" . | indent 4 }}
-{{- if .Values.providers }}
-  metrics.yaml: |
-{{- range .Values.providers }}
-{{- if eq . "istio"}}
-{{ include "metrics.istio" $globalContext | indent 4 }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/testdata/charts/iter8lib/templates/_metrics-common-istio.tpl b/testdata/charts/iter8lib/templates/_metrics-common-istio.tpl
deleted file mode 100644
index 15877ba9..00000000
--- a/testdata/charts/iter8lib/templates/_metrics-common-istio.tpl
+++ /dev/null
@@ -1,116 +0,0 @@
-{{- define "metrics.common.istio" }}        
-{{- if .Values.app}}
-  app="{{.Values.app}}",
-{{- end}}
-{{- if .Values.chart}}
-  chart="{{.Values.chart}}",
-{{- end}}
-{{- if .Values.connection_security_policy}}
-  connection_security_policy="{{.Values.connection_security_policy}}",
-{{- end}}
-{{- if .Values.destination_app}}
-  destination_app="{{.Values.destination_app}}",
-{{- end}}
-{{- if .Values.destination_canonical_revision}}
-  destination_canonical_revision="{{.Values.destination_canonical_revision}}",
-{{- end}}
-{{- if .Values.destination_canonical_service}}
-  destination_canonical_service="{{.Values.destination_canonical_service}}",
-{{- end}}
-{{- if .Values.destination_cluster}}
-  destination_cluster="{{.Values.destination_cluster}}",
-{{- end}}
-{{- if .Values.destination_principal}}
-  destination_principal="{{.Values.destination_principal}}",
-{{- end}}
-{{- if .Values.destination_service}}
-  destination_service="{{.Values.destination_service}}",
-{{- end}}
-{{- if .Values.destination_service_name}}
-  destination_service_name="{{.Values.destination_service_name}}",
-{{- end}}
-{{- if .Values.destination_service_namespace}}
-  destination_service_namespace="{{.Values.destination_service_namespace}}",
-{{- end}}
-{{- if .Values.destination_version}}
-  destination_version="{{.Values.destination_version}}",
-{{- end}}
-{{- if .Values.heritage}}
-  heritage="{{.Values.heritage}}",
-{{- end}}
-{{- if .Values.install_operator_istio_io_owning_resource}}
-  install_operator_istio_io_owning_resource="{{.Values.install_operator_istio_io_owning_resource}}",
-{{- end}}
-{{- if .Values.instance}}
-  instance="{{.Values.instance}}",
-{{- end}}
-{{- if .Values.istio}}
-  istio="{{.Values.istio}}",
-{{- end}}
-{{- if .Values.istio_io_rev}}
-  istio_io_rev="{{.Values.istio_io_rev}}",
-{{- end}}
-{{- if .Values.job}}
-  job="{{.Values.job}}",
-{{- end}}
-{{- if .Values.namespace}}
-  namespace="{{.Values.namespace}}",
-{{- end}}
-{{- if .Values.operator_istio_io_component}}
-  operator_istio_io_component="{{.Values.operator_istio_io_component}}",
-{{- end}}
-{{- if .Values.pod}}
-  pod="{{.Values.pod}}",
-{{- end}}
-{{- if .Values.pod_template_hash}}
-  pod_template_hash="{{.Values.pod_template_hash}}",
-{{- end}}
-{{- if .Values.release}}
-  release="{{.Values.release}}",
-{{- end}}
-{{- if .Values.request_protocol}}
-  request_protocol="{{.Values.request_protocol}}",
-{{- end}}
-{{- if .Values.response_flags}}
-  response_flags="{{.Values.response_flags}}",
-{{- end}}
-{{- if .Values.service_istio_io_canonical_name}}
-  service_istio_io_canonical_name="{{.Values.service_istio_io_canonical_name}}",
-{{- end}}
-{{- if .Values.service_istio_io_canonical_revision}}
-  service_istio_io_canonical_revision="{{.Values.service_istio_io_canonical_revision}}",
-{{- end}}
-{{- if .Values.sidecar_istio_io_inject}}
-  sidecar_istio_io_inject="{{.Values.sidecar_istio_io_inject}}",
-{{- end}}
-{{- if .Values.source_app}}
-  source_app="{{.Values.source_app}}",
-{{- end}}
-{{- if .Values.source_canonical_revision}}
-  source_canonical_revision="{{.Values.source_canonical_revision}}",
-{{- end}}
-{{- if .Values.source_canonical_service}}
-  source_canonical_service="{{.Values.source_canonical_service}}",
-{{- end}}
-{{- if .Values.source_cluster}}
-  source_cluster="{{.Values.source_cluster}}",
-{{- end}}
-{{- if .Values.source_principal}}
-  source_principal="{{.Values.source_principal}}",
-{{- end}}
-{{- if .Values.source_version}}
-  source_version="{{.Values.source_version}}",
-{{- end}}
-{{- if .Values.source_workload}}
-  source_workload="{{.Values.source_workload}}",
-{{- end}}
-{{- if .Values.source_workload_namespace}}
-  source_workload_namespace="{{.Values.source_workload_namespace}}",
-{{- end}}
-{{- if .destination_workload }}
-  destination_workload="{{.destination_workload}}",
-{{- end }}
-{{- if .destination_workload_namespace }}
-  destination_workload_namespace="{{.destination_workload_namespace}}",
-{{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/testdata/charts/iter8lib/templates/_metrics-istio.tpl b/testdata/charts/iter8lib/templates/_metrics-istio.tpl
deleted file mode 100644
index e028f826..00000000
--- a/testdata/charts/iter8lib/templates/_metrics-istio.tpl
+++ /dev/null
@@ -1,206 +0,0 @@
-{{- define "metrics.istio" -}}
-url: {{ .Values.endpoint }}/api/v1/query
-provider: istio
-method: GET
-# Inputs for the template:
-#   app                                       string
-#   chart                                     string
-#   connection_security_policy                string
-#   destination_app                           string
-#   destination_canonical_revision            string
-#   destination_canonical_service             string
-#   destination_cluster                       string
-#   destination_principal                     string
-#   destination_service                       string
-#   destination_service_name                  string
-#   destination_service_namespace             string
-#   destination_version                       string
-#   heritage                                  string
-#   install_operator_istio_io_owning_resource string
-#   instance                                  string
-#   istio                                     string
-#   istio_io_rev                              string
-#   job                                       string
-#   namespace                                 string
-#   operator_istio_io_component               string
-#   pod                                       string
-#   pod_template_hash                         string
-#   release                                   string
-#   request_protocol                          string
-#   response_code                             string
-#   response_flags                            string
-#   service_istio_io_canonical_name           string
-#   service_istio_io_canonical_revision       string
-#   sidecar_istio_io_inject                   string
-#   source_app                                string
-#   source_canonical_revision                 string
-#   source_canonical_service                  string
-#   source_cluster                            string
-#   source_principal                          string
-#   source_version                            string
-#   source_workload                           string
-#   source_workload_namespace                 string
-#
-# Inputs for the metrics (output of template):
-#   destination_workload                      string
-#   destination_workload_namespace            string
-#   startingTime                              string
-#
-# Note: elapsedTimeSeconds is produced by Iter8
-metrics:
-- name: request-count
-  type: counter
-  description: |
-    Number of requests
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_requests_total{
-        {{- if .Values.response_code }}
-          response_code="{{.Values.response_code}}",
-        {{- end }}
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1]
-- name: error-count
-  type: counter
-  description: |
-    Number of non-successful requests
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_requests_total{
-        response_code=~'5..',
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1]
-- name: error-rate
-  type: gauge
-  description: |
-    Percentage of non-successful requests
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_requests_total{
-        response_code=~'5..',
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)/sum(last_over_time(istio_requests_total{
-        {{- if .Values.response_code }}
-          response_code="{{.Values.response_code}}",
-        {{- end }}
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result.[0].value.[1]
-- name: le500ms-latency-percentile
-  type: gauge
-  description: |
-    Less than 500 ms latency
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_request_duration_milliseconds_bucket{
-        le='500',
-        {{- if .Values.response_code }}
-          response_code="{{.Values.response_code}}",
-        {{- end }}
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)/sum(last_over_time(istio_request_duration_milliseconds_bucket{
-        le='+Inf',
-        {{- if .Values.response_code }}
-          response_code="{{.Values.response_code}}",
-        {{- end }}
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1]
-- name: mean-latency
-  type: gauge
-  description: |
-    Mean latency
-  params:
-  - name: query
-    value: |
-      sum(last_over_time(istio_request_duration_milliseconds_sum{
-        {{- if .Values.response_code }}
-          response_code="{{.Values.response_code}}",
-        {{- end }}
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)/sum(last_over_time(istio_requests_total{
-        {{- if .Values.response_code }}
-          response_code="{{.Values.response_code}}",
-        {{- end }}
-        {{- if .Values.reporter }}
-          reporter="{{.Values.reporter}}",
-        {{- end }}
-        {{- include "metrics.common.istio" . | indent 4 }}
-        {{"{{"}}- if .destination_workload {{"}}"}}
-          destination_workload="{{"{{"}}.destination_workload{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-        {{"{{"}}- if .destination_workload_namespace {{"}}"}}
-          destination_workload_namespace="{{"{{"}}.destination_workload_namespace{{"}}"}}",
-        {{"{{"}}- end {{"}}"}}
-      }[{{"{{"}}.elapsedTimeSeconds{{"}}"}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1]
-{{- end }}
\ No newline at end of file
diff --git a/testdata/charts/iter8lib/templates/_task-assess.tpl b/testdata/charts/iter8lib/templates/_task-assess.tpl
deleted file mode 100644
index 3a3f5677..00000000
--- a/testdata/charts/iter8lib/templates/_task-assess.tpl
+++ /dev/null
@@ -1,13 +0,0 @@
-{{- define "task.assess" -}}
-{{- if .Values.SLOs }}
-# task: validate service level objectives for app using
-# the metrics collected in an earlier task
-- task: assess-app-versions
-  with:
-    SLOs:
-    {{- range $key, $value := .Values.SLOs }}
-    - metric: {{ $key | toString }}
-      upperLimit: {{ $value | float64 }}
-    {{- end }}
-{{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/testdata/charts/iter8lib/templates/_task-database.tpl b/testdata/charts/iter8lib/templates/_task-database.tpl
deleted file mode 100644
index 7a429342..00000000
--- a/testdata/charts/iter8lib/templates/_task-database.tpl
+++ /dev/null
@@ -1,17 +0,0 @@
-{{- define "task.database" -}}
-- task: collect-metrics-database
-  with:
-    {{- if .Values.providers }}
-    providers:
-    {{- range .Values.providers }}
-      - {{ . }}
-    {{- end }}
-    {{- end }}
-    versionInfo:
-    - endpoint: {{ required "A valid endpoint value is required!" .Values.endpoint | toString }}
-      destination_workload: {{ required "A valid destination_workload value is required!" .Values.destination_workload | toString }}
-      destination_workload_namespace: {{ required "A valid destination_workload_namespace value is required!" .Values.destination_workload_namespace | toString }}
-      {{- if .Values.startingTime }}
-      startingTime: {{ .Values.startingTime }}
-      {{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/testdata/charts/iter8lib/templates/_task-ready-rbac.tpl b/testdata/charts/iter8lib/templates/_task-ready-rbac.tpl
deleted file mode 100644
index 193ddf1a..00000000
--- a/testdata/charts/iter8lib/templates/_task-ready-rbac.tpl
+++ /dev/null
@@ -1,41 +0,0 @@
-{{- define "task.ready.rbac" -}}
-{{- if not .Values.iter8lib.disable.readinessrbac }}
-{{- if .Values.ready }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: {{ .Release.Name }}-readiness-role
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-rules:
-{{- if .Values.ready.service }}
-- apiGroups: [""]
-  resourceNames: [{{ .Values.ready.service | quote }}]
-  resources: ["services"]
-  verbs: ["get"]
-{{- end }}
-{{- if .Values.ready.deploy }}
-- apiGroups: ["apps"]
-  resourceNames: [{{ .Values.ready.deploy | quote }}]
-  resources: ["deployments"]
-  verbs: ["get"]
-{{- end }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ .Release.Name }}-readiness-rolebinding
-  annotations:
-    iter8.tools/revision: {{ .Release.Revision | quote }}
-subjects:
-- kind: ServiceAccount
-  name: default
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  kind: Role
-  name: {{ .Release.Name }}-readiness-role
-  apiGroup: rbac.authorization.k8s.io
-{{- end }}
-{{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/testdata/charts/iter8lib/values.yaml b/testdata/charts/iter8lib/values.yaml
deleted file mode 100644
index 6b116612..00000000
--- a/testdata/charts/iter8lib/values.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-### iter8Image default iter8 image used for running Kubernetes experiments
-iter8Image: iter8/iter8:0.10
-
-### majorMinor is the minor version of Iter8
-majorMinor: v0.10
-
-### disable Manifests that should be disabled. This is useful for testing purposes.
-### By default, no manifest is disabled.
-### To disable the job manifest, uncomment the next two lines and delete the line after that.
-### disable:
-###   job: true
-### To disable rbac role and rolebinding for readiness, uncomment the next line
-###   readinessrbac: true
-disable: {}
diff --git a/testdata/metrics/test-ce.metrics.yaml b/testdata/metrics/test-ce.metrics.yaml
index 6f054df3..1623ce53 100644
--- a/testdata/metrics/test-ce.metrics.yaml
+++ b/testdata/metrics/test-ce.metrics.yaml
@@ -39,10 +39,10 @@ metrics:
     value: |
       sum(last_over_time(ibm_codeengine_application_requests_total{
         {{- if .ibm_codeengine_revision_name }}
-          ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
+        ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
         {{- end }}
       }[{{.elapsedTimeSeconds}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1]
+  jqExpression: .data.result[0].value[1] | tonumber
 - name: error-count
   type: counter
   description: |
@@ -53,10 +53,10 @@ metrics:
       sum(last_over_time(ibm_codeengine_application_requests_total{
         ibm_codeengine_status!="200",
         {{- if .ibm_codeengine_revision_name }}
-          ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
+        ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
         {{- end }}
       }[{{.elapsedTimeSeconds}}s])) or on() vector(0)
-  jqExpression: .data.result[0].value[1]
+  jqExpression: .data.result[0].value[1] | tonumber
 - name: error-rate
   type: gauge
   description: |
@@ -67,11 +67,11 @@ metrics:
       sum(last_over_time(ibm_codeengine_application_requests_total{
         ibm_codeengine_status!="200",
         {{- if .ibm_codeengine_revision_name }}
-          ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
+        ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
         {{- end }}
       }[{{.elapsedTimeSeconds}}s])) or on() vector(0)/sum(last_over_time(ibm_codeengine_application_requests_total{
         {{- if .ibm_codeengine_revision_name }}
-          ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
+        ibm_codeengine_revision_name="{{.ibm_codeengine_revision_name}}",
         {{- end }}
       }[{{.elapsedTimeSeconds}}s])) or on() vector(0)
   jqExpression: .data.result.[0].value.[1]
\ No newline at end of file
diff --git a/testdata/metrics/test-request-body.metrics.yaml b/testdata/metrics/test-request-body.metrics.yaml
new file mode 100644
index 00000000..ac5dd129
--- /dev/null
+++ b/testdata/metrics/test-request-body.metrics.yaml
@@ -0,0 +1,16 @@
+url: test-database.com/prometheus/api/v1/query
+provider: test-request-body
+method: GET
+# Note: elapsedTimeSeconds is produced by Iter8
+metrics:
+- name: request-count
+  type: counter
+  description: |
+    Number of requests
+  body: |
+    example request body
+  params:
+  - name: query
+    value: |
+      example query parameter
+  jqExpression: .data.result[0].value[1] | tonumber
\ No newline at end of file