Repository: iter8-tools/iter8
Pull request #1184 (2022-03-03 23:18:26)

Add collect database task

Alan-Cha: I see the unit test has failed but it's not clear to me what went wrong. I'll take a closer look tomorrow.
kalantar: @Alan-Cha Do we have some examples of how this task would work in an experiment?  Do we have any example experiments. I looked at the test cases but did not understand how the versionInfo is being defined.

Patch:
diff --git a/base/collect_database.go b/base/collect_database.go
new file mode 100644
index 00000000..75ae7d0f
--- /dev/null
+++ b/base/collect_database.go
@@ -0,0 +1,325 @@
+package base
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/itchyny/gojq"
+	log "github.com/iter8-tools/iter8/base/log"
+
+	"text/template"
+	"time"
+
+	"sigs.k8s.io/yaml"
+)
+
+type CollectDatabaseTemplate struct {
+	Url      string            `json:"url" yaml:"url"`
+	Headers  map[string]string `json:"headers" yaml:"headers"` // TODO: should this be map[string]interface{} instead?
+	Provider string            `json:"provider" yaml:"provider"`
+	Method   string            `json:"method" yaml:"method"`
+	Metrics  []Metric          `json:"metrics" yaml:"metrics"`
+}
+
+type Metric struct {
+	Name         string    `json:"name" yaml:"name"`
+	Description  *string   `json:"description,omitempty" yaml:"description,omitempty"`
+	Type         string    `json:"type" yaml:"type"`
+	Units        *string   `json:"units,omitempty" yaml:"units,omitempty"`
+	Params       *[]Params `json:"params,omitempty" yaml:"params,omitempty"`
+	Body         *string   `json:"body,omitempty" yaml:"body,omitempty"`
+	JqExpression string    `json:"jqExpression" yaml:"jqExpression"`
+}
+
+type Params struct {
+	Name  string `json:"name" yaml:"name"`
+	Value string `json:"value" yaml:"value"`
+}
+
+const startingTimeString = "StartingTime"
+const elapsedTimeString = "ElapsedTime"
+
+// collectDatabaseInputs holds all the inputs for this task
+//
+// Inputs for the template:
+//   ibm_codeengine_application_name string
+//   ibm_codeengine_gateway_instance string
+//   ibm_codeengine_namespace        string
+//   ibm_codeengine_project_name     string
+//   ibm_codeengine_revision_name    string
+//   ibm_codeengine_status           string
+//   ibm_ctype                       string
+//   ibm_location                    string
+//   ibm_scope                       string
+//   ibm_service_instance            string
+//   ibm_service_name                string
+//
+// Inputs for the metrics (output of template):
+//   ibm_codeengine_revision_name string
+//   StartingTime                 int64 (UNIX time stamp)
+//
+// Note: ElapsedTime is produced by Iter8
+type collectDatabaseInputs struct {
+	VersionInfo []map[string]interface{} `json:"versionInfo" yaml:"versionInfo"`
+}
+
+const (
+	// collectDatabaseTaskName is the name of this task which performs load generation and metrics collection for gRPC services.
+	CollectDatabaseTaskName = "collect-metrics-database"
+)
+
+// collectDatabaseTask enables load testing of gRPC services.
+type collectDatabaseTask struct {
+	TaskMeta
+	With collectDatabaseInputs `json:"with" yaml:"with"`
+}
+
+// initializeDefaults sets default values for the collect task
+func (t *collectDatabaseTask) initializeDefaults() {
+}
+
+// validate task inputs
+func (t *collectDatabaseTask) validateInputs() error {
+	return nil
+}
+
+// for a given version info and Experiment, calculate the elapsed time that
+// should be used for queries
+//
+// elapsed time is based on the StartingTime in the version info or the
+// starting time in the Experiment
+func getElapsedTime(versionInfo map[string]interface{}, exp *Experiment) (int64, error) {
+	// ElapsedTime should not be provided by the user
+	if versionInfo[elapsedTimeString] != nil {
+		return 0, errors.New("ElapsedTime should not be provided by the user in VersionInfo: " + fmt.Sprintf("%v", versionInfo))
+	}
+
+	// set StartingTime based on VersionInfo or start of the experiment
+	var startingTime int64
+	if versionInfo[startingTimeString] != nil {
+		rawStartingTime := versionInfo[startingTimeString]
+		switch rawStartingTime := rawStartingTime.(type) {
+		case int64:
+			startingTime = rawStartingTime
+		case int:
+			startingTime = int64(rawStartingTime)
+		default:
+			return 0, errors.New("Cannot integer parse StartingTime from VersionInfo: " + fmt.Sprintf("%v", versionInfo))
+		}
+	} else {
+		startingTime = exp.Result.StartTime.Unix()
+	}
+
+	// calculate the ElapsedTime based on the StartingTime if it has been provided
+	currentTime := time.Now().Unix()
+	return currentTime - startingTime, nil
+}
+
+// construct request to database and return extracted metric value
+//
+// bool return value represents whether the pipeline was able to run to
+// completion (prevents double error statement)
+func queryDatabaseAndGetValue(template CollectDatabaseTemplate, metric Metric) (interface{}, bool) {
+	var requestBody io.Reader
+	if metric.Body != nil {
+		requestBody = strings.NewReader(*metric.Body)
+	} else {
+		requestBody = nil
+	}
+
+	// create a new HTTP request
+	req, err := http.NewRequest(template.Method, template.Url, requestBody)
+	if err != nil {
+		log.Logger.Error("could not create new request for metric ", metric.Name, ": ", err)
+		return nil, false
+	}
+
+	// iterate through headers
+	for headerName, headerValue := range template.Headers {
+		req.Header.Add(headerName, headerValue)
+	}
+	req.Header.Add("Content-Type", "application/json;charset=utf-8")
+
+	// add query params
+	q := req.URL.Query()
+	params := metric.Params
+	for _, param := range *params {
+		q.Add(param.Name, param.Value)
+	}
+	req.URL.RawQuery = q.Encode()
+
+	// send request
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Logger.Error("could not request metric ", metric.Name, ": ", err)
+		return nil, false
+	}
+	defer resp.Body.Close()
+
+	// read response responseBody
+	responseBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Logger.Error("could not read response body for metric ", metric.Name, ": ", err)
+		return nil, false
+	}
+
+	// JSON parse response body
+	var jsonBody interface{}
+	err = json.Unmarshal([]byte(responseBody), &jsonBody)
+	if err != nil {
+		log.Logger.Error("could not JSON parse response body for metric ", metric.Name, ": ", err)
+		return nil, false
+	}
+
+	// perform jq expression
+	query, err := gojq.Parse(metric.JqExpression)
+	if err != nil {
+		log.Logger.Error("could not parse jq expression \""+metric.JqExpression+"\" for metric ", metric.Name, ": ", err)
+		return nil, false
+	}
+	iter := query.Run(jsonBody)
+
+	value, ok := iter.Next()
+	if !ok {
+		log.Logger.Error("could not extract value with jq expression for metric ", metric.Name, ": ", err)
+		return nil, false
+	}
+
+	return value, true
+}
+
+// run executes this task
+func (t *collectDatabaseTask) Run(exp *Experiment) error {
+	// validate inputs
+	var err error
+
+	err = t.validateInputs()
+	if err != nil {
+		return err
+	}
+
+	// initialize defaults
+	t.initializeDefaults()
+
+	// get current directory path
+	path, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	// collect all files paths in current directory that ends with metrics.yaml
+	metricFilePaths := []string{}
+	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() && strings.HasSuffix(path, "metrics.yaml") {
+			metricFilePaths = append(metricFilePaths, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// inputs for this task determine the number of versions participating in the
+	// experiment. Initiate insights with num versions.
+	err = exp.Result.initInsightsWithNumVersions(len(t.With.VersionInfo))
+	if err != nil {
+		return err
+	}
+
+	// collect metrics for all metric files and versionInfos
+	for _, metricFilePath := range metricFilePaths {
+		for i, versionInfo := range t.With.VersionInfo {
+			// add ElapsedTime
+			elapsedTime, err := getElapsedTime(versionInfo, exp)
+			if err != nil {
+				return err
+			}
+			versionInfo[elapsedTimeString] = elapsedTime
+
+			// finalize metrics template
+			template, err := template.ParseFiles(metricFilePath)
+			if err != nil {
+				return err
+			}
+			var buf bytes.Buffer
+			err = template.Execute(&buf, versionInfo)
+			if err != nil {
+				return err
+			}
+			var metrics CollectDatabaseTemplate
+			err = yaml.Unmarshal(buf.Bytes(), &metrics)
+			if err != nil {
+				return err
+			}
+
+			for _, metric := range metrics.Metrics {
+				// perform database query and extract metric value
+				value, ok := queryDatabaseAndGetValue(metrics, metric)
+
+				// check if there were any issues querying database and extracting value
+				if !ok {
+					continue
+				}
+
+				// do not save value if it has no value
+				if value == nil {
+					log.Logger.Error("could not extract non-nil value for metric ", metric.Name, ": ", err)
+					continue
+				}
+
+				// determine metric name
+				pathTokens := strings.Split(metricFilePath, "/")
+				fileNameWithExtension := pathTokens[len(pathTokens)-1]
+				fileNameTokens := strings.Split(fileNameWithExtension, ".metrics.yaml")
+				fileName := fileNameTokens[0]
+				metricName := fileName + "/" + metric.Name
+
+				// determine metric type
+				var metricType MetricType
+				if metric.Type == "gauge" {
+					metricType = GaugeMetricType
+				} else if metric.Type == "counter" {
+					metricType = CounterMetricType
+				}
+
+				// finalize metric data
+				mm := MetricMeta{
+					Description: *metric.Description,
+					Type:        metricType,
+					Units:       metric.Units,
+				}
+
+				// convert value to float
+				valueString := fmt.Sprint(value)
+				floatValue, err := strconv.ParseFloat(valueString, 64)
+				if err != nil {
+					log.Logger.Error("could not parse string \""+valueString+"\" to float:", err)
+					continue
+				}
+
+				err = exp.Result.Insights.updateMetric(metricName, mm, i, floatValue)
+
+				if err != nil {
+					log.Logger.Error("could not add update metric", err)
+					continue
+				}
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/base/collect_http.go b/base/collect_http.go
index 8ab3f6eb..e98dbf85 100644
--- a/base/collect_http.go
+++ b/base/collect_http.go
@@ -127,7 +127,7 @@ func (t *collectHTTPTask) initializeDefaults() {
 	}
 }
 
-//validateInputs for this task
+// validateInputs for this task
 func (t *collectHTTPTask) validateInputs() error {
 	return nil
 }
@@ -149,7 +149,7 @@ func (t *collectHTTPTask) getFortioOptions(j int) (*fhttp.HTTPRunnerOptions, err
 		},
 	}
 
-	//num requests
+	// num requests
 	if t.With.NumRequests != nil {
 		fo.RunnerOptions.Exactly = *t.With.NumRequests
 	}
diff --git a/go.mod b/go.mod
index cac4a79f..898f39f1 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/antonmedv/expr v1.9.0
 	github.com/bojand/ghz v0.106.1
+	github.com/itchyny/gojq v0.12.7 // indirect
 	github.com/jarcoal/httpmock v1.1.0
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/montanaflynn/stats v0.6.6
diff --git a/go.sum b/go.sum
index 1d95c695..cb70b4e9 100644
--- a/go.sum
+++ b/go.sum
@@ -495,9 +495,12 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator v9.30.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -522,6 +525,8 @@ github.com/gobuffalo/packr/v2 v2.8.1 h1:tkQpju6i3EtMXJ9uoF5GT6kB+LMTimDWD8Xvbz6z
 github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
+github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@@ -739,6 +744,12 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/itchyny/gojq v0.12.6 h1:VjaFn59Em2wTxDNGcrRkDK9ZHMNa8IksOgL13sLL4d0=
+github.com/itchyny/gojq v0.12.6/go.mod h1:ZHrkfu7A+RbZLy5J1/JKpS4poEqrzItSTGDItqsfP0A=
+github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
+github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
+github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
+github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
 github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
@@ -848,6 +859,7 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
@@ -1098,6 +1110,8 @@ github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiB
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
 github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8 h1:ajJQhvqPSQFJJ4aV5mDAMx8F7iFi6Dxfo6y62wymLNs=
+github.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8/go.mod h1:Nw/CCOXNyF5JDd6UpYxBwG5WWZ2FOJ/d5QnXL4KQ6vY=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -1558,6 +1572,8 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=


Test patch:
diff --git a/base/collect_database_test.go b/base/collect_database_test.go
new file mode 100644
index 00000000..77397431
--- /dev/null
+++ b/base/collect_database_test.go
@@ -0,0 +1,571 @@
+package base
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"text/template"
+
+	"github.com/jarcoal/httpmock"
+	"github.com/stretchr/testify/assert"
+)
+
+var templatePath string = "../testdata/templates/ce.metrics.tpl"
+var tempMetricsPath string = "test-ce.metrics.yaml"
+
+type collectDatabaseTemplateInput struct {
+	MonitoringEndpoint string `json:"MonitoringEndpoint" yaml:"MonitoringEndpoint"`
+	IAMToken           string `json:"IAMToken" yaml:"IAMToken"`
+	GUID               string `json:"GUID" yaml:"GUID"`
+}
+
+// has to be a map[string]string in order to do input checks in template
+// func executeTemplate(inputs interface{}, templatePath string, writePath string) error {
+func executeTemplate(inputs map[string]interface{}, templatePath string, writePath string) error {
+	template, err := template.ParseFiles(templatePath)
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Create(writePath)
+	if err != nil {
+		return err
+	}
+
+	err = template.Execute(file, inputs)
+	if err != nil {
+		return err
+	}
+
+	err = file.Close()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// test getElapsedTime()
+func TestGetElapsedTime(t *testing.T) {
+	versionInfo := map[string]interface{}{
+		"ibm_service_instance": "version1",
+		"StartingTime":         1000,
+	}
+
+	exp := &Experiment{
+		Tasks:  []Task{},
+		Result: &ExperimentResult{},
+	}
+
+	// this should add a startingTime that will be overwritten by the one in
+	// versionInfo
+	exp.InitResults()
+
+	elapsedTime, _ := getElapsedTime(versionInfo, exp)
+
+	// elapsedTime should be a large number
+	//
+	// if getElapsedTime() used the starting time from the experiment instead of
+	// the one from versionInfo, the elapsed time would be 0 or close to 0
+	assert.Equal(t, elapsedTime > 1000000, true)
+}
+
+// basic test with one version, mimicking Code Engine
+// one version, three successful metrics
+func TestCEOneVersion(t *testing.T) {
+	// create metrics file from template
+	input := &collectDatabaseTemplateInput{
+		MonitoringEndpoint: "test-database.com",
+		IAMToken:           "test-token",
+		GUID:               "test-guid",
+	}
+
+	// convert input to map[string]interface{}
+	var templateInput map[string]interface{}
+	inrec, err := json.Marshal(input)
+	assert.NoError(t, err)
+
+	json.Unmarshal(inrec, &templateInput)
+
+	// create metrics file from template
+	err = executeTemplate(templateInput, templatePath, tempMetricsPath)
+	assert.NoError(t, err)
+
+	// valid collect database task... should succeed
+	ct := &collectDatabaseTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectDatabaseTaskName),
+		},
+		With: collectDatabaseInputs{
+			VersionInfo: []map[string]interface{}{{
+				"ibm_service_instance": "version1",
+			}},
+		},
+	}
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	// request-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645602108.839,
+							"43"
+						]
+					}
+				]
+			}
+		}`))
+
+	// error-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29++%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645648760.725,
+							"6"
+						]
+					}
+				]
+			}
+		}`))
+
+	// error-rate
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29%2Fsum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645043851.825,
+							"0.13953488372093023"
+						]
+					}
+				]
+			}
+		}`))
+
+	exp := &Experiment{
+		Tasks:  []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.InitResults()
+	exp.Result.initInsightsWithNumVersions(1)
+
+	err = ct.Run(exp)
+
+	// test should not fail
+	assert.NoError(t, err)
+
+	// all three metrics should exist and have values
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/request-count"][0], float64(43))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-count"][0], float64(6))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-rate"][0], 0.13953488372093023)
+
+	// delete metrics file
+	os.Remove(tempMetricsPath)
+}
+
+// test with one version and improper authorization, mimicking Code Engine
+// one version, three successful metrics
+func TestCEUnauthorized(t *testing.T) {
+	// create metrics file from template
+	input := &collectDatabaseTemplateInput{
+		MonitoringEndpoint: "test-database.com",
+		IAMToken:           "test-token",
+		GUID:               "test-guid",
+	}
+
+	// convert input to map[string]interface{}
+	var templateInput map[string]interface{}
+	inrec, err := json.Marshal(input)
+	assert.NoError(t, err)
+
+	json.Unmarshal(inrec, &templateInput)
+
+	// create metrics file from template
+	err = executeTemplate(templateInput, templatePath, tempMetricsPath)
+	assert.NoError(t, err)
+
+	ct := &collectDatabaseTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectDatabaseTaskName),
+		},
+		With: collectDatabaseInputs{
+			VersionInfo: []map[string]interface{}{{
+				"ibm_service_instance": "version1",
+			}},
+		},
+	}
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	// request-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(401, `Unauthorized`))
+
+	// error-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29++%0A`,
+		httpmock.NewStringResponder(401, `Unauthorized`))
+
+	// error-rate
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29%2Fsum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(401, `Unauthorized`))
+
+	exp := &Experiment{
+		Tasks:  []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.InitResults()
+	exp.Result.initInsightsWithNumVersions(1)
+
+	err = ct.Run(exp)
+
+	// test should not fail
+	assert.NoError(t, err)
+
+	// no values should be collected because of unauthorized requests
+	assert.Equal(t, len(exp.Result.Insights.NonHistMetricValues[0]), 0)
+
+	// delete metrics file
+	os.Remove(tempMetricsPath)
+}
+
+// test with one version with some values, mimicking Code Engine
+// one version, three successful metrics, one without values
+func TestCESomeValues(t *testing.T) {
+	// create metrics file from template
+	input := &collectDatabaseTemplateInput{
+		MonitoringEndpoint: "test-database.com",
+		IAMToken:           "test-token",
+		GUID:               "test-guid",
+	}
+
+	// convert input to map[string]interface{}
+	var templateInput map[string]interface{}
+	inrec, err := json.Marshal(input)
+	assert.NoError(t, err)
+
+	json.Unmarshal(inrec, &templateInput)
+
+	// create metrics file from template
+	err = executeTemplate(templateInput, templatePath, tempMetricsPath)
+	assert.NoError(t, err)
+
+	ct := &collectDatabaseTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectDatabaseTaskName),
+		},
+		With: collectDatabaseInputs{
+			VersionInfo: []map[string]interface{}{{
+				"ibm_service_instance": "version1",
+			}},
+		},
+	}
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	// request-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": []
+			}
+		}`))
+
+	// error-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29++%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645648760.725,
+							"6"
+						]
+					}
+				]
+			}
+		}`))
+
+	// error-rate
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29%2Fsum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645043851.825,
+							"0.13953488372093023"
+						]
+					}
+				]
+			}
+		}`))
+
+	exp := &Experiment{
+		Tasks:  []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.InitResults()
+	exp.Result.initInsightsWithNumVersions(1)
+
+	err = ct.Run(exp)
+
+	// test should not fail
+	assert.NoError(t, err)
+
+	// two metrics should exist and have values
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-count"][0], float64(6))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-rate"][0], 0.13953488372093023)
+
+	// request-count should not exist because there was no value from response
+	_, ok := exp.Result.Insights.NonHistMetricValues[0]["test-ce/request-count"]
+	assert.Equal(t, ok, false)
+
+	// delete metrics file
+	os.Remove(tempMetricsPath)
+}
+
+// test with two version with some values, mimicking Code Engine
+// two versions, four successful metrics, two without values
+func TestCEMultipleVersions(t *testing.T) {
+	// create metrics file from template
+	input := &collectDatabaseTemplateInput{
+		MonitoringEndpoint: "test-database.com",
+		IAMToken:           "test-token",
+		GUID:               "test-guid",
+	}
+
+	// convert input to map[string]interface{}
+	var templateInput map[string]interface{}
+	inrec, err := json.Marshal(input)
+	assert.NoError(t, err)
+
+	json.Unmarshal(inrec, &templateInput)
+
+	// create metrics file from template
+	err = executeTemplate(templateInput, templatePath, tempMetricsPath)
+	assert.NoError(t, err)
+
+	ct := &collectDatabaseTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectDatabaseTaskName),
+		},
+		With: collectDatabaseInputs{
+			VersionInfo: []map[string]interface{}{{
+				"ibm_service_instance": "version1",
+			}, {
+				"ibm_service_instance": "version2",
+			}},
+		},
+	}
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	// request-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": []
+			}
+		}`))
+
+	// error-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29++%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645648760.725,
+							"6"
+						]
+					}
+				]
+			}
+		}`))
+
+	// error-rate
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29%2Fsum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645043851.825,
+							"0.13953488372093023"
+						]
+					}
+				]
+			}
+		}`))
+
+	exp := &Experiment{
+		Tasks:  []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.InitResults()
+	exp.Result.initInsightsWithNumVersions(2)
+
+	err = ct.Run(exp)
+
+	// test should not fail
+	assert.NoError(t, err)
+
+	// two metrics should exist and have values
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-count"][0], float64(6))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[1]["test-ce/error-count"][0], float64(6))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-rate"][0], 0.13953488372093023)
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[1]["test-ce/error-rate"][0], 0.13953488372093023)
+
+	// request-count should not exist because there was no value from response
+	_, ok := exp.Result.Insights.NonHistMetricValues[0]["test-ce/request-count"]
+	assert.Equal(t, ok, false)
+
+	// delete metrics file
+	os.Remove(tempMetricsPath)
+}
+
+// test with two version with some values, mimicking Code Engine
+// two versions, four successful metrics, two without values
+func TestCEMultipleVersionsAndMetrics(t *testing.T) {
+	// create metrics file from template
+	input := &collectDatabaseTemplateInput{
+		MonitoringEndpoint: "test-database.com",
+		IAMToken:           "test-token",
+		GUID:               "test-guid",
+	}
+
+	// convert input to map[string]interface{}
+	var templateInput map[string]interface{}
+	inrec, err := json.Marshal(input)
+	assert.NoError(t, err)
+
+	json.Unmarshal(inrec, &templateInput)
+
+	// create metrics file from template
+	err = executeTemplate(templateInput, templatePath, tempMetricsPath)
+	assert.NoError(t, err)
+
+	ct := &collectDatabaseTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectDatabaseTaskName),
+		},
+		With: collectDatabaseInputs{
+			VersionInfo: []map[string]interface{}{{
+				"ibm_service_instance": "version1",
+			}, {
+				"ibm_service_instance": "version2",
+			}},
+		},
+	}
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	// request-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": []
+			}
+		}`))
+
+	// error-count
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29++%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645648760.725,
+							"6"
+						]
+					}
+				]
+			}
+		}`))
+
+	// error-rate
+	httpmock.RegisterResponder("GET", `test-database.com/prometheus/api/v1/query?query=sum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A++ibm_codeengine_status%21%3D%22200%22%2C%0A%7D%5B0s%5D%29%29%2Fsum%28last_over_time%28ibm_codeengine_application_requests_total%7B%0A%7D%5B0s%5D%29%29+%0A`,
+		httpmock.NewStringResponder(200, `{
+			"status": "success",
+			"data": {
+				"resultType": "vector",
+				"result": [
+					{
+						"metric": {},
+						"value": [
+							1645043851.825,
+							"0.13953488372093023"
+						]
+					}
+				]
+			}
+		}`))
+
+	exp := &Experiment{
+		Tasks:  []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.InitResults()
+	exp.Result.initInsightsWithNumVersions(2)
+
+	err = ct.Run(exp)
+
+	// test should not fail
+	assert.NoError(t, err)
+
+	// two metrics should exist and have values
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-count"][0], float64(6))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[1]["test-ce/error-count"][0], float64(6))
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[0]["test-ce/error-rate"][0], 0.13953488372093023)
+	assert.Equal(t, exp.Result.Insights.NonHistMetricValues[1]["test-ce/error-rate"][0], 0.13953488372093023)
+
+	// request-count should not exist because there was no value from response
+	_, ok := exp.Result.Insights.NonHistMetricValues[0]["test-ce/request-count"]
+	assert.Equal(t, ok, false)
+
+	// delete metrics file
+	os.Remove(tempMetricsPath)
+}
diff --git a/testdata/templates/ce.metrics.tpl b/testdata/templates/ce.metrics.tpl
new file mode 100644
index 00000000..da50b17d
--- /dev/null
+++ b/testdata/templates/ce.metrics.tpl
@@ -0,0 +1,197 @@
+# endpoint where the monitoring instance is available
+# https://cloud.ibm.com/docs/monitoring?topic=monitoring-endpoints#endpoints_sysdig
+url: {{ .MonitoringEndpoint }}/prometheus/api/v1/query # e.g. https://ca-tor.monitoring.cloud.ibm.com/api/data
+headers:
+  # IAM token
+  # to get the token, run: ibmcloud iam oauth-tokens | grep IAM | cut -d \: -f 2 | sed 's/^ *//'
+  Authorization: Bearer {{ .IAMToken }}
+  # GUID of the IBM Cloud Monitoring instance
+  # to get the GUID, run: ibmcloud resource service-instance <NAME> --output json | jq -r '.[].guid'
+  # https://cloud.ibm.com/docs/monitoring?topic=monitoring-mon-curl
+  IBMInstanceID: {{ .GUID }}
+provider: IBM Cloud Code Engine Sysdig
+method: GET
+# Inputs for the template:
+#   ibm_codeengine_application_name string
+#   ibm_codeengine_gateway_instance string
+#   ibm_codeengine_namespace        string
+#   ibm_codeengine_project_name     string
+#   ibm_codeengine_revision_name    string
+#   ibm_codeengine_status           string
+#   ibm_ctype                       string
+#   ibm_location                    string
+#   ibm_scope                       string
+#   ibm_service_instance            string
+#   ibm_service_name                string
+#
+# Inputs for the metrics (output of template):
+#   ibm_codeengine_revision_name string
+#   StartingTime                 int64 (UNIX time stamp)
+#
+# Note: ElapsedTime is produced by Iter8
+metrics:
+- name: request-count
+  type: counter
+  description: |
+    The number of requests sent to the Code Engine application.
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(ibm_codeengine_application_requests_total{
+        {{- if .ibm_codeengine_application_name }}
+          ibm_codeengine_application_name="{{ .ibm_codeengine_application_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_gateway_instance }}
+          ibm_codeengine_gateway_instance="{{ .ibm_codeengine_gateway_instance}}",
+        {{- end }}
+        {{- if .ibm_codeengine_namespace }}
+          ibm_codeengine_namespace="{{ .ibm_codeengine_namespace}}",
+        {{- end }}
+        {{- if .ibm_codeengine_project_name }}
+          ibm_codeengine_project_name="{{ .ibm_codeengine_project_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_status }}
+          ibm_codeengine_status="{{ .ibm_codeengine_status}}",
+        {{- end }}
+        {{- if .ibm_ctype }}
+          ibm_ctype="{{ .ibm_ctype}}",
+        {{- end }}
+        {{- if .ibm_location }}
+          ibm_location="{{ .ibm_location}}",
+        {{- end }}
+        {{- if .ibm_scope }}
+          ibm_scope="{{ .ibm_scope}}",
+        {{- end }}
+        {{- if .ibm_service_instance }}
+          ibm_service_instance="{{ .ibm_service_instance}}",
+        {{- end }}
+        {{- if .ibm_service_name }}
+          ibm_service_name="{{ .ibm_service_name}}",
+        {{- end }}
+        {{"{{"}}- if .ibm_codeengine_revision_name {{"}}"}}
+          ibm_codeengine_revision_name="{{"{{"}}.ibm_codeengine_revision_name{{"}}"}}",
+        {{"{{"}}- end {{"}}"}}
+      }[{{"{{"}}.ElapsedTime{{"}}"}}s])) 
+  jqExpression: .data.result[0].value[1]
+- name: error-count
+  type: counter
+  description: |
+    The number of non-successful requests sent to the Code Engine application.
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(ibm_codeengine_application_requests_total{
+        ibm_codeengine_status!="200",
+        {{- if .ibm_codeengine_application_name }}
+          ibm_codeengine_application_name="{{ .ibm_codeengine_application_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_gateway_instance }}
+          ibm_codeengine_gateway_instance="{{ .ibm_codeengine_gateway_instance}}",
+        {{- end }}
+        {{- if .ibm_codeengine_namespace }}
+          ibm_codeengine_namespace="{{ .ibm_codeengine_namespace}}",
+        {{- end }}
+        {{- if .ibm_codeengine_project_name }}
+          ibm_codeengine_project_name="{{ .ibm_codeengine_project_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_status }}
+          ibm_codeengine_status="{{ .ibm_codeengine_status}}",
+        {{- end }}
+        {{- if .ibm_ctype }}
+          ibm_ctype="{{ .ibm_ctype}}",
+        {{- end }}
+        {{- if .ibm_location }}
+          ibm_location="{{ .ibm_location}}",
+        {{- end }}
+        {{- if .ibm_scope }}
+          ibm_scope="{{ .ibm_scope}}",
+        {{- end }}
+        {{- if .ibm_service_instance }}
+          ibm_service_instance="{{ .ibm_service_instance}}",
+        {{- end }}
+        {{- if .ibm_service_name }}
+          ibm_service_name="{{ .ibm_service_name}}",
+        {{- end }}
+        {{"{{"}}- if .ibm_codeengine_revision_name {{"}}"}}
+          ibm_codeengine_revision_name="{{"{{"}}.ibm_codeengine_revision_name{{"}}"}}",
+        {{"{{"}}- end {{"}}"}}
+      }[{{"{{"}}.ElapsedTime{{"}}"}}s]))  
+  jqExpression: .data.result[0].value[1]
+- name: error-rate
+  type: gauge
+  description: |
+    The number of non-successful requests sent to the Code Engine application.
+  params:
+  - name: query
+    value: |
+      sum(last_over_time(ibm_codeengine_application_requests_total{
+        ibm_codeengine_status!="200",
+        {{- if .ibm_codeengine_application_name }}
+          ibm_codeengine_application_name="{{ .ibm_codeengine_application_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_gateway_instance }}
+          ibm_codeengine_gateway_instance="{{ .ibm_codeengine_gateway_instance}}",
+        {{- end }}
+        {{- if .ibm_codeengine_namespace }}
+          ibm_codeengine_namespace="{{ .ibm_codeengine_namespace}}",
+        {{- end }}
+        {{- if .ibm_codeengine_project_name }}
+          ibm_codeengine_project_name="{{ .ibm_codeengine_project_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_status }}
+          ibm_codeengine_status="{{ .ibm_codeengine_status}}",
+        {{- end }}
+        {{- if .ibm_ctype }}
+          ibm_ctype="{{ .ibm_ctype}}",
+        {{- end }}
+        {{- if .ibm_location }}
+          ibm_location="{{ .ibm_location}}",
+        {{- end }}
+        {{- if .ibm_scope }}
+          ibm_scope="{{ .ibm_scope}}",
+        {{- end }}
+        {{- if .ibm_service_instance }}
+          ibm_service_instance="{{ .ibm_service_instance}}",
+        {{- end }}
+        {{- if .ibm_service_name }}
+          ibm_service_name="{{ .ibm_service_name}}",
+        {{- end }}
+        {{"{{"}}- if .ibm_codeengine_revision_name {{"}}"}}
+          ibm_codeengine_revision_name="{{"{{"}}.ibm_codeengine_revision_name{{"}}"}}",
+        {{"{{"}}- end {{"}}"}}
+      }[{{"{{"}}.ElapsedTime{{"}}"}}s]))/sum(last_over_time(ibm_codeengine_application_requests_total{
+        {{- if .ibm_codeengine_application_name }}
+          ibm_codeengine_application_name="{{ .ibm_codeengine_application_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_gateway_instance }}
+          ibm_codeengine_gateway_instance="{{ .ibm_codeengine_gateway_instance}}",
+        {{- end }}
+        {{- if .ibm_codeengine_namespace }}
+          ibm_codeengine_namespace="{{ .ibm_codeengine_namespace}}",
+        {{- end }}
+        {{- if .ibm_codeengine_project_name }}
+          ibm_codeengine_project_name="{{ .ibm_codeengine_project_name}}",
+        {{- end }}
+        {{- if .ibm_codeengine_status }}
+          ibm_codeengine_status="{{ .ibm_codeengine_status}}",
+        {{- end }}
+        {{- if .ibm_ctype }}
+          ibm_ctype="{{ .ibm_ctype}}",
+        {{- end }}
+        {{- if .ibm_location }}
+          ibm_location="{{ .ibm_location}}",
+        {{- end }}
+        {{- if .ibm_scope }}
+          ibm_scope="{{ .ibm_scope}}",
+        {{- end }}
+        {{- if .ibm_service_instance }}
+          ibm_service_instance="{{ .ibm_service_instance}}",
+        {{- end }}
+        {{- if .ibm_service_name }}
+          ibm_service_name="{{ .ibm_service_name}}",
+        {{- end }}
+        {{"{{"}}- if .ibm_codeengine_revision_name {{"}}"}}
+          ibm_codeengine_revision_name="{{"{{"}}.ibm_codeengine_revision_name{{"}}"}}",
+        {{"{{"}}- end {{"}}"}}
+      }[{{"{{"}}.ElapsedTime{{"}}"}}s])) 
+  jqExpression: .data.result.[0].value.[1]
\ No newline at end of file