Repository: iter8-tools/iter8
Pull request #1260 (2022-05-17 11:21:09)

Add metrics spec to driver
Signed-off-by: Alan Cha <Alan.cha1@ibm.com>

sriumcp: @Alan-Cha The unit test is failing for unrelated reasons as you pointed out.

In this regard... I have a request.

Could you please remove [lines 17 and 18 -- i.e., the 2nd responder](https://github.com/iter8-tools/iter8/blob/34b2cfe793fd185ddfa6a2391d1fe1a742d88a7f/base/collect_http_test.go#L17) from the test file whenever you update the PR? I think that might fix the issue.

Patch:
diff --git a/action/report/htmlreport.tpl b/action/report/htmlreport.tpl
index 0f14a7ec..fe8e569d 100644
--- a/action/report/htmlreport.tpl
+++ b/action/report/htmlreport.tpl
@@ -58,111 +58,119 @@
       });	
       </script>
 
-      {{- if not (empty .Result.Insights.SLOs) }}  
-      <section class="mt-5">
-        <h3 class="display-6">Service level objectives (SLOs)</h3>
-        <h4 class="display-7 text-muted">Whether or not SLOs are satisfied</h4>
-        <hr>
-        <table class="table">
-          <thead class="thead-light">
-            <tr>
-              <th scope="col">SLO Conditions</th>
-              {{- if ge .Result.Insights.NumVersions 2 }}
-              {{- range until .Result.Insights.NumVersions }}
-              <th scope="col" class="text-center">Version {{ . }} </th>
-              {{- end}}              
-              {{- else }} 
-              <th scope="col" class="text-center">Satisfied</th>
-              {{- end }}
-            </tr>
-          </thead>
-          <tbody>
-              {{- range $ind, $slo := .Result.Insights.SLOs }}
-              <tr scope="row">
-                <td>
-                  {{ if $slo.LowerLimit }} {{- $slo.LowerLimit }} &leq; {{ end -}}
-                  <a href="javascript:void(0)" data-toggle="tooltip" data-placement="top" title="{{ $.MetricDescriptionHTML $slo.Metric }}">
-                    {{ $.MetricWithUnits $slo.Metric }}
-                  </a>
-                  {{ if $slo.UpperLimit }} &leq; {{ $slo.UpperLimit -}} {{- end }}
-                </td>
-                {{- range (index $.Result.Insights.SLOsSatisfied $ind) }}
-                <td class="{{ renderSLOSatisfiedCellClass .  }} text-center">
-                  <i class="far {{ renderSLOSatisfiedHTML . }}"></i>                
-                </td>
+      {{- if .Result.Insights }}
+        {{- if not (empty .Result.Insights.SLOs) }}  
+        <section class="mt-5">
+          <h3 class="display-6">Service level objectives (SLOs)</h3>
+          <h4 class="display-7 text-muted">Whether or not SLOs are satisfied</h4>
+          <hr>
+          <table class="table">
+            <thead class="thead-light">
+              <tr>
+                <th scope="col">SLO Conditions</th>
+                {{- if ge .Result.Insights.NumVersions 2 }}
+                {{- range until .Result.Insights.NumVersions }}
+                <th scope="col" class="text-center">Version {{ . }} </th>
+                {{- end}}              
+                {{- else }} 
+                <th scope="col" class="text-center">Satisfied</th>
                 {{- end }}
               </tr>
-              {{- end}}          
-          </tbody>
-        </table>
-      </section>
-      {{- end }}
+            </thead>
+            <tbody>
+                {{- range $ind, $slo := .Result.Insights.SLOs }}
+                <tr scope="row">
+                  <td>
+                    {{ if $slo.LowerLimit }} {{- $slo.LowerLimit }} &leq; {{ end -}}
+                    <a href="javascript:void(0)" data-toggle="tooltip" data-placement="top" title="{{ $.MetricDescriptionHTML $slo.Metric }}">
+                      {{ $.MetricWithUnits $slo.Metric }}
+                    </a>
+                    {{ if $slo.UpperLimit }} &leq; {{ $slo.UpperLimit -}} {{- end }}
+                  </td>
+                  {{- range (index $.Result.Insights.SLOsSatisfied $ind) }}
+                  <td class="{{ renderSLOSatisfiedCellClass .  }} text-center">
+                    <i class="far {{ renderSLOSatisfiedHTML . }}"></i>                
+                  </td>
+                  {{- end }}
+                </tr>
+                {{- end}}          
+            </tbody>
+          </table>
+        </section>
+        {{- end }}
 
-  		<section class="mt-5">
-        <h3 class="display-6">Metric Histograms</h3>
-        <hr>
+        <section class="mt-5">
+          <h3 class="display-6">Metric Histograms</h3>
+          <hr>
 
-        {{- range $ind, $mn := .SortedVectorMetrics }}
-        <div id="vm-{{ $mn }}"></div>
-        <script>
-          var data = [];
-          {{- range until $.Result.Insights.NumVersions }}
-          data.push({
-            x: {{ $.VectorMetricValue . $mn }},
-            name: "Version {{ . }}",
-            histnorm: "percent", 
-            opacity: 0.5, 
-            type: "histogram"
-          })
-          {{- end }}
+          {{- range $ind, $mn := .SortedVectorMetrics }}
+          <div id="vm-{{ $mn }}"></div>
+          <script>
+            var data = [];
+            {{- range until $.Result.Insights.NumVersions }}
+            data.push({
+              x: {{ $.VectorMetricValue . $mn }},
+              name: "Version {{ . }}",
+              histnorm: "percent", 
+              opacity: 0.5, 
+              type: "histogram"
+            })
+            {{- end }}
 
-          var layout = {
-            bargroupgap: 0.2, 
-            barmode: "overlay", 
-            title: "Histogram of {{ $.MetricWithUnits $mn }}", 
-            xaxis: {title: "{{ $.MetricWithUnits $mn }}"}, 
-            yaxis: {title: "% of observations"}
-          };
-          Plotly.newPlot("vm-{{ $mn }}", data, layout);
-        </script>
-        {{- end }}
-			</section>
+            var layout = {
+              bargroupgap: 0.2, 
+              barmode: "overlay", 
+              title: "Histogram of {{ $.MetricWithUnits $mn }}", 
+              xaxis: {title: "{{ $.MetricWithUnits $mn }}"}, 
+              yaxis: {title: "% of observations"}
+            };
+            Plotly.newPlot("vm-{{ $mn }}", data, layout);
+          </script>
+          {{- end }}
+        </section>
 
-  	  <section class="mt-5">
-        <h3 class="display-6">Latest observed values for metrics</h3>
-        <hr>
-        <table class="table">
-          <thead class="thead-light">
-            <tr>
-              <th scope="col">Metric</th>
-              {{- if ge .Result.Insights.NumVersions 2 }}
-              {{- range until .Result.Insights.NumVersions }}
-              <th scope="col" class="text-center">Version {{ . }} </th>
-              {{- end}}              
-              {{- else }} 
-              <th scope="col" class="text-center">Value</th>
-              {{- end }}
-            </tr>
-          </thead>
-          <tbody>
-              {{- range $ind, $mn := .SortedScalarAndSLOMetrics }}
-              <tr scope="row">
-                <td>
-                  <a href="javascript:void(0)" data-toggle="tooltip" data-placement="top" title="{{ $.MetricDescriptionHTML $mn }}">
-                    {{ $.MetricWithUnits $mn }}
-                  </a>
-                </td>
-                {{- range until $.Result.Insights.NumVersions }}
-                <td class="text-center">
-                {{ $.ScalarMetricValueStr . $mn }}
-                </td>
+        <section class="mt-5">
+          <h3 class="display-6">Latest observed values for metrics</h3>
+          <hr>
+          <table class="table">
+            <thead class="thead-light">
+              <tr>
+                <th scope="col">Metric</th>
+                {{- if ge .Result.Insights.NumVersions 2 }}
+                {{- range until .Result.Insights.NumVersions }}
+                <th scope="col" class="text-center">Version {{ . }} </th>
+                {{- end}}              
+                {{- else }} 
+                <th scope="col" class="text-center">Value</th>
                 {{- end }}
               </tr>
-              {{- end}}          
-          </tbody>
-        </table>
-      </section>
-
+            </thead>
+            <tbody>
+                {{- range $ind, $mn := .SortedScalarAndSLOMetrics }}
+                <tr scope="row">
+                  <td>
+                    <a href="javascript:void(0)" data-toggle="tooltip" data-placement="top" title="{{ $.MetricDescriptionHTML $mn }}">
+                      {{ $.MetricWithUnits $mn }}
+                    </a>
+                  </td>
+                  {{- range until $.Result.Insights.NumVersions }}
+                  <td class="text-center">
+                  {{ $.ScalarMetricValueStr . $mn }}
+                  </td>
+                  {{- end }}
+                </tr>
+                {{- end}}          
+            </tbody>
+          </table>
+        </section>
+      {{- else }}
+        <section class="mt-5">
+          <h3 class="display-6">Metrics-based Insights</h3>
+          <hr>
+          <p> Insights not found in experiment results. You may need to retry this report at a later time. </p>
+        </section>
+      {{- end }}
+      
     </div>
   </body>
 </html>
diff --git a/action/report/textreport.tpl b/action/report/textreport.tpl
index af80345a..59a34555 100644
--- a/action/report/textreport.tpl
+++ b/action/report/textreport.tpl
@@ -7,6 +7,7 @@ Experiment summary:
   Total number of tasks: {{ len .Tasks }}
   Number of completed tasks: {{ .Result.NumCompletedTasks }}
 
+{{- if .Result.Insights }}
 {{- if not (empty .Result.Insights.SLOs) }}
 
 Whether or not service level objectives (SLOs) are satisfied:
@@ -18,4 +19,11 @@ Whether or not service level objectives (SLOs) are satisfied:
 Latest observed values for metrics:
 ***********************************
 
-{{ .PrintMetricsText | indent 2 }}
\ No newline at end of file
+{{ .PrintMetricsText | indent 2 }}
+{{- else }}
+
+Metrics-based Insights:
+***********************
+
+  Insights not found in experiment results. You may need to retry this report at a later time.
+{{- end }}
diff --git a/base/collect_database.go b/base/collect_database.go
index de2dc1b3..a7d8d6ad 100644
--- a/base/collect_database.go
+++ b/base/collect_database.go
@@ -11,7 +11,6 @@ import (
 	"strconv"
 	"strings"
 
-	"text/template"
 	"time"
 
 	"github.com/itchyny/gojq"
@@ -20,47 +19,80 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-// ToDo: Go Doc is needed in this file
+// MetricsSpec specifies the set of metrics that can be obtained from a provider
+type MetricsSpec struct {
+	// Provider is the label/name of the data source
+	Provider string `json:"provider" yaml:"provider"`
 
-type CollectDatabaseTemplate struct {
-	URL      string            `json:"url" yaml:"url"`
-	Headers  map[string]string `json:"headers" yaml:"headers"`
-	Provider string            `json:"provider" yaml:"provider"`
-	Method   string            `json:"method" yaml:"method"`
-	Metrics  []Metric          `json:"metrics" yaml:"metrics"`
+	// URL is the database endpoint
+	URL string `json:"url" yaml:"url"`
+
+	// Method is the HTTP method that needs to be used
+	Method string `json:"method" yaml:"method"`
+
+	// Headers is the set of HTTP headers that need to be sent
+	Headers map[string]string `json:"headers" yaml:"headers"`
+
+	// Metrics is the set of metrics that can be obtained
+	Metrics []Metric `json:"metrics" yaml:"metrics"`
 }
 
+// Metric defines how to obtain a metric
 type Metric struct {
-	Name         string    `json:"name" yaml:"name"`
-	Description  *string   `json:"description,omitempty" yaml:"description,omitempty"`
-	Type         string    `json:"type" yaml:"type"`
-	Units        *string   `json:"units,omitempty" yaml:"units,omitempty"`
-	Params       *[]Params `json:"params,omitempty" yaml:"params,omitempty"`
-	Body         *string   `json:"body,omitempty" yaml:"body,omitempty"`
-	JqExpression string    `json:"jqExpression" yaml:"jqExpression"`
+	// Name is the name of the metric
+	Name string `json:"name" yaml:"name"`
+
+	// Description is the description of the metric
+	Description *string `json:"description,omitempty" yaml:"description,omitempty"`
+
+	// Type is the type of the metric, either gauge or counter
+	Type string `json:"type" yaml:"type"`
+
+	// Units is the unit of the metric, which can be omitted for unitless metrics
+	Units *string `json:"units,omitempty" yaml:"units,omitempty"`
+
+	// Params is the set of HTTP parameters that need to be sent
+	Params *[]Params `json:"params,omitempty" yaml:"params,omitempty"`
+
+	// Body is the HTTP request body that needs to be sent
+	Body *string `json:"body,omitempty" yaml:"body,omitempty"`
+
+	// JqExpression is the jq expression that can extract the value from the HTTP
+	// response
+	JqExpression string `json:"jqExpression" yaml:"jqExpression"`
 }
 
+// Params defines an HTTP parameter
 type Params struct {
-	Name  string `json:"name" yaml:"name"`
+	// Name is the name of the HTTP parameter
+	Name string `json:"name" yaml:"name"`
+
+	// Value is the value of the HTTP parameter
 	Value string `json:"value" yaml:"value"`
 }
 
-const (
-	startingTimeStr       = "startingTime"
-	elapsedTimeSecondsStr = "elapsedTimeSeconds"
-	timeLayout            = "Jan 2, 2006 at 3:04pm (MST)"
-)
-
+// collectDatabaseInputs is the input to the collect-metrics-database task
 type collectDatabaseInputs struct {
-	Providers   []string                 `json:"providers" yaml:"providers"`
+	// Providers is the set of labels/names of the data sources
+	Providers []string `json:"providers" yaml:"providers"`
+
+	// VersionInfo
 	VersionInfo []map[string]interface{} `json:"versionInfo" yaml:"versionInfo"`
 }
 
 const (
 	// CollectDatabaseTaskName is the name of this task which performs load generation and metrics collection for gRPC services.
 	CollectDatabaseTaskName = "collect-metrics-database"
+
 	// experimentMetricsPathSuffix is the name of the metrics spec file
 	experimentMetricsPathSuffix = ".metrics.yaml"
+
+	startingTimeStr = "startingTime"
+
+	elapsedTimeSecondsStr = "elapsedTimeSeconds"
+
+	// timeLayout is an example time layout for startingTime
+	timeLayout = "Jan 2, 2006 at 3:04pm (MST)"
 )
 
 // collectDatabaseTask enables load testing of gRPC services.
@@ -110,7 +142,7 @@ func getElapsedTimeSeconds(versionInfo map[string]interface{}, exp *Experiment)
 //
 // bool return value represents whether the pipeline was able to run to
 // completion (prevents double error statement)
-func queryDatabaseAndGetValue(template CollectDatabaseTemplate, metric Metric) (interface{}, bool) {
+func queryDatabaseAndGetValue(template MetricsSpec, metric Metric) (interface{}, bool) {
 	var requestBody io.Reader
 	if metric.Body != nil {
 		requestBody = strings.NewReader(*metric.Body)
@@ -194,6 +226,10 @@ func (t *collectDatabaseTask) run(exp *Experiment) error {
 		return err
 	}
 
+	if exp.driver == nil {
+		return errors.New("no driver was provided for collect-metrics-database task")
+	}
+
 	// initialize defaults
 	t.initializeDefaults()
 
@@ -214,8 +250,8 @@ func (t *collectDatabaseTask) run(exp *Experiment) error {
 			}
 			versionInfo[elapsedTimeSecondsStr] = elapsedTimeSeconds
 
-			// finalize metrics template
-			template, err := template.ParseFiles(provider + experimentMetricsPathSuffix)
+			// finalize metrics spec
+			template, err := exp.driver.ReadMetricsSpec(provider)
 			if err != nil {
 				return err
 			}
@@ -224,7 +260,7 @@ func (t *collectDatabaseTask) run(exp *Experiment) error {
 			if err != nil {
 				return err
 			}
-			var metrics CollectDatabaseTemplate
+			var metrics MetricsSpec
 			err = yaml.Unmarshal(buf.Bytes(), &metrics)
 			if err != nil {
 				return err
diff --git a/base/experiment.go b/base/experiment.go
index 709ca3ec..282130c0 100644
--- a/base/experiment.go
+++ b/base/experiment.go
@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/antonmedv/expr"
 	log "github.com/iter8-tools/iter8/base/log"
@@ -20,8 +21,10 @@ import (
 type Task interface {
 	// validateInputs for this task
 	validateInputs() error
+
 	// initializeDefaults of the input values to this task
 	initializeDefaults()
+
 	// run this task
 	run(exp *Experiment) error
 }
@@ -33,9 +36,13 @@ type ExperimentSpec []Task
 type Experiment struct {
 	// Tasks is the sequence of tasks that constitute this experiment
 	Tasks ExperimentSpec
+
 	// Result is the current results from this experiment.
 	// The experiment may not have completed in which case results may be partial.
 	Result *ExperimentResult
+
+	// driver enables interacting with experiment result stored externally
+	driver Driver
 }
 
 // ExperimentResult defines the current results from the experiment
@@ -168,7 +175,12 @@ func (s *ExperimentSpec) UnmarshalJSON(data []byte) error {
 				tsk = rt
 			case CollectDatabaseTaskName:
 				cdt := &collectDatabaseTask{}
-				json.Unmarshal(tBytes, cdt)
+				err := json.Unmarshal(tBytes, cdt)
+				if err != nil {
+					e := errors.New("json unmarshal error")
+					log.Logger.WithStackTrace(err.Error()).Error(e)
+					return e
+				}
 				tsk = cdt
 			case CollectHTTPTaskName:
 				cht := &collectHTTPTask{}
@@ -612,10 +624,15 @@ func (in *Insights) GetMetricsInfo(nm string) (*MetricMeta, error) {
 type Driver interface {
 	// ReadSpec reads the experiment spec
 	ReadSpec() (ExperimentSpec, error)
+
 	// ReadResult reads the experiment result
 	ReadResult() (*ExperimentResult, error)
+
 	// WriteResult writes the experiment result
 	WriteResult(r *ExperimentResult) error
+
+	// ReadMetricsSpec reads the metrics spec
+	ReadMetricsSpec(provider string) (*template.Template, error)
 }
 
 // Completed returns true if the experiment is complete
@@ -694,6 +711,7 @@ func (exp *Experiment) SLOs() bool {
 // run the experiment
 func (exp *Experiment) run(driver Driver) error {
 	log.Logger.Debug("experiment run started ...")
+	exp.driver = driver
 	var err error
 	if exp.Result == nil {
 		exp.initResults()
diff --git a/driver/common.go b/driver/common.go
index f7746a15..f3ef8e34 100644
--- a/driver/common.go
+++ b/driver/common.go
@@ -1,6 +1,8 @@
 package driver
 
 import (
+	"text/template"
+
 	"github.com/iter8-tools/iter8/base"
 	"github.com/iter8-tools/iter8/base/log"
 	"sigs.k8s.io/yaml"
@@ -28,6 +30,11 @@ func SpecFromBytes(b []byte) (base.ExperimentSpec, error) {
 	return e, err
 }
 
+// MetricsSpecFromBytes reads metrics spec from bytes
+func MetricsSpecFromBytes(b []byte) (*template.Template, error) {
+	return template.New("metrics-spec").Parse(string(b))
+}
+
 // ResultFromBytes reads experiment result from bytes
 func ResultFromBytes(b []byte) (*base.ExperimentResult, error) {
 	r := &base.ExperimentResult{}
diff --git a/driver/filedriver.go b/driver/filedriver.go
index 06039dc4..5e60dbed 100644
--- a/driver/filedriver.go
+++ b/driver/filedriver.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"io/ioutil"
 	"path"
+	"text/template"
 
 	"github.com/iter8-tools/iter8/base"
 	"github.com/iter8-tools/iter8/base/log"
@@ -26,6 +27,16 @@ func (f *FileDriver) ReadSpec() (base.ExperimentSpec, error) {
 	return SpecFromBytes(b)
 }
 
+// ReadMetricsSpec reads metrics spec from file
+func (f *FileDriver) ReadMetricsSpec(provider string) (*template.Template, error) {
+	b, err := ioutil.ReadFile(path.Join(f.RunDir, provider+ExperimentMetricsPathSuffix))
+	if err != nil {
+		log.Logger.WithStackTrace(err.Error()).Error("unable to read metrics spec")
+		return nil, errors.New("unable to read metrics spec")
+	}
+	return MetricsSpecFromBytes(b)
+}
+
 // ReadResult reads experiment result from file
 func (f *FileDriver) ReadResult() (*base.ExperimentResult, error) {
 	b, err := ioutil.ReadFile(path.Join(f.RunDir, ExperimentResultPath))
diff --git a/driver/kubedriver.go b/driver/kubedriver.go
index 9f39b07f..85fd60c9 100644
--- a/driver/kubedriver.go
+++ b/driver/kubedriver.go
@@ -11,6 +11,7 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"text/template"
 	"time"
 
 	// Import to initialize client auth plugins.
@@ -203,11 +204,6 @@ func (driver *KubeDriver) getExperimentSpecSecret() (s *corev1.Secret, err error
 	return driver.getSecretWithRetry(driver.getSpecSecretName())
 }
 
-// getExperimentResultSecret gets the Kubernetes experiment result secret
-func (driver *KubeDriver) getExperimentResultSecret() (s *corev1.Secret, err error) {
-	return driver.getSecretWithRetry(driver.getResultSecretName())
-}
-
 // ReadSpec creates an ExperimentSpec struct for a Kubernetes experiment
 func (driver *KubeDriver) ReadSpec() (base.ExperimentSpec, error) {
 	s, err := driver.getExperimentSpecSecret()
@@ -225,6 +221,28 @@ func (driver *KubeDriver) ReadSpec() (base.ExperimentSpec, error) {
 	return SpecFromBytes(spec)
 }
 
+// ReadMetricsSpec creates a metrics Template struct for a Kubernetes experiment
+func (driver *KubeDriver) ReadMetricsSpec(provider string) (*template.Template, error) {
+	s, err := driver.getExperimentSpecSecret()
+	if err != nil {
+		return nil, err
+	}
+
+	res, ok := s.Data[provider+ExperimentMetricsPathSuffix]
+	if !ok {
+		err = fmt.Errorf("unable to extract %v metrics spec; result secret has no %v field", provider, provider+ExperimentMetricsPathSuffix)
+		log.Logger.Error(err)
+		return nil, err
+	}
+
+	return MetricsSpecFromBytes(res)
+}
+
+// getExperimentResultSecret gets the Kubernetes experiment result secret
+func (driver *KubeDriver) getExperimentResultSecret() (s *corev1.Secret, err error) {
+	return driver.getSecretWithRetry(driver.getResultSecretName())
+}
+
 // ReadResult creates an ExperimentResult struct for a Kubernetes experiment
 func (driver *KubeDriver) ReadResult() (*base.ExperimentResult, error) {
 	s, err := driver.getExperimentResultSecret()


Test patch:
diff --git a/action/report_test.go b/action/report_test.go
index c41d85a3..85ebdba8 100644
--- a/action/report_test.go
+++ b/action/report_test.go
@@ -23,6 +23,15 @@ func TestLocalReportText(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLocalReportTextNoInsights(t *testing.T) {
+	// fix rOpts
+	rOpts := NewReportOpts(driver.NewFakeKubeDriver(cli.New()))
+	rOpts.RunDir = base.CompletePath("../", "testdata/assertinputs/noinsights")
+
+	err := rOpts.LocalRun(os.Stdout)
+	assert.NoError(t, err)
+}
+
 func TestLocalReportHTML(t *testing.T) {
 	// fix rOpts
 	rOpts := NewReportOpts(driver.NewFakeKubeDriver(cli.New()))
@@ -59,3 +68,12 @@ func TestKubeReportText(t *testing.T) {
 	err := rOpts.KubeRun(os.Stdout)
 	assert.NoError(t, err)
 }
+
+func TestLocalReportHTMLNoInsights(t *testing.T) {
+	// fix rOpts
+	rOpts := NewReportOpts(driver.NewFakeKubeDriver(cli.New()))
+	rOpts.RunDir = base.CompletePath("../", "testdata/assertinputs/noinsights")
+	rOpts.OutputFormat = HTMLOutputFormatKey
+	err := rOpts.LocalRun(os.Stdout)
+	assert.NoError(t, err)
+}
diff --git a/base/collect_database_test.go b/base/collect_database_test.go
index 4a584889..4a71db08 100644
--- a/base/collect_database_test.go
+++ b/base/collect_database_test.go
@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"os"
 	"testing"
+	"text/template"
 
 	"github.com/jarcoal/httpmock"
 	"github.com/stretchr/testify/assert"
@@ -49,17 +50,18 @@ func GoToTempDirectoryAndCopyMetricsFile(t *testing.T, test func()) error {
 	}
 
 	// return to original path
-	defer func() {
+	t.Cleanup(func() {
 		os.Chdir(originalPath)
-	}()
+	})
 
 	// get metrics file
 	srcFile, err := os.Open(metricsDirectory + metricsFileName)
 	if err != nil {
 		return errors.New("could not open metrics file.")
 	}
-
-	defer srcFile.Close()
+	t.Cleanup(func() {
+		srcFile.Close()
+	})
 
 	// go to temp directory
 	destDir := t.TempDir()
@@ -70,7 +72,9 @@ func GoToTempDirectoryAndCopyMetricsFile(t *testing.T, test func()) error {
 	if err != nil {
 		return errors.New("could not create copy of metrics file in temp directory.")
 	}
-	defer destFile.Close()
+	t.Cleanup(func() {
+		destFile.Close()
+	})
 	io.Copy(destFile, srcFile)
 
 	// run test
@@ -104,6 +108,51 @@ func TestGetElapsedTimeSeconds(t *testing.T) {
 	assert.Equal(t, elapsedTimeSeconds > 1000000, true)
 }
 
+// test if a user sets elapsedTimeSeconds getElapsedTimeSeconds()
+func TestSetElapsedTimeSecondsError(t *testing.T) {
+	versionInfo := map[string]interface{}{
+		"ibm_service_instance": "version1",
+		"startingTime":         "Feb 4, 2014 at 6:05pm (PST)",
+		"elapsedTimeSeconds":   "Feb 5, 2014 at 6:05pm (PST)",
+	}
+
+	exp := &Experiment{
+		Tasks:  []Task{},
+		Result: &ExperimentResult{},
+	}
+
+	// this should add a startingTime that will be overwritten by the one in
+	// versionInfo
+	exp.initResults()
+
+	elapsedTimeSeconds, err := getElapsedTimeSeconds(versionInfo, exp)
+
+	assert.Error(t, err)
+	assert.Equal(t, elapsedTimeSeconds, int64(0))
+}
+
+// test if a user sets startingTime incorrectly getElapsedTimeSeconds()
+func TestStartingTimeFormatError(t *testing.T) {
+	versionInfo := map[string]interface{}{
+		"ibm_service_instance": "version1",
+		"startingTime":         "1652935205",
+	}
+
+	exp := &Experiment{
+		Tasks:  []Task{},
+		Result: &ExperimentResult{},
+	}
+
+	// this should add a startingTime that will be overwritten by the one in
+	// versionInfo
+	exp.initResults()
+
+	elapsedTimeSeconds, err := getElapsedTimeSeconds(versionInfo, exp)
+
+	assert.Error(t, err)
+	assert.Equal(t, elapsedTimeSeconds, int64(0))
+}
+
 // basic test with one version, mimicking Code Engine
 // one version, three successful metrics
 func TestCEOneVersion(t *testing.T) {
@@ -190,9 +239,18 @@ func TestCEOneVersion(t *testing.T) {
 				}
 			}`))
 
+		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
 		exp := &Experiment{
 			Tasks:  []Task{ct},
 			Result: &ExperimentResult{},
+			driver: &md,
 		}
 		exp.initResults()
 		exp.Result.initInsightsWithNumVersions(1)
@@ -211,7 +269,6 @@ func TestCEOneVersion(t *testing.T) {
 	})
 
 	assert.NoError(t, err)
-
 }
 
 // test with one version and improper authorization, mimicking Code Engine
@@ -257,9 +314,18 @@ func TestCEUnauthorized(t *testing.T) {
 		httpmock.RegisterResponder("GET", testPromURL+url.QueryEscape(errorRateQuery),
 			httpmock.NewStringResponder(401, `Unauthorized`))
 
+		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
 		exp := &Experiment{
 			Tasks:  []Task{ct},
 			Result: &ExperimentResult{},
+			driver: &md,
 		}
 		exp.initResults()
 		exp.Result.initInsightsWithNumVersions(1)
@@ -354,9 +420,18 @@ func TestCESomeValues(t *testing.T) {
 				}
 			}`))
 
+		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
 		exp := &Experiment{
 			Tasks:  []Task{ct},
 			Result: &ExperimentResult{},
+			driver: &md,
 		}
 		exp.initResults()
 		exp.Result.initInsightsWithNumVersions(1)
@@ -458,9 +533,18 @@ func TestCEMultipleVersions(t *testing.T) {
 				}
 			}`))
 
+		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
 		exp := &Experiment{
 			Tasks:  []Task{ct},
 			Result: &ExperimentResult{},
+			driver: &md,
 		}
 		exp.initResults()
 		exp.Result.initInsightsWithNumVersions(2)
@@ -564,9 +648,18 @@ func TestCEMultipleVersionsAndMetrics(t *testing.T) {
 				}
 			}`))
 
+		template, err := template.ParseFiles(testCe + experimentMetricsPathSuffix)
+
+		assert.NoError(t, err)
+
+		md := mockDriver{
+			metricsTemplate: template,
+		}
+
 		exp := &Experiment{
 			Tasks:  []Task{ct},
 			Result: &ExperimentResult{},
+			driver: &md,
 		}
 		exp.initResults()
 		exp.Result.initInsightsWithNumVersions(2)
diff --git a/base/collect_http_test.go b/base/collect_http_test.go
index 08828144..e7776963 100644
--- a/base/collect_http_test.go
+++ b/base/collect_http_test.go
@@ -14,9 +14,6 @@ func TestRunCollectHTTP(t *testing.T) {
 	httpmock.RegisterResponder("POST", "https://something.com",
 		httpmock.NewStringResponder(200, `[{"id": 1, "name": "My Great Thing"}]`))
 
-	httpmock.RegisterResponder("GET", "https://data.police.uk/api/crimes-street-dates",
-		httpmock.NewStringResponder(200, `[{"my": 1, "great": "payload"}]`))
-
 	// valid collect HTTP task... should succeed
 	ct := &collectHTTPTask{
 		TaskMeta: TaskMeta{
diff --git a/base/experiment_test.go b/base/experiment_test.go
index c7c9a732..657e36a9 100644
--- a/base/experiment_test.go
+++ b/base/experiment_test.go
@@ -3,6 +3,7 @@ package base
 import (
 	"io/ioutil"
 	"testing"
+	"text/template"
 
 	"github.com/iter8-tools/iter8/base/log"
 	"github.com/stretchr/testify/assert"
@@ -23,6 +24,13 @@ func TestReadExperiment(t *testing.T) {
 	err = yaml.Unmarshal(b, es)
 	assert.NoError(t, err)
 	assert.Equal(t, 3, len(*es))
+
+	b, err = ioutil.ReadFile(CompletePath("../testdata", "experiment_db.yaml"))
+	assert.NoError(t, err)
+	es = &ExperimentSpec{}
+	err = yaml.Unmarshal(b, es)
+	assert.NoError(t, err)
+	assert.Equal(t, 4, len(*es))
 }
 func TestRunTask(t *testing.T) {
 	SetupWithMock(t)
@@ -80,7 +88,9 @@ func TestRunExperiment(t *testing.T) {
 		Tasks: *es,
 	}
 
-	err = RunExperiment(&mockDriver{&exp})
+	metricsTemplate := template.Template{}
+
+	err = RunExperiment(&mockDriver{&exp, &metricsTemplate})
 	assert.NoError(t, err)
 
 	yamlBytes, _ := yaml.Marshal(exp.Result)
diff --git a/base/test_helpers.go b/base/test_helpers.go
index 7fedd117..debca82e 100644
--- a/base/test_helpers.go
+++ b/base/test_helpers.go
@@ -2,6 +2,7 @@ package base
 
 import (
 	"testing"
+	"text/template"
 
 	"github.com/jarcoal/httpmock"
 )
@@ -18,6 +19,8 @@ func SetupWithMock(t *testing.T) {
 // mockDriver is a mock driver that can be used to run experiments
 type mockDriver struct {
 	*Experiment
+
+	metricsTemplate *template.Template
 }
 
 // ReadResult enables results to be read from the mock driver
@@ -25,6 +28,11 @@ func (m *mockDriver) ReadResult() (*ExperimentResult, error) {
 	return m.Experiment.Result, nil
 }
 
+// ReadMetricsSpec enables metrics spec to be read from the mock driver
+func (m *mockDriver) ReadMetricsSpec(provider string) (*template.Template, error) {
+	return m.metricsTemplate, nil
+}
+
 // WriteResult enables results to be written from the mock driver
 func (m *mockDriver) WriteResult(r *ExperimentResult) error {
 	m.Experiment.Result = r
diff --git a/cmd/test_helpers.go b/cmd/test_helpers.go
index e69cabc5..1d29f497 100644
--- a/cmd/test_helpers.go
+++ b/cmd/test_helpers.go
@@ -56,7 +56,7 @@ func runTestActionCmd(t *testing.T, tests []cmdTestCase) {
 	// fixed time
 	log.Logger.SetFormatter(testFormatter{log.Logger.Formatter})
 	t.Helper()
-	defer resetEnv()()
+	t.Cleanup(resetEnv())
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/driver/filedriver_test.go b/driver/filedriver_test.go
index db03d269..eafb7ac9 100644
--- a/driver/filedriver_test.go
+++ b/driver/filedriver_test.go
@@ -19,3 +19,36 @@ func TestLocalRun(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, exp.Completed() && exp.NoFailure() && exp.SLOs())
 }
+
+func TestFileDriverReadMetricsSpec(t *testing.T) {
+	base.SetupWithMock(t)
+
+	fd := FileDriver{
+		RunDir: base.CompletePath("../", "testdata/metrics"),
+	}
+
+	metrics, err := fd.ReadMetricsSpec("test-ce")
+
+	assert.NoError(t, err)
+	assert.NotNil(t, metrics)
+}
+
+func TestFileDriverReadError(t *testing.T) {
+	base.SetupWithMock(t)
+
+	fd := FileDriver{
+		RunDir: ".",
+	}
+
+	spec, err := fd.ReadSpec()
+	assert.Error(t, err)
+	assert.Nil(t, spec)
+
+	metrics, err := fd.ReadMetricsSpec("test-ce")
+	assert.Error(t, err)
+	assert.Nil(t, metrics)
+
+	result, err := fd.ReadResult()
+	assert.Error(t, err)
+	assert.Nil(t, result)
+}
diff --git a/driver/kubedriver_test.go b/driver/kubedriver_test.go
index 7834fc56..3518ab6e 100644
--- a/driver/kubedriver_test.go
+++ b/driver/kubedriver_test.go
@@ -16,6 +16,10 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const (
+	testCe = "test-ce"
+)
+
 func TestKOps(t *testing.T) {
 	kd := NewKubeDriver(cli.New()) // we will ignore this value
 	assert.NotNil(t, kd)
@@ -137,3 +141,25 @@ func TestLogs(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "fake logs", str)
 }
+
+func TestKubeDriverReadMetricsSpec(t *testing.T) {
+	base.SetupWithMock(t)
+
+	kd := NewFakeKubeDriver(cli.New())
+	kd.revision = 1
+	metricsSpecPath := testCe + ExperimentMetricsPathSuffix
+
+	byteArray, _ := ioutil.ReadFile(base.CompletePath("../testdata/metrics", metricsSpecPath))
+	kd.Clientset.CoreV1().Secrets("default").Create(context.TODO(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "default-spec",
+			Namespace: "default",
+		},
+		StringData: map[string]string{metricsSpecPath: string(byteArray)},
+	}, metav1.CreateOptions{})
+
+	metricsSpec, err := kd.ReadMetricsSpec(testCe)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, metricsSpec)
+}
diff --git a/testdata/assertinputs/noinsights/experiment.yaml b/testdata/assertinputs/noinsights/experiment.yaml
new file mode 100644
index 00000000..22548f22
--- /dev/null
+++ b/testdata/assertinputs/noinsights/experiment.yaml
@@ -0,0 +1,31 @@
+# task 1: generate HTTP requests for application URL
+# collect Iter8's built-in HTTP latency and error-related metrics
+- task: gen-load-and-collect-metrics-http
+  with:
+    duration: 2s
+    errorRanges:
+    - lower: 500
+    versionInfo:
+    - url: https://httpbin.org/get
+# task 2: validate service level objectives for app using
+# the metrics collected in the above task
+- task: assess-app-versions
+  with:
+    SLOs:
+    - metric: "http/error-rate"
+      upperLimit: 0
+    - metric: "http/latency-mean"
+      upperLimit: 500
+    - metric: "http/latency-p50"
+      upperLimit: 1000
+    - metric: "http/latency-p50.0"
+      upperLimit: 1000
+    - metric: "http/latency-p95.0"
+      upperLimit: 2500
+    - metric: "http/latency-p99"
+      upperLimit: 5000
+# tasks 3 & 4: print if SLOs are satisfied or not
+- if: SLOs()
+  run: echo "SLOs satisfied"
+- if: not SLOs()
+  run: echo "SLOs not satisfied"
diff --git a/testdata/assertinputs/noinsights/result.yaml b/testdata/assertinputs/noinsights/result.yaml
new file mode 100644
index 00000000..0558b9a4
--- /dev/null
+++ b/testdata/assertinputs/noinsights/result.yaml
@@ -0,0 +1,4 @@
+failure: false
+iter8Version: v0.10
+numCompletedTasks: 4
+startTime: "2022-03-16T10:22:58.540897-04:00"
diff --git a/testdata/experiment_db.yaml b/testdata/experiment_db.yaml
new file mode 100644
index 00000000..fc5b1c20
--- /dev/null
+++ b/testdata/experiment_db.yaml
@@ -0,0 +1,21 @@
+# task 1: collect metrics
+- task: collect-metrics-database
+  with:
+    providers:
+    # will look metrics file kfserving.metrics.yaml
+    - kfserving
+    versionInfo:
+      - namespace_name: ns-candidate
+        startingTime: Jan 2, 2006 at 3:04pm (MST)
+# task 2: validate service level objectives for app using
+# the metrics collected in the above task
+- task: assess-app-versions
+  with:
+    SLOs:
+    - metric: "kfserving/request-count"
+      upperLimit: 0
+# tasks 3 & 4: print if SLOs are satisfied or not
+- if: SLOs()
+  run: echo "SLOs satisfied"
+- if: not SLOs()
+  run: echo "SLOs not satisfied"
diff --git a/testdata/metrics/test-ce.metrics.yaml b/testdata/metrics/test-ce.metrics.yaml
index df81400b..6f054df3 100644
--- a/testdata/metrics/test-ce.metrics.yaml
+++ b/testdata/metrics/test-ce.metrics.yaml
@@ -9,7 +9,7 @@ headers:
   # to get the GUID, run: ibmcloud resource service-instance <NAME> --output json | jq -r '.[].guid'
   # https://cloud.ibm.com/docs/monitoring?topic=monitoring-mon-curl
   IBMInstanceID: test-guid
-provider: ce
+provider: test-ce
 method: GET
 # Inputs for the template:
 #   ibm_codeengine_application_name string
diff --git a/testdata/output/kreport.txt b/testdata/output/kreport.txt
index c47ea708..872d874f 100644
--- a/testdata/output/kreport.txt
+++ b/testdata/output/kreport.txt
@@ -39,3 +39,4 @@ Latest observed values for metrics:
   http/latency-stddev (msec) |62.82
   http/request-count         |16.00
   
+
diff --git a/testdata/output/report.txt b/testdata/output/report.txt
index c47ea708..872d874f 100644
--- a/testdata/output/report.txt
+++ b/testdata/output/report.txt
@@ -39,3 +39,4 @@ Latest observed values for metrics:
   http/latency-stddev (msec) |62.82
   http/request-count         |16.00
   
+