Repository: iter8-tools/iter8
Pull request #1409 (2023-02-27 17:24:56)

Multiple endpoints


Patch:
diff --git a/base/collect_grpc.go b/base/collect_grpc.go
index b4daf0d1..69b4b808 100644
--- a/base/collect_grpc.go
+++ b/base/collect_grpc.go
@@ -2,9 +2,11 @@ package base
 
 import (
 	"errors"
+	"fmt"
 	"time"
 
 	"github.com/bojand/ghz/runner"
+	"github.com/imdario/mergo"
 	log "github.com/iter8-tools/iter8/base/log"
 	gd "github.com/mcuadros/go-defaults"
 )
@@ -28,16 +30,22 @@ const (
 	insecureDefault = true
 )
 
+// collectHTTPInputs contain the inputs to the metrics collection task to be executed.
 type collectGRPCInputs struct {
 	runner.Config
+
 	// Warmup indicates if task execution is for warmup purposes; if so the results will be ignored
 	Warmup *bool `json:"warmup,omitempty" yaml:"warmup,omitempty"`
+
+	// Endpoints is used to define multiple endpoints to test
+	Endpoints map[string]runner.Config `json:"endpoints" yaml:"endpoints"`
 }
 
 // collectGRPCTask enables load testing of gRPC services.
 type collectGRPCTask struct {
 	// TaskMeta has fields common to all tasks
 	TaskMeta
+
 	// With contains the inputs to this task
 	With collectGRPCInputs `json:"with" yaml:"with"`
 }
@@ -64,25 +72,72 @@ func (t *collectGRPCTask) validateInputs() error {
 }
 
 // resultForVersion collects gRPC test result for a given version
-func (t *collectGRPCTask) resultForVersion() (*runner.Report, error) {
+func (t *collectGRPCTask) resultForVersion() (map[string]*runner.Report, error) {
 	// the main idea is to run ghz with proper options
 
-	opts := runner.WithConfig(&t.With.Config)
+	var err error
+	results := map[string]*runner.Report{}
+
+	if len(t.With.Endpoints) > 0 {
+		log.Logger.Trace("multiple endpoints")
+		for endpointID, endpoint := range t.With.Endpoints {
+			endpoint := endpoint // prevent implicit memory aliasing
+			log.Logger.Trace(fmt.Sprintf("endpoint: %s", endpointID))
+
+			// default from baseline
+			call := t.With.Call
+			if endpoint.Call != "" {
+				call = endpoint.Call
+			}
+
+			host := t.With.Host
+			if endpoint.Host != "" {
+				host = endpoint.Host
+			}
 
-	// todo: supply all the allowed options
-	igr, err := runner.Run(t.With.Call, t.With.Host, opts)
-	if err != nil {
-		e := errors.New("ghz run failed")
-		log.Logger.WithStackTrace(err.Error()).Error(e)
-		if igr == nil {
-			e = errors.New("failed to get results since ghz run was aborted")
-			log.Logger.Error(e)
+			// merge endpoint options with baseline options
+			if err := mergo.Merge(&endpoint, t.With.Config); err != nil {
+				log.Logger.Error(fmt.Sprintf("could not merge Fortio options for endpoint \"%s\"", endpointID))
+				return nil, err
+			}
+			eOpts := runner.WithConfig(&endpoint) // endpoint options
+
+			igr, err := runner.Run(call, host, eOpts)
+			if err != nil {
+				e := fmt.Errorf("ghz run failed for endpoint \"%s\"", endpointID)
+				log.Logger.WithStackTrace(err.Error()).Error(e)
+				if igr == nil {
+					e = fmt.Errorf("failed to get results for endpoint \"%s\" since ghz run was aborted", endpointID)
+					log.Logger.Error(e)
+				}
+				return nil, e
+			}
+			log.Logger.Trace("ran ghz gRPC test")
+			log.Logger.Trace(igr.ErrorDist)
+
+			results[gRPCMetricPrefix+"-"+endpointID] = igr
+		}
+	} else {
+		// TODO: supply all the allowed options
+		opts := runner.WithConfig(&t.With.Config)
+
+		igr, err := runner.Run(t.With.Call, t.With.Host, opts)
+		if err != nil {
+			e := errors.New("ghz run failed")
+			log.Logger.WithStackTrace(err.Error()).Error(e)
+			if igr == nil {
+				e = errors.New("failed to get results since ghz run was aborted")
+				log.Logger.Error(e)
+			}
+			return nil, e
 		}
-		return nil, e
+		log.Logger.Trace("ran ghz gRPC test")
+		log.Logger.Trace(igr.ErrorDist)
+
+		results[gRPCMetricPrefix] = igr
 	}
-	log.Logger.Trace("ran ghz gRPC test")
-	log.Logger.Trace(igr.ErrorDist)
-	return igr, err
+
+	return results, err
 }
 
 // latencySample extracts a latency sample from ghz result details
@@ -129,10 +184,10 @@ func (t *collectGRPCTask) run(exp *Experiment) error {
 	in := exp.Result.Insights
 
 	// 4. Populate all metrics collected by this task
-	if data != nil { // assuming there is some raw ghz result to process
+	for provider, data := range data {
 		// populate grpc request count
 		// todo: this logic breaks for looped experiments. Fix when we get to loops.
-		m := gRPCMetricPrefix + "/" + gRPCRequestCountMetricName
+		m := provider + "/" + gRPCRequestCountMetricName
 		mm := MetricMeta{
 			Description: "number of gRPC requests sent",
 			Type:        CounterMetricType,
@@ -149,7 +204,7 @@ func (t *collectGRPCTask) run(exp *Experiment) error {
 
 		// populate count
 		// todo: This logic breaks for looped experiments. Fix when we get to loops.
-		m = gRPCMetricPrefix + "/" + gRPCErrorCountMetricName
+		m = provider + "/" + gRPCErrorCountMetricName
 		mm = MetricMeta{
 			Description: "number of responses that were errors",
 			Type:        CounterMetricType,
@@ -160,7 +215,7 @@ func (t *collectGRPCTask) run(exp *Experiment) error {
 
 		// populate rate
 		// todo: This logic breaks for looped experiments. Fix when we get to loops.
-		m = gRPCMetricPrefix + "/" + gRPCErrorRateMetricName
+		m = provider + "/" + gRPCErrorRateMetricName
 		rc := float64(data.Count)
 		if rc != 0 {
 			mm = MetricMeta{
@@ -173,7 +228,7 @@ func (t *collectGRPCTask) run(exp *Experiment) error {
 		}
 
 		// populate latency sample
-		m = gRPCMetricPrefix + "/" + gRPCLatencySampleMetricName
+		m = provider + "/" + gRPCLatencySampleMetricName
 		mm = MetricMeta{
 			Description: "gRPC Latency Sample",
 			Type:        SampleMetricType,
@@ -184,5 +239,6 @@ func (t *collectGRPCTask) run(exp *Experiment) error {
 			return err
 		}
 	}
+
 	return nil
 }
diff --git a/base/collect_http.go b/base/collect_http.go
index 74166580..0e7d2e92 100644
--- a/base/collect_http.go
+++ b/base/collect_http.go
@@ -9,6 +9,7 @@ import (
 	"fortio.org/fortio/fhttp"
 	"fortio.org/fortio/periodic"
 	"fortio.org/fortio/stats"
+	"github.com/imdario/mergo"
 	log "github.com/iter8-tools/iter8/base/log"
 )
 
@@ -20,8 +21,8 @@ type errorRange struct {
 	Upper *int `json:"upper,omitempty" yaml:"upper,omitempty"`
 }
 
-// collectHTTPInputs contain the inputs to the metrics collection task to be executed.
-type collectHTTPInputs struct {
+// endpoint contains the inputs for one endpoint
+type endpoint struct {
 	// NumRequests is the number of requests to be sent to the app. Default value is 100.
 	NumRequests *int64 `json:"numRequests,omitempty" yaml:"numRequests,omitempty"`
 	// Duration of this task. Specified in the Go duration string format (example, 5s). If both duration and numRequests are specified, then duration is ignored.
@@ -50,6 +51,14 @@ type collectHTTPInputs struct {
 	Warmup *bool `json:"warmup,omitempty" yaml:"warmup,omitempty"`
 }
 
+// collectHTTPInputs contain the inputs to the metrics collection task to be executed.
+type collectHTTPInputs struct {
+	endpoint
+
+	// Endpoints is used to define multiple endpoints to test
+	Endpoints map[string]endpoint `json:"endpoints" yaml:"endpoints"`
+}
+
 const (
 	// CollectHTTPTaskName is the name of this task which performs load generation and metrics collection.
 	CollectHTTPTaskName = "http"
@@ -149,32 +158,32 @@ func (t *collectHTTPTask) validateInputs() error {
 }
 
 // getFortioOptions constructs Fortio's HTTP runner options based on collect task inputs
-func (t *collectHTTPTask) getFortioOptions() (*fhttp.HTTPRunnerOptions, error) {
+func getFortioOptions(c endpoint) (*fhttp.HTTPRunnerOptions, error) {
 	// basic runner
 	fo := &fhttp.HTTPRunnerOptions{
 		RunnerOptions: periodic.RunnerOptions{
 			RunType:     "Iter8 load test",
-			QPS:         float64(*t.With.QPS),
-			NumThreads:  *t.With.Connections,
-			Percentiles: t.With.Percentiles,
+			QPS:         float64(*c.QPS),
+			NumThreads:  *c.Connections,
+			Percentiles: c.Percentiles,
 			Out:         io.Discard,
 		},
 		HTTPOptions: fhttp.HTTPOptions{
-			URL: t.With.URL,
+			URL: c.URL,
 		},
-		AllowInitialErrors: *t.With.AllowInitialErrors,
+		AllowInitialErrors: *c.AllowInitialErrors,
 	}
 
 	// num requests
-	if t.With.NumRequests != nil {
-		fo.RunnerOptions.Exactly = *t.With.NumRequests
+	if c.NumRequests != nil {
+		fo.RunnerOptions.Exactly = *c.NumRequests
 	}
 
 	// add duration
 	var duration time.Duration
 	var err error
-	if t.With.Duration != nil {
-		duration, err = time.ParseDuration(*t.With.Duration)
+	if c.Duration != nil {
+		duration, err = time.ParseDuration(*c.Duration)
 		if err == nil {
 			fo.RunnerOptions.Duration = duration
 		} else {
@@ -184,14 +193,14 @@ func (t *collectHTTPTask) getFortioOptions() (*fhttp.HTTPRunnerOptions, error) {
 	}
 
 	// content type & payload
-	if t.With.ContentType != nil {
-		fo.ContentType = *t.With.ContentType
+	if c.ContentType != nil {
+		fo.ContentType = *c.ContentType
 	}
-	if t.With.PayloadStr != nil {
-		fo.Payload = []byte(*t.With.PayloadStr)
+	if c.PayloadStr != nil {
+		fo.Payload = []byte(*c.PayloadStr)
 	}
-	if t.With.PayloadFile != nil {
-		b, err := os.ReadFile(*t.With.PayloadFile)
+	if c.PayloadFile != nil {
+		b, err := os.ReadFile(*c.PayloadFile)
 		if err != nil {
 			return nil, err
 		}
@@ -199,7 +208,7 @@ func (t *collectHTTPTask) getFortioOptions() (*fhttp.HTTPRunnerOptions, error) {
 	}
 
 	// headers
-	for key, value := range t.With.Headers {
+	for key, value := range c.Headers {
 		if err = fo.AddAndValidateExtraHeader(key + ":" + value); err != nil {
 			log.Logger.WithStackTrace("unable to add header").Error(err)
 			return nil, err
@@ -210,24 +219,70 @@ func (t *collectHTTPTask) getFortioOptions() (*fhttp.HTTPRunnerOptions, error) {
 }
 
 // getFortioResults collects Fortio run results
-func (t *collectHTTPTask) getFortioResults() (*fhttp.HTTPRunnerResults, error) {
+// func (t *collectHTTPTask) getFortioResults() (*fhttp.HTTPRunnerResults, error) {
+// key is the metric prefix
+func (t *collectHTTPTask) getFortioResults() (map[string]*fhttp.HTTPRunnerResults, error) {
 	// the main idea is to run Fortio with proper options
 
-	fo, err := t.getFortioOptions()
-	if err != nil {
-		return nil, err
-	}
-	log.Logger.Trace("got fortio options")
-	log.Logger.Trace("URL: ", fo.URL)
-	ifr, err := fhttp.RunHTTPTest(fo)
-	if err != nil {
-		log.Logger.WithStackTrace(err.Error()).Error("fortio failed")
-		if ifr == nil {
-			log.Logger.Error("failed to get results since fortio run was aborted")
+	var err error
+	results := map[string]*fhttp.HTTPRunnerResults{}
+	if len(t.With.Endpoints) > 0 {
+		log.Logger.Trace("multiple endpoints")
+		for endpointID, endpoint := range t.With.Endpoints {
+			endpoint := endpoint // prevent implicit memory aliasing
+			log.Logger.Trace(fmt.Sprintf("endpoint: %s", endpointID))
+
+			// merge endpoint config with baseline config
+			if err := mergo.Merge(&endpoint, t.With.endpoint); err != nil {
+				log.Logger.Error(fmt.Sprintf("could not merge Fortio options for endpoint \"%s\"", endpointID))
+				return nil, err
+			}
+
+			efo, err := getFortioOptions(endpoint)
+			if err != nil {
+				log.Logger.Error(fmt.Sprintf("could not get Fortio options for endpoint \"%s\"", endpointID))
+				return nil, err
+			}
+
+			log.Logger.Trace("got fortio options")
+			log.Logger.Trace("URL: ", efo.URL)
+
+			ifr, err := fhttp.RunHTTPTest(efo)
+			if err != nil {
+				log.Logger.WithStackTrace(err.Error()).Error("fortio failed")
+				if ifr == nil {
+					log.Logger.Error("failed to get results since fortio run was aborted")
+				}
+			}
+
+			log.Logger.Trace("ran fortio http test")
+
+			results[httpMetricPrefix+"-"+endpointID] = ifr
 		}
+	} else {
+		fo, err := getFortioOptions(t.With.endpoint)
+		if err != nil {
+			log.Logger.Error("could not get Fortio options")
+			return nil, err
+		}
+
+		log.Logger.Trace("got fortio options")
+		log.Logger.Trace("URL: ", fo.URL)
+
+		ifr, err := fhttp.RunHTTPTest(fo)
+		if err != nil {
+			log.Logger.WithStackTrace(err.Error()).Error("fortio failed")
+			if ifr == nil {
+				log.Logger.Error("failed to get results since fortio run was aborted")
+			}
+		}
+
+		log.Logger.Trace("ran fortio http test")
+
+		results[httpMetricPrefix] = ifr
 	}
-	log.Logger.Trace("ran fortio http test")
-	return ifr, err
+
+	return results, err
 }
 
 // run executes this task
@@ -259,9 +314,9 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 	}
 	in := exp.Result.Insights
 
-	if data != nil {
+	for provider, data := range data {
 		// request count
-		m := httpMetricPrefix + "/" + builtInHTTPRequestCountID
+		m := provider + "/" + builtInHTTPRequestCountID
 		mm := MetricMeta{
 			Description: "number of requests sent",
 			Type:        CounterMetricType,
@@ -278,7 +333,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 			}
 		}
 		// error count
-		m = httpMetricPrefix + "/" + builtInHTTPErrorCountID
+		m = provider + "/" + builtInHTTPErrorCountID
 		mm = MetricMeta{
 			Description: "number of responses that were errors",
 			Type:        CounterMetricType,
@@ -288,7 +343,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 		}
 
 		// error-rate
-		m = httpMetricPrefix + "/" + builtInHTTPErrorRateID
+		m = provider + "/" + builtInHTTPErrorRateID
 		rc := float64(data.DurationHistogram.Count)
 		if rc != 0 {
 			mm = MetricMeta{
@@ -301,7 +356,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 		}
 
 		// mean-latency
-		m = httpMetricPrefix + "/" + builtInHTTPLatencyMeanID
+		m = provider + "/" + builtInHTTPLatencyMeanID
 		mm = MetricMeta{
 			Description: "mean of observed latency values",
 			Type:        GaugeMetricType,
@@ -312,7 +367,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 		}
 
 		// stddev-latency
-		m = httpMetricPrefix + "/" + builtInHTTPLatencyStdDevID
+		m = provider + "/" + builtInHTTPLatencyStdDevID
 		mm = MetricMeta{
 			Description: "standard deviation of observed latency values",
 			Type:        GaugeMetricType,
@@ -323,7 +378,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 		}
 
 		// min-latency
-		m = httpMetricPrefix + "/" + builtInHTTPLatencyMinID
+		m = provider + "/" + builtInHTTPLatencyMinID
 		mm = MetricMeta{
 			Description: "minimum of observed latency values",
 			Type:        GaugeMetricType,
@@ -334,7 +389,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 		}
 
 		// max-latency
-		m = httpMetricPrefix + "/" + builtInHTTPLatencyMaxID
+		m = provider + "/" + builtInHTTPLatencyMaxID
 		mm = MetricMeta{
 			Description: "maximum of observed latency values",
 			Type:        GaugeMetricType,
@@ -346,7 +401,7 @@ func (t *collectHTTPTask) run(exp *Experiment) error {
 
 		// percentiles
 		for _, p := range data.DurationHistogram.Percentiles {
-			m = fmt.Sprintf("%v/%v%v", httpMetricPrefix, builtInHTTPLatencyPercentilePrefix, p.Percentile)
+			m = fmt.Sprintf("%v/%v%v", provider, builtInHTTPLatencyPercentilePrefix, p.Percentile)
 			mm = MetricMeta{
 				Description: fmt.Sprintf("%v-th percentile of observed latency values", p.Percentile),
 				Type:        GaugeMetricType,
diff --git a/go.mod b/go.mod
index 0bf91709..e74c9ae3 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/antonmedv/expr v1.9.0
 	github.com/bojand/ghz v0.111.0
-	github.com/imdario/mergo v0.3.12
+	github.com/imdario/mergo v0.3.13
 	github.com/itchyny/gojq v0.12.7
 	github.com/jarcoal/httpmock v1.1.0
 	github.com/mattn/go-shellwords v1.0.12
diff --git a/go.sum b/go.sum
index bd8de7f8..8c48f61a 100644
--- a/go.sum
+++ b/go.sum
@@ -401,6 +401,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -1170,6 +1172,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=


Test patch:
diff --git a/base/collect_grpc_test.go b/base/collect_grpc_test.go
index 5ec1daf7..9a007817 100644
--- a/base/collect_grpc_test.go
+++ b/base/collect_grpc_test.go
@@ -13,6 +13,13 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const (
+	unary         = "unary"
+	server        = "server"
+	client        = "client"
+	bidirectional = "bidirectional"
+)
+
 // Credit: Several of the tests in this file are based on
 // https://github.com/bojand/ghz/blob/master/runner/run_test.go
 func TestRunCollectGRPCUnary(t *testing.T) {
@@ -29,12 +36,14 @@ func TestRunCollectGRPCUnary(t *testing.T) {
 		TaskMeta: TaskMeta{
 			Task: StringPointer(CollectGRPCTaskName),
 		},
-		With: collectGRPCInputs{Config: runner.Config{
-			Data: map[string]interface{}{"name": "bob"},
-			Call: "helloworld.Greeter.SayHello",
-			Host: internal.LocalHostPort,
+		With: collectGRPCInputs{
+			Config: runner.Config{
+				Data: map[string]interface{}{"name": "bob"},
+				Call: "helloworld.Greeter.SayHello",
+				Host: internal.LocalHostPort,
+			},
 		},
-		}}
+	}
 
 	log.Logger.Debug("dial timeout before defaulting... ", ct.With.DialTimeout.String())
 
@@ -70,6 +79,84 @@ func TestRunCollectGRPCUnary(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+// Credit: Several of the tests in this file are based on
+// https://github.com/bojand/ghz/blob/master/runner/run_test.go
+func TestRunCollectGRPCEndpoints(t *testing.T) {
+	_ = os.Chdir(t.TempDir())
+	callType := helloworld.Unary
+	gs, s, err := internal.StartServer(false)
+	if err != nil {
+		assert.FailNow(t, err.Error())
+	}
+	t.Cleanup(s.Stop)
+
+	// valid collect GRPC task... should succeed
+	ct := &collectGRPCTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectGRPCTaskName),
+		},
+		With: collectGRPCInputs{
+			Config: runner.Config{
+				Host: internal.LocalHostPort,
+			},
+			Endpoints: map[string]runner.Config{
+				unary: {
+					Data: map[string]interface{}{"name": "bob"},
+					Call: "helloworld.Greeter.SayHello",
+				},
+				server: {
+					Data: map[string]interface{}{"name": "bob"},
+					Call: "helloworld.Greeter.SayHelloCS",
+				},
+				client: {
+					Data: map[string]interface{}{"name": "bob"},
+					Call: "helloworld.Greeter.SayHellos",
+				},
+				bidirectional: {
+					Data: map[string]interface{}{"name": "bob"},
+					Call: "helloworld.Greeter.SayHelloBidi",
+				},
+			},
+		},
+	}
+
+	log.Logger.Debug("dial timeout before defaulting... ", ct.With.DialTimeout.String())
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.initResults(1)
+	err = ct.run(exp)
+
+	log.Logger.Debug("dial timeout after defaulting... ", ct.With.DialTimeout.String())
+
+	assert.NoError(t, err)
+	assert.Equal(t, exp.Result.Insights.NumVersions, 1)
+
+	count := gs.GetCount(callType)
+	assert.Equal(t, 200, count)
+
+	grpcMethods := []string{unary, server, client, bidirectional}
+	for _, method := range grpcMethods {
+		mm, err := exp.Result.Insights.GetMetricsInfo(gRPCMetricPrefix + "-" + method + "/" + gRPCErrorCountMetricName)
+		assert.NotNil(t, mm)
+		assert.NoError(t, err)
+
+		mm, err = exp.Result.Insights.GetMetricsInfo(gRPCMetricPrefix + "-" + method + "/" + gRPCLatencySampleMetricName)
+		assert.NotNil(t, mm)
+		assert.NoError(t, err)
+
+		mm, err = exp.Result.Insights.GetMetricsInfo(gRPCMetricPrefix + "-" + method + "/" + gRPCLatencySampleMetricName + "/" + string(MaxAggregator))
+		assert.NotNil(t, mm)
+		assert.NoError(t, err)
+
+		mm, err = exp.Result.Insights.GetMetricsInfo(gRPCMetricPrefix + "-" + method + "/" + gRPCLatencySampleMetricName + "/" + PercentileAggregatorPrefix + "50")
+		assert.NotNil(t, mm)
+		assert.NoError(t, err)
+	}
+}
+
 func TestMockGRPCWithSLOsAndPercentiles(t *testing.T) {
 	_ = os.Chdir(t.TempDir())
 	callType := helloworld.Unary
@@ -84,17 +171,19 @@ func TestMockGRPCWithSLOsAndPercentiles(t *testing.T) {
 		TaskMeta: TaskMeta{
 			Task: StringPointer(CollectGRPCTaskName),
 		},
-		With: collectGRPCInputs{Config: runner.Config{
-			N:           100,
-			RPS:         20,
-			C:           1,
-			Timeout:     runner.Duration(20 * time.Second),
-			Data:        map[string]interface{}{"name": "bob"},
-			DialTimeout: runner.Duration(20 * time.Second),
-			Call:        "helloworld.Greeter.SayHello",
-			Host:        internal.LocalHostPort,
+		With: collectGRPCInputs{
+			Config: runner.Config{
+				N:           100,
+				RPS:         20,
+				C:           1,
+				Timeout:     runner.Duration(20 * time.Second),
+				Data:        map[string]interface{}{"name": "bob"},
+				DialTimeout: runner.Duration(20 * time.Second),
+				Call:        "helloworld.Greeter.SayHello",
+				Host:        internal.LocalHostPort,
+			},
 		},
-		}}
+	}
 
 	at := &assessTask{
 		TaskMeta: TaskMeta{
diff --git a/base/collect_http_test.go b/base/collect_http_test.go
index cb0092b6..3c35b808 100644
--- a/base/collect_http_test.go
+++ b/base/collect_http_test.go
@@ -1,21 +1,44 @@
 package base
 
 import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"testing"
 
-	"github.com/jarcoal/httpmock"
+	"fortio.org/fortio/fhttp"
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	endpoint1 = "endpoint1"
+	endpoint2 = "endpoint2"
+
+	foo  = "foo"
+	bar  = "bar"
+	from = "from"
+)
+
 func TestRunCollectHTTP(t *testing.T) {
-	_ = os.Chdir(t.TempDir())
-	httpmock.Activate()
-	t.Cleanup(httpmock.DeactivateAndReset)
+	mux, addr := fhttp.DynamicHTTPServer(false)
+
+	// /foo/ handler
+	called := false // ensure that the /foo/ handler is called
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		called = true
+		data, _ := io.ReadAll(r.Body)
+		testData, _ := os.ReadFile(CompletePath("../", "testdata/payload/ukpolice.json"))
+
+		// assert that PayloadFile is working
+		assert.True(t, bytes.Equal(data, testData))
+
+		w.WriteHeader(200)
+	}
+	mux.HandleFunc("/"+foo, handler)
 
-	// Exact URL match
-	httpmock.RegisterResponder("POST", "https://something.com",
-		httpmock.NewStringResponder(200, `[{"id": 1, "name": "My Great Thing"}]`))
+	baseURL := fmt.Sprintf("http://localhost:%d/", addr.Port)
 
 	// valid collect HTTP task... should succeed
 	ct := &collectHTTPTask{
@@ -23,10 +46,12 @@ func TestRunCollectHTTP(t *testing.T) {
 			Task: StringPointer(CollectHTTPTaskName),
 		},
 		With: collectHTTPInputs{
-			Duration:    StringPointer("1s"),
-			PayloadFile: StringPointer(CompletePath("../", "testdata/payload/ukpolice.json")),
-			Headers:     map[string]string{},
-			URL:         "https://something.com",
+			endpoint: endpoint{
+				Duration:    StringPointer("1s"),
+				PayloadFile: StringPointer(CompletePath("../", "testdata/payload/ukpolice.json")),
+				Headers:     map[string]string{},
+				URL:         baseURL + foo,
+			},
 		},
 	}
 
@@ -37,6 +62,7 @@ func TestRunCollectHTTP(t *testing.T) {
 	exp.initResults(1)
 	err := ct.run(exp)
 	assert.NoError(t, err)
+	assert.True(t, called) // ensure that the /foo/ handler is called
 	assert.Equal(t, exp.Result.Insights.NumVersions, 1)
 
 	mm, err := exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "/" + builtInHTTPLatencyMeanID)
@@ -47,3 +73,193 @@ func TestRunCollectHTTP(t *testing.T) {
 	assert.NotNil(t, mm)
 	assert.NoError(t, err)
 }
+
+// Multiple endpoints are provided
+// Test both the /foo/ and /bar/ endpoints
+// Test both endpoints have their respective header values
+func TestRunCollectHTTPMultipleEndpoints(t *testing.T) {
+	mux, addr := fhttp.DynamicHTTPServer(false)
+
+	// /foo/ handler
+	fooCalled := false // ensure that the /foo/ handler is called
+	fooHandler := func(w http.ResponseWriter, r *http.Request) {
+		fooCalled = true
+
+		// assert "from" header has value "foo"
+		assert.Equal(t, foo, r.Header.Get(from))
+
+		w.WriteHeader(200)
+	}
+	mux.HandleFunc("/"+foo, fooHandler)
+
+	// /bar/ handler
+	barCalled := false // ensure that the /foo/ handler is called
+	barHandler := func(w http.ResponseWriter, r *http.Request) {
+		barCalled = true
+
+		// assert "from" header has value "bar"
+		assert.Equal(t, bar, r.Header.Get(from))
+
+		w.WriteHeader(200)
+	}
+	mux.HandleFunc("/"+bar, barHandler)
+
+	baseURL := fmt.Sprintf("http://localhost:%d/", addr.Port)
+
+	// valid collect HTTP task... should succeed
+	ct := &collectHTTPTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectHTTPTaskName),
+		},
+		With: collectHTTPInputs{
+			endpoint: endpoint{
+				Duration: StringPointer("1s"),
+			},
+			Endpoints: map[string]endpoint{
+				endpoint1: {
+					URL: baseURL + foo,
+					Headers: map[string]string{
+						from: foo,
+					},
+				},
+				endpoint2: {
+					URL: baseURL + bar,
+					Headers: map[string]string{
+						from: bar,
+					},
+				},
+			},
+		},
+	}
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.initResults(1)
+	err := ct.run(exp)
+	assert.NoError(t, err)
+	assert.True(t, fooCalled) // ensure that the /foo/ handler is called
+	assert.True(t, barCalled) // ensure that the /bar/ handler is called
+	assert.Equal(t, exp.Result.Insights.NumVersions, 1)
+
+	mm, err := exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint1 + "/" + builtInHTTPLatencyMeanID)
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+
+	mm, err = exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint1 + "/" + builtInHTTPLatencyPercentilePrefix + "50")
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+
+	mm, err = exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint2 + "/" + builtInHTTPLatencyMeanID)
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+
+	mm, err = exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint2 + "/" + builtInHTTPLatencyPercentilePrefix + "50")
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+}
+
+// Multiple endpoints are provided but they share one URL
+// Test that the base-level URL is provided to each endpoint
+// Make multiple calls to the same URL but with different headers
+func TestRunCollectHTTPSingleEndpointMultipleCalls(t *testing.T) {
+	mux, addr := fhttp.DynamicHTTPServer(false)
+
+	// handler
+	fooCalled := false // ensure that foo header is provided
+	barCalled := false // ensure that bar header is provided
+	fooHandler := func(w http.ResponseWriter, r *http.Request) {
+		from := r.Header.Get(from)
+		if from == foo {
+			fooCalled = true
+		} else if from == bar {
+			barCalled = true
+		}
+
+		w.WriteHeader(200)
+	}
+	mux.HandleFunc("/", fooHandler)
+
+	baseURL := fmt.Sprintf("http://localhost:%d/", addr.Port)
+
+	// valid collect HTTP task... should succeed
+	ct := &collectHTTPTask{
+		TaskMeta: TaskMeta{
+			Task: StringPointer(CollectHTTPTaskName),
+		},
+		With: collectHTTPInputs{
+			endpoint: endpoint{
+				Duration: StringPointer("1s"),
+				URL:      baseURL,
+			},
+			Endpoints: map[string]endpoint{
+				endpoint1: {
+					Headers: map[string]string{
+						from: foo,
+					},
+				},
+				endpoint2: {
+					Headers: map[string]string{
+						from: bar,
+					},
+				},
+			},
+		},
+	}
+
+	exp := &Experiment{
+		Spec:   []Task{ct},
+		Result: &ExperimentResult{},
+	}
+	exp.initResults(1)
+	err := ct.run(exp)
+	assert.NoError(t, err)
+	assert.True(t, fooCalled) // ensure that the /foo/ handler is called
+	assert.True(t, barCalled) // ensure that the /bar/ handler is called
+	assert.Equal(t, exp.Result.Insights.NumVersions, 1)
+
+	mm, err := exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint1 + "/" + builtInHTTPLatencyMeanID)
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+
+	mm, err = exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint1 + "/" + builtInHTTPLatencyPercentilePrefix + "50")
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+
+	mm, err = exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint2 + "/" + builtInHTTPLatencyMeanID)
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+
+	mm, err = exp.Result.Insights.GetMetricsInfo(httpMetricPrefix + "-" + endpoint2 + "/" + builtInHTTPLatencyPercentilePrefix + "50")
+	assert.NotNil(t, mm)
+	assert.NoError(t, err)
+}
+
+func TestErrorCode(t *testing.T) {
+	task := collectHTTPTask{}
+	assert.True(t, task.errorCode(-1))
+
+	// if no lower limit (check upper)
+	upper := 10
+	task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{
+		Upper: &upper,
+	})
+	assert.True(t, task.errorCode(5))
+
+	// if no upper limit (check lower)
+	task.With.ErrorRanges = []errorRange{}
+	lower := 1
+	task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{
+		Lower: &lower,
+	})
+	assert.True(t, task.errorCode(5))
+
+	// if both limits are present (check both)
+	task.With.ErrorRanges = []errorRange{}
+	task.With.ErrorRanges = append(task.With.ErrorRanges, errorRange{
+		Upper: &upper,
+		Lower: &lower,
+	})
+	assert.True(t, task.errorCode(5))
+}
diff --git a/base/experiment_test.go b/base/experiment_test.go
index a277a4c1..49cb4ea8 100644
--- a/base/experiment_test.go
+++ b/base/experiment_test.go
@@ -51,9 +51,11 @@ func TestRunningTasks(t *testing.T) {
 			Task: StringPointer(CollectHTTPTaskName),
 		},
 		With: collectHTTPInputs{
-			Duration: StringPointer("1s"),
-			Headers:  map[string]string{},
-			URL:      "https://httpbin.org/get",
+			endpoint: endpoint{
+				Duration: StringPointer("1s"),
+				Headers:  map[string]string{},
+				URL:      "https://httpbin.org/get",
+			},
 		},
 	}
 
diff --git a/base/mock_qs_test.go b/base/mock_qs_test.go
index 7b6fe89b..a2ad326b 100644
--- a/base/mock_qs_test.go
+++ b/base/mock_qs_test.go
@@ -21,9 +21,11 @@ func TestMockQuickStartWithSLOs(t *testing.T) {
 			Task: StringPointer(CollectHTTPTaskName),
 		},
 		With: collectHTTPInputs{
-			Duration: StringPointer("2s"),
-			Headers:  map[string]string{},
-			URL:      testURL,
+			endpoint: endpoint{
+				Duration: StringPointer("2s"),
+				Headers:  map[string]string{},
+				URL:      testURL,
+			},
 		},
 	}
 
@@ -70,9 +72,11 @@ func TestMockQuickStartWithSLOsAndPercentiles(t *testing.T) {
 			Task: StringPointer(CollectHTTPTaskName),
 		},
 		With: collectHTTPInputs{
-			Duration: StringPointer("1s"),
-			Headers:  map[string]string{},
-			URL:      testURL,
+			endpoint: endpoint{
+				Duration: StringPointer("1s"),
+				Headers:  map[string]string{},
+				URL:      testURL,
+			},
 		},
 	}