Repository: knative/serving
Pull request #11910 (2021-09-03 10:57:02)

Speed up concurrency state handler tests
<!-- Please include the 'why' behind your changes if no issue exists -->
## Proposed Changes

As per title, I was looking if our unit test runtime had regressed and these tests stood out. They take ~5s at HEAD. This reworks them a bit so that they can pass as quickly as the machine permits (0.011s on my machine) while maintaining stable behavior at any speed (such is the hope anyway).

/assign @psschwei @julz 


knative-prow-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/knative/serving/pull/11910#" title="Author self-approved">markusthoemmes</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=knative%2Fserving).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[pkg/queue/OWNERS](https://github.com/knative/serving/blob/main/pkg/queue/OWNERS)~~ [markusthoemmes]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
codecov[bot]: # [Codecov](https://codecov.io/gh/knative/serving/pull/11910?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative) Report
> Merging [#11910](https://codecov.io/gh/knative/serving/pull/11910?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative) (fb770b3) into [main](https://codecov.io/gh/knative/serving/commit/cefe5ed204e2ec0d1c179e82da360443dea84b08?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative) (cefe5ed) will **not change** coverage.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/knative/serving/pull/11910/graphs/tree.svg?width=650&height=150&src=pr&token=MPpDfAj4Ui&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative)](https://codecov.io/gh/knative/serving/pull/11910?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative)

```diff
@@           Coverage Diff           @@
##             main   #11910   +/-   ##
=======================================
  Coverage   87.69%   87.69%           
=======================================
  Files         197      197           
  Lines        9467     9467           
=======================================
  Hits         8302     8302           
  Misses        901      901           
  Partials      264      264           
```



------

[Continue to review full report at Codecov](https://codecov.io/gh/knative/serving/pull/11910?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/knative/serving/pull/11910?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative). Last update [cefe5ed...fb770b3](https://codecov.io/gh/knative/serving/pull/11910?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=knative).


Patch:
diff --git a/cmd/queue/main.go b/cmd/queue/main.go
index 4448c3202..d012a0c32 100644
--- a/cmd/queue/main.go
+++ b/cmd/queue/main.go
@@ -294,7 +294,9 @@ func buildServer(ctx context.Context, env config, healthState *health.State, rp
 	metricsSupported := supportsMetrics(ctx, logger, env)
 	tracingEnabled := env.TracingConfigBackend != tracingconfig.None
 	concurrencyStateEnabled := env.ConcurrencyStateEndpoint != ""
-	timeout := time.Duration(env.RevisionTimeoutSeconds) * time.Second
+	firstByteTimeout := time.Duration(env.RevisionTimeoutSeconds) * time.Second
+	// hardcoded to always disable idle timeout for now, will expose this later
+	var idleTimeout time.Duration
 
 	// Create queue handler chain.
 	// Note: innermost handlers are specified first, ie. the last handler in the chain will be executed first.
@@ -308,7 +310,7 @@ func buildServer(ctx context.Context, env config, healthState *health.State, rp
 	}
 	composedHandler = queue.ProxyHandler(breaker, stats, tracingEnabled, composedHandler)
 	composedHandler = queue.ForwardedShimHandler(composedHandler)
-	composedHandler = handler.NewTimeToFirstByteTimeoutHandler(composedHandler, "request timeout", timeout)
+	composedHandler = handler.NewTimeoutHandler(composedHandler, "request timeout", firstByteTimeout, idleTimeout)
 
 	if metricsSupported {
 		composedHandler = requestMetricsHandler(logger, composedHandler, env)
@@ -317,7 +319,7 @@ func buildServer(ctx context.Context, env config, healthState *health.State, rp
 		composedHandler = tracing.HTTPSpanMiddleware(composedHandler)
 	}
 
-	composedHandler = health.ProbeHandler(healthState, rp.ProbeContainer, rp.IsAggressive(), tracingEnabled, composedHandler)
+	composedHandler = health.ProbeHandler(healthState, rp.ProbeContainer, tracingEnabled, composedHandler)
 	composedHandler = network.NewProbeHandler(composedHandler)
 	// We might want sometimes capture the probes/healthchecks in the request
 	// logs. Hence we need to have RequestLogHandler to be the first one.
diff --git a/config/core/200-roles/addressable-resolvers-clusterrole.yaml b/config/core/200-roles/addressable-resolvers-clusterrole.yaml
index 0cca728b7..66714a2ed 100644
--- a/config/core/200-roles/addressable-resolvers-clusterrole.yaml
+++ b/config/core/200-roles/addressable-resolvers-clusterrole.yaml
@@ -27,7 +27,6 @@ aggregationRule:
   clusterRoleSelectors:
   - matchLabels:
       duck.knative.dev/addressable: "true"
-rules: [] # Rules are automatically filled in by the controller manager.
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/config/core/200-serviceaccount.yaml b/config/core/200-serviceaccount.yaml
index 850a20465..539302bb6 100644
--- a/config/core/200-serviceaccount.yaml
+++ b/config/core/200-serviceaccount.yaml
@@ -35,7 +35,6 @@ aggregationRule:
   clusterRoleSelectors:
   - matchLabels:
       serving.knative.dev/controller: "true"
-rules: [] # Rules are automatically filled in by the controller manager.
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/go.mod b/go.mod
index a8e7afb89..fed20b37c 100644
--- a/go.mod
+++ b/go.mod
@@ -45,9 +45,9 @@ require (
 	k8s.io/code-generator v0.21.4
 	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7
 	k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 // indirect
-	knative.dev/caching v0.0.0-20210831064556-77b5d6ab7525
+	knative.dev/caching v0.0.0-20210903022457-553240b1b365
 	knative.dev/hack v0.0.0-20210806075220-815cd312d65c
-	knative.dev/networking v0.0.0-20210902120205-ee93e22aa557
-	knative.dev/pkg v0.0.0-20210902202457-7a61343b36ca
+	knative.dev/networking v0.0.0-20210903170058-7c11d237f014
+	knative.dev/pkg v0.0.0-20210906105443-bb433c98147d
 	sigs.k8s.io/yaml v1.2.0
 )
diff --git a/go.sum b/go.sum
index 97a38091a..bef35c30a 100644
--- a/go.sum
+++ b/go.sum
@@ -1526,15 +1526,16 @@ k8s.io/legacy-cloud-providers v0.21.0/go.mod h1:bNxo7gDg+PGkBmT/MFZswLTWdSWK9kAl
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ=
 k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-knative.dev/caching v0.0.0-20210831064556-77b5d6ab7525 h1:dy2VJqa2gCVGhU9rNXu59W3pmiAexCKoh03Un+K3L/g=
-knative.dev/caching v0.0.0-20210831064556-77b5d6ab7525/go.mod h1:m+qT5SoXDbEm0DqBw9LTheZIi/UlQ5YqSsvjgFSom1k=
+knative.dev/caching v0.0.0-20210903022457-553240b1b365 h1:p8jEwTrExGkeEL3ytXCOKG4CHAyJM/lWitVzqxt00ZI=
+knative.dev/caching v0.0.0-20210903022457-553240b1b365/go.mod h1:J2o0f0nayAT6ZaXy9yo4GyEcoPyJndMrh+4X5kQTQ5w=
 knative.dev/hack v0.0.0-20210806075220-815cd312d65c h1:nOXoDWAAItwr4o0dp3nHr6skgpVD4IvME/UX84YNl5k=
 knative.dev/hack v0.0.0-20210806075220-815cd312d65c/go.mod h1:PHt8x8yX5Z9pPquBEfIj0X66f8iWkWfR0S/sarACJrI=
-knative.dev/networking v0.0.0-20210902120205-ee93e22aa557 h1:Wuss+xFFkH4xitrk6n2YV75Q7HzkVcF+HiDt+ptXfHg=
-knative.dev/networking v0.0.0-20210902120205-ee93e22aa557/go.mod h1:dJaJfTZQlZqqtvKA59ihb5Ndl7KtCn6PW7VVr+fls5Q=
-knative.dev/pkg v0.0.0-20210830224055-82f3a9f1c5bc/go.mod h1:jMSqkNMsrzuy+XR4Yr/BMy7SDVbUOl3KKB6+5MR+ZU8=
-knative.dev/pkg v0.0.0-20210902202457-7a61343b36ca h1:1tsBZgj1FheRcDQrIReteUiU/afjTZG8j5hTM3iBkJQ=
+knative.dev/networking v0.0.0-20210903170058-7c11d237f014 h1:uZ6dLmPDjV4FmvZAtEhUAIKehFjHdfDQfeSzcov2o0c=
+knative.dev/networking v0.0.0-20210903170058-7c11d237f014/go.mod h1:DldEflDGWAychEH7SzPj1MeJRZaugxOIbdXaIYHvEz0=
 knative.dev/pkg v0.0.0-20210902202457-7a61343b36ca/go.mod h1:jMSqkNMsrzuy+XR4Yr/BMy7SDVbUOl3KKB6+5MR+ZU8=
+knative.dev/pkg v0.0.0-20210903055557-dec98b426132/go.mod h1:jMSqkNMsrzuy+XR4Yr/BMy7SDVbUOl3KKB6+5MR+ZU8=
+knative.dev/pkg v0.0.0-20210906105443-bb433c98147d h1:foZbYLUl8w0CXoxO66qsluinmUzzxUiYZHBV3Kgt3ks=
+knative.dev/pkg v0.0.0-20210906105443-bb433c98147d/go.mod h1:jMSqkNMsrzuy+XR4Yr/BMy7SDVbUOl3KKB6+5MR+ZU8=
 pgregory.net/rapid v0.3.3 h1:jCjBsY4ln4Atz78QoBWxUEvAHaFyNDQg9+WU62aCn1U=
 pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/pkg/autoscaler/metrics/stats_scraper.go b/pkg/autoscaler/metrics/stats_scraper.go
index daeadede1..4a0a95c6b 100644
--- a/pkg/autoscaler/metrics/stats_scraper.go
+++ b/pkg/autoscaler/metrics/stats_scraper.go
@@ -137,11 +137,26 @@ var client = &http.Client{
 	Transport: keepAliveTransport,
 }
 
+// meshMode determines whethe we should attempt to sample for stats via the K8s
+// service, or whether we should attempt to directly gather stats from pods
+// (which is more efficient, but not possible when mesh is enabled).
+type meshMode int
+
+const (
+	// meshModeEnabled causes the service scraper to always sample the k8s service for scraping.
+	meshModeEnabled meshMode = iota
+	// meshModeDisabled causes the service scraper to always use pod IPs directly for scraping.
+	meshModeDisabled
+	// meshModeAuto first attempts direct pod IP scraping, and then falls back to service scraping if needed.
+	meshModeAuto
+)
+
 // serviceScraper scrapes Revision metrics via a K8S service by sampling. Which
 // pod to be picked up to serve the request is decided by K8S. Please see
 // https://kubernetes.io/docs/concepts/services-networking/network-policies/
 // for details.
 type serviceScraper struct {
+	meshMode     meshMode
 	directClient scrapeClient
 	meshClient   scrapeClient
 
@@ -161,7 +176,7 @@ func NewStatsScraper(metric *autoscalingv1alpha1.Metric, revisionName string, po
 	usePassthroughLb bool, logger *zap.SugaredLogger) StatsScraper {
 	directClient := newHTTPScrapeClient(client)
 	meshClient := newHTTPScrapeClient(noKeepaliveClient)
-	return newServiceScraperWithClient(metric, revisionName, podAccessor, usePassthroughLb, directClient, meshClient, logger)
+	return newServiceScraperWithClient(metric, revisionName, podAccessor, usePassthroughLb, meshModeAuto, directClient, meshClient, logger)
 }
 
 func newServiceScraperWithClient(
@@ -169,6 +184,7 @@ func newServiceScraperWithClient(
 	revisionName string,
 	podAccessor resources.PodAccessor,
 	usePassthroughLb bool,
+	meshMode meshMode,
 	directClient, meshClient scrapeClient,
 	logger *zap.SugaredLogger) *serviceScraper {
 	svcName := metric.Labels[serving.ServiceLabelKey]
@@ -177,6 +193,7 @@ func newServiceScraperWithClient(
 	ctx := metrics.RevisionContext(metric.ObjectMeta.Namespace, svcName, cfgName, revisionName)
 
 	return &serviceScraper{
+		meshMode:         meshMode,
 		directClient:     directClient,
 		meshClient:       meshClient,
 		host:             metric.Spec.ScrapeTarget + "." + metric.ObjectMeta.Namespace,
@@ -209,30 +226,32 @@ func (s *serviceScraper) Scrape(window time.Duration) (stat Stat, err error) {
 		pkgmetrics.RecordBatch(s.statsCtx, scrapeTimeM.M(float64(scrapeTime.Milliseconds())))
 	}()
 
-	if s.podsAddressable || s.usePassthroughLb {
-		stat, err := s.scrapePods(window)
-		// Return here if some pods were scraped, but not enough or if we're using a
-		// passthrough loadbalancer and want no fallback to service-scrape logic.
-		if !errors.Is(err, errDirectScrapingNotAvailable) || s.usePassthroughLb {
-			return stat, err
+	switch s.meshMode {
+	case meshModeEnabled:
+		s.logger.Debug("Scraping via service due to meshMode setting")
+		return s.scrapeService(window)
+	case meshModeDisabled:
+		s.logger.Debug("Scraping pods directly due to meshMode setting")
+		return s.scrapePods(window)
+	default:
+		if s.podsAddressable || s.usePassthroughLb {
+			stat, err := s.scrapePods(window)
+			// Return here if some pods were scraped, but not enough or if we're using a
+			// passthrough loadbalancer and want no fallback to service-scrape logic.
+			if !errors.Is(err, errDirectScrapingNotAvailable) || s.usePassthroughLb {
+				return stat, err
+			}
+			// Else fall back to service scrape.
 		}
-		// Else fall back to service scrape.
-	}
-	readyPodsCount, err := s.podAccessor.ReadyCount()
-	if err != nil {
-		return emptyStat, ErrFailedGetEndpoints
-	}
-	if readyPodsCount == 0 {
-		return emptyStat, nil
-	}
-	stat, err = s.scrapeService(window, readyPodsCount)
-	if err == nil && s.podsAddressable {
-		s.logger.Info("Direct pod scraping off, service scraping, on")
-		// If err == nil, this means that we failed to scrape all pods, but service worked
-		// thus it is probably a mesh case.
-		s.podsAddressable = false
+		stat, err = s.scrapeService(window)
+		if err == nil && s.podsAddressable {
+			s.logger.Info("Direct pod scraping off, service scraping, on")
+			// If err == nil, this means that we failed to scrape all pods, but service worked
+			// thus it is probably a mesh case.
+			s.podsAddressable = false
+		}
+		return stat, err
 	}
-	return stat, err
 }
 
 func (s *serviceScraper) scrapePods(window time.Duration) (Stat, error) {
@@ -354,9 +373,16 @@ func computeAverages(results <-chan Stat, sample, total float64) Stat {
 
 // scrapeService scrapes the metrics using service endpoint
 // as its target, rather than individual pods.
-func (s *serviceScraper) scrapeService(window time.Duration, readyPods int) (Stat, error) {
-	frpc := float64(readyPods)
+func (s *serviceScraper) scrapeService(window time.Duration) (Stat, error) {
+	readyPods, err := s.podAccessor.ReadyCount()
+	if err != nil {
+		return emptyStat, ErrFailedGetEndpoints
+	}
+	if readyPods == 0 {
+		return emptyStat, nil
+	}
 
+	frpc := float64(readyPods)
 	sampleSizeF := populationMeanSampleSize(frpc)
 	sampleSize := int(sampleSizeF)
 	oldStatCh := make(chan Stat, sampleSize)
diff --git a/pkg/http/handler/timeout.go b/pkg/http/handler/timeout.go
index a7958bcf2..5f4077516 100644
--- a/pkg/http/handler/timeout.go
+++ b/pkg/http/handler/timeout.go
@@ -28,14 +28,16 @@ import (
 	"knative.dev/pkg/websocket"
 )
 
-type timeToFirstByteTimeoutHandler struct {
-	handler http.Handler
-	timeout time.Duration
-	body    string
+type timeoutHandler struct {
+	handler          http.Handler
+	firstByteTimeout time.Duration
+	idleTimeout      time.Duration
+	body             string
 }
 
-// NewTimeToFirstByteTimeoutHandler returns a Handler that runs `h` with the
-// given timeout in which the first byte of the response must be written.
+// NewTimeoutHandler returns a Handler that runs `h` with the
+// given timeout in which the first byte of the response must be written,
+// and with the given idle timeout
 //
 // The new Handler calls h.ServeHTTP to handle each request, but if a
 // call runs for longer than its time limit, the handler responds with
@@ -49,15 +51,16 @@ type timeToFirstByteTimeoutHandler struct {
 // https://golang.org/pkg/net/http/#Handler.
 //
 // The implementation is largely inspired by http.TimeoutHandler.
-func NewTimeToFirstByteTimeoutHandler(h http.Handler, msg string, timeout time.Duration) http.Handler {
-	return &timeToFirstByteTimeoutHandler{
-		handler: h,
-		body:    msg,
-		timeout: timeout,
+func NewTimeoutHandler(h http.Handler, msg string, firstByteTimeout time.Duration, idleTimeout time.Duration) http.Handler {
+	return &timeoutHandler{
+		handler:          h,
+		body:             msg,
+		firstByteTimeout: firstByteTimeout,
+		idleTimeout:      idleTimeout,
 	}
 }
 
-func (h *timeToFirstByteTimeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+func (h *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithCancel(r.Context())
 	defer cancel()
 
@@ -75,9 +78,25 @@ func (h *timeToFirstByteTimeoutHandler) ServeHTTP(w http.ResponseWriter, r *http
 		h.handler.ServeHTTP(tw, r.WithContext(ctx))
 	}()
 
-	timeout := getTimer(h.timeout)
-	var timeoutDrained bool
-	defer func() { putTimer(timeout, timeoutDrained) }()
+	firstByteTimeout := getTimer(h.firstByteTimeout)
+	var firstByteTimeoutDrained bool
+	defer func() {
+		putTimer(firstByteTimeout, firstByteTimeoutDrained)
+	}()
+
+	var idleTimeout *time.Timer
+	var idleTimeoutDrained bool
+	if h.idleTimeout > 0 {
+		idleTimeout = getTimer(h.idleTimeout)
+		defer func() {
+			putTimer(idleTimeout, idleTimeoutDrained)
+		}()
+	}
+	var idleTimeoutCh <-chan time.Time
+	if idleTimeout != nil {
+		idleTimeoutCh = idleTimeout.C
+	}
+
 	for {
 		select {
 		case p, ok := <-done:
@@ -85,11 +104,18 @@ func (h *timeToFirstByteTimeoutHandler) ServeHTTP(w http.ResponseWriter, r *http
 				panic(p)
 			}
 			return
-		case <-timeout.C:
-			timeoutDrained = true
-			if tw.timeoutAndWriteError(h.body) {
+		case <-firstByteTimeout.C:
+			firstByteTimeoutDrained = true
+			if tw.tryFirstByteTimeoutAndWriteError(h.body) {
+				return
+			}
+		case <-idleTimeoutCh:
+			timedOut, timeToNextTimeout := tw.tryIdleTimeoutAndWriteError(time.Now(), h.idleTimeout, h.body)
+			if timedOut {
+				idleTimeoutDrained = true
 				return
 			}
+			idleTimeout.Reset(timeToNextTimeout)
 		}
 	}
 }
@@ -104,9 +130,9 @@ func (h *timeToFirstByteTimeoutHandler) ServeHTTP(w http.ResponseWriter, r *http
 type timeoutWriter struct {
 	w http.ResponseWriter
 
-	mu        sync.Mutex
-	timedOut  bool
-	wroteOnce bool
+	mu            sync.Mutex
+	timedOut      bool
+	lastWriteTime time.Time
 }
 
 var _ http.Flusher = (*timeoutWriter)(nil)
@@ -141,7 +167,7 @@ func (tw *timeoutWriter) Write(p []byte) (int, error) {
 		return 0, http.ErrHandlerTimeout
 	}
 
-	tw.wroteOnce = true
+	tw.lastWriteTime = time.Now()
 	return tw.w.Write(p)
 }
 
@@ -151,31 +177,54 @@ func (tw *timeoutWriter) WriteHeader(code int) {
 	if tw.timedOut {
 		return
 	}
-	tw.wroteOnce = true
+	tw.lastWriteTime = time.Now()
 	tw.w.WriteHeader(code)
 }
 
-// timeoutAndWriteError writes an error to the responsewriter if
+// tryFirstByteTimeoutAndWriteError writes an error to the responsewriter if
 // nothing has been written to the writer before. Returns whether
 // an error was written or not.
 //
 // If this writes an error, all subsequent calls to Write will
 // result in http.ErrHandlerTimeout.
-func (tw *timeoutWriter) timeoutAndWriteError(msg string) bool {
+func (tw *timeoutWriter) tryFirstByteTimeoutAndWriteError(msg string) bool {
 	tw.mu.Lock()
 	defer tw.mu.Unlock()
 
-	if !tw.wroteOnce {
-		tw.w.WriteHeader(http.StatusGatewayTimeout)
-		io.WriteString(tw.w, msg)
-
-		tw.timedOut = true
+	if tw.lastWriteTime.IsZero() {
+		tw.timeoutAndWriteError(msg)
 		return true
 	}
 
 	return false
 }
 
+// tryIdleTimeoutAndWriteError writes an error to the responsewriter if
+// nothing has been written to the writer within the idleTimeout period. Returns whether
+// an error was written or not and time left to the next timeout
+//
+// If this writes an error, all subsequent calls to Write will
+// result in http.ErrHandlerTimeout.
+func (tw *timeoutWriter) tryIdleTimeoutAndWriteError(curTime time.Time, idleTimeout time.Duration, msg string) (timedOut bool, timeToNextTimeout time.Duration) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	timeSinceLastWrite := curTime.Sub(tw.lastWriteTime)
+	if timeSinceLastWrite >= idleTimeout {
+		tw.timeoutAndWriteError(msg)
+		return true, 0
+	}
+
+	return false, idleTimeout - timeSinceLastWrite
+}
+
+func (tw *timeoutWriter) timeoutAndWriteError(msg string) {
+	tw.w.WriteHeader(http.StatusGatewayTimeout)
+	io.WriteString(tw.w, msg)
+
+	tw.timedOut = true
+}
+
 var timerPool sync.Pool
 
 func getTimer(timeout time.Duration) *time.Timer {
diff --git a/pkg/queue/health/handler.go b/pkg/queue/health/handler.go
index 7bca57f51..201187c22 100644
--- a/pkg/queue/health/handler.go
+++ b/pkg/queue/health/handler.go
@@ -29,7 +29,7 @@ const badProbeTemplate = "unexpected probe header value: "
 
 // ProbeHandler returns a http.HandlerFunc that responds to health checks if the
 // knative network probe header is passed, and otherwise delegates to the next handler.
-func ProbeHandler(healthState *State, prober func() bool, isAggressive bool, tracingEnabled bool, next http.Handler) http.HandlerFunc {
+func ProbeHandler(healthState *State, prober func() bool, tracingEnabled bool, next http.Handler) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		ph := network.KnativeProbeHeader(r)
 
diff --git a/pkg/queue/readiness/probe.go b/pkg/queue/readiness/probe.go
index 100f16670..2d8e0e162 100644
--- a/pkg/queue/readiness/probe.go
+++ b/pkg/queue/readiness/probe.go
@@ -97,8 +97,8 @@ func NewProbeWithHTTP2AutoDetection(v1p *corev1.Probe) *Probe {
 	}
 }
 
-// IsAggressive indicates whether the Knative probe with aggressive retries should be used.
-func (p *Probe) IsAggressive() bool {
+// shouldProbeAggressively indicates whether the Knative probe with aggressive retries should be used.
+func (p *Probe) shouldProbeAggressively() bool {
 	return p.PeriodSeconds == 0
 }
 
@@ -155,7 +155,7 @@ func (p *Probe) probeContainerImpl() bool {
 }
 
 func (p *Probe) doProbe(probe func(time.Duration) error) error {
-	if !p.IsAggressive() {
+	if !p.shouldProbeAggressively() {
 		return probe(time.Duration(p.TimeoutSeconds) * time.Second)
 	}
 
diff --git a/pkg/reconciler/revision/resources/queue.go b/pkg/reconciler/revision/resources/queue.go
index c5f3e09c7..e3331de66 100644
--- a/pkg/reconciler/revision/resources/queue.go
+++ b/pkg/reconciler/revision/resources/queue.go
@@ -236,38 +236,44 @@ func makeQueueContainer(rev *v1.Revision, cfg *config.Config) (*corev1.Container
 
 	container := rev.Spec.GetContainer()
 
-	// During startup we want to poll the container faster than Kubernetes will
-	// allow, so we use an ExecProbe which starts immediately and then polls
-	// every 25ms. We encode the original probe as JSON in an environment
-	// variable for this probe to use.
-	userProbe := container.ReadinessProbe.DeepCopy()
-	applyReadinessProbeDefaultsForExec(userProbe, userPort)
-	execProbe := makeStartupExecProbe(userProbe, cfg.Deployment.ProgressDeadline)
-	userProbeJSON, err := readiness.EncodeProbe(userProbe)
-	if err != nil {
-		return nil, fmt.Errorf("failed to serialize readiness probe: %w", err)
-	}
+	var httpProbe, execProbe *corev1.Probe
+	var userProbeJSON string
+	if container.ReadinessProbe != nil {
+		// During startup we want to poll the container faster than Kubernetes will
+		// allow, so we use an ExecProbe which starts immediately and then polls
+		// every 25ms. We encode the original probe as JSON in an environment
+		// variable for this probe to use.
+		userProbe := container.ReadinessProbe.DeepCopy()
+		applyReadinessProbeDefaultsForExec(userProbe, userPort)
+		execProbe = makeStartupExecProbe(userProbe, cfg.Deployment.ProgressDeadline)
+
+		var err error
+		userProbeJSON, err = readiness.EncodeProbe(userProbe)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize readiness probe: %w", err)
+		}
 
-	// After startup we'll directly use the same http health check endpoint the
-	// execprobe would have used (which will then check the user container).
-	// Unlike the StartupProbe, we don't need to override any of the other settings
-	// except period here. See below.
-	httpProbe := container.ReadinessProbe.DeepCopy()
-	httpProbe.Handler = corev1.Handler{
-		HTTPGet: &corev1.HTTPGetAction{
-			Port: intstr.FromInt(int(servingPort.ContainerPort)),
-			HTTPHeaders: []corev1.HTTPHeader{{
-				Name:  network.ProbeHeaderName,
-				Value: queue.Name,
-			}},
-		},
-	}
+		// After startup we'll directly use the same http health check endpoint the
+		// execprobe would have used (which will then check the user container).
+		// Unlike the StartupProbe, we don't need to override any of the other settings
+		// except period here. See below.
+		httpProbe = container.ReadinessProbe.DeepCopy()
+		httpProbe.Handler = corev1.Handler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Port: intstr.FromInt(int(servingPort.ContainerPort)),
+				HTTPHeaders: []corev1.HTTPHeader{{
+					Name:  network.ProbeHeaderName,
+					Value: queue.Name,
+				}},
+			},
+		}
 
-	// Default PeriodSeconds to 1 if not set to make for the quickest possible startup
-	// time.
-	// TODO(#10973): Remove this once we're on K8s 1.21
-	if httpProbe.PeriodSeconds == 0 {
-		httpProbe.PeriodSeconds = 1
+		// Default PeriodSeconds to 1 if not set to make for the quickest possible startup
+		// time.
+		// TODO(#10973): Remove this once we're on K8s 1.21
+		if httpProbe.PeriodSeconds == 0 {
+			httpProbe.PeriodSeconds = 1
+		}
 	}
 
 	c := &corev1.Container{
diff --git a/pkg/reconciler/route/resources/ingress.go b/pkg/reconciler/route/resources/ingress.go
index 1b2f7e09e..e081b3f69 100644
--- a/pkg/reconciler/route/resources/ingress.go
+++ b/pkg/reconciler/route/resources/ingress.go
@@ -19,7 +19,9 @@ package resources
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/davecgh/go-spew/spew"
 	"go.uber.org/zap"
@@ -193,16 +195,9 @@ func makeIngressSpec(
 		}
 	}
 
-	var httpOption netv1alpha1.HTTPOption
-
-	switch config.FromContext(ctx).Network.HTTPProtocol {
-	case network.HTTPEnabled:
-		httpOption = netv1alpha1.HTTPOptionEnabled
-	case network.HTTPRedirected:
-		httpOption = netv1alpha1.HTTPOptionRedirected
-	// This will be deprecated soon
-	case network.HTTPDisabled:
-		httpOption = ""
+	httpOption, err := getHTTPOption(ctx, r.Annotations)
+	if err != nil {
+		return netv1alpha1.IngressSpec{}, err
 	}
 
 	return netv1alpha1.IngressSpec{
@@ -212,6 +207,32 @@ func makeIngressSpec(
 	}, nil
 }
 
+func getHTTPOption(ctx context.Context, annotations map[string]string) (netv1alpha1.HTTPOption, error) {
+	if len(annotations) != 0 && annotations[networking.HTTPOptionAnnotationKey] != "" {
+		annotation := annotations[networking.HTTPOptionAnnotationKey]
+		switch strings.ToLower(annotation) {
+		case "enabled":
+			return netv1alpha1.HTTPOptionEnabled, nil
+		case "redirected":
+			return netv1alpha1.HTTPOptionRedirected, nil
+		default:
+			return "", fmt.Errorf("incorrect HTTPOption annotation:" + annotation)
+		}
+	}
+
+	switch config.FromContext(ctx).Network.HTTPProtocol {
+	case network.HTTPEnabled:
+		return netv1alpha1.HTTPOptionEnabled, nil
+	case network.HTTPRedirected:
+		return netv1alpha1.HTTPOptionRedirected, nil
+	// This will be deprecated soon
+	case network.HTTPDisabled:
+		return "", nil
+	default:
+		return "", nil
+	}
+}
+
 func getChallengeHosts(challenges []netv1alpha1.HTTP01Challenge) map[string]netv1alpha1.HTTP01Challenge {
 	c := make(map[string]netv1alpha1.HTTP01Challenge, len(challenges))
 
diff --git a/vendor/knative.dev/pkg/webhook/json/decode.go b/vendor/knative.dev/pkg/webhook/json/decode.go
index 9e206a10b..b62caec61 100644
--- a/vendor/knative.dev/pkg/webhook/json/decode.go
+++ b/vendor/knative.dev/pkg/webhook/json/decode.go
@@ -81,11 +81,7 @@ func Decode(bites []byte, target interface{}, disallowUnknownFields bool) error
 		bytes.NewReader(metaSuffix),
 	))
 
-	if err := dec.Decode(target); err != nil {
-		return err
-	}
-
-	return nil
+	return dec.Decode(target)
 }
 
 func findMetadataOffsets(bites []byte) (start, end int64, err error) {
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 73877e941..9584e443d 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -1017,7 +1017,7 @@ k8s.io/utils/buffer
 k8s.io/utils/integer
 k8s.io/utils/pointer
 k8s.io/utils/trace
-# knative.dev/caching v0.0.0-20210831064556-77b5d6ab7525
+# knative.dev/caching v0.0.0-20210903022457-553240b1b365
 ## explicit
 knative.dev/caching/config
 knative.dev/caching/pkg/apis/caching
@@ -1042,7 +1042,7 @@ knative.dev/caching/pkg/client/listers/caching/v1alpha1
 ## explicit
 knative.dev/hack
 knative.dev/hack/shell
-# knative.dev/networking v0.0.0-20210902120205-ee93e22aa557
+# knative.dev/networking v0.0.0-20210903170058-7c11d237f014
 ## explicit
 knative.dev/networking/config
 knative.dev/networking/pkg
@@ -1072,7 +1072,7 @@ knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/serve
 knative.dev/networking/pkg/client/listers/networking/v1alpha1
 knative.dev/networking/pkg/ingress
 knative.dev/networking/pkg/prober
-# knative.dev/pkg v0.0.0-20210902202457-7a61343b36ca
+# knative.dev/pkg v0.0.0-20210906105443-bb433c98147d
 ## explicit
 knative.dev/pkg/apiextensions/storageversion
 knative.dev/pkg/apiextensions/storageversion/cmd/migrate


Test patch:
diff --git a/cmd/queue/main_test.go b/cmd/queue/main_test.go
index 71bc00067..27ebcbf18 100644
--- a/cmd/queue/main_test.go
+++ b/cmd/queue/main_test.go
@@ -149,7 +149,7 @@ func TestQueueTraceSpans(t *testing.T) {
 				h := queue.ProxyHandler(breaker, network.NewRequestStats(time.Now()), true /*tracingEnabled*/, proxy)
 				h(writer, req)
 			} else {
-				h := health.ProbeHandler(healthState, tc.prober, true /* isAggressive*/, true /*tracingEnabled*/, nil)
+				h := health.ProbeHandler(healthState, tc.prober, true /*tracingEnabled*/, nil)
 				req.Header.Set(network.ProbeHeaderName, tc.requestHeader)
 				h(writer, req)
 			}
diff --git a/pkg/autoscaler/metrics/stats_scraper_test.go b/pkg/autoscaler/metrics/stats_scraper_test.go
index 2a31c813d..d1557a460 100644
--- a/pkg/autoscaler/metrics/stats_scraper_test.go
+++ b/pkg/autoscaler/metrics/stats_scraper_test.go
@@ -141,7 +141,7 @@ func TestPodDirectScrapeSuccess(t *testing.T) {
 	})
 
 	client := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
 
 	// No pods at all.
 	if stat, err := scraper.Scrape(defaultMetric.Spec.StableWindow); err != nil {
@@ -175,7 +175,7 @@ func TestPodDirectScrapeSomeFailButSuccess(t *testing.T) {
 	makePods(ctx, "pods-", 5, metav1.Now())
 
 	client := newTestScrapeClient(testStats, []error{nil, nil, errors.New("okay"), nil, nil})
-	scraper := serviceScraperForTest(ctx, t, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
 	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
 		t.Fatal("Unexpected error from scraper.Scrape():", err)
@@ -218,7 +218,7 @@ func TestPodDirectScrapeAllFailWithMeshError(t *testing.T) {
 	})
 	makePods(ctx, "pods-", 4, metav1.Now())
 
-	scraper := serviceScraperForTest(ctx, t, direct, mesh, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, direct, mesh, true /*podsAddressable*/, false /*passthroughLb*/)
 	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
 		t.Fatal("Unexpected error from scraper.Scrape():", err)
@@ -235,6 +235,77 @@ func TestPodDirectScrapeAllFailWithMeshError(t *testing.T) {
 	}
 }
 
+func TestPodDirectScrapeAllFailWithMeshErrorNoFallbackInDirectMode(t *testing.T) {
+	testStats := testStatsWithTime(4, youngPodCutOffDuration.Seconds() /*youngest*/)
+	meshErr := scrapeError{
+		error:       errors.New("just meshing with you"),
+		mightBeMesh: true,
+	}
+	direct := newTestScrapeClient(testStats, []error{
+		// Pods fail, and they're all potentially mesh-related errors.
+		meshErr, meshErr, meshErr, meshErr,
+	})
+	mesh := newTestScrapeClient(testStats, []error{
+		// Service succeeds.
+		nil, nil, nil, nil,
+	})
+	ctx, cancel, informers := SetupFakeContextWithCancel(t)
+	wf, err := RunAndSyncInformers(ctx, informers...)
+	if err != nil {
+		cancel()
+		t.Fatal("Failed to start informers:", err)
+	}
+	t.Cleanup(func() {
+		cancel()
+		wf()
+	})
+	makePods(ctx, "pods-", 4, metav1.Now())
+
+	scraper := serviceScraperForTest(ctx, t, meshModeDisabled, direct, mesh, true /*podsAddressable*/, false /*passthroughLb*/)
+	if _, err := scraper.Scrape(defaultMetric.Spec.StableWindow); err == nil {
+		t.Fatal("Expected scrape to fail even though all pods return mesh-compatible errors since meshMode is disabled")
+	}
+}
+
+func TestPodAlwaysServiceScrapeInMeshMode(t *testing.T) {
+	testStats := testStatsWithTime(4, youngPodCutOffDuration.Seconds() /*youngest*/)
+	notMeshErr := scrapeError{
+		error:       errors.New("just meshing with you"),
+		mightBeMesh: false,
+	}
+	direct := newTestScrapeClient(testStats, []error{
+		// Not mesh errors, but we still expect to service scrape
+		notMeshErr, notMeshErr, notMeshErr, notMeshErr,
+	})
+	mesh := newTestScrapeClient(testStats, []error{
+		// Service succeeds.
+		nil, nil, nil, nil,
+	})
+	ctx, cancel, informers := SetupFakeContextWithCancel(t)
+	wf, err := RunAndSyncInformers(ctx, informers...)
+	if err != nil {
+		cancel()
+		t.Fatal("Failed to start informers:", err)
+	}
+	t.Cleanup(func() {
+		cancel()
+		wf()
+	})
+	makePods(ctx, "pods-", 4, metav1.Now())
+
+	scraper := serviceScraperForTest(ctx, t, meshModeEnabled, direct, mesh, true /*podsAddressable*/, false /*passthroughLb*/)
+	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
+	if err != nil {
+		t.Fatal("Unexpected error from scraper.Scrape():", err)
+	}
+
+	// Checking one of the metrics is enough here.
+	if got.AverageConcurrentRequests != 20.0 {
+		t.Errorf("stat.AverageConcurrentRequests=%v, want %v",
+			got.AverageConcurrentRequests, 20.0)
+	}
+}
+
 func TestPodDirectScrapeSomeFailWithNonMeshError(t *testing.T) {
 	meshErr := scrapeError{
 		error:       errors.New("what a mesh"),
@@ -258,7 +329,7 @@ func TestPodDirectScrapeSomeFailWithNonMeshError(t *testing.T) {
 	makePods(ctx, "pods-", 4, metav1.Now())
 
 	client := newTestScrapeClient(testStats, []error{meshErr, nonMeshErr, meshErr, meshErr})
-	scraper := serviceScraperForTest(ctx, t, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err == nil {
 		t.Fatal("Expected an error")
@@ -283,7 +354,7 @@ func TestPodDirectScrapePodsExhausted(t *testing.T) {
 	makePods(ctx, "pods-", 4, metav1.Now())
 
 	client := newTestScrapeClient(testStats, []error{nil, nil, errors.New("okay"), nil})
-	scraper := serviceScraperForTest(ctx, t, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, nil /* mesh not used */, true /*podsAddressable*/, false /*passthroughLb*/)
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err == nil {
 		t.Fatal("Expected an error")
@@ -309,7 +380,7 @@ func TestScrapeReportStatWhenAllCallsSucceed(t *testing.T) {
 
 	// Scrape will set a timestamp bigger than this.
 	client := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
 
 	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
@@ -344,7 +415,7 @@ func TestScrapeAllPodsYoungPods(t *testing.T) {
 	})
 	makePods(ctx, "pods-", numP, metav1.Now())
 
-	scraper := serviceScraperForTest(ctx, t, direct, mesh, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, direct, mesh, false /*podsAddressable*/, false /*passthroughLb*/)
 	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
 		t.Fatal("Unexpected error from scraper.Scrape():", err)
@@ -377,7 +448,7 @@ func TestScrapeAllPodsOldPods(t *testing.T) {
 	makePods(ctx, "pods-", numP, metav1.Now())
 	direct := newTestScrapeClient(testStats, []error{errDirectScrapingNotAvailable}) // fall back to service scrape
 	mesh := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, direct, mesh, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, direct, mesh, false /*podsAddressable*/, false /*passthroughLb*/)
 	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
 		t.Fatal("Unexpected error from scraper.Scrape():", err)
@@ -413,7 +484,7 @@ func TestScrapeSomePodsOldPods(t *testing.T) {
 	// Scrape will set a timestamp bigger than this.
 	direct := newTestScrapeClient(testStats, []error{errDirectScrapingNotAvailable}) // fall back to service scrape
 	mesh := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, direct, mesh, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, direct, mesh, false /*podsAddressable*/, false /*passthroughLb*/)
 
 	got, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
@@ -443,7 +514,7 @@ func TestScrapeReportErrorCannotFindEnoughPods(t *testing.T) {
 	makePods(ctx, "pods-", 2, metav1.Now())
 
 	client := newTestScrapeClient(testStats[2:], []error{nil})
-	scraper := serviceScraperForTest(ctx, t, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
 
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err == nil {
@@ -469,7 +540,7 @@ func TestScrapeReportErrorIfAnyFails(t *testing.T) {
 	// 1 success and 10 failures so one scrape fails permanently through retries.
 	client := newTestScrapeClient(testStats, []error{nil, errTest, errTest,
 		errTest, errTest, errTest, errTest, errTest, errTest, errTest, errTest})
-	scraper := serviceScraperForTest(ctx, t, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
 
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if !errors.Is(err, errTest) {
@@ -481,7 +552,7 @@ func TestScrapeDoNotScrapeIfNoPodsFound(t *testing.T) {
 	ctx, cancel, _ := SetupFakeContextWithCancel(t)
 	t.Cleanup(cancel)
 	client := newTestScrapeClient(testStats, nil)
-	scraper := serviceScraperForTest(ctx, t, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, false /*podsAddressable*/, false /*passthroughLb*/)
 
 	stat, err := scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
@@ -515,7 +586,7 @@ func TestMixedPodShuffle(t *testing.T) {
 	t.Log("WantScrapes", wantScrapes)
 
 	client := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, client, client, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, true /*podsAddressable*/, false /*passthroughLb*/)
 
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
@@ -558,7 +629,7 @@ func TestOldPodShuffle(t *testing.T) {
 	t.Log("WantScrapes", wantScrapes)
 
 	client := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, client, client, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, true /*podsAddressable*/, false /*passthroughLb*/)
 
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
@@ -632,7 +703,7 @@ func TestOldPodsFallback(t *testing.T) {
 		}
 		return r
 	}())
-	scraper := serviceScraperForTest(ctx, t, client, client, true /*podsAddressable*/, false /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, client, true /*podsAddressable*/, false /*passthroughLb*/)
 
 	_, err = scraper.Scrape(defaultMetric.Spec.StableWindow)
 	if err != nil {
@@ -672,7 +743,7 @@ func TestPodDirectPassthroughScrapeSuccess(t *testing.T) {
 	})
 
 	client := newTestScrapeClient(testStats, []error{nil})
-	scraper := serviceScraperForTest(ctx, t, client, nil /* mesh not used */, true /*podsAddressable*/, true /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, client, nil /* mesh not used */, true /*podsAddressable*/, true /*passthroughLb*/)
 
 	makePods(ctx, "pods-", 3, metav1.Now())
 	if _, err := scraper.Scrape(defaultMetric.Spec.StableWindow); err != nil {
@@ -706,7 +777,7 @@ func TestPodDirectPassthroughScrapeNoneSucceed(t *testing.T) {
 	})
 	makePods(ctx, "pods-", 4, metav1.Now())
 
-	scraper := serviceScraperForTest(ctx, t, direct, mesh, true /*podsAddressable*/, true /*passthroughLb*/)
+	scraper := serviceScraperForTest(ctx, t, meshModeAuto, direct, mesh, true /*podsAddressable*/, true /*passthroughLb*/)
 
 	// No fallback to service scraping expected.
 	if _, err = scraper.Scrape(defaultMetric.Spec.StableWindow); err == nil {
@@ -718,14 +789,14 @@ func TestPodDirectPassthroughScrapeNoneSucceed(t *testing.T) {
 	}
 }
 
-func serviceScraperForTest(ctx context.Context, t *testing.T, directClient, meshClient scrapeClient,
+func serviceScraperForTest(ctx context.Context, t *testing.T, meshMode meshMode, directClient, meshClient scrapeClient,
 	podsAddressable bool, usePassthroughLb bool) *serviceScraper {
 	metric := testMetric()
 	accessor := resources.NewPodAccessor(
 		fakepodsinformer.Get(ctx).Lister(),
 		testNamespace, testRevision)
 	logger := logtesting.TestLogger(t)
-	ss := newServiceScraperWithClient(metric, testRevision, accessor, usePassthroughLb, directClient, meshClient, logger)
+	ss := newServiceScraperWithClient(metric, testRevision, accessor, usePassthroughLb, meshMode, directClient, meshClient, logger)
 	ss.podsAddressable = podsAddressable
 	return ss
 }
diff --git a/pkg/http/handler/timeout_test.go b/pkg/http/handler/timeout_test.go
index 001d3c0cf..a0cab6e67 100644
--- a/pkg/http/handler/timeout_test.go
+++ b/pkg/http/handler/timeout_test.go
@@ -26,11 +26,12 @@ import (
 	"time"
 )
 
-func TestTimeoutWriterAllowsForAdditionalWrites(t *testing.T) {
+func TestTimeoutWriterAllowsForAdditionalWritesBeforeTimeout(t *testing.T) {
 	recorder := httptest.NewRecorder()
 	handler := &timeoutWriter{w: recorder}
 	handler.WriteHeader(http.StatusOK)
-	handler.timeoutAndWriteError("error")
+	handler.tryFirstByteTimeoutAndWriteError("error")
+	handler.tryIdleTimeoutAndWriteError(time.Now(), 10*time.Second, "error")
 	if _, err := io.WriteString(handler, "test"); err != nil {
 		t.Fatalf("handler.Write() = %v, want no error", err)
 	}
@@ -54,7 +55,7 @@ func TestTimeoutWriterDoesntFlushAfterTimeout(t *testing.T) {
 	}
 }
 
-func TestTimeoutWriterFlushesAfterTimeout(t *testing.T) {
+func TestTimeoutWriterFlushesBeforeTimeout(t *testing.T) {
 	recorder := httptest.NewRecorder()
 	handler := &timeoutWriter{w: recorder}
 	handler.Flush()
@@ -73,24 +74,68 @@ func TestTimeoutWriterErrorsWriteAfterTimeout(t *testing.T) {
 	}
 }
 
+type timeoutHandlerTestScenario struct {
+	name             string
+	firstByteTimeout time.Duration
+	idleTimeout      time.Duration
+	handler          func(mux *sync.Mutex, writeErrors chan error) http.Handler
+	timeoutMessage   string
+	wantStatus       int
+	wantBody         string
+	wantWriteError   bool
+	wantPanic        bool
+}
+
+func testTimeoutScenario(t *testing.T, scenarios []timeoutHandlerTestScenario) {
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+
+			var reqMux sync.Mutex
+			writeErrors := make(chan error, 1)
+			rr := httptest.NewRecorder()
+			handler := NewTimeoutHandler(scenario.handler(&reqMux, writeErrors), scenario.timeoutMessage, scenario.firstByteTimeout, scenario.idleTimeout)
+
+			defer func() {
+				if scenario.wantPanic {
+					if recovered := recover(); recovered != http.ErrAbortHandler { //nolint // False positive for errors.Is check
+						t.Errorf("Recover = %v, want: %v", recovered, http.ErrAbortHandler)
+					}
+				}
+			}()
+
+			reqMux.Lock() // Will cause an inner 'Lock' to block. ServeHTTP will exit early if the call times out.
+			handler.ServeHTTP(rr, req)
+			reqMux.Unlock() // Allows the inner 'Lock' to go through to complete potential writes.
+
+			if status := rr.Code; status != scenario.wantStatus {
+				t.Errorf("Handler returned wrong status code: got %v want %v", status, scenario.wantStatus)
+			}
+
+			if rr.Body.String() != scenario.wantBody {
+				t.Errorf("Handler returned unexpected body: got %q want %q", rr.Body.String(), scenario.wantBody)
+			}
+
+			if scenario.wantWriteError {
+				if err := <-writeErrors; !errors.Is(err, http.ErrHandlerTimeout) {
+					t.Error("Expected a timeout error, got", err)
+				}
+			}
+		})
+	}
+}
+
 func TestTimeToFirstByteTimeoutHandler(t *testing.T) {
 	const (
 		immediateTimeout = 0 * time.Millisecond
 		longTimeout      = 1 * time.Minute // Super long, not supposed to hit this.
+		noIdleTimeout    = 0 * time.Millisecond
 	)
 
-	tests := []struct {
-		name           string
-		timeout        time.Duration
-		handler        func(mux *sync.Mutex, writeErrors chan error) http.Handler
-		timeoutMessage string
-		wantStatus     int
-		wantBody       string
-		wantWriteError bool
-		wantPanic      bool
-	}{{
-		name:    "all good",
-		timeout: longTimeout,
+	scenarios := []timeoutHandlerTestScenario{{
+		name:             "all good",
+		firstByteTimeout: longTimeout,
+		idleTimeout:      noIdleTimeout,
 		handler: func(*sync.Mutex, chan error) http.Handler {
 			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.Write([]byte("hi"))
@@ -99,8 +144,9 @@ func TestTimeToFirstByteTimeoutHandler(t *testing.T) {
 		wantStatus: http.StatusOK,
 		wantBody:   "hi",
 	}, {
-		name:    "custom timeout message",
-		timeout: immediateTimeout,
+		name:             "custom timeout message",
+		firstByteTimeout: immediateTimeout,
+		idleTimeout:      noIdleTimeout,
 		handler: func(mux *sync.Mutex, writeErrors chan error) http.Handler {
 			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				mux.Lock()
@@ -114,8 +160,9 @@ func TestTimeToFirstByteTimeoutHandler(t *testing.T) {
 		wantBody:       "request timeout",
 		wantWriteError: true,
 	}, {
-		name:    "propagate panic",
-		timeout: longTimeout,
+		name:             "propagate panic",
+		firstByteTimeout: longTimeout,
+		idleTimeout:      noIdleTimeout,
 		handler: func(*sync.Mutex, chan error) http.Handler {
 			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				panic(http.ErrAbortHandler)
@@ -125,11 +172,13 @@ func TestTimeToFirstByteTimeoutHandler(t *testing.T) {
 		wantBody:   "request timeout",
 		wantPanic:  true,
 	}, {
-		name:    "timeout before panic",
-		timeout: immediateTimeout,
-		handler: func(*sync.Mutex, chan error) http.Handler {
+		name:             "timeout before panic",
+		firstByteTimeout: immediateTimeout,
+		idleTimeout:      noIdleTimeout,
+		handler: func(mux *sync.Mutex, _ chan error) http.Handler {
 			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				time.Sleep(1 * time.Second)
+				mux.Lock()
+				defer mux.Unlock()
 				panic(http.ErrAbortHandler)
 			})
 		},
@@ -138,45 +187,127 @@ func TestTimeToFirstByteTimeoutHandler(t *testing.T) {
 		wantBody:       "request timeout",
 		wantPanic:      false,
 	}}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			req := httptest.NewRequest(http.MethodGet, "/", nil)
-
-			var reqMux sync.Mutex
-			writeErrors := make(chan error, 1)
-			rr := httptest.NewRecorder()
-			handler := NewTimeToFirstByteTimeoutHandler(test.handler(&reqMux, writeErrors), test.timeoutMessage, test.timeout)
-
-			defer func() {
-				if test.wantPanic {
-					if recovered := recover(); recovered != http.ErrAbortHandler { //nolint // False positive for errors.Is check
-						t.Errorf("Recover = %v, want: %v", recovered, http.ErrAbortHandler)
-					}
-				}
-			}()
+	testTimeoutScenario(t, scenarios)
+}
 
-			reqMux.Lock() // Will cause an inner 'Lock' to block. ServeHTTP will exit early if the call times out.
-			handler.ServeHTTP(rr, req)
-			reqMux.Unlock() // Allows the inner 'Lock' to go through to complete potential writes.
+func TestIdleTimeoutHandler(t *testing.T) {
+	const (
+		noIdleTimeout        = 0 * time.Millisecond
+		immediateIdleTimeout = 1 * time.Millisecond // 0 would disable the timeout
+		shortIdleTimeout     = 100 * time.Millisecond
+		longIdleTimeout      = 1 * time.Minute // Super long, not supposed to hit this.
+		longFirstByteTimeout = 1 * time.Minute // Super long, not supposed to hit this.
+	)
 
-			if status := rr.Code; status != test.wantStatus {
-				t.Errorf("Handler returned wrong status code: got %v want %v", status, test.wantStatus)
-			}
+	scenarios := []timeoutHandlerTestScenario{{
+		name:             "all good",
+		firstByteTimeout: longFirstByteTimeout,
+		idleTimeout:      longIdleTimeout,
+		handler: func(*sync.Mutex, chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte("hi"))
+			})
+		},
+		wantStatus: http.StatusOK,
+		wantBody:   "hi",
+	}, {
+		name:             "custom timeout message",
+		idleTimeout:      immediateIdleTimeout,
+		firstByteTimeout: longFirstByteTimeout,
+		handler: func(mux *sync.Mutex, writeErrors chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mux.Lock()
+				defer mux.Unlock()
+				_, werr := w.Write([]byte("hi"))
+				writeErrors <- werr
+			})
+		},
+		timeoutMessage: "request timeout",
+		wantStatus:     http.StatusGatewayTimeout,
+		wantBody:       "request timeout",
+		wantWriteError: true,
+	}, {
+		name:             "propagate panic",
+		idleTimeout:      longIdleTimeout,
+		firstByteTimeout: longFirstByteTimeout,
+		handler: func(*sync.Mutex, chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				panic(http.ErrAbortHandler)
+			})
+		},
+		wantStatus: http.StatusGatewayTimeout,
+		wantBody:   "request timeout",
+		wantPanic:  true,
+	}, {
+		name:             "timeout before panic",
+		idleTimeout:      immediateIdleTimeout,
+		firstByteTimeout: longFirstByteTimeout,
+		handler: func(mux *sync.Mutex, _ chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				mux.Lock()
+				defer mux.Unlock()
+				panic(http.ErrAbortHandler)
+			})
+		},
+		timeoutMessage: "request timeout",
+		wantStatus:     http.StatusGatewayTimeout,
+		wantBody:       "request timeout",
+		wantPanic:      false,
+	}, {
+		name:             "successful writes prevent timeout",
+		idleTimeout:      shortIdleTimeout,
+		firstByteTimeout: longFirstByteTimeout,
+		handler: func(*sync.Mutex, chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte("foo"))
+				time.Sleep(shortIdleTimeout / 3)
+				w.Write([]byte("bar"))
+				time.Sleep(shortIdleTimeout / 3)
+				w.Write([]byte("baz"))
+				time.Sleep(shortIdleTimeout / 3)
+				w.Write([]byte("test"))
+				time.Sleep(shortIdleTimeout / 3)
+			})
+		},
+		wantStatus: http.StatusOK,
+		wantBody:   "foobarbaztest",
+		wantPanic:  false,
+	}, {
+		name:             "can still timeout after a successful write",
+		idleTimeout:      shortIdleTimeout,
+		firstByteTimeout: longFirstByteTimeout,
+		handler: func(mux *sync.Mutex, _ chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte("foo"))
+				mux.Lock()
+				defer mux.Unlock()
+				panic(http.ErrAbortHandler)
+			})
+		},
+		timeoutMessage: "request timeout",
+		wantStatus:     http.StatusOK,
+		wantBody:       "foorequest timeout",
+		wantPanic:      false,
+	}, {
+		name:             "no idle timeout",
+		idleTimeout:      noIdleTimeout,
+		firstByteTimeout: longFirstByteTimeout,
+		handler: func(*sync.Mutex, chan error) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				panic(http.ErrAbortHandler)
+			})
+		},
+		wantStatus: http.StatusGatewayTimeout,
+		wantBody:   "request timeout",
+		wantPanic:  true,
+	},
+	}
 
-			if rr.Body.String() != test.wantBody {
-				t.Errorf("Handler returned unexpected body: got %q want %q", rr.Body.String(), test.wantBody)
-			}
+	testTimeoutScenario(t, scenarios)
 
-			if test.wantWriteError {
-				if err := <-writeErrors; !errors.Is(err, http.ErrHandlerTimeout) {
-					t.Error("Expected a timeout error, got", err)
-				}
-			}
-		})
-	}
 }
 
-func BenchmarkTimeToFirstByteTimeoutHandler(b *testing.B) {
+func BenchmarkTimeoutHandler(b *testing.B) {
 	writes := [][]byte{[]byte("this"), []byte("is"), []byte("a"), []byte("test")}
 	baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(200)
@@ -184,7 +315,7 @@ func BenchmarkTimeToFirstByteTimeoutHandler(b *testing.B) {
 			w.Write(write)
 		}
 	})
-	handler := NewTimeToFirstByteTimeoutHandler(baseHandler, "test", 10*time.Minute)
+	handler := NewTimeoutHandler(baseHandler, "test", 10*time.Minute, 10*time.Minute)
 	req := httptest.NewRequest(http.MethodPost, "http://example.com", nil)
 
 	b.Run("sequential", func(b *testing.B) {
diff --git a/pkg/queue/concurrency_state_test.go b/pkg/queue/concurrency_state_test.go
index a0cc29ea1..85a84f4db 100644
--- a/pkg/queue/concurrency_state_test.go
+++ b/pkg/queue/concurrency_state_test.go
@@ -19,114 +19,124 @@ package queue
 import (
 	"net/http"
 	"net/http/httptest"
-	"strconv"
-	"sync"
 	"testing"
 	"time"
 
 	"go.uber.org/atomic"
+	"k8s.io/apimachinery/pkg/util/wait"
 	pkglogging "knative.dev/pkg/logging"
+	ltesting "knative.dev/pkg/logging/testing"
 
 	network "knative.dev/networking/pkg"
 )
 
 func TestConcurrencyStateHandler(t *testing.T) {
-	tests := []struct {
-		name            string
-		pauses, resumes int64
-		events          map[time.Duration]time.Duration // start time => req length
-	}{{
-		name:    "single request",
-		pauses:  1,
-		resumes: 1,
-		events: map[time.Duration]time.Duration{
-			1 * time.Second: 2 * time.Second,
-		},
-	}, {
-		name:    "overlapping requests",
-		pauses:  1,
-		resumes: 1,
-		events: map[time.Duration]time.Duration{
-			25 * time.Millisecond: 100 * time.Millisecond,
-			75 * time.Millisecond: 200 * time.Millisecond,
-		},
-	}, {
-		name:    "subsumbed request",
-		pauses:  1,
-		resumes: 1,
-		events: map[time.Duration]time.Duration{
-			25 * time.Millisecond: 300 * time.Millisecond,
-			75 * time.Millisecond: 200 * time.Millisecond,
-		},
-	}, {
-		name:    "start stop start",
-		pauses:  2,
-		resumes: 2,
-		events: map[time.Duration]time.Duration{
-			25 * time.Millisecond:  300 * time.Millisecond,
-			75 * time.Millisecond:  200 * time.Millisecond,
-			850 * time.Millisecond: 300 * time.Millisecond,
-			900 * time.Millisecond: 400 * time.Millisecond,
-		},
-	}}
+	paused := atomic.NewInt64(0)
+	resumed := atomic.NewInt64(0)
 
-	logger, _ := pkglogging.NewLogger("", "error")
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			paused := atomic.NewInt64(0)
-			pause := func() {
-				paused.Inc()
-			}
+	handler := func(w http.ResponseWriter, r *http.Request) {}
+	logger := ltesting.TestLogger(t)
+	h := ConcurrencyStateHandler(logger, http.HandlerFunc(handler), func() { paused.Inc() }, func() { resumed.Inc() })
 
-			resumed := atomic.NewInt64(0)
-			resume := func() {
-				resumed.Inc()
-			}
+	h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://target", nil))
+	if got, want := pollFor(paused, 1), int64(1); got != want {
+		t.Errorf("Pause was called %d times, want %d times", got, want)
+	}
 
-			delegated := atomic.NewInt64(0)
-			delegate := func(w http.ResponseWriter, r *http.Request) {
-				wait, err := strconv.Atoi(r.Header.Get("wait"))
-				if err != nil {
-					panic(err)
-				}
+	if got, want := pollFor(resumed, 1), int64(1); got != want {
+		t.Errorf("Resume was called %d times, want %d times", got, want)
+	}
 
-				time.Sleep(time.Duration(wait))
-				delegated.Inc()
-			}
+	h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://target", nil))
+	if got, want := pollFor(paused, 2), int64(2); got != want {
+		t.Errorf("Pause was called %d times, want %d times", got, want)
+	}
 
-			h := ConcurrencyStateHandler(logger, http.HandlerFunc(delegate), pause, resume)
-
-			var wg sync.WaitGroup
-			wg.Add(len(tt.events))
-			for delay, length := range tt.events {
-				length := length
-				time.AfterFunc(delay, func() {
-					w := httptest.NewRecorder()
-					r := httptest.NewRequest("GET", "http://target", nil)
-					r.Header.Set("wait", strconv.FormatInt(int64(length), 10))
-					h.ServeHTTP(w, r)
-					wg.Done()
-				})
-			}
+	if got, want := pollFor(resumed, 2), int64(2); got != want {
+		t.Errorf("Resume was called %d times, want %d times", got, want)
+	}
+}
 
-			wg.Wait()
-			// Allow last update to finish (otherwise values are off, though this doesn't show
-			// as a race condition when running `go test -race `
-			// TODO Less hacky fix for this
-			time.Sleep(100 * time.Microsecond)
+func TestConcurrencyStateHandlerParallelSubsumed(t *testing.T) {
+	paused := atomic.NewInt64(0)
+	resumed := atomic.NewInt64(0)
 
-			if got, want := paused.Load(), tt.pauses; got != want {
-				t.Errorf("expected to be paused %d times, but was paused %d times", want, got)
-			}
+	req1 := make(chan struct{})
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("req") == "1" {
+			req1 <- struct{}{} // to know it's here.
+			req1 <- struct{}{} // to make it wait.
+		}
+	}
+	logger := ltesting.TestLogger(t)
+	h := ConcurrencyStateHandler(logger, http.HandlerFunc(handler), func() { paused.Inc() }, func() { resumed.Inc() })
 
-			if got, want := delegated.Load(), int64(len(tt.events)); got != want {
-				t.Errorf("expected to be delegated %d times, but delegated %d times", want, got)
-			}
+	go func() {
+		defer func() { req1 <- struct{}{} }()
+		h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://target?req=1", nil))
+	}()
 
-			if got, want := resumed.Load(), tt.resumes; got != want {
-				t.Errorf("expected to be resumed %d times, but was resumed %d times", want, got)
-			}
-		})
+	<-req1 // Wait for req1 to arrive.
+
+	// Send a second request, which can pass through.
+	h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://target", nil))
+
+	<-req1 // Allow req1 to pass.
+	<-req1 // Wait for req1 to finish.
+
+	if got, want := pollFor(paused, 1), int64(1); got != want {
+		t.Errorf("Pause was called %d times, want %d times", got, want)
+	}
+
+	if got, want := pollFor(resumed, 1), int64(1); got != want {
+		t.Errorf("Resume was called %d times, want %d times", got, want)
+	}
+}
+
+func TestConcurrencyStateHandlerParallelOverlapping(t *testing.T) {
+	paused := atomic.NewInt64(0)
+	resumed := atomic.NewInt64(0)
+
+	req1 := make(chan struct{})
+	req2 := make(chan struct{})
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("req") == "1" {
+			req1 <- struct{}{} // to know it's here.
+			req1 <- struct{}{} // to make it wait.
+		} else {
+			req2 <- struct{}{} // to know it's here.
+			req2 <- struct{}{} // to make it wait.
+		}
+	}
+	logger := ltesting.TestLogger(t)
+	h := ConcurrencyStateHandler(logger, http.HandlerFunc(handler), func() { paused.Inc() }, func() { resumed.Inc() })
+
+	go func() {
+		defer func() { req1 <- struct{}{} }()
+		h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://target?req=1", nil))
+	}()
+
+	<-req1 // Wait for req1 to arrive.
+
+	go func() {
+		defer func() { req2 <- struct{}{} }()
+		h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://target?req=2", nil))
+	}()
+
+	<-req2 // Wait for req2 to arrive
+
+	<-req1 // Allow req1 to pass.
+	<-req1 // Wait for req1 to finish.
+
+	<-req2 // Allow req2 to pass.
+	<-req2 // Wait for req2 to finish.
+
+	if got, want := pollFor(paused, 1), int64(1); got != want {
+		t.Errorf("Pause was called %d times, want %d times", got, want)
+	}
+
+	if got, want := pollFor(resumed, 1), int64(1); got != want {
+		t.Errorf("Resume was called %d times, want %d times", got, want)
 	}
 }
 
@@ -195,3 +205,12 @@ func BenchmarkConcurrencyStateProxyHandler(b *testing.B) {
 		reportTicker.Stop()
 	}
 }
+
+func pollFor(val *atomic.Int64, want int64) int64 {
+	var lastVal int64
+	wait.PollImmediate(1*time.Millisecond, 1*time.Second, func() (bool, error) {
+		lastVal = val.Load()
+		return lastVal == want, nil
+	})
+	return lastVal
+}
diff --git a/pkg/queue/health/handler_test.go b/pkg/queue/health/handler_test.go
index dd66fd2a2..e26bf0b1a 100644
--- a/pkg/queue/health/handler_test.go
+++ b/pkg/queue/health/handler_test.go
@@ -80,7 +80,7 @@ func TestProbeHandler(t *testing.T) {
 				req.Header.Set(network.ProbeHeaderName, tc.requestHeader)
 			}
 
-			h := ProbeHandler(healthState, tc.prober, true /* isAggressive*/, true /*tracingEnabled*/, incHandler)
+			h := ProbeHandler(healthState, tc.prober, true /*tracingEnabled*/, incHandler)
 			h(writer, req)
 
 			if got, want := writer.Code, tc.wantCode; got != want {
diff --git a/pkg/reconciler/revision/background_test.go b/pkg/reconciler/revision/background_test.go
index ed852655f..5a04f06b9 100644
--- a/pkg/reconciler/revision/background_test.go
+++ b/pkg/reconciler/revision/background_test.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"reflect"
 	"testing"
 	"time"
@@ -223,7 +224,8 @@ func TestRateLimitPerItem(t *testing.T) {
 		return "", errors.New("failed")
 	}
 
-	queue := workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(50*time.Millisecond, 5*time.Second))
+	baseDelay := 50 * time.Millisecond
+	queue := workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(baseDelay, 5*time.Second))
 
 	enqueue := make(chan struct{})
 	subject := newBackgroundResolver(logger, resolver, queue, func(types.NamespacedName) {
@@ -238,38 +240,38 @@ func TestRateLimitPerItem(t *testing.T) {
 		<-done
 	}()
 
-	start := time.Now()
 	revision := rev("rev", "img1", "img2")
-	for i := 0; i < 5; i++ {
+	for i := 0; i < 3; i++ {
 		subject.Clear(types.NamespacedName{Name: revision.Name, Namespace: revision.Namespace})
+		start := time.Now()
 		resolution, err := subject.Resolve(revision, k8schain.Options{ServiceAccountName: "san"}, sets.NewString("skip"), 0)
 		if err != nil || resolution != nil {
 			t.Fatalf("Expected Resolve to be nil, nil but got %v, %v", resolution, err)
 		}
 
 		<-enqueue
+
 		_, err = subject.Resolve(revision, k8schain.Options{ServiceAccountName: "san"}, sets.NewString("skip"), 0)
 		if err == nil {
 			t.Fatalf("Expected Resolve to fail")
 		}
-	}
 
-	if took := time.Since(start); took < 600*time.Millisecond {
-		// Per-item time is 50ms, so after 5 cycles of back-off should take at least 800ms.
-		// (Otherwise will take only ~250ms)
-		t.Fatal("Expected resolves to take longer than 600ms, but took", took)
+		latency := time.Since(start)
+		expected := time.Duration(math.Pow(2, float64(i))) * baseDelay
+		if latency < expected {
+			t.Fatalf("latency = %s, want at least %s", latency, expected)
+		}
 	}
 
 	t.Run("Does not affect other revisions", func(t *testing.T) {
 		start := time.Now()
-
 		resolution, err := subject.Resolve(rev("another-revision", "img1", "img2"), k8schain.Options{ServiceAccountName: "san"}, sets.NewString("skip"), 0)
 		if err != nil || resolution != nil {
 			t.Fatalf("Expected Resolve to be nil, nil but got %v, %v", resolution, err)
 		}
 
 		<-enqueue
-		if took := time.Since(start); took > 500*time.Millisecond {
+		if took := time.Since(start); took > baseDelay*2*2 {
 			t.Fatal("Expected per-item limit not to affect other revisions, but took", took)
 		}
 	})
@@ -277,14 +279,14 @@ func TestRateLimitPerItem(t *testing.T) {
 	t.Run("Forget clears per-item rate limit", func(t *testing.T) {
 		subject.Forget(types.NamespacedName{Name: revision.Name, Namespace: revision.Namespace})
 
-		start = time.Now()
+		start := time.Now()
 		resolution, err := subject.Resolve(revision, k8schain.Options{ServiceAccountName: "san"}, sets.NewString("skip"), 0)
 		if err != nil || resolution != nil {
 			t.Fatalf("Expected Resolve to be nil, nil but got %v, %v", resolution, err)
 		}
 
 		<-enqueue
-		if took := time.Since(start); took > 500*time.Millisecond {
+		if took := time.Since(start); took > baseDelay*2*2 {
 			t.Fatal("Expected Forget to remove revision from rate limiter, but took", took)
 		}
 	})
diff --git a/pkg/reconciler/revision/resources/deploy_test.go b/pkg/reconciler/revision/resources/deploy_test.go
index 209a44546..1de81e6f4 100644
--- a/pkg/reconciler/revision/resources/deploy_test.go
+++ b/pkg/reconciler/revision/resources/deploy_test.go
@@ -1087,6 +1087,27 @@ func TestMakePodSpec(t *testing.T) {
 			},
 			withAppendedVolumes(varLogVolume),
 		),
+	}, {
+		name: "with no readiness probe",
+		rev: revision("bar", "foo",
+			withContainers([]corev1.Container{{
+				Image: "busybox",
+				Name:  servingContainerName,
+			}}),
+		),
+		want: podSpec(
+			[]corev1.Container{
+				servingContainer(func(container *corev1.Container) {
+					container.StartupProbe = nil
+					container.ReadinessProbe = nil
+				}),
+				queueContainer(
+					withEnvVar("SERVING_READINESS_PROBE", ""),
+					func(container *corev1.Container) {
+						container.StartupProbe = nil
+						container.ReadinessProbe = nil
+					}),
+			}),
 	}}
 
 	for _, test := range tests {
@@ -1111,12 +1132,6 @@ var quantityComparer = cmp.Comparer(func(x, y resource.Quantity) bool {
 	return x.Cmp(y) == 0
 })
 
-func TestMissingProbeError(t *testing.T) {
-	if _, err := MakeDeployment(revision("bar", "foo"), revConfig()); err == nil {
-		t.Error("expected error from MakeDeployment")
-	}
-}
-
 func TestMakeDeployment(t *testing.T) {
 	tests := []struct {
 		name      string
diff --git a/pkg/reconciler/route/resources/ingress_test.go b/pkg/reconciler/route/resources/ingress_test.go
index 2a302fca1..f955e0b08 100644
--- a/pkg/reconciler/route/resources/ingress_test.go
+++ b/pkg/reconciler/route/resources/ingress_test.go
@@ -1045,6 +1045,50 @@ func TestMakeIngressWithTLS(t *testing.T) {
 	}
 }
 
+func TestMakeIngressWithHTTPOption(t *testing.T) {
+	targets := map[string]traffic.RevisionTargets{}
+	ingressClass := "foo-ingress"
+	tls := []netv1alpha1.IngressTLS{{
+		Hosts:      []string{"*.default.domain.com"},
+		SecretName: "secret",
+	}}
+	tests := []struct {
+		name                 string
+		httpOptionAnnotation string
+		wantOption           netv1alpha1.HTTPOption
+		wantError            bool
+	}{{
+		name:                 "Route annotation HTTPOption enabled",
+		httpOptionAnnotation: "Enabled",
+		wantOption:           netv1alpha1.HTTPOptionEnabled,
+	}, {
+		name:       "No HTTPOption annotation",
+		wantOption: netv1alpha1.HTTPOptionRedirected,
+	}, {
+		name:                 "Incorrect HTTPOption annotation",
+		httpOptionAnnotation: "INCORRECT",
+		wantError:            true,
+	}}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			annotations := map[string]string{
+				networking.HTTPOptionAnnotationKey: tc.httpOptionAnnotation,
+			}
+			r := Route(ns, "test-route", WithURL, WithRouteAnnotation(annotations))
+			got, err := MakeIngress(testContextWithHTTPOption(), r, &traffic.Config{Targets: targets}, tls, ingressClass)
+			if (err != nil) != tc.wantError {
+				t.Fatalf("MakeIngress() error = %v, WantErr %v", err, tc.wantError)
+			}
+			if tc.wantError {
+				return
+			}
+			if diff := cmp.Diff(tc.wantOption, got.Spec.HTTPOption); diff != "" {
+				t.Error("Unexpected Ingress (-want, +got):", diff)
+			}
+		})
+	}
+}
+
 func TestMakeIngressTLS(t *testing.T) {
 	cert := &netv1alpha1.Certificate{
 		ObjectMeta: metav1.ObjectMeta{
@@ -1250,3 +1294,9 @@ func testContext() context.Context {
 		Defaults: configDefaults,
 	}), cfg)
 }
+
+func testContextWithHTTPOption() context.Context {
+	cfg := testConfig()
+	cfg.Network.HTTPProtocol = network.HTTPRedirected
+	return config.ToContext(context.Background(), cfg)
+}
diff --git a/test/adding_tests.md b/test/adding_tests.md
index 2cd90f2b9..fbbf720c4 100644
--- a/test/adding_tests.md
+++ b/test/adding_tests.md
@@ -133,7 +133,7 @@ _, err := pkgTest.WaitForEndpointState(
 	clients.KubeClient,
 	logger,
 	updatedRoute.Status.URL.URL(),
-	pkgTest.EventuallyMatchesBody(expectedText),
+	spoof.IsStatusOK,
 	"SomeDescription",
 	test.ServingFlags.ResolvableDomain)
 if err != nil {
diff --git a/test/conformance/api/v1/blue_green_test.go b/test/conformance/api/v1/blue_green_test.go
index fe55a5f15..0124f6f69 100644
--- a/test/conformance/api/v1/blue_green_test.go
+++ b/test/conformance/api/v1/blue_green_test.go
@@ -134,7 +134,7 @@ func TestBlueGreenRoute(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		greenURL,
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"CheckSuccessfulResponse",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/conformance/api/v1/generatename_test.go b/test/conformance/api/v1/generatename_test.go
index c7337cb23..d94fb2dac 100644
--- a/test/conformance/api/v1/generatename_test.go
+++ b/test/conformance/api/v1/generatename_test.go
@@ -98,7 +98,7 @@ func canServeRequests(t *testing.T, clients *test.Clients, route *v1.Route) erro
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText)),
 		"CheckEndpointToServeText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))
diff --git a/test/conformance/api/v1/resources_test.go b/test/conformance/api/v1/resources_test.go
index b5baa0889..7f35686c1 100644
--- a/test/conformance/api/v1/resources_test.go
+++ b/test/conformance/api/v1/resources_test.go
@@ -68,7 +68,7 @@ func TestCustomResourcesLimits(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK)),
+		spoof.MatchesAllOf(spoof.IsStatusOK),
 		"ResourceTestServesText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))
diff --git a/test/conformance/api/v1/revision_timeout_test.go b/test/conformance/api/v1/revision_timeout_test.go
index 7879f021b..3266e8d80 100644
--- a/test/conformance/api/v1/revision_timeout_test.go
+++ b/test/conformance/api/v1/revision_timeout_test.go
@@ -124,7 +124,7 @@ func TestRevisionTimeout(t *testing.T) {
 				clients.KubeClient,
 				t.Logf,
 				serviceURL,
-				v1test.RetryingRouteInconsistency(spoof.IsOneOfStatusCodes(http.StatusOK, http.StatusGatewayTimeout)),
+				spoof.IsOneOfStatusCodes(http.StatusOK, http.StatusGatewayTimeout),
 				"CheckSuccessfulResponse",
 				test.ServingFlags.ResolvableDomain,
 				test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/conformance/api/v1/route_test.go b/test/conformance/api/v1/route_test.go
index 8d7875b68..f0a5dae9f 100644
--- a/test/conformance/api/v1/route_test.go
+++ b/test/conformance/api/v1/route_test.go
@@ -45,7 +45,7 @@ func assertResourcesUpdatedWhenRevisionIsReady(t *testing.T, clients *test.Clien
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, pkgtest.EventuallyMatchesBody(expectedText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(expectedText)),
 		"CheckEndpointToServeText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))
diff --git a/test/conformance/api/v1/service_test.go b/test/conformance/api/v1/service_test.go
index 539e159a2..3a03ef833 100644
--- a/test/conformance/api/v1/service_test.go
+++ b/test/conformance/api/v1/service_test.go
@@ -36,6 +36,54 @@ import (
 	rtesting "knative.dev/serving/pkg/testing/v1"
 )
 
+// TestServiceCreateListAndDelete tests Creation, Listing and Deletion for Service resources.
+//   This test doesn't validate the Data Plane, it is just to check the Control Plane resources and their APIs
+func TestServiceCreateListAndDelete(t *testing.T) {
+	t.Parallel()
+	clients := test.Setup(t)
+
+	names := test.ResourceNames{
+		Service: test.ObjectNameForTest(t),
+		Image:   test.PizzaPlanet1,
+	}
+
+	// Clean up on test failure or interrupt
+	test.EnsureTearDown(t, clients, &names)
+
+	// Setup initial Service
+	if _, err := v1test.CreateServiceReady(t, clients, &names); err != nil {
+		t.Fatalf("Failed to create initial Service %v: %v", names.Service, err)
+	}
+
+	// Validate State after Creation
+	if err := validateControlPlane(t, clients, names, "1"); err != nil {
+		t.Error(err)
+	}
+
+	list, err := v1test.GetServices(clients)
+	if err != nil {
+		t.Fatal("Listing Services failed")
+	}
+	if len(list.Items) < 1 {
+		t.Fatal("Listing should return at least one Service")
+	}
+	var serviceFound = false
+	for _, service := range list.Items {
+		t.Logf("Service Returned: %s", service.Name)
+		if service.Name == names.Service {
+			serviceFound = true
+		}
+	}
+	if !serviceFound {
+		t.Fatal("The Service that was previously created was not found by listing all Services.")
+	}
+	t.Logf("Deleting Service: %s", names.Service)
+	if err := v1test.DeleteService(clients, names.Service); err != nil {
+		t.Fatal("Error deleting Service")
+	}
+
+}
+
 // TestServiceCreateAndUpdate tests both Creation and Update paths for a service. The test performs a series of Update/Validate steps to ensure that
 // the service transitions as expected during each step.
 // Currently the test performs the following updates:
diff --git a/test/conformance/api/v1/single_threaded_test.go b/test/conformance/api/v1/single_threaded_test.go
index bab28aa7b..1dc141da6 100644
--- a/test/conformance/api/v1/single_threaded_test.go
+++ b/test/conformance/api/v1/single_threaded_test.go
@@ -67,7 +67,7 @@ func TestSingleConcurrency(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"CheckSuccessfulResponse",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/conformance/api/v1/util.go b/test/conformance/api/v1/util.go
index 0a329fdf0..0ae094aa2 100644
--- a/test/conformance/api/v1/util.go
+++ b/test/conformance/api/v1/util.go
@@ -50,7 +50,7 @@ func checkForExpectedResponses(ctx context.Context, t testing.TB, clients *test.
 	if err != nil {
 		return err
 	}
-	_, err = client.Poll(req, v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesAllBodies(expectedResponses...))))
+	_, err = client.Poll(req, spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesAllBodies(expectedResponses...)))
 	return err
 }
 
@@ -133,7 +133,7 @@ func validateDataPlane(t testing.TB, clients *test.Clients, names test.ResourceN
 		clients.KubeClient,
 		t.Logf,
 		names.URL,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(expectedText)),
 		"WaitForEndpointToServeText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))
diff --git a/test/conformance/api/v1/volumes_test.go b/test/conformance/api/v1/volumes_test.go
index a9da9d1cf..f449f6514 100644
--- a/test/conformance/api/v1/volumes_test.go
+++ b/test/conformance/api/v1/volumes_test.go
@@ -476,7 +476,7 @@ func TestProjectedServiceAccountToken(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		names.URL,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, isJWT)),
+		spoof.MatchesAllOf(spoof.IsStatusOK, isJWT),
 		"CheckEndpointToServeTheToken",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/conformance/api/v1alpha1/domain_mapping_test.go b/test/conformance/api/v1alpha1/domain_mapping_test.go
index 063b29d68..ab8cd7d44 100644
--- a/test/conformance/api/v1alpha1/domain_mapping_test.go
+++ b/test/conformance/api/v1alpha1/domain_mapping_test.go
@@ -113,7 +113,7 @@ func TestDomainMapping(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1)),
 		"CheckSuccessfulResponse",
 		resolvableCustomDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
@@ -180,7 +180,7 @@ func TestDomainMapping(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1)),
 		"CheckSuccessfulResponse",
 		resolvableCustomDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
@@ -213,7 +213,7 @@ func TestDomainMapping(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText2))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText2)),
 		"CheckSuccessfulResponse",
 		resolvableCustomDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/conformance/api/v1beta1/domain_mapping_test.go b/test/conformance/api/v1beta1/domain_mapping_test.go
index 27bf9c265..28c114495 100644
--- a/test/conformance/api/v1beta1/domain_mapping_test.go
+++ b/test/conformance/api/v1beta1/domain_mapping_test.go
@@ -113,7 +113,7 @@ func TestDomainMapping(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1)),
 		"WaitForSuccessfulResponse",
 		resolvableCustomDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
@@ -180,7 +180,7 @@ func TestDomainMapping(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText1)),
 		"WaitForSuccessfulResponse",
 		resolvableCustomDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
@@ -213,7 +213,7 @@ func TestDomainMapping(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		endpoint,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText2))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.PizzaPlanetText2)),
 		"WaitForSuccessfulResponse",
 		resolvableCustomDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/conformance/runtime/readiness_probe_test.go b/test/conformance/runtime/readiness_probe_test.go
index 17cc52aed..b2157d7ce 100644
--- a/test/conformance/runtime/readiness_probe_test.go
+++ b/test/conformance/runtime/readiness_probe_test.go
@@ -130,7 +130,7 @@ func TestProbeRuntime(t *testing.T) {
 					clients.KubeClient,
 					t.Logf,
 					url,
-					v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText))),
+					spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText)),
 					"readinessIsReady",
 					test.ServingFlags.ResolvableDomain,
 					test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
@@ -235,7 +235,7 @@ func waitReadyThenStartFailing(t *testing.T, clients *test.Clients, names test.R
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText)),
 		"readinessIsReady",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/conformance/runtime/util.go b/test/conformance/runtime/util.go
index ff70464fc..1b1d3923a 100644
--- a/test/conformance/runtime/util.go
+++ b/test/conformance/runtime/util.go
@@ -60,7 +60,7 @@ func fetchRuntimeInfo(
 		clients.KubeClient,
 		t.Logf,
 		objects.Service.Status.URL.URL(),
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"RuntimeInfo",
 		test.ServingFlags.ResolvableDomain,
 		append(reqOpts, test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))...)
diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh
index cdf763ddb..4f839dd6a 100755
--- a/test/e2e-tests.sh
+++ b/test/e2e-tests.sh
@@ -71,9 +71,9 @@ fi
 
 toggle_feature autocreateClusterDomainClaims true config-network || fail_test
 go_test_e2e -timeout=30m \
- ./test/conformance/api/... \
- ./test/conformance/runtime/... \
- ./test/e2e \
+  ./test/conformance/api/... \
+  ./test/conformance/runtime/... \
+  ./test/e2e \
   ${parallelism} \
   ${TEST_OPTIONS} || failed=1
 toggle_feature autocreateClusterDomainClaims false config-network || fail_test
@@ -103,7 +103,7 @@ kubectl replace cm "config-gc" -n ${SYSTEM_NAMESPACE} -f ${TMP_DIR}/config-gc.ya
 go_test_e2e -timeout=20m -parallel=300 ./test/scale ${TEST_OPTIONS} || failed=1
 
 # Run HPA tests
-go_test_e2e -timeout=15m -tags=hpa ./test/e2e ${TEST_OPTIONS} || failed=1
+go_test_e2e -timeout=30m -tags=hpa ./test/e2e ${TEST_OPTIONS} || failed=1
 
 # Run emptyDir tests with alpha enabled avoiding any issues with the testing options guard above
 toggle_feature kubernetes.podspec-volumes-emptydir Enabled
@@ -130,4 +130,32 @@ fi
 # Remove the kail log file if the test flow passes.
 # This is for preventing too many large log files to be uploaded to GCS in CI.
 rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt"
+
+header "Collecting performance data"
+
+cat <<EOF | ko apply -f -
+apiVersion: serving.knative.dev/v1
+kind: Service
+metadata:
+  name: podspeed
+spec:
+  template:
+    metadata:
+      annotations:
+        autoscaling.knative.dev/minScale: "1"
+    spec:
+      containers:
+      - image: ko://knative.dev/serving/test/test_images/helloworld
+EOF
+
+kubectl wait ksvc/podspeed --for=condition=Ready
+
+template=$(mktemp)
+kubectl get pods -lserving.knative.dev/service=podspeed -ojson | jq '.items[0]' > "$template"
+
+run_go_tool github.com/markusthoemmes/podspeed/cmd/podspeed@358209f podspeed --prepull -pods 100 -template "$template" > "${ARTIFACTS}/pod-bringup-performance.txt"
+cat "${ARTIFACTS}/pod-bringup-performance.txt"
+
+kubectl delete ksvc podspeed
+
 success
diff --git a/test/e2e/activator_test.go b/test/e2e/activator_test.go
index 050f227cb..9683b9ad7 100644
--- a/test/e2e/activator_test.go
+++ b/test/e2e/activator_test.go
@@ -74,7 +74,7 @@ func TestActivatorOverload(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		resources.Route.Status.URL.URL(),
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"WaitForSuccessfulResponse",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go
index b800f15a6..4f83d4ef3 100644
--- a/test/e2e/autoscale.go
+++ b/test/e2e/autoscale.go
@@ -239,7 +239,7 @@ func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization
 		clients.KubeClient,
 		t.Logf,
 		names.URL,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK)),
+		spoof.MatchesAllOf(spoof.IsStatusOK),
 		"CheckingEndpointAfterCreate",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/autoscale_hpa_test.go b/test/e2e/autoscale_hpa_test.go
index 22bdf6261..b61f1f17e 100644
--- a/test/e2e/autoscale_hpa_test.go
+++ b/test/e2e/autoscale_hpa_test.go
@@ -46,19 +46,35 @@ const (
 	cpuTarget             = 75
 	targetPods            = 5
 	concurrency           = 10
-	scaleUpTimeout        = 3 * time.Minute
-	scaleToMinimumTimeout = 10 * time.Minute // 5 minutes is the default window for hpa to calculate if should scale down
+	scaleUpTimeout        = 10 * time.Minute
+	scaleToMinimumTimeout = 15 * time.Minute // 5 minutes is the default window for hpa to calculate if should scale down
 	minPods               = 1.0
 	maxPods               = 10.0
 	primeNum              = 1000000
+	memoryTarget          = 200 // 200Mib
+	memoryTargetPods      = 2
+	bloatNum              = 40
+
+	vegetaParamPrime = "prime"
+	vegetaParamBloat = "bloat"
 )
 
+func TestHPAAutoscaleUpDownUpMem(t *testing.T) {
+	t.Skip("#11944: Skipped because of excessive flakiness")
+
+	ctx := setupHPASvc(t, autoscaling.Memory, memoryTarget)
+	test.EnsureTearDown(t, ctx.Clients(), ctx.Names())
+	assertMemoryHPAAutoscaleUpToNumPods(ctx, memoryTargetPods, time.After(scaleUpTimeout), true /* quick */)
+	assertScaleDownToOne(ctx)
+	assertMemoryHPAAutoscaleUpToNumPods(ctx, memoryTargetPods, time.After(scaleUpTimeout), true /* quick */)
+}
+
 func TestHPAAutoscaleUpDownUp(t *testing.T) {
 	ctx := setupHPASvc(t, autoscaling.CPU, cpuTarget)
 	test.EnsureTearDown(t, ctx.Clients(), ctx.Names())
-	assertHPAAutoscaleUpToNumPods(ctx, targetPods, time.After(scaleUpTimeout), true /* quick */)
+	assertCPUHPAAutoscaleUpToNumPods(ctx, targetPods, time.After(scaleUpTimeout), true /* quick */)
 	assertScaleDownToOne(ctx)
-	assertHPAAutoscaleUpToNumPods(ctx, targetPods, time.After(scaleUpTimeout), true /* quick */)
+	assertCPUHPAAutoscaleUpToNumPods(ctx, targetPods, time.After(scaleUpTimeout), true /* quick */)
 }
 
 func setupHPASvc(t *testing.T, metric string, target int) *TestContext {
@@ -96,7 +112,7 @@ func setupHPASvc(t *testing.T, metric string, target int) *TestContext {
 		clients.KubeClient,
 		t.Logf,
 		names.URL,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK)),
+		spoof.MatchesAllOf(spoof.IsStatusOK),
 		"CheckingEndpointAfterCreate",
 		false,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
@@ -120,13 +136,32 @@ func setupHPASvc(t *testing.T, metric string, target int) *TestContext {
 	}
 }
 
-func assertHPAAutoscaleUpToNumPods(ctx *TestContext, targetPods float64, done <-chan time.Time, quick bool) {
+func assertCPUHPAAutoscaleUpToNumPods(ctx *TestContext, targetPods float64, done <-chan time.Time, quick bool) {
+	ctx.t.Helper()
+
+	stopChan := make(chan struct{})
+	var grp errgroup.Group
+	grp.Go(func() error {
+		return generateTrafficAtFixedConcurrencyWithLoad(ctx, concurrency, vegetaParamPrime, primeNum, stopChan)
+	})
+
+	grp.Go(func() error {
+		defer close(stopChan)
+		return checkPodScale(ctx, targetPods, minPods, maxPods, done, quick)
+	})
+
+	if err := grp.Wait(); err != nil {
+		ctx.t.Fatal(err)
+	}
+}
+
+func assertMemoryHPAAutoscaleUpToNumPods(ctx *TestContext, targetPods float64, done <-chan time.Time, quick bool) {
 	ctx.t.Helper()
 
 	stopChan := make(chan struct{})
 	var grp errgroup.Group
 	grp.Go(func() error {
-		return generateTrafficAtFixedConcurrencyWithCPULoad(ctx, concurrency, stopChan)
+		return generateTrafficAtFixedConcurrencyWithLoad(ctx, concurrency, vegetaParamBloat, bloatNum, stopChan)
 	})
 
 	grp.Go(func() error {
@@ -139,14 +174,14 @@ func assertHPAAutoscaleUpToNumPods(ctx *TestContext, targetPods float64, done <-
 	}
 }
 
-func generateTrafficAtFixedConcurrencyWithCPULoad(ctx *TestContext, concurrency int, stopChan chan struct{}) error {
+func generateTrafficAtFixedConcurrencyWithLoad(ctx *TestContext, concurrency int, vegetaParam string, vegetaValue int, stopChan chan struct{}) error {
 	pacer := vegeta.ConstantPacer{} // Sends requests as quickly as possible, capped by MaxWorkers below.
 	attacker := vegeta.NewAttacker(
 		vegeta.Timeout(0), // No timeout is enforced at all.
 		vegeta.Workers(uint64(concurrency)),
 		vegeta.MaxWorkers(uint64(concurrency)))
 	target, err := getVegetaTarget(
-		ctx.clients.KubeClient, ctx.resources.Route.Status.URL.URL().Hostname(), pkgTest.Flags.IngressEndpoint, test.ServingFlags.ResolvableDomain, "prime", primeNum)
+		ctx.clients.KubeClient, ctx.resources.Route.Status.URL.URL().Hostname(), pkgTest.Flags.IngressEndpoint, test.ServingFlags.ResolvableDomain, vegetaParam, vegetaValue)
 	if err != nil {
 		return fmt.Errorf("error creating vegeta target: %w", err)
 	}
@@ -211,7 +246,7 @@ func waitForScaleToOne(t *testing.T, deploymentName string, clients *test.Client
 }
 
 func waitForHPAState(t *testing.T, name, namespace string, clients *test.Clients) error {
-	return wait.PollImmediate(time.Second, 10*time.Minute, func() (bool, error) {
+	return wait.PollImmediate(time.Second, 15*time.Minute, func() (bool, error) {
 		hpa, err := clients.KubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(namespace).Get(context.Background(), name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
diff --git a/test/e2e/destroypod_test.go b/test/e2e/destroypod_test.go
index 6782d6925..ff9d1a1bc 100644
--- a/test/e2e/destroypod_test.go
+++ b/test/e2e/destroypod_test.go
@@ -95,7 +95,7 @@ func TestDestroyPodInflight(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		routeURL,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(timeoutExpectedOutput))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(timeoutExpectedOutput)),
 		"TimeoutAppServesText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
@@ -179,7 +179,7 @@ func TestDestroyPodTimely(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		routeURL,
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"RouteServes",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
@@ -263,7 +263,7 @@ func TestDestroyPodWithRequests(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		routeURL,
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"RouteServes",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/domainmapping/domain_mapping_test.go b/test/e2e/domainmapping/domain_mapping_test.go
index 814456e09..e29fb5f92 100644
--- a/test/e2e/domainmapping/domain_mapping_test.go
+++ b/test/e2e/domainmapping/domain_mapping_test.go
@@ -157,7 +157,7 @@ func TestBYOCertificate(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		&url.URL{Scheme: "HTTPS", Host: host},
-		pkgTest.EventuallyMatchesBody(test.PizzaPlanetText1),
+		spoof.MatchesBody(test.PizzaPlanetText1),
 		"DomainMappingBYOSSLCert",
 		resolvableCustomDomain,
 		trustSelfSigned,
diff --git a/test/e2e/egress_traffic_test.go b/test/e2e/egress_traffic_test.go
index 15f2b524b..839285fb3 100644
--- a/test/e2e/egress_traffic_test.go
+++ b/test/e2e/egress_traffic_test.go
@@ -65,7 +65,7 @@ func TestEgressTraffic(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"HTTPProxy",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/emptydir_test.go b/test/e2e/emptydir_test.go
index 7ee00692b..290941935 100644
--- a/test/e2e/emptydir_test.go
+++ b/test/e2e/emptydir_test.go
@@ -73,7 +73,7 @@ func TestEmptyDirVolume(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.EmptyDirText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.EmptyDirText)),
 		"EmptyDirText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go
index f0b2be793..0f24479d6 100644
--- a/test/e2e/grpc_test.go
+++ b/test/e2e/grpc_test.go
@@ -358,7 +358,7 @@ func testGRPC(t *testing.T, f grpcTest, fopts ...rtesting.ServiceOption) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.IsStatusOK),
+		spoof.IsStatusOK,
 		"gRPCPingReadyToServe",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/helloworld_test.go b/test/e2e/helloworld_test.go
index 3802489db..b0451dee3 100644
--- a/test/e2e/helloworld_test.go
+++ b/test/e2e/helloworld_test.go
@@ -61,7 +61,7 @@ func TestHelloWorld(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText)),
 		"HelloWorldServesText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
@@ -114,7 +114,7 @@ func TestQueueSideCarResourceLimit(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText)),
 		"HelloWorldServesText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/http2_test.go b/test/e2e/http2_test.go
index b84ec9567..132e66dc7 100644
--- a/test/e2e/http2_test.go
+++ b/test/e2e/http2_test.go
@@ -59,7 +59,7 @@ func TestHelloHTTP2WithPortNameH2C(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloHTTP2Text))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloHTTP2Text)),
 		"HelloHTTP2ServesTextOnH2C",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
@@ -99,7 +99,7 @@ func TestHelloHTTP2WithEmptyPortName(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsOneOfStatusCodes(http.StatusUpgradeRequired))),
+		spoof.MatchesAllOf(spoof.IsOneOfStatusCodes(http.StatusUpgradeRequired)),
 		"HelloHTTP2ServesTextWithEmptyPort",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/logging_test.go b/test/e2e/logging_test.go
index 32ef67b08..6070a29b2 100644
--- a/test/e2e/logging_test.go
+++ b/test/e2e/logging_test.go
@@ -90,7 +90,7 @@ func TestRequestLogs(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		resources.Route.Status.URL.URL(),
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText)),
 		"CheckEndpointToServeText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))
diff --git a/test/e2e/multicontainer_test.go b/test/e2e/multicontainer_test.go
index aa2be274a..c1b5158a3 100644
--- a/test/e2e/multicontainer_test.go
+++ b/test/e2e/multicontainer_test.go
@@ -72,7 +72,7 @@ func TestMultiContainer(t *testing.T) {
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.MultiContainerResponse))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.MultiContainerResponse)),
 		"MulticontainerServesExpectedText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/namespace_test.go b/test/e2e/namespace_test.go
index d9dd221d1..c27076632 100644
--- a/test/e2e/namespace_test.go
+++ b/test/e2e/namespace_test.go
@@ -39,7 +39,7 @@ func checkResponse(t *testing.T, clients *test.Clients, names test.ResourceNames
 		clients.KubeClient,
 		t.Logf,
 		names.URL,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(expectedText)),
 		"WaitForEndpointToServeText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/scale.go b/test/e2e/scale.go
index ca78f9559..a334ba56e 100644
--- a/test/e2e/scale.go
+++ b/test/e2e/scale.go
@@ -188,7 +188,7 @@ func ScaleToWithin(t *testing.T, scale int, duration time.Duration, latencies La
 					clients.KubeClient,
 					t.Logf,
 					url,
-					v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText), abortOnTimeout(ctx))),
+					spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.HelloWorldText), abortOnTimeout(ctx)),
 					"CheckEndpointToServeText",
 					test.ServingFlags.ResolvableDomain,
 					test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS))
diff --git a/test/e2e/service_to_service_test.go b/test/e2e/service_to_service_test.go
index 3954dc804..beb64f1b1 100644
--- a/test/e2e/service_to_service_test.go
+++ b/test/e2e/service_to_service_test.go
@@ -132,7 +132,7 @@ func testProxyToHelloworld(t *testing.T, clients *test.Clients, helloworldURL *u
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.Retrying(spoof.MatchesAllOf(spoof.IsStatusOK, pkgTest.EventuallyMatchesBody(helloworldResponse)), http.StatusBadGateway)),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(helloworldResponse)),
 		"HTTPProxy",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/e2e/tagheader/tag_header_based_routing_test.go b/test/e2e/tagheader/tag_header_based_routing_test.go
index 1442c5e46..430ef63e5 100644
--- a/test/e2e/tagheader/tag_header_based_routing_test.go
+++ b/test/e2e/tagheader/tag_header_based_routing_test.go
@@ -120,7 +120,7 @@ func TestTagHeaderBasedRouting(t *testing.T) {
 				clients.KubeClient,
 				t.Logf,
 				objects.Service.Status.URL.URL(),
-				v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(tt.wantResponse))),
+				spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(tt.wantResponse)),
 				"WaitForSuccessfulResponse",
 				test.ServingFlags.ResolvableDomain,
 				test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/ha/ha.go b/test/ha/ha.go
index 74efaf853..22bf20366 100644
--- a/test/ha/ha.go
+++ b/test/ha/ha.go
@@ -68,7 +68,7 @@ func assertServiceEventuallyWorks(t *testing.T, clients *test.Clients, names tes
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(expectedText)),
 		"CheckEndpointToServeText",
 		resolvabledomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS),
diff --git a/test/upgrade/upgrade.go b/test/upgrade/upgrade.go
index 3f055bbea..73210af22 100644
--- a/test/upgrade/upgrade.go
+++ b/test/upgrade/upgrade.go
@@ -54,7 +54,7 @@ func assertServiceResourcesUpdated(t testing.TB, clients *test.Clients, names te
 		clients.KubeClient,
 		t.Logf,
 		url,
-		v1test.RetryingRouteInconsistency(spoof.MatchesAllOf(spoof.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedText))),
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(expectedText)),
 		"WaitForEndpointToServeText",
 		test.ServingFlags.ResolvableDomain,
 		test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)); err != nil {
diff --git a/test/v1/route.go b/test/v1/route.go
index 170a7b2a9..a39aa3e1a 100644
--- a/test/v1/route.go
+++ b/test/v1/route.go
@@ -26,7 +26,6 @@ import (
 	"knative.dev/pkg/ptr"
 	"knative.dev/pkg/reconciler"
 	"knative.dev/pkg/test/logging"
-	"knative.dev/pkg/test/spoof"
 	v1 "knative.dev/serving/pkg/apis/serving/v1"
 	rtesting "knative.dev/serving/pkg/testing/v1"
 	"knative.dev/serving/test"
@@ -124,14 +123,6 @@ func IsRouteFailed(r *v1.Route) (bool, error) {
 	return r.IsFailed(), nil
 }
 
-// RetryingRouteInconsistency retries common requests seen when creating a new route
-func RetryingRouteInconsistency(innerCheck spoof.ResponseChecker) spoof.ResponseChecker {
-	return func(resp *spoof.Response) (bool, error) {
-		// If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped.
-		return innerCheck(resp)
-	}
-}
-
 // AllRouteTrafficAtRevision will check the revision that route r is routing
 // traffic to and return true if 100% of the traffic is routing to revisionName.
 func AllRouteTrafficAtRevision(names test.ResourceNames) func(r *v1.Route) (bool, error) {
diff --git a/test/v1/service.go b/test/v1/service.go
index bf96d0c80..65a761f57 100644
--- a/test/v1/service.go
+++ b/test/v1/service.go
@@ -57,6 +57,21 @@ func validateCreatedServiceStatus(clients *test.Clients, names *test.ResourceNam
 	})
 }
 
+// GetServices gets a list of services.
+func GetServices(clients *test.Clients) (list *v1.ServiceList, err error) {
+	return list, reconciler.RetryTestErrors(func(int) (err error) {
+		list, err = clients.ServingClient.Services.List(context.Background(), metav1.ListOptions{})
+		return err
+	})
+}
+
+// DeleteService deletes a service.
+func DeleteService(clients *test.Clients, serviceName string) (err error) {
+	return reconciler.RetryTestErrors(func(int) (err error) {
+		return clients.ServingClient.Services.Delete(context.Background(), serviceName, metav1.DeleteOptions{})
+	})
+}
+
 // GetResourceObjects obtains the services resources from the k8s API server.
 func GetResourceObjects(clients *test.Clients, names test.ResourceNames) (*ResourceObjects, error) {
 	routeObject, err := clients.ServingClient.Routes.Get(context.Background(), names.Route, metav1.GetOptions{})
diff --git a/third_party/cert-manager-latest/net-certmanager.yaml b/third_party/cert-manager-latest/net-certmanager.yaml
index 7a137989a..330118649 100644
--- a/third_party/cert-manager-latest/net-certmanager.yaml
+++ b/third_party/cert-manager-latest/net-certmanager.yaml
@@ -18,7 +18,7 @@ metadata:
   # These are the permissions needed by the `cert-manager` `Certificate` implementation.
   name: knative-serving-certmanager
   labels:
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     serving.knative.dev/controller: "true"
     networking.knative.dev/certificate-provider: cert-manager
 rules:
@@ -49,7 +49,7 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: config.webhook.net-certmanager.networking.internal.knative.dev
   labels:
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
 webhooks:
   - admissionReviewVersions:
@@ -88,7 +88,7 @@ metadata:
   name: net-certmanager-webhook-certs
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
 
 ---
@@ -112,7 +112,7 @@ metadata:
   name: config-certmanager
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
 data:
   _example: |
@@ -159,7 +159,7 @@ metadata:
   name: net-certmanager-controller
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
 spec:
   selector:
@@ -171,14 +171,14 @@ spec:
         cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
       labels:
         app: net-certmanager-controller
-        serving.knative.dev/release: "v20210830-92cb1c13"
+        serving.knative.dev/release: "v20210908-dd2ff643"
     spec:
       serviceAccountName: controller
       containers:
         - name: controller
           # This is the Go import path for the binary that is containerized
           # and substituted here.
-          image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/controller@sha256:9869d8cf9458aa73b8448ae7af11b9d18c797307d7cbe6fcf30944b27764da38
+          image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/controller@sha256:af23e53aa6f0cc54e9ec78d751c6a35cdb23855287840d3fa287c15619727ee7
           resources:
             requests:
               cpu: 30m
@@ -216,7 +216,7 @@ kind: Service
 metadata:
   labels:
     app: net-certmanager-controller
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
   name: net-certmanager-controller
   namespace: knative-serving
@@ -253,7 +253,7 @@ metadata:
   name: net-certmanager-webhook
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
 spec:
   selector:
@@ -267,14 +267,14 @@ spec:
       labels:
         app: net-certmanager-webhook
         role: net-certmanager-webhook
-        serving.knative.dev/release: "v20210830-92cb1c13"
+        serving.knative.dev/release: "v20210908-dd2ff643"
     spec:
       serviceAccountName: controller
       containers:
         - name: webhook
           # This is the Go import path for the binary that is containerized
           # and substituted here.
-          image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/webhook@sha256:b26545e005631c96ae50922baf6fb492f01c2bfc33505f0d4646de06f67818de
+          image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/webhook@sha256:61883f82fe743dd7a2f044550640d6ac31da696da9e3f08a371b55b7d61d0928
           resources:
             requests:
               cpu: 20m
@@ -333,7 +333,7 @@ metadata:
   namespace: knative-serving
   labels:
     role: net-certmanager-webhook
-    serving.knative.dev/release: "v20210830-92cb1c13"
+    serving.knative.dev/release: "v20210908-dd2ff643"
     networking.knative.dev/certificate-provider: cert-manager
 spec:
   ports:
diff --git a/third_party/contour-latest/contour.yaml b/third_party/contour-latest/contour.yaml
index c03dbf234..e2cc30313 100644
--- a/third_party/contour-latest/contour.yaml
+++ b/third_party/contour-latest/contour.yaml
@@ -50,8 +50,9 @@ data:
     #   determine which XDS Server implementation to utilize in Contour.
     #   xds-server-type: contour
     #
-    # Specify the gateway-api Gateway Contour should watch.
+    # Specify the Gateway API configuration.
     # gateway:
+    #   controllerName: projectcontour.io/projectcontour/contour
     #   name: contour
     #   namespace: contour-external
     #
@@ -89,9 +90,18 @@ data:
     # leaderelection:
     #   configmap-name: leader-elect
     #   configmap-namespace: contour-external
+    ####
+    # ExternalName Services are disabled by default due to CVE-2021-XXXXX
+    # You can re-enable them by setting this setting to `true`.
+    # This is not recommended without understanding the security implications.
+    # Please see the advisory at https://github.com/projectcontour/contour/security/advisories/GHSA-5ph6-qq5x-7jwc for the details.
+    enableExternalNameService: true
+    ##
     ### Logging options
     # Default setting
     accesslog-format: envoy
+    # The default access log format is defined by Envoy but it can be customized by setting following variable.
+    # accesslog-format-string: "...\n"
     # To enable JSON logging in Envoy
     # accesslog-format: json
     # The default fields that will be logged are specified below.
@@ -145,6 +155,8 @@ data:
     #   Configure the number of additional ingress proxy hops from the
     #   right side of the x-forwarded-for HTTP header to trust.
     #   num-trusted-hops: 0
+    #   Configure the port used to access the Envoy Admin interface.
+    #   admin-port: 9001
     #
     # Configure an optional global rate limit service.
     # rateLimitService:
@@ -186,7 +198,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.5.0
   creationTimestamp: null
   name: extensionservices.projectcontour.io
   labels:
@@ -296,7 +308,7 @@ spec:
                   description: UpstreamValidation defines how to verify the backend service's certificate
                   properties:
                     caSecret:
-                      description: Name of the Kubernetes secret be used to validate the certificate presented by the backend
+                      description: Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend
                       type: string
                     subjectName:
                       description: Key which is expected to be present in the 'subjectAltName' of the presented certificate
@@ -443,7 +455,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.5.0
   creationTimestamp: null
   name: httpproxies.projectcontour.io
   labels:
@@ -522,6 +534,9 @@ spec:
                                 notexact:
                                   description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value.
                                   type: string
+                                notpresent:
+                                  description: NotPresent specifies that condition is true when the named header is not present. Note that setting NotPresent to false does not make the condition true if the named header is present.
+                                  type: boolean
                                 present:
                                   description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent.
                                   type: boolean
@@ -543,6 +558,9 @@ spec:
                       - name
                     type: object
                   type: array
+                ingressClassName:
+                  description: IngressClassName optionally specifies the ingress class to use for this HTTPProxy. This replaces the deprecated `kubernetes.io/ingress.class` annotation. For backwards compatibility, when that annotation is set, it is given precedence over this field.
+                  type: string
                 routes:
                   description: Routes are the ingress routes. If TCPProxy is present, Routes is ignored.
                   items:
@@ -583,6 +601,9 @@ spec:
                                 notexact:
                                   description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value.
                                   type: string
+                                notpresent:
+                                  description: NotPresent specifies that condition is true when the named header is not present. Note that setting NotPresent to false does not make the condition true if the named header is present.
+                                  type: boolean
                                 present:
                                   description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent.
                                   type: boolean
@@ -718,6 +739,49 @@ spec:
                                                 minLength: 1
                                                 type: string
                                             type: object
+                                          requestHeaderValueMatch:
+                                            description: RequestHeaderValueMatch defines a descriptor entry that's populated if the request's headers match a set of 1+ match criteria. The descriptor key is "header_match", and the descriptor value is static.
+                                            properties:
+                                              expectMatch:
+                                                default: true
+                                                description: ExpectMatch defines whether the request must positively match the match criteria in order to generate a descriptor entry (i.e. true), or not match the match criteria in order to generate a descriptor entry (i.e. false). The default is true.
+                                                type: boolean
+                                              headers:
+                                                description: Headers is a list of 1+ match criteria to apply against the request to determine whether to populate the descriptor entry or not.
+                                                items:
+                                                  description: HeaderMatchCondition specifies how to conditionally match against HTTP headers. The Name field is required, but only one of the remaining fields should be be provided.
+                                                  properties:
+                                                    contains:
+                                                      description: Contains specifies a substring that must be present in the header value.
+                                                      type: string
+                                                    exact:
+                                                      description: Exact specifies a string that the header value must be equal to.
+                                                      type: string
+                                                    name:
+                                                      description: Name is the name of the header to match against. Name is required. Header names are case insensitive.
+                                                      type: string
+                                                    notcontains:
+                                                      description: NotContains specifies a substring that must not be present in the header value.
+                                                      type: string
+                                                    notexact:
+                                                      description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value.
+                                                      type: string
+                                                    notpresent:
+                                                      description: NotPresent specifies that condition is true when the named header is not present. Note that setting NotPresent to false does not make the condition true if the named header is present.
+                                                      type: boolean
+                                                    present:
+                                                      description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent.
+                                                      type: boolean
+                                                  required:
+                                                    - name
+                                                  type: object
+                                                minItems: 1
+                                                type: array
+                                              value:
+                                                description: Value defines the value of the descriptor entry.
+                                                minLength: 1
+                                                type: string
+                                            type: object
                                         type: object
                                       minItems: 1
                                       type: array
@@ -948,7 +1012,7 @@ spec:
                               description: UpstreamValidation defines how to verify the backend service's certificate
                               properties:
                                 caSecret:
-                                  description: Name of the Kubernetes secret be used to validate the certificate presented by the backend
+                                  description: Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend
                                   type: string
                                 subjectName:
                                   description: Key which is expected to be present in the 'subjectAltName' of the presented certificate
@@ -1138,7 +1202,7 @@ spec:
                             description: UpstreamValidation defines how to verify the backend service's certificate
                             properties:
                               caSecret:
-                                description: Name of the Kubernetes secret be used to validate the certificate presented by the backend
+                                description: Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend
                                 type: string
                               subjectName:
                                 description: Key which is expected to be present in the 'subjectAltName' of the presented certificate
@@ -1162,7 +1226,7 @@ spec:
                   description: Virtualhost appears at most once. If it is present, the object is considered to be a "root" HTTPProxy.
                   properties:
                     authorization:
-                      description: This field configures an extension service to perform authorization for this virtual host. Authorization can only be configured on virtual hosts that have TLS enabled. If the TLS configuration requires client certificate /validation, the client certificate is always included in the authentication check request.
+                      description: This field configures an extension service to perform authorization for this virtual host. Authorization can only be configured on virtual hosts that have TLS enabled. If the TLS configuration requires client certificate validation, the client certificate is always included in the authentication check request.
                       properties:
                         authPolicy:
                           description: AuthPolicy sets a default authorization policy for client requests. This policy will be used unless overridden by individual routes.
@@ -1286,6 +1350,49 @@ spec:
                                               minLength: 1
                                               type: string
                                           type: object
+                                        requestHeaderValueMatch:
+                                          description: RequestHeaderValueMatch defines a descriptor entry that's populated if the request's headers match a set of 1+ match criteria. The descriptor key is "header_match", and the descriptor value is static.
+                                          properties:
+                                            expectMatch:
+                                              default: true
+                                              description: ExpectMatch defines whether the request must positively match the match criteria in order to generate a descriptor entry (i.e. true), or not match the match criteria in order to generate a descriptor entry (i.e. false). The default is true.
+                                              type: boolean
+                                            headers:
+                                              description: Headers is a list of 1+ match criteria to apply against the request to determine whether to populate the descriptor entry or not.
+                                              items:
+                                                description: HeaderMatchCondition specifies how to conditionally match against HTTP headers. The Name field is required, but only one of the remaining fields should be be provided.
+                                                properties:
+                                                  contains:
+                                                    description: Contains specifies a substring that must be present in the header value.
+                                                    type: string
+                                                  exact:
+                                                    description: Exact specifies a string that the header value must be equal to.
+                                                    type: string
+                                                  name:
+                                                    description: Name is the name of the header to match against. Name is required. Header names are case insensitive.
+                                                    type: string
+                                                  notcontains:
+                                                    description: NotContains specifies a substring that must not be present in the header value.
+                                                    type: string
+                                                  notexact:
+                                                    description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value.
+                                                    type: string
+                                                  notpresent:
+                                                    description: NotPresent specifies that condition is true when the named header is not present. Note that setting NotPresent to false does not make the condition true if the named header is present.
+                                                    type: boolean
+                                                  present:
+                                                    description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent.
+                                                    type: boolean
+                                                required:
+                                                  - name
+                                                type: object
+                                              minItems: 1
+                                              type: array
+                                            value:
+                                              description: Value defines the value of the descriptor entry.
+                                              minLength: 1
+                                              type: string
+                                          type: object
                                       type: object
                                     minItems: 1
                                     type: array
@@ -1345,14 +1452,15 @@ spec:
                       description: If present the fields describes TLS properties of the virtual host. The SNI names that will be matched on are described in fqdn, the tls.secretName secret must contain a certificate that itself contains a name that matches the FQDN.
                       properties:
                         clientValidation:
-                          description: "ClientValidation defines how to verify the client certificate when an external client establishes a TLS connection to Envoy. \n This setting: \n 1. Enables TLS client certificate validation. 2. Requires clients to present a TLS certificate (i.e. not optional validation). 3. Specifies how the client certificate will be validated."
+                          description: "ClientValidation defines how to verify the client certificate when an external client establishes a TLS connection to Envoy. \n This setting: \n 1. Enables TLS client certificate validation. 2. Specifies how the client certificate will be validated (i.e.    validation required or skipped). \n Note: Setting client certificate validation to be skipped should be only used in conjunction with an external authorization server that performs client validation as Contour will ensure client certificates are passed along."
                           properties:
                             caSecret:
-                              description: Name of a Kubernetes secret that contains a CA certificate bundle. The client certificate must validate against the certificates in the bundle.
+                              description: Name of a Kubernetes secret that contains a CA certificate bundle. The client certificate must validate against the certificates in the bundle. If specified and SkipClientCertValidation is true, client certificates will be required on requests.
                               minLength: 1
                               type: string
-                          required:
-                            - caSecret
+                            skipClientCertValidation:
+                              description: SkipClientCertValidation disables downstream client certificate validation. Defaults to false. This field is intended to be used in conjunction with external authorization in order to enable the external authorization server to validate client certificates. When this field is set to true, client certificates are requested but not verified by Envoy. If CACertificate is specified, client certificates are required on requests, but not verified. If external authorization is in use, they are presented to the external authorization server.
+                              type: boolean
                           type: object
                         enableFallbackCertificate:
                           description: EnableFallbackCertificate defines if the vhost should allow a default certificate to be applied which handles all requests which don't match the SNI defined in this vhost.
@@ -1553,7 +1661,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.5.0
   creationTimestamp: null
   name: tlscertificatedelegations.projectcontour.io
   labels:
@@ -1783,7 +1891,7 @@ rules:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: contour-certgen-v1.14.0
+  name: contour-certgen-v1.18.1
   namespace: contour-external
   labels:
     networking.knative.dev/ingress-provider: contour
@@ -1796,7 +1904,7 @@ spec:
     spec:
       containers:
         - name: contour
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           command:
             - contour
@@ -1903,13 +2011,28 @@ rules:
       - networking.x-k8s.io
     resources:
       - backendpolicies
+      - gatewayclasses
       - gateways
       - httproutes
+      - tcproutes
       - tlsroutes
+      - udproutes
     verbs:
       - get
       - list
       - watch
+  - apiGroups:
+      - networking.x-k8s.io
+    resources:
+      - backendpolicies/status
+      - gatewayclasses/status
+      - gateways/status
+      - httproutes/status
+      - tcproutes/status
+      - tlsroutes/status
+      - udproutes/status
+    verbs:
+      - update
   - apiGroups:
       - projectcontour.io
     resources:
@@ -2032,7 +2155,7 @@ spec:
             - --config-path=/config/contour.yaml
           command:
             - contour
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           name: contour
           ports:
@@ -2121,7 +2244,7 @@ spec:
           args:
             - envoy
             - shutdown-manager
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           lifecycle:
             preStop:
@@ -2137,6 +2260,9 @@ spec:
             initialDelaySeconds: 3
             periodSeconds: 10
           name: shutdown-manager
+          volumeMounts:
+            - name: envoy-admin
+              mountPath: /admin
         - args:
             - -c
             - /config/envoy.json
@@ -2145,7 +2271,7 @@ spec:
             - --log-level info
           command:
             - envoy
-          image: docker.io/envoyproxy/envoy:v1.17.1
+          image: docker.io/envoyproxy/envoy:v1.19.1
           imagePullPolicy: IfNotPresent
           name: envoy
           env:
@@ -2179,6 +2305,8 @@ spec:
             - name: envoycert
               mountPath: /certs
               readOnly: true
+            - name: envoy-admin
+              mountPath: /admin
           lifecycle:
             preStop:
               httpGet:
@@ -2198,7 +2326,7 @@ spec:
             - --envoy-key-file=/certs/tls.key
           command:
             - contour
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           name: envoy-initconfig
           volumeMounts:
@@ -2216,6 +2344,8 @@ spec:
       serviceAccountName: envoy
       terminationGracePeriodSeconds: 300
       volumes:
+        - name: envoy-admin
+          emptyDir: {}
         - name: envoy-config
           emptyDir: {}
         - name: envoycert
@@ -2275,8 +2405,9 @@ data:
     #   determine which XDS Server implementation to utilize in Contour.
     #   xds-server-type: contour
     #
-    # Specify the gateway-api Gateway Contour should watch.
+    # Specify the Gateway API configuration.
     # gateway:
+    #   controllerName: projectcontour.io/projectcontour/contour
     #   name: contour
     #   namespace: contour-internal
     #
@@ -2314,9 +2445,18 @@ data:
     # leaderelection:
     #   configmap-name: leader-elect
     #   configmap-namespace: contour-internal
+    ####
+    # ExternalName Services are disabled by default due to CVE-2021-XXXXX
+    # You can re-enable them by setting this setting to `true`.
+    # This is not recommended without understanding the security implications.
+    # Please see the advisory at https://github.com/projectcontour/contour/security/advisories/GHSA-5ph6-qq5x-7jwc for the details.
+    enableExternalNameService: true
+    ##
     ### Logging options
     # Default setting
     accesslog-format: envoy
+    # The default access log format is defined by Envoy but it can be customized by setting following variable.
+    # accesslog-format-string: "...\n"
     # To enable JSON logging in Envoy
     # accesslog-format: json
     # The default fields that will be logged are specified below.
@@ -2370,6 +2510,8 @@ data:
     #   Configure the number of additional ingress proxy hops from the
     #   right side of the x-forwarded-for HTTP header to trust.
     #   num-trusted-hops: 0
+    #   Configure the port used to access the Envoy Admin interface.
+    #   admin-port: 9001
     #
     # Configure an optional global rate limit service.
     # rateLimitService:
@@ -2450,7 +2592,7 @@ rules:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: contour-certgen-v1.14.0
+  name: contour-certgen-v1.18.1
   namespace: contour-internal
   labels:
     networking.knative.dev/ingress-provider: contour
@@ -2463,7 +2605,7 @@ spec:
     spec:
       containers:
         - name: contour
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           command:
             - contour
@@ -2575,7 +2717,7 @@ spec:
             - --config-path=/config/contour.yaml
           command:
             - contour
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           name: contour
           ports:
@@ -2664,7 +2806,7 @@ spec:
           args:
             - envoy
             - shutdown-manager
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           lifecycle:
             preStop:
@@ -2680,6 +2822,9 @@ spec:
             initialDelaySeconds: 3
             periodSeconds: 10
           name: shutdown-manager
+          volumeMounts:
+            - name: envoy-admin
+              mountPath: /admin
         - args:
             - -c
             - /config/envoy.json
@@ -2688,7 +2833,7 @@ spec:
             - --log-level info
           command:
             - envoy
-          image: docker.io/envoyproxy/envoy:v1.17.1
+          image: docker.io/envoyproxy/envoy:v1.19.1
           imagePullPolicy: IfNotPresent
           name: envoy
           env:
@@ -2722,6 +2867,8 @@ spec:
             - name: envoycert
               mountPath: /certs
               readOnly: true
+            - name: envoy-admin
+              mountPath: /admin
           lifecycle:
             preStop:
               httpGet:
@@ -2741,7 +2888,7 @@ spec:
             - --envoy-key-file=/certs/tls.key
           command:
             - contour
-          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:792af35e7250d38bd8c42862c907f00fd13e89e08d8e164929da859eee180f75
+          image: gcr.io/knative-nightly/github.com/projectcontour/contour/cmd/contour@sha256:4421d04bd44212495295c0ab744b6b912f9beb985b5bee1ffc56741cadee1af0
           imagePullPolicy: IfNotPresent
           name: envoy-initconfig
           volumeMounts:
@@ -2759,6 +2906,8 @@ spec:
       serviceAccountName: envoy
       terminationGracePeriodSeconds: 300
       volumes:
+        - name: envoy-admin
+          emptyDir: {}
         - name: envoy-config
           emptyDir: {}
         - name: envoycert
diff --git a/third_party/contour-latest/net-contour.yaml b/third_party/contour-latest/net-contour.yaml
index e28be42fa..7b553522e 100644
--- a/third_party/contour-latest/net-contour.yaml
+++ b/third_party/contour-latest/net-contour.yaml
@@ -33,7 +33,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: contour
-    serving.knative.dev/release: "v20210826-0fc8f4d7"
+    serving.knative.dev/release: "v20210908-2aa41854"
 data:
   _example: |
     ################################
@@ -103,7 +103,7 @@ spec:
         - name: controller
           # This is the Go import path for the binary that is containerized
           # and substituted here.
-          image: gcr.io/knative-nightly/knative.dev/net-contour/cmd/controller@sha256:4ba33b6789e3603ea5a853ea7bb2781557668ed04aef8f84070c0d34a94d0d76
+          image: gcr.io/knative-nightly/knative.dev/net-contour/cmd/controller@sha256:edf9301ee0be34bc422b492faf3ba66b7e066b4230bf2249fd0c331f203d9820
           resources:
             requests:
               cpu: 40m
diff --git a/third_party/istio-latest/net-istio.yaml b/third_party/istio-latest/net-istio.yaml
index 008c16267..5d7c4b18f 100644
--- a/third_party/istio-latest/net-istio.yaml
+++ b/third_party/istio-latest/net-istio.yaml
@@ -1,4 +1,4 @@
-# Generated when HEAD was f9454e196ca2485c81c3c0067375d1666ea0aafd
+# Generated when HEAD was 6793a7353f4c20c3d912a86280d9eef7b80932a4
 #
 # Copyright 2019 The Knative Authors
 #
@@ -20,7 +20,7 @@ metadata:
   # These are the permissions needed by the Istio Ingress implementation.
   name: knative-serving-istio
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     serving.knative.dev/controller: "true"
     networking.knative.dev/ingress-provider: istio
 rules:
@@ -50,7 +50,7 @@ metadata:
   name: knative-ingress-gateway
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -87,7 +87,7 @@ metadata:
   name: knative-local-gateway
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -106,7 +106,7 @@ metadata:
   name: knative-local-gateway
   namespace: istio-system
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
     experimental.istio.io/disable-gateway-port-translation: "true"
 spec:
@@ -139,7 +139,7 @@ metadata:
   name: config-istio
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 data:
   _example: |
@@ -200,7 +200,7 @@ metadata:
   name: "webhook"
   namespace: "knative-serving"
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -216,7 +216,7 @@ metadata:
   name: "domainmapping-webhook"
   namespace: "knative-serving"
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -232,7 +232,7 @@ metadata:
   name: "net-istio-webhook"
   namespace: "knative-serving"
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -263,7 +263,7 @@ metadata:
   name: net-istio-controller
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -279,14 +279,14 @@ spec:
         sidecar.istio.io/inject: "false"
       labels:
         app: net-istio-controller
-        serving.knative.dev/release: "v20210830-f9454e19"
+        serving.knative.dev/release: "v20210908-6793a735"
     spec:
       serviceAccountName: controller
       containers:
         - name: controller
           # This is the Go import path for the binary that is containerized
           # and substituted here.
-          image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/controller@sha256:c45bab239ec732191592960f36d771482875b39c364299893787e29fe5aeb804
+          image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/controller@sha256:2f398fac34d00c0db1ff7212fd226cd7b315e726def887436600516dc7dc41a2
           resources:
             requests:
               cpu: 30m
@@ -343,7 +343,7 @@ metadata:
   name: net-istio-webhook
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -357,14 +357,14 @@ spec:
       labels:
         app: net-istio-webhook
         role: net-istio-webhook
-        serving.knative.dev/release: "v20210830-f9454e19"
+        serving.knative.dev/release: "v20210908-6793a735"
     spec:
       serviceAccountName: controller
       containers:
         - name: webhook
           # This is the Go import path for the binary that is containerized
           # and substituted here.
-          image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/webhook@sha256:6888de59803678ae597eec6bcb3fd001349ffb0ac54050d106c19071c7a79106
+          image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/webhook@sha256:63bd5e0bdff6df2e05c5d04d2638fc2b56bb229246be8c569bc7edb9a2466e74
           resources:
             requests:
               cpu: 20m
@@ -417,7 +417,7 @@ metadata:
   name: net-istio-webhook-certs
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 
 ---
@@ -442,7 +442,7 @@ metadata:
   namespace: knative-serving
   labels:
     role: net-istio-webhook
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 spec:
   ports:
@@ -479,7 +479,7 @@ kind: MutatingWebhookConfiguration
 metadata:
   name: webhook.istio.networking.internal.knative.dev
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 webhooks:
   - admissionReviewVersions:
@@ -516,7 +516,7 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: config.webhook.istio.networking.internal.knative.dev
   labels:
-    serving.knative.dev/release: "v20210830-f9454e19"
+    serving.knative.dev/release: "v20210908-6793a735"
     networking.knative.dev/ingress-provider: istio
 webhooks:
   - admissionReviewVersions:
diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml
index af34ae97f..ace6fbaff 100644
--- a/third_party/kourier-latest/kourier.yaml
+++ b/third_party/kourier-latest/kourier.yaml
@@ -18,7 +18,7 @@ metadata:
   name: kourier-system
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 
 ---
 # Copyright 2020 The Knative Authors
@@ -42,7 +42,7 @@ metadata:
   namespace: kourier-system
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 data:
   envoy-bootstrap.yaml: |
     dynamic_resources:
@@ -145,7 +145,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 data:
   _example: |
     ################################
@@ -190,7 +190,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -199,7 +199,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 rules:
   - apiGroups: [""]
     resources: ["events"]
@@ -226,7 +226,7 @@ metadata:
   name: net-kourier
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -258,7 +258,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 spec:
   replicas: 1
   selector:
@@ -270,7 +270,7 @@ spec:
         app: net-kourier-controller
     spec:
       containers:
-        - image: gcr.io/knative-nightly/knative.dev/net-kourier/cmd/kourier@sha256:32e25873aabb392f6194141453f771b82e0c4a43851bf70180be3ca92abd14fc
+        - image: gcr.io/knative-nightly/knative.dev/net-kourier/cmd/kourier@sha256:4939ab1b75c42ad179f835ed89ea33808c4bec07687361134c35c21212a8b700
           name: controller
           env:
             - name: CERTS_SECRET_NAMESPACE
@@ -309,7 +309,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 spec:
   ports:
     - name: grpc-xds
@@ -342,7 +342,7 @@ metadata:
   namespace: kourier-system
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 spec:
   strategy:
     type: RollingUpdate
@@ -417,7 +417,7 @@ metadata:
   namespace: kourier-system
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 spec:
   ports:
     - name: http2
@@ -439,7 +439,7 @@ metadata:
   namespace: kourier-system
   labels:
     networking.knative.dev/ingress-provider: kourier
-    serving.knative.dev/release: "v20210901-8b94798b"
+    serving.knative.dev/release: "v20210908-1f4223a8"
 spec:
   ports:
     - name: http2
diff --git a/vendor/knative.dev/pkg/test/spoof/spoof.go b/vendor/knative.dev/pkg/test/spoof/spoof.go
index aa3a90f1a..f2baa4edf 100644
--- a/vendor/knative.dev/pkg/test/spoof/spoof.go
+++ b/vendor/knative.dev/pkg/test/spoof/spoof.go
@@ -68,6 +68,14 @@ type ResponseChecker func(resp *Response) (done bool, err error)
 // If an error should be retried, it should return true and the wrapped error to explain why to retry.
 type ErrorRetryChecker func(e error) (retry bool, err error)
 
+// ResponseRetryChecker is used to determine if a response should be retried or not.
+// If a response should be retried, it should return true and an error to explain why to retry.
+//
+// This is distinct from ResponseChecker in that it shall be used to retry responses,
+// where the HTTP request was technically successful (it returned something) but indicates
+// an error (e.g. the overload page of a loadbalancer).
+type ResponseRetryChecker func(resp *Response) (retry bool, err error)
+
 // SpoofingClient is a minimal HTTP client wrapper that spoofs the domain of requests
 // for non-resolvable domains.
 type SpoofingClient struct {
@@ -147,16 +155,16 @@ func ResolveEndpoint(ctx context.Context, kubeClientset kubernetes.Interface, do
 // Do dispatches to the underlying http.Client.Do, spoofing domains as needed
 // and transforming the http.Response into a spoof.Response.
 // Each response is augmented with "ZipkinTraceID" header that identifies the zipkin trace corresponding to the request.
-func (sc *SpoofingClient) Do(req *http.Request, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
+func (sc *SpoofingClient) Do(req *http.Request, errorRetryCheckers ...interface{}) (*Response, error) {
 	return sc.Poll(req, func(*Response) (bool, error) { return true, nil }, errorRetryCheckers...)
 }
 
 // Poll executes an http request until it satisfies the inState condition or, if there's an error,
 // none of the error retry checkers permit a retry.
 // If no retry checkers are specified `DefaultErrorRetryChecker` will be used.
-func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
-	if len(errorRetryCheckers) == 0 {
-		errorRetryCheckers = []ErrorRetryChecker{DefaultErrorRetryChecker}
+func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker, checkers ...interface{}) (*Response, error) {
+	if len(checkers) == 0 {
+		checkers = []interface{}{ErrorRetryChecker(DefaultErrorRetryChecker)}
 	}
 
 	var resp *Response
@@ -166,11 +174,13 @@ func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker, error
 		defer span.End()
 		rawResp, err := sc.Client.Do(req.WithContext(traceContext))
 		if err != nil {
-			for _, checker := range errorRetryCheckers {
-				retry, newErr := checker(err)
-				if retry {
-					sc.Logf("Retrying %s: %v", req.URL.String(), newErr)
-					return false, nil
+			for _, checker := range checkers {
+				if ec, ok := checker.(ErrorRetryChecker); ok {
+					retry, newErr := ec(err)
+					if retry {
+						sc.Logf("Retrying %s: %v", req.URL.String(), newErr)
+						return false, nil
+					}
 				}
 			}
 			sc.Logf("NOT Retrying %s: %v", req.URL.String(), err)
@@ -190,6 +200,20 @@ func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker, error
 			Header:     rawResp.Header,
 			Body:       body,
 		}
+
+		// This is distinct from inState in that it allows us to uniformly check for
+		// error responses to retry HTTP requests that have technically been successful,
+		// but haven't reached their destination (e.g. got a loadbalancer overload page).
+		for _, checker := range checkers {
+			if rc, ok := checker.(ResponseRetryChecker); ok {
+				retry, newErr := rc(resp)
+				if retry {
+					sc.Logf("Retrying %s: %v", req.URL.String(), newErr)
+					return false, nil
+				}
+			}
+		}
+
 		return inState(resp)
 	})
 
@@ -295,8 +319,8 @@ func (sc *SpoofingClient) endpointState(
 	return f(req, inState)
 }
 
-func (sc *SpoofingClient) Check(req *http.Request, inState ResponseChecker, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
-	resp, err := sc.Do(req, errorRetryCheckers...)
+func (sc *SpoofingClient) Check(req *http.Request, inState ResponseChecker, checkers ...interface{}) (*Response, error) {
+	resp, err := sc.Do(req, checkers...)
 	if err != nil {
 		return nil, err
 	}