Repository: knative/serving
Pull request #9233 (2020-08-31 19:03:57)

Forwarder can forward metrics via IP address
<!--
Request Prow to automatically lint any go code in this PR:

/lint
-->

For #2930. The feature track doc: https://docs.google.com/document/d/1FyjB1GIJW259nMaNOc7nlQVjvgAZHlQ-fD6tbfUsaz8/edit#heading=h.n8a530nnrb


<!-- Please include the 'why' behind your changes if no issue exists -->
## Proposed Changes

* Enrich the forwarder with the ability to forward metrics to other autoscaler via pod IP address.

**Release Note**

<!-- Enter your extended release note in the below block. If the PR requires
additional action from users switching to the new release, include the string
"action required". If no release note is required, write "NONE". -->

```release-note
NONE
```


knative-prow-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/knative/serving/pull/9233#" title="Author self-approved">yanweiguo</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **markusthoemmes**
You can assign the PR to them by writing `/assign @markusthoemmes` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=knative%2Fserving).

<details open>
Needs approval from an approver in each of these files:

- **[pkg/autoscaler/OWNERS](https://github.com/knative/serving/blob/master/pkg/autoscaler/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["markusthoemmes"]} -->
yanweiguo: /assign @vagababov @julz 
knative-metrics-robot: The following is the coverage report on the affected files.
Say `/test pull-knative-serving-go-coverage` to re-run this coverage report

File | Old Coverage | New Coverage | Delta
---- |:------------:|:------------:|:-----:
[pkg/autoscaler/statforwarder/forwarder.go](https://storage.cloud.google.com/knative-prow/pr-logs/pull/knative_serving/9233/pull-knative-serving-go-coverage/1300509765968859136/artifacts/line-cov.html#file0) | 92.2% | 86.0% | -6.2


Patch:
diff --git a/.github/workflows/knative-go-build.yaml b/.github/workflows/knative-go-build.yaml
new file mode 100644
index 000000000..81b9b0084
--- /dev/null
+++ b/.github/workflows/knative-go-build.yaml
@@ -0,0 +1,49 @@
+# Copyright 2020 The Knative Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build
+
+on:
+  push:
+    branches: [ 'master', 'release-*' ]
+  pull_request:
+    branches: [ 'master', 'release-*' ]
+
+jobs:
+
+  build:
+    name: Build
+    strategy:
+      matrix:
+        go-version: [1.15.x]
+        platform: [ubuntu-latest]
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Build
+        run: go build -v ./...
diff --git a/.github/workflows/knative-stale.yaml b/.github/workflows/knative-stale.yaml
index 9eb667bff..795eadf33 100644
--- a/.github/workflows/knative-stale.yaml
+++ b/.github/workflows/knative-stale.yaml
@@ -1,7 +1,7 @@
 name: 'Close stale'
 on:
   schedule:
-  - cron: '* * * * *' # Runs everyday
+  - cron: '0 1 * * *' # Everyday at 1AM
 
 jobs:
   stale:
@@ -14,7 +14,7 @@ jobs:
         stale-issue-message: |-
           This issue is stale because it has been open for 90 days with no
           activity. It will automatically close after 30 more days of
-          inactivity. Reopen the issue with `/reopen`.Mark the issue as
+          inactivity. Reopen the issue with `/reopen`. Mark the issue as
           fresh by adding the comment `/remove-lifecycle stale`.
         stale-issue-label: 'lifecycle/stale'
         exempt-issue-labels: 'lifecycle/frozen'
@@ -22,7 +22,7 @@ jobs:
         stale-pr-message: |-
           This Pull Request is stale because it has been open for 90 days with
           no activity. It will automatically close after 30 more days of
-          inactivity. Reopen with `/reopen`.Mark as fresh by adding the
+          inactivity. Reopen with `/reopen`. Mark as fresh by adding the
           comment `/remove-lifecycle stale`.
         stale-pr-label: 'lifecycle/stale'
         exempt-pr-labels: 'lifecycle/frozen'
diff --git a/.github/workflows/knative-style.yaml b/.github/workflows/knative-style.yaml
new file mode 100644
index 000000000..e6dca76c3
--- /dev/null
+++ b/.github/workflows/knative-style.yaml
@@ -0,0 +1,57 @@
+# Copyright 2020 The Knative Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Code Style
+
+on:
+  push:
+    branches: [ 'master', 'release-*' ]
+  pull_request:
+    branches: [ 'master', 'release-*' ]
+
+jobs:
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Set up Go 1.15.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15.x
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      # TODO: add prettier step
+      # TODO: add goimports step
+
+      - name: Go Format
+        shell: bash
+        run: |
+          gofmt -s -w $(find -path './vendor' -prune -o -path './third_party' -prune -o -type f -name '*.go' -print)
+
+      - name: Verify
+        shell: bash
+        run: |
+          if [[ $(git diff-index --name-only HEAD --) ]]; then
+              echo "Found diffs in:"
+              git diff-index --name-only HEAD --
+              echo "${{ github.repository }} is out of style. Please run go fmt."
+              exit 1
+          fi
+          echo "${{ github.repository }} is formatted correctly."
diff --git a/.github/workflows/knative-verify.yaml b/.github/workflows/knative-verify.yaml
new file mode 100644
index 000000000..6128969f3
--- /dev/null
+++ b/.github/workflows/knative-verify.yaml
@@ -0,0 +1,74 @@
+# Copyright 2020 The Knative Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Verify
+
+on:
+  push:
+    branches: [ 'master', 'release-*' ]
+  pull_request:
+    branches: [ 'master', 'release-*' ]
+
+jobs:
+
+  verify:
+    name: Verify Deps and Codegen
+    strategy:
+      matrix:
+        go-version: [1.15.x]
+        platform: [ubuntu-latest]
+
+    runs-on: ${{ matrix.platform }}
+
+    env:
+      GOPATH: ${{ github.workspace }}
+
+    steps:
+
+    - name: Set up Go ${{ matrix.go-version }}
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+      id: go
+
+    - name: Install Dependencies
+      run: |
+        go get github.com/google/ko/cmd/ko
+        go get github.com/google/go-licenses
+
+    - name: Check out code onto GOPATH
+      uses: actions/checkout@v2
+      with:
+        path: ./src/knative.dev/${{ github.event.repository.name }}
+
+    - name: Update Codegen
+      shell: bash
+      run: |
+        pushd ./src/knative.dev/${{ github.event.repository.name }}
+        ./hack/update-codegen.sh
+        popd
+
+    - name: Verify
+      shell: bash
+      run: |
+        pushd ./src/knative.dev/${{ github.event.repository.name }}
+        if [[ -z "$(git status --porcelain)" ]]; then
+            echo "${{ github.repository }} up to date."
+        else
+            repoDiff=$(git diff-index --name-only HEAD --)
+            echo "Found diffs in: $repoDiff"
+            echo "${{ github.repository }} is out of date. Please run hack/update-codegen.sh"
+            exit 1
+        fi
+        popd
diff --git a/config/core/deployments/activator.yaml b/config/core/deployments/activator.yaml
index af55ede92..4f4f1cbaf 100644
--- a/config/core/deployments/activator.yaml
+++ b/config/core/deployments/activator.yaml
@@ -87,14 +87,21 @@ spec:
         - name: h2c
           containerPort: 8013
 
-        readinessProbe: &probe
+        readinessProbe:
           httpGet:
             port: 8012
             httpHeaders:
             - name: k-kubelet-probe
               value: "activator"
           failureThreshold: 12
-        livenessProbe: *probe
+        livenessProbe:
+          httpGet:
+            port: 8012
+            httpHeaders:
+            - name: k-kubelet-probe
+              value: "activator"
+          failureThreshold: 12
+          initialDelaySeconds: 15
 
       # The activator (often) sits on the dataplane, and may proxy long (e.g.
       # streaming, websockets) requests.  We give a long grace period for the
diff --git a/config/core/deployments/autoscaler.yaml b/config/core/deployments/autoscaler.yaml
index b33b4387c..69688c2a0 100644
--- a/config/core/deployments/autoscaler.yaml
+++ b/config/core/deployments/autoscaler.yaml
@@ -82,13 +82,19 @@ spec:
         - name: websocket
           containerPort: 8080
 
-        readinessProbe: &probe
+        readinessProbe:
           httpGet:
             port: 8080
             httpHeaders:
             - name: k-kubelet-probe
               value: "autoscaler"
-        livenessProbe: *probe
+        livenessProbe:
+          httpGet:
+            port: 8080
+            httpHeaders:
+            - name: k-kubelet-probe
+              value: "autoscaler"
+          failureThreshold: 6
 
 ---
 apiVersion: v1
diff --git a/config/core/deployments/webhook.yaml b/config/core/deployments/webhook.yaml
index 836bde85a..a9d41868d 100644
--- a/config/core/deployments/webhook.yaml
+++ b/config/core/deployments/webhook.yaml
@@ -86,7 +86,7 @@ spec:
         - name: https-webhook
           containerPort: 8443
 
-        readinessProbe: &probe
+        readinessProbe:
           periodSeconds: 1
           httpGet:
             scheme: HTTPS
@@ -94,7 +94,15 @@ spec:
             httpHeaders:
             - name: k-kubelet-probe
               value: "webhook"
-        livenessProbe: *probe
+        livenessProbe:
+          periodSeconds: 1
+          httpGet:
+            scheme: HTTPS
+            port: 8443
+            httpHeaders:
+            - name: k-kubelet-probe
+              value: "webhook"
+          failureThreshold: 6
 
       # Our webhook should gracefully terminate by lame ducking first, set this to a sufficiently
       # high value that we respect whatever value it has configured for the lame duck grace period.
diff --git a/config/post-install/default-domain.yaml b/config/post-install/default-domain.yaml
index d6ae9c144..733428bc1 100644
--- a/config/post-install/default-domain.yaml
+++ b/config/post-install/default-domain.yaml
@@ -38,10 +38,13 @@ spec:
         ports:
         - name: http
           containerPort: 8080
-        readinessProbe: &probe
+        readinessProbe:
           httpGet:
             port: 8080
-        livenessProbe: *probe
+        livenessProbe:
+          httpGet:
+            port: 8080
+          failureThreshold: 6
         env:
         - name: POD_NAME
           valueFrom:
diff --git a/go.mod b/go.mod
index bf7fb8b6b..388eafb4b 100644
--- a/go.mod
+++ b/go.mod
@@ -31,10 +31,10 @@ require (
 	k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
 	k8s.io/code-generator v0.18.6
 	k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29
-	knative.dev/caching v0.0.0-20200828162208-ccf6aa330912
-	knative.dev/networking v0.0.0-20200828165907-5809e67909aa
-	knative.dev/pkg v0.0.0-20200831094608-e9da10bc8a18
-	knative.dev/test-infra v0.0.0-20200828211307-9d4372c9b1c7
+	knative.dev/caching v0.0.0-20200831164016-8993b9e0e6f6
+	knative.dev/networking v0.0.0-20200901153316-80398434dcd4
+	knative.dev/pkg v0.0.0-20200901172015-edd1b5e0bebf
+	knative.dev/test-infra v0.0.0-20200831235415-fac473dda98b
 )
 
 replace (
diff --git a/go.sum b/go.sum
index 3b58d5f4a..68ae9b4ad 100644
--- a/go.sum
+++ b/go.sum
@@ -1797,12 +1797,12 @@ k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 h1:v8ud2Up6QK1lNOKFgiIVrZdMg7Mpm
 k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg=
 knative.dev/caching v0.0.0-20200116200605-67bca2c83dfa/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg=
-knative.dev/caching v0.0.0-20200828162208-ccf6aa330912 h1:Xf7jOtNExPVZzGjGqhG2bcXe7u053T+w0dJvmAVwd6s=
-knative.dev/caching v0.0.0-20200828162208-ccf6aa330912/go.mod h1:YyZ0aH2jK3h1j9UeyNycr3f28F0hXk4+yIy24hItzcs=
+knative.dev/caching v0.0.0-20200831164016-8993b9e0e6f6 h1:tS93pEA8jCt8onqnXE3LoE1X6qTHdZVJ3QgF6UoE9jg=
+knative.dev/caching v0.0.0-20200831164016-8993b9e0e6f6/go.mod h1:5V2/0AagUm0RX4CsCDOY9FwLRnnF2c5IaaA+kqXbqAc=
 knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g=
 knative.dev/eventing-contrib v0.11.2/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g=
-knative.dev/networking v0.0.0-20200828165907-5809e67909aa h1:xBcjIPQBBcAWrI3rx4hS2MuxTB17D/KTeFpz+tYz3jM=
-knative.dev/networking v0.0.0-20200828165907-5809e67909aa/go.mod h1:Qkx7FwNtPTQ5MMrhSoSOOYQelIs+2m8zF3pKf1OiidA=
+knative.dev/networking v0.0.0-20200901153316-80398434dcd4 h1:hdZOOdXg94MiBVZWJdgjO/JKoiC8kIdoMI9ChHv/sQc=
+knative.dev/networking v0.0.0-20200901153316-80398434dcd4/go.mod h1:9ZMxo5uevxp5T+e7djbo+jH4KXK6Lja9rPS0u86+GHw=
 knative.dev/pkg v0.0.0-20191101194912-56c2594e4f11/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q=
 knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q=
 knative.dev/pkg v0.0.0-20200207155214-fef852970f43/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q=
@@ -1811,10 +1811,10 @@ knative.dev/pkg v0.0.0-20200505191044-3da93ebb24c2/go.mod h1:Q6sL35DdGs8hIQZKdaC
 knative.dev/pkg v0.0.0-20200515002500-16d7b963416f/go.mod h1:tMOHGbxtRz8zYFGEGpV/bpoTEM1o89MwYFC4YJXl3GY=
 knative.dev/pkg v0.0.0-20200528142800-1c6815d7e4c9/go.mod h1:QgNZTxnwpB/oSpNcfnLVlw+WpEwwyKAvJlvR3hgeltA=
 knative.dev/pkg v0.0.0-20200711004937-22502028e31a/go.mod h1:AqAJV6rYi8IGikDjJ/9ZQd9qKdkXVlesVnVjwx62YB8=
-knative.dev/pkg v0.0.0-20200827231407-8d478d55c0e6 h1:Wj8q6aj02ht5vimP6wLKfxG3I50v637wXxoyt0oLnTE=
-knative.dev/pkg v0.0.0-20200827231407-8d478d55c0e6/go.mod h1:vOVUMtkIWK+hq1+KwlLYBSYVjz5SNIP1vMj3/5SNGZw=
 knative.dev/pkg v0.0.0-20200831094608-e9da10bc8a18 h1:BJTbbN0/J0BY76tzHE6/oko4lw6GEeahMrZUfigySyc=
 knative.dev/pkg v0.0.0-20200831094608-e9da10bc8a18/go.mod h1:vOVUMtkIWK+hq1+KwlLYBSYVjz5SNIP1vMj3/5SNGZw=
+knative.dev/pkg v0.0.0-20200901172015-edd1b5e0bebf h1:4sLAMPCGqJVocPMIGNwrh1NYsTbmefvWudnOKl5UZLo=
+knative.dev/pkg v0.0.0-20200901172015-edd1b5e0bebf/go.mod h1:q+4+Cm768P6vsAvsD9J+cZ1hoy4aHyHSfRTvaFyPd3g=
 knative.dev/test-infra v0.0.0-20200407185800-1b88cb3b45a5/go.mod h1:xcdUkMJrLlBswIZqL5zCuBFOC22WIPMQoVX1L35i0vQ=
 knative.dev/test-infra v0.0.0-20200505052144-5ea2f705bb55/go.mod h1:WqF1Azka+FxPZ20keR2zCNtiQA1MP9ZB4BH4HuI+SIU=
 knative.dev/test-infra v0.0.0-20200513011557-d03429a76034/go.mod h1:aMif0KXL4g19YCYwsy4Ocjjz5xgPlseYV+B95Oo4JGE=
@@ -1822,10 +1822,10 @@ knative.dev/test-infra v0.0.0-20200519015156-82551620b0a9/go.mod h1:A5b2OAXTOeHT
 knative.dev/test-infra v0.0.0-20200707183444-aed09e56ddc7/go.mod h1:RjYAhXnZqeHw9+B0zsbqSPlae0lCvjekO/nw5ZMpLCs=
 knative.dev/test-infra v0.0.0-20200825022047-cb4bb218c5e5 h1:iqbpe5rccMW2r1Mu1Prw4gJ9xi7yqahmtxzckTUHBeI=
 knative.dev/test-infra v0.0.0-20200825022047-cb4bb218c5e5/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
-knative.dev/test-infra v0.0.0-20200828024406-bf34ea55d769 h1:4F657CFFjQ81XPe/hOSj2OoCA9G0GUVmgW/5FWAy6vM=
-knative.dev/test-infra v0.0.0-20200828024406-bf34ea55d769/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
 knative.dev/test-infra v0.0.0-20200828211307-9d4372c9b1c7 h1:yIzao6i9Hu51SCdpyqBEnhpt7G12FqFrxHK6ZnQnu8o=
 knative.dev/test-infra v0.0.0-20200828211307-9d4372c9b1c7/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
+knative.dev/test-infra v0.0.0-20200831235415-fac473dda98b h1:jafH61XDC6EkogEyR70h/sjYLx2Nf3qKVzPP2MCqEQ4=
+knative.dev/test-infra v0.0.0-20200831235415-fac473dda98b/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
diff --git a/pkg/apis/serving/v1/revision_lifecycle.go b/pkg/apis/serving/v1/revision_lifecycle.go
index 0130bf30a..95347d3c7 100644
--- a/pkg/apis/serving/v1/revision_lifecycle.go
+++ b/pkg/apis/serving/v1/revision_lifecycle.go
@@ -194,8 +194,12 @@ func (rs *RevisionStatus) PropagateAutoscalerStatus(ps *av1alpha1.PodAutoscalerS
 		// See #8922 for details. When we try to scale to 0, we force the Deployment's
 		// Progress status to become `true`, since successful scale down means
 		// progress has been achieved.
+		// There's the possibility of the revision reconciler reconciling PA before
+		// the ServiceName is populated, and therefore even though we will mark
+		// ScaleTargetInitialized down the road, we would have marked resources
+		// unavailable here, and have no way of recovering later.
 		// If the ResourcesAvailable is already false, don't override the message.
-		if !ps.IsScaleTargetInitialized() && resUnavailable {
+		if !ps.IsScaleTargetInitialized() && resUnavailable && ps.ServiceName != "" {
 			rs.MarkResourcesAvailableFalse(ReasonProgressDeadlineExceeded,
 				"Initial scale was never achieved")
 		}
diff --git a/pkg/autoscaler/statforwarder/forwarder.go b/pkg/autoscaler/statforwarder/forwarder.go
index f6b850a4a..a05837f4c 100644
--- a/pkg/autoscaler/statforwarder/forwarder.go
+++ b/pkg/autoscaler/statforwarder/forwarder.go
@@ -18,9 +18,11 @@ package statforwarder
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
+	gorillawebsocket "github.com/gorilla/websocket"
 	"go.uber.org/zap"
 
 	coordinationv1 "k8s.io/api/coordination/v1"
@@ -39,6 +41,8 @@ import (
 	"knative.dev/pkg/controller"
 	"knative.dev/pkg/hash"
 	"knative.dev/pkg/system"
+	"knative.dev/pkg/websocket"
+	asmetrics "knative.dev/serving/pkg/autoscaler/metrics"
 )
 
 const (
@@ -49,27 +53,43 @@ const (
 	retryInterval      = 100 * time.Millisecond
 )
 
+// statProcessor is a function to process a single StatMessage.
+type statProcessor func(sm asmetrics.StatMessage)
+
+// bucketProcessor includes the information about how to process
+// the StatMessage owned by a bucket.
+type bucketProcessor struct {
+	// holder is the HolderIdentity for a bucket from the Lease.
+	holder string
+	// conn is the WebSocket connection to the holder pod.
+	conn *websocket.ManagedConnection
+	// proc is the function to process the StatMessage owned by the bucket.
+	proc statProcessor
+}
+
 // Forwarder does the following things:
 // 1. Watches the change of Leases for Autoscaler buckets. Stores the
 //    Lease -> IP mapping.
 // 2. Creates/updates the corresponding K8S Service and Endpoints.
-// 3. Can be used to forward the metrics falling in a bucket based on
-//    the holder IP. (This is a TODO)
+// 3. Can be used to forward the metrics owned by a bucket based on
+//    the holder IP.
 type Forwarder struct {
 	selfIP          string
 	logger          *zap.SugaredLogger
 	kc              kubernetes.Interface
 	endpointsLister corev1listers.EndpointsLister
+	// `accept` is the function to process a StatMessage which doesn't need
+	// to be forwarded.
+	accept statProcessor
 	// bs is the BucketSet including all Autoscaler buckets.
 	bs *hash.BucketSet
-	// Lock for leaseHolders.
-	leaseLock sync.RWMutex
-	// leaseHolders stores the Lease -> IP relationships.
-	leaseHolders map[string]string
+	// processorsLock is the lock for processors.
+	processorsLock sync.RWMutex
+	processors     map[string]*bucketProcessor
 }
 
 // New creates a new Forwarder.
-func New(ctx context.Context, logger *zap.SugaredLogger, kc kubernetes.Interface, selfIP string, bs *hash.BucketSet) *Forwarder {
+func New(ctx context.Context, logger *zap.SugaredLogger, kc kubernetes.Interface, selfIP string, bs *hash.BucketSet, accept statProcessor) *Forwarder {
 	ns := system.Namespace()
 	bkts := bs.Buckets()
 	endpointsInformer := endpointsinformer.Get(ctx)
@@ -79,7 +99,8 @@ func New(ctx context.Context, logger *zap.SugaredLogger, kc kubernetes.Interface
 		kc:              kc,
 		endpointsLister: endpointsInformer.Lister(),
 		bs:              bs,
-		leaseHolders:    make(map[string]string, len(bkts)),
+		processors:      make(map[string]*bucketProcessor, len(bkts)),
+		accept:          accept,
 	}
 
 	leaseInformer := leaseinformer.Get(ctx)
@@ -113,7 +134,7 @@ func (f *Forwarder) filterFunc(namespace string) func(interface{}) bool {
 			return false
 		}
 
-		if h, ok := f.getHolder(l.Name); ok && h == *l.Spec.HolderIdentity {
+		if p := f.getProcessor(l.Name); p != nil && p.holder == *l.Spec.HolderIdentity {
 			// Already up-to-date.
 			return false
 		}
@@ -134,7 +155,7 @@ func (f *Forwarder) leaseUpdated(obj interface{}) {
 	}
 
 	holder := *l.Spec.HolderIdentity
-	f.setHolder(n, holder)
+	f.setProcessor(n, f.createProcessor(n, holder))
 
 	if holder != f.selfIP {
 		// Skip creating/updating Service and Endpoints if not the leader.
@@ -253,15 +274,79 @@ func (f *Forwarder) createOrUpdateEndpoints(ns, n string) error {
 	return nil
 }
 
-func (f *Forwarder) getHolder(name string) (string, bool) {
-	f.leaseLock.RLock()
-	defer f.leaseLock.RUnlock()
-	result, ok := f.leaseHolders[name]
-	return result, ok
+func (f *Forwarder) getProcessor(bkt string) *bucketProcessor {
+	f.processorsLock.RLock()
+	defer f.processorsLock.RUnlock()
+	return f.processors[bkt]
 }
 
-func (f *Forwarder) setHolder(name, holder string) {
-	f.leaseLock.Lock()
-	defer f.leaseLock.Unlock()
-	f.leaseHolders[name] = holder
+func (f *Forwarder) setProcessor(bkt string, p *bucketProcessor) {
+	f.processorsLock.Lock()
+	defer f.processorsLock.Unlock()
+	f.processors[bkt] = p
+}
+
+func (f *Forwarder) createProcessor(bkt, holder string) *bucketProcessor {
+	if holder == f.selfIP {
+		return &bucketProcessor{
+			holder: holder,
+			proc: func(sm asmetrics.StatMessage) {
+				l := f.logger.With(zap.String("revision", sm.Key.String()))
+				l.Debug("Accept stat as owner of bucket ", bkt)
+				if f.accept != nil {
+					f.accept(sm)
+				}
+			},
+		}
+	}
+
+	// TODO(9235): Currently we use IP directly. This won't work if there
+	// is mesh. Need to fall back to connection via Service.
+	dns := fmt.Sprintf("ws://%s:%d", holder, autoscalerPort)
+	f.logger.Info("Connecting to Autoscaler bucket at ", dns)
+	c := websocket.NewDurableSendingConnection(dns, f.logger)
+	return &bucketProcessor{
+		holder: holder,
+		conn:   c,
+		proc: func(sm asmetrics.StatMessage) {
+			l := f.logger.With(zap.String("revision", sm.Key.String()))
+			l.Debugf("Forward stat of bucket %s to the holder %s", bkt, holder)
+			wsms := asmetrics.ToWireStatMessages([]asmetrics.StatMessage{sm})
+			b, err := wsms.Marshal()
+			if err != nil {
+				l.Errorw("Error while marshaling stats", zap.Error(err))
+				return
+			}
+
+			if err := c.SendRaw(gorillawebsocket.BinaryMessage, b); err != nil {
+				l.Errorw("Error while sending stats", zap.Error(err))
+			}
+		},
+	}
+}
+
+// Process calls Forwarder.accept if the pod where this Forwarder is running is the owner
+// of the given StatMessage. Otherwise it forwards the given StatMessage to the right
+// owner pod. If it can't find the owner, it drops the StatMessage.
+func (f *Forwarder) Process(sm asmetrics.StatMessage) {
+	rev := sm.Key.String()
+	l := f.logger.With(zap.String("revision", rev))
+	bkt := f.bs.Owner(rev)
+
+	p := f.getProcessor(bkt)
+	if p == nil {
+		l.Warnf("Can't find the owner for Rev %s. Dropping its stat.", rev)
+		return
+	}
+
+	p.proc(sm)
+}
+
+// Cancel is the function to call when terminating a Forwarder.
+func (f *Forwarder) Cancel() {
+	for _, p := range f.processors {
+		if p.conn != nil {
+			p.conn.Shutdown()
+		}
+	}
 }
diff --git a/pkg/queue/breaker.go b/pkg/queue/breaker.go
index 20c9a312f..1a4bcab8c 100644
--- a/pkg/queue/breaker.go
+++ b/pkg/queue/breaker.go
@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"sync"
 
 	"go.uber.org/atomic"
 )
@@ -177,131 +176,143 @@ func (b *Breaker) Capacity() int {
 	return b.sem.Capacity()
 }
 
-// newSemaphore creates a semaphore with the desired maximal and initial capacity.
-// Maximal capacity is the size of the buffered channel, it defines maximum number of tokens
-// in the rotation. Attempting to add more capacity then the max will result in error.
-// Initial capacity is the initial number of free tokens.
+// newSemaphore creates a semaphore with the desired initial capacity.
 func newSemaphore(maxCapacity, initialCapacity int) *semaphore {
 	queue := make(chan struct{}, maxCapacity)
 	sem := &semaphore{queue: queue}
-	if initialCapacity > 0 {
-		sem.updateCapacity(initialCapacity)
-	}
+	sem.updateCapacity(initialCapacity)
 	return sem
 }
 
-// semaphore is an implementation of a semaphore based on Go channels.
-// The presence of elements in the `queue` buffered channel correspond to available tokens.
-// Hence the max number of tokens to hand out equals to the size of the channel.
-// `capacity` defines the current number of tokens in the rotation.
+// semaphore is an implementation of a semaphore based on packed integers and a channel.
+// state is an uint64 that has two uint32s packed into it: capacity and inFlight. The
+// former specifies how many request are allowed at any given time into the semaphore
+// while the latter refers to the currently in-flight requests.
+// Packing them both into one uint64 allows us to optimize access semantics using atomic
+// operations, which can't be guaranteed on 2 individual values.
+// The channel is merely used as a vehicle to be able to "wake up" individual goroutines
+// if capacity becomes free. It's not consistently used in accordance to actual capacity
+// but is rather a communication vehicle to ensure waiting routines are properly woken
+// up.
 type semaphore struct {
-	queue    chan struct{}
-	reducers int
-	capacity int
-	mux      sync.RWMutex
+	state atomic.Uint64
+	queue chan struct{}
 }
 
-// tryAcquire receives the token from the semaphore if there's one
-// otherwise an error is returned.
+// tryAcquire receives a token from the semaphore if there is one otherwise returns false.
 func (s *semaphore) tryAcquire() bool {
-	select {
-	case <-s.queue:
-		return true
-	default:
-		return false
+	for {
+		old := s.state.Load()
+		capacity, in := unpack(old)
+		if in >= capacity {
+			return false
+		}
+		in++
+		if s.state.CAS(old, pack(capacity, in)) {
+			return true
+		}
 	}
 }
 
-// acquire receives the token from the semaphore, potentially blocking.
+// acquire acquires capacity from the semaphore.
 func (s *semaphore) acquire(ctx context.Context) error {
-	select {
-	case <-s.queue:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
+	for {
+		old := s.state.Load()
+		capacity, in := unpack(old)
+
+		if in >= capacity {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-s.queue:
+			}
+			// Force reload state.
+			continue
+		}
+
+		in++
+		if s.state.CAS(old, pack(capacity, in)) {
+			return nil
+		}
 	}
 }
 
-// release potentially puts the token back to the queue.
-// If the semaphore capacity was reduced in between and is not yet reflected,
-// we remove the tokens from the rotation instead of returning them back.
+// release releases capacity in the semaphore.
+// If the semaphore capacity was reduced in between and as a result inFlight is greater
+// than capacity, we don't wake up goroutines as they'd not get any capacity anyway.
 func (s *semaphore) release() error {
-	s.mux.Lock()
-	defer s.mux.Unlock()
+	for {
+		old := s.state.Load()
+		capacity, in := unpack(old)
 
-	if s.reducers > 0 {
-		s.capacity--
-		s.reducers--
-		return nil
-	}
+		if in == 0 {
+			return ErrRelease
+		}
 
-	// We want to make sure releasing a token is always non-blocking.
-	select {
-	case s.queue <- struct{}{}:
-		return nil
-	default:
-		// This only happens if release is called more often than acquire.
-		return ErrRelease
+		in--
+		if s.state.CAS(old, pack(capacity, in)) {
+			if in < capacity {
+				select {
+				case s.queue <- struct{}{}:
+				default:
+					// We generate more wakeups than we might need as we don't know
+					// how many goroutines are waiting here. It is therefore okay
+					// to drop the poke on the floor here as this case would mean we
+					// have enough wakeups to wake up as many goroutines as this semaphore
+					// can take, which is guaranteed to be enough.
+				}
+			}
+			return nil
+		}
 	}
 }
 
-// updateCapacity updates the capacity of the semaphore to the desired
-// size.
+// updateCapacity updates the capacity of the semaphore to the desired size.
 func (s *semaphore) updateCapacity(size int) error {
 	if size < 0 || size > cap(s.queue) {
 		return ErrUpdateCapacity
 	}
 
-	s.mux.Lock()
-	defer s.mux.Unlock()
-
-	if s.effectiveCapacity() == size {
-		return nil
-	}
+	s64 := uint64(size)
+	for {
+		old := s.state.Load()
+		capacity, in := unpack(old)
 
-	// Add capacity until we reach size, potentially consuming
-	// outstanding reducers first.
-	for s.effectiveCapacity() < size {
-		if s.reducers > 0 {
-			s.reducers--
-		} else {
-			select {
-			case s.queue <- struct{}{}:
-				s.capacity++
-			default:
-				// This indicates that we're operating close to
-				// MaxCapacity and returned more tokens than we
-				// acquired.
-				return ErrUpdateCapacity
-			}
+		if capacity == s64 {
+			// Nothing to do, exit early.
+			return nil
 		}
-	}
 
-	// Reduce capacity until we reach size, potentially adding
-	// new reducers if the queue channel is empty because of
-	// requests in-flight.
-	for s.effectiveCapacity() > size {
-		select {
-		case <-s.queue:
-			s.capacity--
-		default:
-			s.reducers++
+		if s.state.CAS(old, pack(s64, in)) {
+			if s64 > capacity {
+				for i := uint64(0); i < s64-capacity; i++ {
+					select {
+					case s.queue <- struct{}{}:
+					default:
+						// See comment in `release` for explanation of this case.
+					}
+				}
+			}
+			return nil
 		}
 	}
-
-	return nil
 }
 
-// effectiveCapacity is the capacity with reducers taken into account.
-// `mux` must be held to call it.
-func (s *semaphore) effectiveCapacity() int {
-	return s.capacity - s.reducers
+// Capacity is the capacity of the semaphore.
+func (s *semaphore) Capacity() int {
+	capacity, _ := unpack(s.state.Load())
+	return int(capacity)
 }
 
-// Capacity is the effective capacity after taking reducers into account.
-func (s *semaphore) Capacity() int {
-	s.mux.RLock()
-	defer s.mux.RUnlock()
+// unpack takes an uint64 and returns two uint32 (as uint64) comprised of the leftmost
+// and the rightmost bits respectively.
+func unpack(in uint64) (uint64, uint64) {
+	return in >> 32, in & 0xffffffff
+}
 
-	return s.effectiveCapacity()
+// pack takes two uint32 (as uint64 to avoid casting) and packs them into a single uint64
+// at the leftmost and the rightmost bits respectively.
+// It's up to the caller to ensure that left and right actually fit into 32 bit.
+func pack(left, right uint64) uint64 {
+	return left<<32 | right
 }
diff --git a/pkg/reconciler/autoscaling/kpa/kpa.go b/pkg/reconciler/autoscaling/kpa/kpa.go
index 2a777c429..92dbdfad4 100644
--- a/pkg/reconciler/autoscaling/kpa/kpa.go
+++ b/pkg/reconciler/autoscaling/kpa/kpa.go
@@ -141,8 +141,8 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, pa *pav1alpha1.PodAutosc
 	if err != nil {
 		return fmt.Errorf("error getting pod counts %s: %w", sks.Status.PrivateServiceName, err)
 	}
+
 	// If SKS is not ready — ensure we're not becoming ready.
-	// TODO: see if we can perhaps propagate the SKS state to computing active status.
 	if sks.IsReady() {
 		logger.Debug("SKS is ready, marking SKS status ready")
 		pa.Status.MarkSKSReady()
@@ -243,7 +243,7 @@ func computeActiveCondition(ctx context.Context, pa *pav1alpha1.PodAutoscaler, p
 	// In this case we'll be in the NoTraffic/inactive state.
 	// TODO(taragu): remove after 0.19
 	alreadyScaledDownSuccessfully := minReady > 0 && pa.Status.GetCondition(pav1alpha1.PodAutoscalerConditionActive).Reason == noTrafficReason
-	if pc.ready >= minReady || alreadyScaledDownSuccessfully {
+	if (pc.ready >= minReady || alreadyScaledDownSuccessfully) && pa.Status.ServiceName != "" {
 		pa.Status.MarkScaleTargetInitialized()
 	}
 
diff --git a/pkg/reconciler/route/resources/labels/labels.go b/pkg/reconciler/route/resources/labels/labels.go
index 3a77f6017..c6cbeee38 100644
--- a/pkg/reconciler/route/resources/labels/labels.go
+++ b/pkg/reconciler/route/resources/labels/labels.go
@@ -22,8 +22,8 @@ import (
 )
 
 // IsObjectLocalVisibility returns whether an ObjectMeta is of cluster-local visibility
-func IsObjectLocalVisibility(meta v1.ObjectMeta) bool {
-	return meta.Labels != nil && meta.Labels[serving.VisibilityLabelKey] != ""
+func IsObjectLocalVisibility(meta *v1.ObjectMeta) bool {
+	return meta.Labels[serving.VisibilityLabelKey] != ""
 }
 
 // SetVisibility sets the visibility on an ObjectMeta
diff --git a/pkg/reconciler/route/traffic/traffic.go b/pkg/reconciler/route/traffic/traffic.go
index c4389dcdc..9f695f6fa 100644
--- a/pkg/reconciler/route/traffic/traffic.go
+++ b/pkg/reconciler/route/traffic/traffic.go
@@ -116,7 +116,7 @@ func (t *Config) GetRevisionTrafficTargets(ctx context.Context, r *v1.Route) ([]
 
 			labels.SetVisibility(meta, t.Visibility[tt.Tag] == netv1alpha1.IngressVisibilityClusterLocal)
 
-			// http is currently the only supported scheme
+			// HTTP is currently the only supported scheme.
 			fullDomain, err := domains.DomainNameFromTemplate(ctx, *meta, hostname)
 			if err != nil {
 				return nil, err
diff --git a/pkg/reconciler/route/visibility/visibility.go b/pkg/reconciler/route/visibility/visibility.go
index d6f8892d1..08665b0bc 100644
--- a/pkg/reconciler/route/visibility/visibility.go
+++ b/pkg/reconciler/route/visibility/visibility.go
@@ -99,7 +99,7 @@ func (b *Resolver) GetVisibility(ctx context.Context, route *v1.Route) (map[stri
 		ttVisibility := netv1alpha1.IngressVisibilityExternalIP
 		// Is there a visibility setting on the placeholder Service?
 		if svc, ok := services[hostname]; ok {
-			if labels.IsObjectLocalVisibility(svc.ObjectMeta) {
+			if labels.IsObjectLocalVisibility(&svc.ObjectMeta) {
 				ttVisibility = netv1alpha1.IngressVisibilityClusterLocal
 			}
 		}
diff --git a/pkg/reconciler/serverlessservice/serverlessservice.go b/pkg/reconciler/serverlessservice/serverlessservice.go
index 163e19b4d..1f9ffaa19 100644
--- a/pkg/reconciler/serverlessservice/serverlessservice.go
+++ b/pkg/reconciler/serverlessservice/serverlessservice.go
@@ -29,7 +29,6 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"
 	corev1listers "k8s.io/client-go/listers/core/v1"
@@ -39,6 +38,7 @@ import (
 	netv1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
 	"knative.dev/pkg/apis/duck"
 	"knative.dev/pkg/hash"
+	"knative.dev/pkg/kmeta"
 	"knative.dev/pkg/logging"
 	pkgreconciler "knative.dev/pkg/reconciler"
 	"knative.dev/pkg/system"
@@ -291,38 +291,6 @@ func (r *reconciler) reconcilePublicEndpoints(ctx context.Context, sks *netv1alp
 	return nil
 }
 
-func (r *reconciler) privateService(sks *netv1alpha1.ServerlessService) (*corev1.Service, error) {
-	// The code below is for backwards compatibility, when we had
-	// GenerateName for the private services.
-	svcs, err := r.serviceLister.Services(sks.Namespace).List(labels.SelectorFromSet(labels.Set{
-		networking.SKSLabelKey:    sks.Name,
-		networking.ServiceTypeKey: string(networking.ServiceTypePrivate),
-	}))
-	if err != nil {
-		return nil, err
-	}
-	switch l := len(svcs); l {
-	case 0:
-		return nil, apierrs.NewNotFound(corev1.Resource("Services"), sks.Name)
-	case 1:
-		return svcs[0], nil
-	default:
-		// We encountered more than one. Keep the one that is in the SKS status and delete the others.
-		var ret *corev1.Service
-		for _, s := range svcs {
-			if s.Name == sks.Status.PrivateServiceName {
-				ret = s
-				continue
-			}
-			// If we don't control it, don't delete it.
-			if metav1.IsControlledBy(s, sks) {
-				r.kubeclient.CoreV1().Services(sks.Namespace).Delete(s.Name, &metav1.DeleteOptions{})
-			}
-		}
-		return ret, nil
-	}
-}
-
 func (r *reconciler) reconcilePrivateService(ctx context.Context, sks *netv1alpha1.ServerlessService) error {
 	logger := logging.FromContext(ctx)
 
@@ -331,7 +299,8 @@ func (r *reconciler) reconcilePrivateService(ctx context.Context, sks *netv1alph
 		return fmt.Errorf("error retrieving deployment selector spec: %w", err)
 	}
 
-	svc, err := r.privateService(sks)
+	sn := kmeta.ChildName(sks.Name, "-private")
+	svc, err := r.serviceLister.Services(sks.Namespace).Get(sn)
 	if apierrs.IsNotFound(err) {
 		logger.Info("SKS has no private service; creating.")
 		sks.Status.MarkEndpointsNotReady("CreatingPrivateService")
@@ -354,8 +323,10 @@ func (r *reconciler) reconcilePrivateService(ctx context.Context, sks *netv1alph
 		want.Spec.Selector = tmpl.Spec.Selector
 
 		if !equality.Semantic.DeepEqual(svc.Spec, want.Spec) {
+			// Spec has only public fields and cmp can't panic here.
+			logger.Debug("Private service diff(-want,+got):", cmp.Diff(want.Spec, svc.Spec))
 			sks.Status.MarkEndpointsNotReady("UpdatingPrivateService")
-			logger.Infof("Private K8s Service changed %s; reconciling: ", svc.Name)
+			logger.Info("Reconciling a changed private K8s Service  ", svc.Name)
 			if _, err = r.kubeclient.CoreV1().Services(sks.Namespace).Update(want); err != nil {
 				return fmt.Errorf("failed to update private K8s Service: %w", err)
 			}
diff --git a/third_party/cert-manager-0.12.0/net-certmanager.yaml b/third_party/cert-manager-0.12.0/net-certmanager.yaml
index 8936c6eaf..288ec241a 100644
--- a/third_party/cert-manager-0.12.0/net-certmanager.yaml
+++ b/third_party/cert-manager-0.12.0/net-certmanager.yaml
@@ -18,7 +18,7 @@ metadata:
   # These are the permissions needed by the `cert-manager` `Certificate` implementation.
   name: knative-serving-certmanager
   labels:
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
     serving.knative.dev/controller: "true"
     networking.knative.dev/certificate-provider: cert-manager
 rules:
@@ -49,7 +49,7 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: config.webhook.net-certmanager.networking.internal.knative.dev
   labels:
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
 webhooks:
 - admissionReviewVersions:
   - v1beta1
@@ -86,7 +86,7 @@ metadata:
   name: net-certmanager-webhook-certs
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
 
 ---
 # Copyright 2020 The Knative Authors
@@ -109,7 +109,7 @@ metadata:
   name: config-certmanager
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
     networking.knative.dev/certificate-provider: cert-manager
 data:
   _example: |
@@ -156,7 +156,7 @@ metadata:
   name: networking-certmanager
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
     networking.knative.dev/certificate-provider: cert-manager
 spec:
   selector:
@@ -168,14 +168,14 @@ spec:
         cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
       labels:
         app: networking-certmanager
-        serving.knative.dev/release: "v20200831-8a5392f"
+        serving.knative.dev/release: "v20200901-f33fbfc"
     spec:
       serviceAccountName: controller
       containers:
       - name: networking-certmanager
         # This is the Go import path for the binary that is containerized
         # and substituted here.
-        image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/controller@sha256:8e3e77b4bf6cc2353498388cb92b28ba2c6e66c45794a59094145c366f63563e
+        image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/controller@sha256:bd3a8a2dad97d9b339e6cfc6a6903a840a46ab5d4db7b134634a380749233a4a
         resources:
           requests:
             cpu: 30m
@@ -208,7 +208,7 @@ kind: Service
 metadata:
   labels:
     app: networking-certmanager
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
     networking.knative.dev/certificate-provider: cert-manager
   name: networking-certmanager
   namespace: knative-serving
@@ -245,7 +245,7 @@ metadata:
   name: net-certmanager-webhook
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
 spec:
   selector:
     matchLabels:
@@ -258,14 +258,14 @@ spec:
       labels:
         app: net-certmanager-webhook
         role: net-certmanager-webhook
-        serving.knative.dev/release: "v20200831-8a5392f"
+        serving.knative.dev/release: "v20200901-f33fbfc"
     spec:
       serviceAccountName: controller
       containers:
       - name: webhook
         # This is the Go import path for the binary that is containerized
         # and substituted here.
-        image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/webhook@sha256:e705f98bc7e3594c932d14acc1088846ece6085e156173cb02e9895d8955a205
+        image: gcr.io/knative-nightly/knative.dev/net-certmanager/cmd/webhook@sha256:5e4203ee7a2196f00abe25393423753591bb0e8dde94175db547672e1b62f436
         resources:
           requests:
             cpu: 20m
@@ -319,7 +319,7 @@ metadata:
   namespace: knative-serving
   labels:
     role: net-certmanager-webhook
-    serving.knative.dev/release: "v20200831-8a5392f"
+    serving.knative.dev/release: "v20200901-f33fbfc"
 spec:
   ports:
   - # Define metrics and profiling for them to be accessible within service meshes.
diff --git a/third_party/net-istio.yaml b/third_party/net-istio.yaml
index 3c7d2bf63..eb56c2f86 100644
--- a/third_party/net-istio.yaml
+++ b/third_party/net-istio.yaml
@@ -18,7 +18,7 @@ metadata:
   # These are the permissions needed by the Istio Ingress implementation.
   name: knative-serving-istio
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     serving.knative.dev/controller: "true"
     networking.knative.dev/ingress-provider: istio
 rules:
@@ -48,7 +48,7 @@ metadata:
   name: knative-ingress-gateway
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -85,7 +85,7 @@ metadata:
   name: cluster-local-gateway
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -104,7 +104,7 @@ metadata:
   name: knative-local-gateway
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -123,7 +123,7 @@ metadata:
   name: knative-local-gateway
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   type: ClusterIP
@@ -181,7 +181,7 @@ kind: MutatingWebhookConfiguration
 metadata:
   name: webhook.istio.networking.internal.knative.dev
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 webhooks:
 - admissionReviewVersions:
@@ -217,7 +217,7 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: config.webhook.istio.networking.internal.knative.dev
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 webhooks:
 - admissionReviewVersions:
@@ -255,7 +255,7 @@ metadata:
   name: istio-webhook-certs
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 
 ---
@@ -279,7 +279,7 @@ metadata:
   name: config-istio
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 data:
   _example: |
@@ -349,7 +349,7 @@ metadata:
   name: networking-istio
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -365,14 +365,14 @@ spec:
         sidecar.istio.io/inject: "false"
       labels:
         app: networking-istio
-        serving.knative.dev/release: "v20200831-cfa0ad5"
+        serving.knative.dev/release: "v20200901-7ef0bc2"
     spec:
       serviceAccountName: controller
       containers:
       - name: networking-istio
         # This is the Go import path for the binary that is containerized
         # and substituted here.
-        image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/controller@sha256:0b5d610527b7e5df00d02f1e2d8d8451fce055e787412bb6d566573486f777d6
+        image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/controller@sha256:2a358a6d975af98abf2dc4005b24884e9a7ddb9ad20744c2982ac070a2827705
         resources:
           requests:
             cpu: 30m
@@ -424,7 +424,7 @@ metadata:
   name: istio-webhook
   namespace: knative-serving
   labels:
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   selector:
@@ -438,14 +438,14 @@ spec:
       labels:
         app: istio-webhook
         role: istio-webhook
-        serving.knative.dev/release: "v20200831-cfa0ad5"
+        serving.knative.dev/release: "v20200901-7ef0bc2"
     spec:
       serviceAccountName: controller
       containers:
       - name: webhook
         # This is the Go import path for the binary that is containerized
         # and substituted here.
-        image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/webhook@sha256:73f91373ef04b6059617fcfec956265444d700e70383ceaed5024c8881203ac5
+        image: gcr.io/knative-nightly/knative.dev/net-istio/cmd/webhook@sha256:9be3454ec441bf05b97e2e29e539e88cb688c68d1d12847f5255c3f0da7b6f13
         resources:
           requests:
             cpu: 20m
@@ -499,7 +499,7 @@ metadata:
   namespace: knative-serving
   labels:
     role: istio-webhook
-    serving.knative.dev/release: "v20200831-cfa0ad5"
+    serving.knative.dev/release: "v20200901-7ef0bc2"
     networking.knative.dev/ingress-provider: istio
 spec:
   ports:
diff --git a/vendor/modules.txt b/vendor/modules.txt
index a2d4df22a..addc5b08e 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -1040,7 +1040,7 @@ k8s.io/utils/buffer
 k8s.io/utils/integer
 k8s.io/utils/pointer
 k8s.io/utils/trace
-# knative.dev/caching v0.0.0-20200828162208-ccf6aa330912
+# knative.dev/caching v0.0.0-20200831164016-8993b9e0e6f6
 ## explicit
 knative.dev/caching/config
 knative.dev/caching/pkg/apis/caching
@@ -1061,7 +1061,7 @@ knative.dev/caching/pkg/client/injection/informers/caching/v1alpha1/image/fake
 knative.dev/caching/pkg/client/injection/informers/factory
 knative.dev/caching/pkg/client/injection/informers/factory/fake
 knative.dev/caching/pkg/client/listers/caching/v1alpha1
-# knative.dev/networking v0.0.0-20200828165907-5809e67909aa
+# knative.dev/networking v0.0.0-20200901153316-80398434dcd4
 ## explicit
 knative.dev/networking/pkg
 knative.dev/networking/pkg/apis/networking
@@ -1088,7 +1088,7 @@ knative.dev/networking/pkg/client/injection/informers/networking/v1alpha1/server
 knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/serverlessservice
 knative.dev/networking/pkg/client/listers/networking/v1alpha1
 knative.dev/networking/pkg/ingress
-# knative.dev/pkg v0.0.0-20200831094608-e9da10bc8a18
+# knative.dev/pkg v0.0.0-20200901172015-edd1b5e0bebf
 ## explicit
 knative.dev/pkg/apiextensions/storageversion
 knative.dev/pkg/apiextensions/storageversion/cmd/migrate
@@ -1214,7 +1214,7 @@ knative.dev/pkg/webhook/resourcesemantics/conversion
 knative.dev/pkg/webhook/resourcesemantics/defaulting
 knative.dev/pkg/webhook/resourcesemantics/validation
 knative.dev/pkg/websocket
-# knative.dev/test-infra v0.0.0-20200828211307-9d4372c9b1c7
+# knative.dev/test-infra v0.0.0-20200831235415-fac473dda98b
 ## explicit
 knative.dev/test-infra/scripts
 # sigs.k8s.io/yaml v1.2.0


Test patch:
diff --git a/.github/workflows/knative-go-test.yaml b/.github/workflows/knative-go-test.yaml
new file mode 100644
index 000000000..53985abd7
--- /dev/null
+++ b/.github/workflows/knative-go-test.yaml
@@ -0,0 +1,46 @@
+# Copyright 2020 The Knative Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Test
+
+on:
+  push:
+    branches: [ 'master', 'release-*' ]
+  pull_request:
+    branches: [ 'master', 'release-*' ]
+
+jobs:
+
+  test:
+    name: Test
+    strategy:
+      matrix:
+        go-version: [1.15.x]
+        platform: [ubuntu-latest]
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: go test -race ./...
diff --git a/pkg/apis/serving/v1/revision_lifecycle_test.go b/pkg/apis/serving/v1/revision_lifecycle_test.go
index 3544b9d8e..97a3cf4d0 100644
--- a/pkg/apis/serving/v1/revision_lifecycle_test.go
+++ b/pkg/apis/serving/v1/revision_lifecycle_test.go
@@ -611,6 +611,7 @@ func TestPropagateAutoscalerStatusNoProgress(t *testing.T) {
 
 	// PodAutoscaler is not ready and initial scale was never attained.
 	r.PropagateAutoscalerStatus(&av1alpha1.PodAutoscalerStatus{
+		ServiceName: "testRevision",
 		Status: duckv1.Status{
 			Conditions: duckv1.Conditions{{
 				Type:   av1alpha1.PodAutoscalerConditionReady,
diff --git a/pkg/autoscaler/statforwarder/forwarder_test.go b/pkg/autoscaler/statforwarder/forwarder_test.go
index 40855c49f..ebd256737 100644
--- a/pkg/autoscaler/statforwarder/forwarder_test.go
+++ b/pkg/autoscaler/statforwarder/forwarder_test.go
@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	ktesting "k8s.io/client-go/testing"
@@ -40,6 +41,7 @@ import (
 	rtesting "knative.dev/pkg/reconciler/testing"
 	"knative.dev/pkg/system"
 	_ "knative.dev/pkg/system/testing"
+	asmetrics "knative.dev/serving/pkg/autoscaler/metrics"
 )
 
 const (
@@ -61,9 +63,12 @@ var (
 			HolderIdentity: &testIP1,
 		},
 	}
+	// A statProcessor doing nothing.
+	noOp = func(sm asmetrics.StatMessage) {}
 )
 
 func TestForwarderReconcile(t *testing.T) {
+	t.Parallel()
 	ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t)
 	kubeClient := fakekubeclient.Get(ctx)
 	endpoints := fakeendpointsinformer.Get(ctx)
@@ -81,8 +86,8 @@ func TestForwarderReconcile(t *testing.T) {
 		waitInformers()
 	})
 
-	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs)
-	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP2, testBs)
+	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs, noOp)
+	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP2, testBs, noOp)
 
 	kubeClient.CoordinationV1().Leases(testNs).Create(testLease)
 	lease.Informer().GetIndexer().Add(testLease)
@@ -153,6 +158,7 @@ func TestForwarderReconcile(t *testing.T) {
 }
 
 func TestForwarderRetryOnSvcCreationFailure(t *testing.T) {
+	t.Parallel()
 	ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t)
 	kubeClient := fakekubeclient.Get(ctx)
 	endpoints := fakeendpointsinformer.Get(ctx)
@@ -170,7 +176,7 @@ func TestForwarderRetryOnSvcCreationFailure(t *testing.T) {
 		waitInformers()
 	})
 
-	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs)
+	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs, noOp)
 
 	svcCreation := 0
 	retried := make(chan struct{})
@@ -195,6 +201,7 @@ func TestForwarderRetryOnSvcCreationFailure(t *testing.T) {
 }
 
 func TestForwarderRetryOnEndpointsCreationFailure(t *testing.T) {
+	t.Parallel()
 	ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t)
 	kubeClient := fakekubeclient.Get(ctx)
 	endpoints := fakeendpointsinformer.Get(ctx)
@@ -212,7 +219,7 @@ func TestForwarderRetryOnEndpointsCreationFailure(t *testing.T) {
 		waitInformers()
 	})
 
-	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs)
+	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs, noOp)
 
 	endpointsCreation := 0
 	retried := make(chan struct{})
@@ -237,6 +244,7 @@ func TestForwarderRetryOnEndpointsCreationFailure(t *testing.T) {
 }
 
 func TestForwarderRetryOnEndpointsUpdateFailure(t *testing.T) {
+	t.Parallel()
 	ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t)
 	kubeClient := fakekubeclient.Get(ctx)
 	endpoints := fakeendpointsinformer.Get(ctx)
@@ -254,7 +262,7 @@ func TestForwarderRetryOnEndpointsUpdateFailure(t *testing.T) {
 		waitInformers()
 	})
 
-	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs)
+	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs, noOp)
 
 	endpointsUpdate := 0
 	retried := make(chan struct{})
@@ -287,6 +295,7 @@ func TestForwarderRetryOnEndpointsUpdateFailure(t *testing.T) {
 }
 
 func TestForwarderSkipReconciling(t *testing.T) {
+	t.Parallel()
 	ns := system.Namespace()
 	ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t)
 	kubeClient := fakekubeclient.Get(ctx)
@@ -305,7 +314,7 @@ func TestForwarderSkipReconciling(t *testing.T) {
 		waitInformers()
 	})
 
-	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs)
+	New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, testBs, noOp)
 
 	svcCreated := make(chan struct{})
 	kubeClient.PrependReactor("create", "services",
@@ -376,3 +385,88 @@ func TestForwarderSkipReconciling(t *testing.T) {
 		})
 	}
 }
+
+func TestProcess(t *testing.T) {
+	t.Parallel()
+	ctx, cancel, _ := rtesting.SetupFakeContextWithCancel(t)
+	kubeClient := fakekubeclient.Get(ctx)
+	endpoints := fakeendpointsinformer.Get(ctx)
+	service := fakeserviceinformer.Get(ctx)
+	lease := fakeleaseinformer.Get(ctx)
+
+	waitInformers, err := controller.RunInformers(
+		ctx.Done(), endpoints.Informer(), service.Informer(), lease.Informer())
+	if err != nil {
+		t.Fatal("Failed to start informers:", err)
+	}
+
+	t.Cleanup(func() {
+		cancel()
+		waitInformers()
+	})
+
+	acceptCount := 0
+	accept := func(sm asmetrics.StatMessage) {
+		acceptCount++
+	}
+	f := New(ctx, zap.NewNop().Sugar(), kubeClient, testIP1, hash.NewBucketSet(sets.NewString(bucket1, bucket2)), accept)
+
+	stat1 := asmetrics.StatMessage{
+		Key: types.NamespacedName{
+			Namespace: testNs,
+			Name:      "succulent", // Mapped to bucket1
+		},
+	}
+	stat2 := asmetrics.StatMessage{
+		Key: types.NamespacedName{
+			Namespace: testNs,
+			Name:      "plant", // Mapped to bucket2
+		},
+	}
+
+	// A Forward without any leadership information should process without error.
+	f.Process(stat1)
+
+	kubeClient.CoordinationV1().Leases(testNs).Create(testLease)
+	lease.Informer().GetIndexer().Add(testLease)
+	anotherLease := &coordinationv1.Lease{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      bucket2,
+			Namespace: testNs,
+		},
+		Spec: coordinationv1.LeaseSpec{
+			HolderIdentity: &testIP2,
+		},
+	}
+	kubeClient.CoordinationV1().Leases(testNs).Create(anotherLease)
+	lease.Informer().GetIndexer().Add(anotherLease)
+
+	// Wait for the forwarder to become the leader for bucket1.
+	if err := wait.PollImmediate(100*time.Millisecond, 2*time.Second, func() (bool, error) {
+		p1 := f.getProcessor(bucket1)
+		p2 := f.getProcessor(bucket2)
+		return p1 != nil && p2 != nil && p1.holder == testIP1 && p2.holder == testIP2 && p1.conn == nil && p2.conn != nil, nil
+	}); err != nil {
+		t.Fatalf("Timeout waiting f.processors got updated")
+	}
+
+	forwardCount := 0
+	// Override the proc so we do not actually send via WebSocket.
+	f.processors[bucket2].proc = func(sm asmetrics.StatMessage) {
+		forwardCount++
+	}
+
+	f.Process(stat1)
+	f.Process(stat2)
+	f.Process(stat2)
+
+	if got, want := acceptCount, 1; got != want {
+		t.Errorf("acceptCount = %d, want = %d", got, want)
+	}
+	if got, want := forwardCount, 2; got != want {
+		t.Errorf("forwardCount = %d, want = %d", got, want)
+	}
+
+	// Make sure Cancel can be called without crash.
+	f.Cancel()
+}
diff --git a/pkg/queue/breaker_test.go b/pkg/queue/breaker_test.go
index 1a73e8a2d..394d3853a 100644
--- a/pkg/queue/breaker_test.go
+++ b/pkg/queue/breaker_test.go
@@ -94,7 +94,7 @@ func TestBreakerOverloadMixed(t *testing.T) {
 	// Bring breaker to capacity.
 	reqs.request()
 	// This happens in go-routine, so spin.
-	for len(b.sem.queue) > 0 {
+	for _, in := unpack(b.sem.state.Load()); in != 1; _, in = unpack(b.sem.state.Load()) {
 		time.Sleep(time.Millisecond * 2)
 	}
 	_, rr := b.Reserve(context.Background())
@@ -263,7 +263,7 @@ func TestSemaphoreAcquireHasCapacity(t *testing.T) {
 
 	sem := newSemaphore(1, 0)
 	tryAcquire(sem, gotChan)
-	sem.release() // Allows 1 acquire
+	sem.updateCapacity(1) // Allows 1 acquire
 
 	for i := 0; i < want; i++ {
 		select {
@@ -293,30 +293,6 @@ func TestSemaphoreRelease(t *testing.T) {
 	}
 }
 
-func TestSemaphoreReleasesSeveralReducers(t *testing.T) {
-	const wantAfterFirstrelease = 1
-	const wantAfterSecondrelease = 0
-	sem := newSemaphore(2, 2)
-	sem.acquire(context.Background())
-	sem.acquire(context.Background())
-	sem.updateCapacity(0)
-	sem.release()
-	if got := sem.Capacity(); got != wantAfterSecondrelease {
-		t.Errorf("Capacity = %d, want: %d", got, wantAfterSecondrelease)
-	}
-	if sem.reducers != wantAfterFirstrelease {
-		t.Errorf("sem.reducers = %d, want: %d", sem.reducers, wantAfterFirstrelease)
-	}
-
-	sem.release()
-	if got := sem.Capacity(); got != wantAfterSecondrelease {
-		t.Errorf("Capacity = %d, want: %d", got, wantAfterSecondrelease)
-	}
-	if sem.reducers != wantAfterSecondrelease {
-		t.Errorf("sem.reducers = %d, want: %d", sem.reducers, wantAfterSecondrelease)
-	}
-}
-
 func TestSemaphoreUpdateCapacity(t *testing.T) {
 	const initialCapacity = 1
 	sem := newSemaphore(3, initialCapacity)
@@ -330,40 +306,6 @@ func TestSemaphoreUpdateCapacity(t *testing.T) {
 	}
 }
 
-// Test the case when we add more capacity then the number of waiting reducers
-func TestSemaphoreUpdateCapacityLessThenReducers(t *testing.T) {
-	const initialCapacity = 2
-	sem := newSemaphore(2, initialCapacity)
-	sem.acquire(context.Background())
-	sem.acquire(context.Background())
-	sem.updateCapacity(initialCapacity - 2)
-	if got, want := sem.reducers, 2; got != want {
-		t.Errorf("sem.reducers = %d, want: %d", got, want)
-	}
-	sem.release()
-	sem.release()
-	sem.release()
-	if got, want := sem.reducers, 0; got != want {
-		t.Errorf("sem.reducers = %d, want: %d", got, want)
-	}
-}
-
-func TestSemaphoreUpdateCapacityConsumingReducers(t *testing.T) {
-	const initialCapacity = 2
-	sem := newSemaphore(2, initialCapacity)
-	sem.acquire(context.Background())
-	sem.acquire(context.Background())
-	sem.updateCapacity(initialCapacity - 2)
-	if got, want := sem.reducers, 2; got != want {
-		t.Errorf("sem.reducers = %d, want: %d", got, want)
-	}
-
-	sem.updateCapacity(initialCapacity)
-	if got, want := sem.reducers, 0; got != want {
-		t.Errorf("sem.reducers = %d, want: %d", got, want)
-	}
-}
-
 func TestSemaphoreUpdateCapacityOverflow(t *testing.T) {
 	sem := newSemaphore(2, 0)
 	if err := sem.updateCapacity(3); err != ErrUpdateCapacity {
@@ -379,14 +321,6 @@ func TestSemaphoreUpdateCapacityOutOfBound(t *testing.T) {
 	}
 }
 
-func TestSemaphoreUpdateCapacityBrokenState(t *testing.T) {
-	sem := newSemaphore(1, 0)
-	sem.release() // This Release is not paired with an acquire
-	if err := sem.updateCapacity(1); err != ErrUpdateCapacity {
-		t.Errorf("updateCapacity = %v, want: %v", err, ErrUpdateCapacity)
-	}
-}
-
 func TestSemaphoreUpdateCapacityDoNothing(t *testing.T) {
 	sem := newSemaphore(1, 1)
 	if err := sem.updateCapacity(1); err != nil {
@@ -394,6 +328,17 @@ func TestSemaphoreUpdateCapacityDoNothing(t *testing.T) {
 	}
 }
 
+func TestPackUnpack(t *testing.T) {
+	wantL := uint64(256)
+	wantR := uint64(513)
+
+	gotL, gotR := unpack(pack(wantL, wantR))
+
+	if gotL != wantL || gotR != wantR {
+		t.Fatalf("Got %d, %d want %d, %d", gotL, gotR, wantL, wantR)
+	}
+}
+
 func tryAcquire(sem *semaphore, gotChan chan struct{}) {
 	go func() {
 		// blocking until someone puts the token into the semaphore
diff --git a/pkg/reconciler/autoscaling/kpa/kpa_test.go b/pkg/reconciler/autoscaling/kpa/kpa_test.go
index d3bb678d3..24419ec38 100644
--- a/pkg/reconciler/autoscaling/kpa/kpa_test.go
+++ b/pkg/reconciler/autoscaling/kpa/kpa_test.go
@@ -1048,7 +1048,7 @@ func TestReconcile(t *testing.T) {
 			kpa(testNamespace, testRevision, withScales(0, -1), WithReachabilityReachable,
 				WithPAMetricsService(privateSvc), WithPASKSNotReady(noPrivateServiceName)),
 			// SKS won't be ready bc no ready endpoints, but private service name will be populated.
-			sks(testNamespace, testRevision, WithDeployRef(deployName), WithPrivateService),
+			sks(testNamespace, testRevision, WithDeployRef(deployName), WithPrivateService, WithPubService),
 			metric(testNamespace, testRevision),
 			deploy(testNamespace, testRevision, func(d *appsv1.Deployment) {
 				d.Spec.Replicas = ptr.Int32(0)
@@ -1056,6 +1056,30 @@ func TestReconcile(t *testing.T) {
 		},
 		WantStatusUpdates: []clientgotesting.UpdateActionImpl{{
 			Object: kpa(testNamespace, testRevision, markScaleTargetInitialized,
+				WithNoTraffic(noTrafficReason, "The target is not receiving traffic."),
+				withScales(0, -1), WithReachabilityReachable,
+				WithPAMetricsService(privateSvc), WithObservedGeneration(1),
+				WithPASKSNotReady(""), WithPAStatusService(testRevision),
+			),
+		}},
+	}, {
+		Name: "initial scale zero: sks ServiceName empty",
+		Key:  key,
+		Ctx: context.WithValue(context.WithValue(context.Background(), asConfigKey{}, initialScaleZeroASConfig()), deciderKey{},
+			decider(testNamespace, testRevision, -1, /* desiredScale */
+				0 /* ebc */, scaling.MinActivators)),
+		Objects: []runtime.Object{
+			kpa(testNamespace, testRevision, withScales(0, -1), WithReachabilityReachable,
+				WithPAMetricsService(privateSvc), WithPASKSNotReady(noPrivateServiceName)),
+			// SKS won't be ready bc no ready endpoints, but private service name will be populated.
+			sks(testNamespace, testRevision, WithDeployRef(deployName), WithPrivateService),
+			metric(testNamespace, testRevision),
+			deploy(testNamespace, testRevision, func(d *appsv1.Deployment) {
+				d.Spec.Replicas = ptr.Int32(0)
+			}),
+		},
+		WantStatusUpdates: []clientgotesting.UpdateActionImpl{{
+			Object: kpa(testNamespace, testRevision,
 				WithNoTraffic(noTrafficReason, "The target is not receiving traffic."),
 				withScales(0, -1), WithReachabilityReachable,
 				WithPAMetricsService(privateSvc), WithObservedGeneration(1),
diff --git a/pkg/reconciler/revision/table_test.go b/pkg/reconciler/revision/table_test.go
index 530f4a935..a4040546c 100644
--- a/pkg/reconciler/revision/table_test.go
+++ b/pkg/reconciler/revision/table_test.go
@@ -319,7 +319,8 @@ func TestReconcile(t *testing.T) {
 				WithK8sServiceName("something-in-the-way"), WithLogURL,
 				WithRevisionObservedGeneration(1)),
 			pa("foo", "pa-inactive",
-				WithNoTraffic("NoTraffic", "This thing is inactive.")),
+				WithNoTraffic("NoTraffic", "This thing is inactive."),
+				WithPAStatusService("pa-inactive")),
 			readyDeploy(deploy(t, "foo", "pa-inactive")),
 			image("foo", "pa-inactive"),
 		},
@@ -330,7 +331,26 @@ func TestReconcile(t *testing.T) {
 				// is inactive, we should see the following change.
 				MarkInactive("NoTraffic", "This thing is inactive."), WithRevisionObservedGeneration(1),
 				MarkResourcesUnavailable(v1.ReasonProgressDeadlineExceeded,
-					"Initial scale was never achieved")),
+					"Initial scale was never achieved"), WithK8sServiceName("pa-inactive")),
+		}},
+		Key: "foo/pa-inactive",
+	}, {
+		Name: "pa is not ready with initial scale zero, but ServiceName still empty, so not marking resources available false",
+		Objects: []runtime.Object{
+			Revision("foo", "pa-inactive", allUnknownConditions,
+				WithK8sServiceName("something-in-the-way"), WithLogURL,
+				WithRevisionObservedGeneration(1)),
+			pa("foo", "pa-inactive",
+				WithNoTraffic("NoTraffic", "This thing is inactive."), WithPAStatusService("")),
+			readyDeploy(deploy(t, "foo", "pa-inactive")),
+			image("foo", "pa-inactive"),
+		},
+		WantStatusUpdates: []clientgotesting.UpdateActionImpl{{
+			// We should not mark resources unavailable if ServiceName is empty
+			Object: Revision("foo", "pa-inactive",
+				WithLogURL, withDefaultContainerStatuses(), allUnknownConditions,
+				MarkInactive("NoTraffic", "This thing is inactive."),
+				WithRevisionObservedGeneration(1)),
 		}},
 		Key: "foo/pa-inactive",
 	}, {
diff --git a/pkg/reconciler/route/resources/labels/labels_test.go b/pkg/reconciler/route/resources/labels/labels_test.go
index 24e16c502..a5f8a68bd 100644
--- a/pkg/reconciler/route/resources/labels/labels_test.go
+++ b/pkg/reconciler/route/resources/labels/labels_test.go
@@ -24,47 +24,83 @@ import (
 	"knative.dev/serving/pkg/apis/serving"
 )
 
-func TestDeleteLabel(t *testing.T) {
+func TestIsObjectLocalVisibility(t *testing.T) {
 	tests := []struct {
-		name     string
-		meta     *v1.ObjectMeta
-		key      string
-		expected v1.ObjectMeta
-	}{
-		{
-			name:     "No labels in object meta",
-			meta:     &v1.ObjectMeta{},
-			key:      "key",
-			expected: v1.ObjectMeta{},
+		name string
+		meta *v1.ObjectMeta
+		want bool
+	}{{
+		name: "nil",
+		meta: &v1.ObjectMeta{},
+	}, {
+		name: "empty labels",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{},
+		},
+	}, {
+		name: "no matching labels",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{"frankie-goes": "to-hollywood"},
 		},
-		{
-			name: "No matching key",
-			meta: &v1.ObjectMeta{
-				Labels: map[string]string{"some label": "some value"},
-			},
-			key: "unknown",
-			expected: v1.ObjectMeta{
-				Labels: map[string]string{"some label": "some value"},
-			},
+	}, {
+		name: "false",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{serving.VisibilityLabelKey: ""},
 		},
-		{
-			name: "Has matching key",
-			meta: &v1.ObjectMeta{
-				Labels: map[string]string{"some label": "some value"},
-			},
-			key: "some label",
-			expected: v1.ObjectMeta{
-				Labels: map[string]string{},
-			},
+	}, {
+		name: "true",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{serving.VisibilityLabelKey: "set"},
 		},
+		want: true,
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got, want := IsObjectLocalVisibility(test.meta), test.want; got != want {
+				t.Errorf("IsObjectLocalVisibility = %v, want: %v", got, want)
+			}
+		})
 	}
+}
+
+func TestDeleteLabel(t *testing.T) {
+	tests := []struct {
+		name string
+		meta *v1.ObjectMeta
+		key  string
+		want v1.ObjectMeta
+	}{{
+		name: "No labels in object meta",
+		meta: &v1.ObjectMeta{},
+		key:  "key",
+		want: v1.ObjectMeta{},
+	}, {
+		name: "No matching key",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{"some label": "some value"},
+		},
+		key: "unknown",
+		want: v1.ObjectMeta{
+			Labels: map[string]string{"some label": "some value"},
+		},
+	}, {
+		name: "Has matching key",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{"some label": "some value"},
+		},
+		key: "some label",
+		want: v1.ObjectMeta{
+			Labels: map[string]string{},
+		},
+	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			DeleteLabel(tt.meta, tt.key)
 
-			if !cmp.Equal(tt.expected, *tt.meta) {
+			if !cmp.Equal(tt.want, *tt.meta) {
 				t.Errorf("DeleteLabel (-want, +got) = %v",
-					cmp.Diff(tt.expected, *tt.meta))
+					cmp.Diff(tt.want, *tt.meta))
 			}
 		})
 	}
@@ -76,48 +112,43 @@ func TestSetLabel(t *testing.T) {
 		meta  *v1.ObjectMeta
 		key   string
 		value string
-
-		expected v1.ObjectMeta
-	}{
-		{
-			name:  "No labels in object meta",
-			meta:  &v1.ObjectMeta{},
-			key:   "key",
-			value: "value",
-			expected: v1.ObjectMeta{
-				Labels: map[string]string{"key": "value"},
-			},
+		want  v1.ObjectMeta
+	}{{
+		name:  "No labels in object meta",
+		meta:  &v1.ObjectMeta{},
+		key:   "key",
+		value: "value",
+		want: v1.ObjectMeta{
+			Labels: map[string]string{"key": "value"},
 		},
-		{
-			name: "Empty labels",
-			meta: &v1.ObjectMeta{
-				Labels: map[string]string{},
-			},
-			key:   "key",
-			value: "value",
-			expected: v1.ObjectMeta{
-				Labels: map[string]string{"key": "value"},
-			},
+	}, {
+		name: "Empty labels",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{},
 		},
-		{
-			name: "Conflicting labels",
-			meta: &v1.ObjectMeta{
-				Labels: map[string]string{"key": "old value"},
-			},
-			key:   "key",
-			value: "new value",
-			expected: v1.ObjectMeta{
-				Labels: map[string]string{"key": "new value"},
-			},
+		key:   "key",
+		value: "value",
+		want: v1.ObjectMeta{
+			Labels: map[string]string{"key": "value"},
 		},
-	}
+	}, {
+		name: "Conflicting labels",
+		meta: &v1.ObjectMeta{
+			Labels: map[string]string{"key": "old value"},
+		},
+		key:   "key",
+		value: "new value",
+		want: v1.ObjectMeta{
+			Labels: map[string]string{"key": "new value"},
+		},
+	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			SetLabel(tt.meta, tt.key, tt.value)
 
-			if !cmp.Equal(tt.expected, *tt.meta) {
+			if !cmp.Equal(tt.want, *tt.meta) {
 				t.Errorf("DeleteLabel (-want, +got) = %v",
-					cmp.Diff(tt.expected, *tt.meta))
+					cmp.Diff(tt.want, *tt.meta))
 			}
 		})
 	}
@@ -128,17 +159,16 @@ func TestSetVisibility(t *testing.T) {
 		name           string
 		meta           *v1.ObjectMeta
 		isClusterLocal bool
-		expected       v1.ObjectMeta
+		want           v1.ObjectMeta
 	}{{
 		name:           "Set cluster local true",
 		meta:           &v1.ObjectMeta{},
 		isClusterLocal: true,
-		expected:       v1.ObjectMeta{Labels: map[string]string{serving.VisibilityLabelKey: serving.VisibilityClusterLocal}},
+		want:           v1.ObjectMeta{Labels: map[string]string{serving.VisibilityLabelKey: serving.VisibilityClusterLocal}},
 	}, {
-		name:           "Set cluster local false",
-		meta:           &v1.ObjectMeta{Labels: map[string]string{serving.VisibilityLabelKey: serving.VisibilityClusterLocal}},
-		isClusterLocal: false,
-		expected:       v1.ObjectMeta{Labels: map[string]string{}},
+		name: "Set cluster local false",
+		meta: &v1.ObjectMeta{Labels: map[string]string{serving.VisibilityLabelKey: serving.VisibilityClusterLocal}},
+		want: v1.ObjectMeta{Labels: map[string]string{}},
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/pkg/reconciler/route/route_test.go b/pkg/reconciler/route/route_test.go
index 2628d2a85..00b2d7d5a 100644
--- a/pkg/reconciler/route/route_test.go
+++ b/pkg/reconciler/route/route_test.go
@@ -1375,7 +1375,7 @@ func TestUpdateDomainConfigMap(t *testing.T) {
 				if err != nil {
 					return false, err
 				}
-				return !cmp.Equal(r.Status, route.Status), nil
+				return !cmp.Equal(r.Status, route.Status) && r.Generation == r.Status.ObservedGeneration, nil
 			}); err != nil {
 				t.Fatal("Failed to see route initial reconcile propagation:", err)
 			}
@@ -1385,43 +1385,56 @@ func TestUpdateDomainConfigMap(t *testing.T) {
 			// fake cache is going to be changed. Now, this update sometimes races with
 			// the CM update and since the update below is a noop (the same object
 			// is applied), nothing happens and the test fails (for one of the tests).
-			route = route.DeepCopy()
-			route.Generation++
-			tc.apply(route, watcher)
-			fakerouteinformer.Get(ctx).Informer().GetIndexer().Add(route)
-			if _, err := routeClient.Update(route); err != nil {
-				t.Fatal("Route.Update() =", err)
-			}
-
-			// Ensure we have the proper version in the informers.
-			if err := wait.PollImmediate(10*time.Millisecond, 3*time.Second, func() (bool, error) {
-				r, err := rl.Get(route.Name)
-				return r != nil && r.Generation == route.Generation, err
-			}); err != nil {
-				t.Error("Failed to see informers get the new Route version:", err)
-			}
+			// So we'll retry the operation several times.
+			timeout := time.After(10 * time.Second)
+			wantDomain := fmt.Sprintf("%s.%s.%s", route.Name, route.Namespace, tc.expectedDomainSuffix)
+			for i := 1; ; i++ {
+				select {
+				case <-timeout:
+					t.Fatal("Timed out trying to see the propagation")
+				default:
+					t.Log("Starting iteration", i)
+				}
+				route, _ := rl.Get(route.Name)
+				route = route.DeepCopy()
+				route.Generation++
+				tc.apply(route, watcher)
+				if _, err := routeClient.Update(route); err != nil {
+					t.Fatal("Route.Update() =", err)
+				}
 
-			// Now that we know the exact version the reconciler is going to see in the
-			// informers, let's reconcile.
-			if err := ctl.Reconciler.Reconcile(context.Background(), KeyOrDie(route)); err != nil {
-				t.Fatal("Reconcile() =", err)
-			}
+				// Ensure we have the proper version in the informers.
+				if err := wait.PollImmediate(10*time.Millisecond, 3*time.Second, func() (bool, error) {
+					r, err := rl.Get(route.Name)
+					return r != nil && r.Generation == route.Generation, err
+				}); err != nil {
+					t.Fatal("Failed to see informers get the new Route version:", err)
+				}
 
-			wantDomain := fmt.Sprintf("%s.%s.%s", route.Name, route.Namespace, tc.expectedDomainSuffix)
-			var gotDomain string
-			if err := wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (bool, error) {
-				r, err := routeClient.Get(route.Name, metav1.GetOptions{})
-				if err != nil {
-					return false, err
+				// Now that we know the exact version the reconciler is going to see in the
+				// informers, let's reconcile.
+				if err := ctl.Reconciler.Reconcile(context.Background(), KeyOrDie(route)); err != nil {
+					t.Fatal("Reconcile() =", err)
 				}
-				// Wait for the object to reconcile and the domain to propagate.
-				if r.Status.ObservedGeneration != route.Generation && r.Status.URL == nil {
-					return false, nil
+
+				var gotDomain string
+				if err := wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (bool, error) {
+					r, err := routeClient.Get(route.Name, metav1.GetOptions{})
+					if err != nil {
+						return false, err
+					}
+					// Wait for the object to reconcile and the domain to propagate.
+					if r.Status.ObservedGeneration != route.Generation && r.Status.URL == nil {
+						return false, nil
+					}
+					gotDomain = r.Status.URL.Host
+					return gotDomain == wantDomain, nil
+				}); err != nil {
+					t.Logf("Failed to see route domain propagation, got %s, want %s: %v", gotDomain, wantDomain, err)
+				} else {
+					t.Log("Success")
+					break
 				}
-				gotDomain = r.Status.URL.Host
-				return gotDomain == wantDomain, nil
-			}); err != nil {
-				t.Fatalf("Failed to see route domain propagation, got %s, want %s: %v", gotDomain, wantDomain, err)
 			}
 		})
 	}
diff --git a/pkg/reconciler/route/table_test.go b/pkg/reconciler/route/table_test.go
index 048aa2dbc..db5224bf4 100644
--- a/pkg/reconciler/route/table_test.go
+++ b/pkg/reconciler/route/table_test.go
@@ -1750,7 +1750,6 @@ func TestReconcile(t *testing.T) {
 				Namespace: "default",
 				Verb:      "delete",
 				Resource: schema.GroupVersionResource{
-					Group:    "core",
 					Version:  "v1",
 					Resource: "services",
 				},
@@ -1789,7 +1788,6 @@ func TestReconcile(t *testing.T) {
 				Namespace: "default",
 				Verb:      "delete",
 				Resource: schema.GroupVersionResource{
-					Group:    "core",
 					Version:  "v1",
 					Resource: "services",
 				},
diff --git a/pkg/reconciler/serverlessservice/serverlessservice_test.go b/pkg/reconciler/serverlessservice/serverlessservice_test.go
index 894531c02..0032df8d9 100644
--- a/pkg/reconciler/serverlessservice/serverlessservice_test.go
+++ b/pkg/reconciler/serverlessservice/serverlessservice_test.go
@@ -33,7 +33,14 @@ import (
 	"knative.dev/serving/pkg/client/injection/ducks/autoscaling/v1alpha1/podscalable"
 	_ "knative.dev/serving/pkg/client/injection/ducks/autoscaling/v1alpha1/podscalable/fake"
 
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	clientgotesting "k8s.io/client-go/testing"
+
 	"knative.dev/networking/pkg/apis/networking"
 	nv1a1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
 	sksreconciler "knative.dev/networking/pkg/client/injection/reconciler/networking/v1alpha1/serverlessservice"
@@ -45,14 +52,6 @@ import (
 	"knative.dev/serving/pkg/reconciler/serverlessservice/resources"
 	presources "knative.dev/serving/pkg/resources"
 
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	clientgotesting "k8s.io/client-go/testing"
-
 	. "knative.dev/pkg/reconciler/testing"
 	. "knative.dev/serving/pkg/reconciler/testing/v1"
 	. "knative.dev/serving/pkg/testing"
@@ -185,34 +184,6 @@ func TestReconcile(t *testing.T) {
 		WantUpdates: []clientgotesting.UpdateActionImpl{{
 			Object: endpointspub("steady", "to-proxy", WithSubsets, withFilteredPorts(networking.BackendHTTPPort)),
 		}},
-	}, {
-		Name: "many-private-services",
-		Key:  "many/privates",
-		Objects: []runtime.Object{
-			SKS("many", "privates", markHappy, WithPubService, WithPrivateService,
-				WithDeployRef("bar")),
-			deploy("many", "bar"),
-			svcpub("many", "privates"),
-			svcpriv("many", "privates"),
-			svcpriv("many", "privates", svcWithName("privates-brutality-is-here")),
-			svcpriv("many", "privates", svcWithName("privates-uncharacteristically-pretty"),
-				WithK8sSvcOwnersRemoved), // unowned, should remain.
-			endpointspub("many", "privates", WithSubsets, withFilteredPorts(networking.BackendHTTPPort)),
-			endpointspriv("many", "privates", WithSubsets),
-			activatorEndpoints(WithSubsets),
-		},
-		WantDeletes: []clientgotesting.DeleteActionImpl{{
-			ActionImpl: clientgotesting.ActionImpl{
-				Namespace: "many",
-				Verb:      "delete",
-				Resource: schema.GroupVersionResource{
-					Group:    "core",
-					Version:  "v1",
-					Resource: "services",
-				},
-			},
-			Name: "privates-brutality-is-here",
-		}},
 	}, {
 		Name: "user changes public svc",
 		Key:  "public/svc-change",
diff --git a/test/conformance/api/v1/util.go b/test/conformance/api/v1/util.go
index 129004fe9..5c2b94e9c 100644
--- a/test/conformance/api/v1/util.go
+++ b/test/conformance/api/v1/util.go
@@ -33,7 +33,7 @@ import (
 	v1test "knative.dev/serving/test/v1"
 )
 
-func waitForExpectedResponse(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, expectedResponse string) error {
+func checkForExpectedResponses(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, expectedResponses ...string) error {
 	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, url.Hostname(), test.ServingFlags.ResolvableDomain, test.AddRootCAtoTransport(t.Logf, clients, test.ServingFlags.Https))
 	if err != nil {
 		return err
@@ -42,7 +42,7 @@ func waitForExpectedResponse(t pkgTest.TLegacy, clients *test.Clients, url *url.
 	if err != nil {
 		return err
 	}
-	_, err = client.Poll(req, v1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedResponse))))
+	_, err = client.Poll(req, v1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesAllBodies(expectedResponses...))))
 	return err
 }
 
@@ -59,19 +59,15 @@ func validateDomains(t pkgTest.TLegacy, clients *test.Clients, baseDomain *url.U
 	// We don't have a good way to check if the route is updated so we will wait until a subdomain has
 	// started returning at least one expected result to key that we should validate percentage splits.
 	// In order for tests to succeed reliably, we need to make sure that all domains succeed.
-	for _, resp := range baseExpected {
-		// Check for each of the responses we expect from the base domain.
-		resp := resp
-		g.Go(func() error {
-			t.Log("Waiting for route to update", baseDomain)
-			return waitForExpectedResponse(t, clients, baseDomain, resp)
-		})
-	}
+	g.Go(func() error {
+		t.Log("Checking updated route", baseDomain)
+		return checkForExpectedResponses(t, clients, baseDomain, baseExpected...)
+	})
 	for i, s := range subdomains {
 		i, s := i, s
 		g.Go(func() error {
-			t.Log("Waiting for route to update", s)
-			return waitForExpectedResponse(t, clients, s, targetsExpected[i])
+			t.Log("Checking updated route tags", s)
+			return checkForExpectedResponses(t, clients, s, targetsExpected[i])
 		})
 	}
 	if err := g.Wait(); err != nil {
diff --git a/test/conformance/api/v1alpha1/util.go b/test/conformance/api/v1alpha1/util.go
index 77fc870d9..25f56c027 100644
--- a/test/conformance/api/v1alpha1/util.go
+++ b/test/conformance/api/v1alpha1/util.go
@@ -33,7 +33,7 @@ import (
 	v1a1test "knative.dev/serving/test/v1alpha1"
 )
 
-func waitForExpectedResponse(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, expectedResponse string) error {
+func checkForExpectedResponses(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, expectedResponses ...string) error {
 	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, url.Hostname(), test.ServingFlags.ResolvableDomain, test.AddRootCAtoTransport(t.Logf, clients, test.ServingFlags.Https))
 	if err != nil {
 		return err
@@ -42,7 +42,7 @@ func waitForExpectedResponse(t pkgTest.TLegacy, clients *test.Clients, url *url.
 	if err != nil {
 		return err
 	}
-	_, err = client.Poll(req, v1a1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedResponse))))
+	_, err = client.Poll(req, v1a1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesAllBodies(expectedResponses...))))
 	return err
 }
 
@@ -59,19 +59,15 @@ func validateDomains(t pkgTest.TLegacy, clients *test.Clients, baseDomain *url.U
 	// We don't have a good way to check if the route is updated so we will wait until a subdomain has
 	// started returning at least one expected result to key that we should validate percentage splits.
 	// In order for tests to succeed reliably, we need to make sure that all domains succeed.
-	for _, resp := range baseExpected {
-		// Check for each of the responses we expect from the base domain.
-		resp := resp
-		g.Go(func() error {
-			t.Log("Waiting for route to update", baseDomain)
-			return waitForExpectedResponse(t, clients, baseDomain, resp)
-		})
-	}
+	g.Go(func() error {
+		t.Log("Checking updated route", baseDomain)
+		return checkForExpectedResponses(t, clients, baseDomain, baseExpected...)
+	})
 	for i, s := range subdomains {
 		i, s := i, s
 		g.Go(func() error {
-			t.Log("Waiting for route to update", s)
-			return waitForExpectedResponse(t, clients, s, targetsExpected[i])
+			t.Log("Checking updated route tags", s)
+			return checkForExpectedResponses(t, clients, s, targetsExpected[i])
 		})
 	}
 	if err := g.Wait(); err != nil {
diff --git a/test/conformance/api/v1beta1/util.go b/test/conformance/api/v1beta1/util.go
index 909e6ca7c..cd1796b91 100644
--- a/test/conformance/api/v1beta1/util.go
+++ b/test/conformance/api/v1beta1/util.go
@@ -33,7 +33,7 @@ import (
 	v1b1test "knative.dev/serving/test/v1beta1"
 )
 
-func waitForExpectedResponse(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, expectedResponse string) error {
+func checkForExpectedResponses(t pkgTest.TLegacy, clients *test.Clients, url *url.URL, expectedResponses ...string) error {
 	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, url.Hostname(), test.ServingFlags.ResolvableDomain, test.AddRootCAtoTransport(t.Logf, clients, test.ServingFlags.Https))
 	if err != nil {
 		return err
@@ -42,7 +42,7 @@ func waitForExpectedResponse(t pkgTest.TLegacy, clients *test.Clients, url *url.
 	if err != nil {
 		return err
 	}
-	_, err = client.Poll(req, v1b1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.EventuallyMatchesBody(expectedResponse))))
+	_, err = client.Poll(req, v1b1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesAllBodies(expectedResponses...))))
 	return err
 }
 
@@ -59,19 +59,15 @@ func validateDomains(t pkgTest.TLegacy, clients *test.Clients, baseDomain *url.U
 	// We don't have a good way to check if the route is updated so we will wait until a subdomain has
 	// started returning at least one expected result to key that we should validate percentage splits.
 	// In order for tests to succeed reliably, we need to make sure that all domains succeed.
-	for _, resp := range baseExpected {
-		// Check for each of the responses we expect from the base domain.
-		resp := resp
-		g.Go(func() error {
-			t.Log("Waiting for route to update", baseDomain)
-			return waitForExpectedResponse(t, clients, baseDomain, resp)
-		})
-	}
+	g.Go(func() error {
+		t.Log("Checking updated route", baseDomain)
+		return checkForExpectedResponses(t, clients, baseDomain, baseExpected...)
+	})
 	for i, s := range subdomains {
 		i, s := i, s
 		g.Go(func() error {
-			t.Log("Waiting for route to update", s)
-			return waitForExpectedResponse(t, clients, s, targetsExpected[i])
+			t.Log("Checking updated route tags", s)
+			return checkForExpectedResponses(t, clients, s, targetsExpected[i])
 		})
 	}
 	if err := g.Wait(); err != nil {
diff --git a/test/e2e-common.sh b/test/e2e-common.sh
index 1426b02ff..f9d766343 100644
--- a/test/e2e-common.sh
+++ b/test/e2e-common.sh
@@ -165,7 +165,7 @@ function build_knative_from_source() {
   eval "${ENV_OUTPUT}"
 }
 
-# Installs Knative Serving in the current cluster, and waits for it to be ready.
+# Installs Knative Serving in the current cluster.
 # If no parameters are passed, installs the current source-based build, unless custom
 # YAML files were passed using the --custom-yamls flag.
 # Parameters: $1 - Knative Serving version "HEAD" or "latest-release". Default is "HEAD".
@@ -186,7 +186,7 @@ function install_knative_serving() {
   done
 }
 
-# Installs Knative Serving in the current cluster, and waits for it to be ready.
+# Installs Knative Serving in the current cluster.
 # If no parameters are passed, installs the current source-based build.
 # Parameters: $1 - Knative Serving version "HEAD" or "latest-release".
 #             $2 - Knative Monitoring YAML file (optional)
diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go
index 3564f4ec1..ddb73b12c 100644
--- a/test/e2e/autoscale.go
+++ b/test/e2e/autoscale.go
@@ -287,7 +287,7 @@ func numberOfReadyPods(ctx *testContext) (float64, error) {
 	return float64(resources.ReadyAddressCount(eps)), nil
 }
 
-func checkPodScale(ctx *testContext, targetPods, minPods, maxPods float64, duration time.Duration) error {
+func checkPodScale(ctx *testContext, targetPods, minPods, maxPods float64, duration time.Duration, quick bool) error {
 	// Short-circuit traffic generation once we exit from the check logic.
 	done := time.After(duration)
 	ticker := time.NewTicker(2 * time.Second)
@@ -311,8 +311,14 @@ func checkPodScale(ctx *testContext, targetPods, minPods, maxPods float64, durat
 			// A quick test succeeds when the number of pods scales up to `targetPods`
 			// (and, for sanity check, no more than `maxPods`).
 			if got >= targetPods && got <= maxPods {
-				ctx.t.Logf("Got %v replicas, reached target of %v, exiting early", got, targetPods)
-				return nil
+				if quick {
+					// A quick test succeeds when the number of pods scales up to `targetPods`
+					// (and, for sanity check, no more than `maxPods`).
+					if got >= targetPods && got <= maxPods {
+						ctx.t.Logf("Quick Mode: got %v >= %v", got, targetPods)
+						return nil
+					}
+				}
 			}
 			if minPods < targetPods-1 {
 				// Increase `minPods`, but leave room to reduce flakiness.
@@ -363,7 +369,7 @@ func assertAutoscaleUpToNumPods(ctx *testContext, curPods, targetPods float64, d
 
 	grp.Go(func() error {
 		defer close(stopChan)
-		return checkPodScale(ctx, targetPods, minPods, maxPods, duration)
+		return checkPodScale(ctx, targetPods, minPods, maxPods, duration, quick)
 	})
 
 	if err := grp.Wait(); err != nil {
diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go
index 6d0baf6cb..6f52305b6 100644
--- a/test/e2e/grpc_test.go
+++ b/test/e2e/grpc_test.go
@@ -259,7 +259,7 @@ func assertGRPCAutoscaleUpToNumPods(ctx *testContext, curPods, targetPods float6
 
 	grp.Go(func() error {
 		defer close(stopChan)
-		return checkPodScale(ctx, targetPods, minPods, maxPods, duration)
+		return checkPodScale(ctx, targetPods, minPods, maxPods, duration, true /* quick */)
 	})
 
 	if err := grp.Wait(); err != nil {
diff --git a/third_party/contour-latest/net-contour.yaml b/third_party/contour-latest/net-contour.yaml
index a45ea95e0..f5d03123c 100644
--- a/third_party/contour-latest/net-contour.yaml
+++ b/third_party/contour-latest/net-contour.yaml
@@ -35,7 +35,7 @@ metadata:
   namespace: knative-serving
   labels:
     networking.knative.dev/ingress-provider: contour
-    serving.knative.dev/release: "v20200831-3e31139c"
+    serving.knative.dev/release: "v20200901-a95fcd90"
 data:
   _example: |
     ################################
@@ -100,7 +100,7 @@ spec:
       - name: controller
         # This is the Go import path for the binary that is containerized
         # and substituted here.
-        image: gcr.io/knative-nightly/knative.dev/net-contour/cmd/controller@sha256:b9e61aa34de4e6790714c5f8aeabc0dc2771f67a51e9d77552631e3cec8500c2
+        image: gcr.io/knative-nightly/knative.dev/net-contour/cmd/controller@sha256:c276b6b4d2b21ac0455df8d2d6be7b24efea2186eaa94832f92624df4aa3956f
         resources:
           requests:
             cpu: 40m
diff --git a/third_party/kourier-latest/kourier.yaml b/third_party/kourier-latest/kourier.yaml
index f02421966..c5e95ab5e 100644
--- a/third_party/kourier-latest/kourier.yaml
+++ b/third_party/kourier-latest/kourier.yaml
@@ -104,7 +104,7 @@ spec:
         app: 3scale-kourier-control
     spec:
       containers:
-      - image: gcr.io/knative-nightly/knative.dev/net-kourier/cmd/kourier@sha256:2350368681819293a7b32dbd2205acb041f2a361f226a238c40d37da0b926826
+      - image: gcr.io/knative-nightly/knative.dev/net-kourier/cmd/kourier@sha256:8e9860c5d7ea46c959d6cedd11e46947e7e75afeabee0712511d26a507110d3f
         imagePullPolicy: Always
         name: kourier-control
         env:
diff --git a/vendor/knative.dev/pkg/metrics/metricstest/metricstest.go b/vendor/knative.dev/pkg/metrics/metricstest/metricstest.go
index ca0c82ec2..f63e34a52 100644
--- a/vendor/knative.dev/pkg/metrics/metricstest/metricstest.go
+++ b/vendor/knative.dev/pkg/metrics/metricstest/metricstest.go
@@ -91,10 +91,10 @@ func CheckDistributionData(t test.T, name string, wantTags map[string]string, ex
 			t.Error("reporter count wrong", "metric", name, "got", s.Count, "want", expectedCount)
 		}
 		if s.Min != expectedMin {
-			t.Error("reporter count wrong", "metric", name, "got", s.Min, "want", expectedMin)
+			t.Error("reporter min wrong", "metric", name, "got", s.Min, "want", expectedMin)
 		}
 		if s.Max != expectedMax {
-			t.Error("reporter count wrong", "metric", name, "got", s.Max, "want", expectedMax)
+			t.Error("reporter max wrong", "metric", name, "got", s.Max, "want", expectedMax)
 		}
 	}
 }
diff --git a/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go b/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go
index d0185015d..a40cc67cb 100644
--- a/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go
+++ b/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go
@@ -90,6 +90,32 @@ func Setup(org, repo, githubTokenPath string, dryrun bool) (*IssueHandler, error
 	return &IssueHandler{client: ghc, config: conf}, nil
 }
 
+// createNewIssue will create a new issue, and add perfLabel for it.
+func (gih *IssueHandler) createNewIssue(title, body string) (*github.Issue, error) {
+	var newIssue *github.Issue
+	if err := helpers.Run(
+		fmt.Sprintf("creating issue %q in %q", title, gih.config.repo),
+		func() error {
+			var err error
+			newIssue, err = gih.client.CreateIssue(gih.config.org, gih.config.repo, title, body)
+			return err
+		},
+		gih.config.dryrun,
+	); nil != err {
+		return nil, err
+	}
+	if err := helpers.Run(
+		fmt.Sprintf("adding perf label for issue %q in %q", title, gih.config.repo),
+		func() error {
+			return gih.client.AddLabelsToIssue(gih.config.org, gih.config.repo, *newIssue.Number, []string{perfLabel})
+		},
+		gih.config.dryrun,
+	); nil != err {
+		return nil, err
+	}
+	return newIssue, nil
+}
+
 // CreateIssueForTest will try to add an issue with the given testName and description.
 // If there is already an issue related to the test, it will try to update that issue.
 func (gih *IssueHandler) CreateIssueForTest(testName, desc string) error {
@@ -140,32 +166,6 @@ func (gih *IssueHandler) CreateIssueForTest(testName, desc string) error {
 	return nil
 }
 
-// createNewIssue will create a new issue, and add perfLabel for it.
-func (gih *IssueHandler) createNewIssue(title, body string) (*github.Issue, error) {
-	var newIssue *github.Issue
-	if err := helpers.Run(
-		fmt.Sprintf("creating issue %q in %q", title, gih.config.repo),
-		func() error {
-			var err error
-			newIssue, err = gih.client.CreateIssue(gih.config.org, gih.config.repo, title, body)
-			return err
-		},
-		gih.config.dryrun,
-	); nil != err {
-		return nil, err
-	}
-	if err := helpers.Run(
-		fmt.Sprintf("adding perf label for issue %q in %q", title, gih.config.repo),
-		func() error {
-			return gih.client.AddLabelsToIssue(gih.config.org, gih.config.repo, *newIssue.Number, []string{perfLabel})
-		},
-		gih.config.dryrun,
-	); nil != err {
-		return nil, err
-	}
-	return newIssue, nil
-}
-
 // CloseIssueForTest will try to close the issue for the given testName.
 // If there is no issue related to the test or the issue is already closed, the function will do nothing.
 func (gih *IssueHandler) CloseIssueForTest(testName string) error {
diff --git a/vendor/knative.dev/pkg/test/request.go b/vendor/knative.dev/pkg/test/request.go
index 4157ac33c..127d4371d 100644
--- a/vendor/knative.dev/pkg/test/request.go
+++ b/vendor/knative.dev/pkg/test/request.go
@@ -24,8 +24,10 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"knative.dev/pkg/test/logging"
 	"knative.dev/pkg/test/spoof"
 )
@@ -83,6 +85,33 @@ func IsStatusOK(resp *spoof.Response) (bool, error) {
 	return IsOneOfStatusCodes(http.StatusOK)(resp)
 }
 
+// MatchesAllBodies checks that the *first* response body matches the "expected" body, otherwise failing.
+func MatchesAllBodies(all ...string) spoof.ResponseChecker {
+
+	var m sync.Mutex
+	seen := make(sets.String, len(all))
+
+	return func(resp *spoof.Response) (bool, error) {
+		for _, expected := range all {
+			if !strings.Contains(string(resp.Body), expected) {
+				// See if the next one matches.
+				continue
+			}
+
+			// Protect our set
+			m.Lock()
+			defer m.Unlock()
+			seen.Insert(expected)
+
+			// Stop once we've seen them all.
+			return seen.HasAll(all...), nil
+		}
+
+		// Returning (true, err) causes SpoofingClient.Poll to fail.
+		return true, fmt.Errorf("body = %s, want one of: %s", string(resp.Body), all)
+	}
+}
+
 // MatchesBody checks that the *first* response body matches the "expected" body, otherwise failing.
 func MatchesBody(expected string) spoof.ResponseChecker {
 	return func(resp *spoof.Response) (bool, error) {