Repository: kubernetes-sigs/cluster-api-provider-azure
Pull request #1635 (2021-08-20 18:12:37)

update templates to use 8GB of etcd db data
 <!-- If this is your first PR, welcome! Please make sure you read the [contributing guidelines](../CONTRIBUTING.md). -->

 <!-- Please label this pull request according to what type of issue you are addressing (see ../CONTRIBUTING.md) -->
**What type of PR is this?**

<!--
Add one of the following kinds:
/kind feature
/kind bug
/kind api-change
/kind cleanup
/kind deprecation
/kind design
/kind documentation
/kind failing-test
/kind flake
/kind other
-->

/kind other

^-- Test configuration change

**What this PR does / why we need it**:

This PR updates the base etcd config for all cluster templates to use an 8 GB data size configuration. This is based on two things:

1) the 2 GB data default can be problematic (i.e., too small) for large clusters
2) 8 GB is the recommended maximum to use (there are ways to use more data but for now using the recommended max seems good and 4x compared to the current 2 GB size is a non-trivial increase)

See:

https://etcd.io/docs/v3.4/dev-guide/limit/

I think the guiding motivation of this change is to help unlock large cluster scenarios by rigorously testing this configuration recommendation. Because we are already using a 256 GB data disk volume for etcd in these templates, the additional etcd data size does not require any changes there.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

_Please confirm that if this PR changes any image versions, then that's the sole change this PR makes._

**TODOs**:
<!-- Put an "X" character inside the brackets of each completed task. Some may be optional depending on the PR. -->

- [ ] squashed commits
- [ ] includes documentation
- [ ] adds unit tests

**Release note**:
<!--  Write your release note:
1. Enter your extended release note in the below block. If the PR requires additional action from users switching to the new release, include the string "action required".
2. If no release note is required, just write "NONE".
-->
```release-note
cluster templates now use 8 GB of etcd db data (was 2 GB)
```



Patch:
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 8bef598ce..73b6880b3 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -6,8 +6,6 @@ about: Tell us about a problem you are experiencing.
 
 /kind bug
 
-[Before submitting an issue, have you checked the [Troubleshooting Guide](https://capz.sigs.k8s.io/topics/troubleshooting.html)?]
-
 **What steps did you take and what happened:**
 [A clear and concise description of what the bug is.]
 
diff --git a/.github/workflows/cover.yaml b/.github/workflows/cover.yaml
index 5b5a441ab..7ac446d8a 100644
--- a/.github/workflows/cover.yaml
+++ b/.github/workflows/cover.yaml
@@ -7,10 +7,8 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-go@v2
-      with:
-        go-version: '^1.17'
+    - uses: actions/checkout@master
+    - run: "bash hack/install-go.sh"
     - run: "PATH=/usr/local/go/bin:$PATH make test-cover"
     - uses: codecov/codecov-action@v1
       with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
deleted file mode 100644
index 0c12e659c..000000000
--- a/.github/workflows/release.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
-on:
-  push:
-    # Sequence of patterns matched against refs/tags
-    tags:
-    - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-name: release
-
-jobs:
-  build:
-    name: tag release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set env
-        run:  echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV
-      - name: checkout code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Install go
-        uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17'
-      - name: generate release artifacts
-        run: |
-          make release
-      - name: generate release notes
-        run: |
-          make release-notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: out/*
-          body_path: release-notes-${{ env.RELEASE_TAG }}.md
diff --git a/Makefile b/Makefile
index fdcafc86d..77e6d3b60 100644
--- a/Makefile
+++ b/Makefile
@@ -405,11 +405,8 @@ set-manifest-pull-policy:
 ## Release
 ## --------------------------------------
 
-RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null)
-PREVIOUS_TAG ?= $(shell git describe --abbrev=0 $(RELEASE_TAG)^ 2>/dev/null)
-RELEASE_DIR ?= out
-GIT_REPO_NAME ?= cluster-api-provider-azure
-GIT_ORG_NAME ?= kubernetes-sigs
+RELEASE_TAG := $(shell git describe --abbrev=0 2>/dev/null)
+RELEASE_DIR := out
 
 $(RELEASE_DIR):
 	mkdir -p $(RELEASE_DIR)/
@@ -463,7 +460,7 @@ release-alias-tag: # Adds the tag to the last build tag.
 
 .PHONY: release-notes
 release-notes: $(RELEASE_NOTES)
-	$(RELEASE_NOTES) --org $(GIT_ORG_NAME) --repo $(GIT_REPO_NAME)  --start-rev $(PREVIOUS_TAG) --end-rev $(RELEASE_TAG) --output release-notes-$(RELEASE_TAG).md
+	$(RELEASE_NOTES)
 
 ## --------------------------------------
 ## Development
diff --git a/api/v1alpha4/azurecluster_validation.go b/api/v1alpha4/azurecluster_validation.go
index aa19715e8..0d26f67cd 100644
--- a/api/v1alpha4/azurecluster_validation.go
+++ b/api/v1alpha4/azurecluster_validation.go
@@ -238,7 +238,7 @@ func validateSubnetCIDR(subnetCidrBlocks []string, vnetCidrBlocks []string, fldP
 		}
 
 		if !found {
-			allErrs = append(allErrs, field.Invalid(fldPath, subnetCidr, fmt.Sprintf("subnet CIDR not in vnet address space: %s", vnetCidrBlocks)))
+			allErrs = append(allErrs, field.Invalid(fldPath, subnetCidr, "subnet CIDR not in vnet CIDR range"))
 		}
 	}
 
diff --git a/api/v1alpha4/azurecluster_webhook.go b/api/v1alpha4/azurecluster_webhook.go
index 6af90486c..d092952ee 100644
--- a/api/v1alpha4/azurecluster_webhook.go
+++ b/api/v1alpha4/azurecluster_webhook.go
@@ -87,7 +87,7 @@ func (c *AzureCluster) ValidateUpdate(oldRaw runtime.Object) error {
 	if !reflect.DeepEqual(c.Spec.AzureEnvironment, old.Spec.AzureEnvironment) {
 		allErrs = append(allErrs,
 			field.Invalid(field.NewPath("spec", "AzureEnvironment"),
-				c.Spec.AzureEnvironment, "field is immutable"),
+				c.Spec.ResourceGroup, "field is immutable"),
 		)
 	}
 
diff --git a/docs/book/src/developers/development.md b/docs/book/src/developers/development.md
index 9ccea8002..22ba9b400 100644
--- a/docs/book/src/developers/development.md
+++ b/docs/book/src/developers/development.md
@@ -47,28 +47,23 @@
    - Get the latest patch version for go v1.16.
 2. Install [jq][jq]
    - `brew install jq` on macOS.
-   - `sudo apt install jq` on Windows + WSL2
+   - `chocolatey install jq` on Windows.
    - `sudo apt install jq` on Ubuntu Linux.
 3. Install [gettext][gettext] package
    - `brew install gettext && brew link --force gettext` on macOS.
-   - `sudo apt install gettext` on Windows + WSL2.
+   - [install instructions][gettextwindows] on Windows.
    - `sudo apt install gettext` on Ubuntu Linux.
 4. Install [KIND][kind]
    - `GO111MODULE="on" go get sigs.k8s.io/kind@v0.9.0`.
 5. Install [Kustomize][kustomize]
    - `brew install kustomize` on macOS.
-   - [install instructions](https://kubectl.docs.kubernetes.io/installation/kustomize/) on Windows + WSL2.
-   - [install instructions][kustomizelinux] on Linux.
+   - `choco install kustomize` on Windows.
+   - [install instructions][kustomizelinux] on Linux
 6. Install Python 3.x or 2.7.x, if neither is already installed.
 7. Install make.
-   - `brew install make` on MacOS.
-   - `sudo apt install make` on Windows + WSL2.
-   - `sudo apt install make` on Linux.
 8. Install [timeout][timeout]
    - `brew install coreutils` on macOS.
 
-When developing on Windows, it is suggested to set up the project on Windows + WSL2 and the file should be checked out on as wsl file system for better results.
-
 ### Get the source
 
 ```shell
@@ -220,11 +215,7 @@ The cluster-api management components that are deployed are configured at the `/
 
 #### Deploying a workload cluster
 
-⚠️ Note that when developing with `tilt` as described above, some `clusterctl` commands won't work. Specifically, `clusterctl config` and `clusterctl generate` may fail. These commands expect specific releases of CAPI and CAPZ to be installed, but the `tilt` environment dynamically updates and installs these components from your local code. `clusterctl get kubeconfig` will still work, however.
-
-After your kind management cluster is up and running with Tilt, you can [configure workload cluster settings](#customizing-the-cluster-deployment) and deploy a workload cluster by opening the `tilt` web UI and clicking the clockwise arrow icon ⟳ on a resource listed, such as "aks-aad," "ipv6," or "windows."
-
-Or you can deploy a workload cluster with the following command:
+After your kind management cluster is up and running with Tilt, you can [configure workload cluster settings](#customizing-the-cluster-deployment) and deploy a workload cluster with the following:
 
 ```bash
 make create-workload-cluster
diff --git a/docs/book/src/developers/releasing.md b/docs/book/src/developers/releasing.md
index 74c96f139..27a59b184 100644
--- a/docs/book/src/developers/releasing.md
+++ b/docs/book/src/developers/releasing.md
@@ -1,47 +1,55 @@
 # Release Process
 
 ## Change milestone
-
-- Create a new GitHub milestone for the next release
-- Change milestone applier so new changes can be applied to the appropriate release
-  - Open a PR in https://github.com/kubernetes/test-infra to change this [line](https://github.com/kubernetes/test-infra/blob/25db54eb9d52e08c16b3601726d8f154f8741025/config/prow/plugins.yaml#L344)
-    - Example PR: https://github.com/kubernetes/test-infra/pull/16827
-
-## Create a tag
-
-- Identify a known good commit on the main branch
-- Fast-forward the release branch to the selected commit. :warning: Always release from the release branch and not from main!
-  - `git checkout release-0.x`
-  - `git fetch upstream`
-  - `git merge --ff-only upstream/main`
-  - `git push`
-- Create tag with git
-  - `export RELEASE_TAG=v0.4.6` (the tag of the release to be cut)
-  - `git tag -s ${RELEASE_TAG} -m "${RELEASE_TAG}"`
-  - `-s` creates a signed tag, you must have a GPG key [added to your GitHub account](https://docs.github.com/en/enterprise/2.16/user/github/authenticating-to-github/generating-a-new-gpg-key)
-  - `git push upstream ${RELEASE_TAG}`
-
-This will automatically trigger a [Github Action](https://github.com/kubernetes-sigs/cluster-api-provider-azure/actions) to create a draft release.
+ -  Create a new GitHub milestone for the next release
+ - Change milestone applier so new changes can be applied to the appropriate release
+      - Open a PR in https://github.com/kubernetes/test-infra to change this [line](https://github.com/kubernetes/test-infra/blob/25db54eb9d52e08c16b3601726d8f154f8741025/config/prow/plugins.yaml#L344)
+        - Example PR: https://github.com/kubernetes/test-infra/pull/16827
+
+## Prepare branch, tag and release notes
+ - Identify a known good commit on the main branch
+ - Fast-forward the release branch to the selected commit. :warning: Always release from the release branch and not from main!
+   - `git checkout release-0.x`
+   - `git fetch upstream`
+   - `git merge --ff-only upstream/main`
+   - `git push`
+ - Create tag with git
+   - `export RELEASE_TAG=v0.4.6` (the tag of the release to be cut)
+   - `git tag -s ${RELEASE_TAG} -m "${RELEASE_TAG}"`
+	 - `-s` creates a signed tag, you must have a GPG key [added to your GitHub account](https://docs.github.com/en/enterprise/2.16/user/github/authenticating-to-github/generating-a-new-gpg-key)
+   - `git push upstream ${RELEASE_TAG}`
+ - Update the file `metadata.yaml` if is a major or minor release
+ - `make release` from repo, this will create the release artifacts in the `out/` folder
+ - Install the `release-notes` tool according to [instructions](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md)
+ - Export GITHUB_TOKEN
+ - Run the release-notes tool with the appropriate commits. Commits range from the first commit after the previous release to the new release commit.
+  ```
+  release-notes --github-org kubernetes-sigs --github-repo cluster-api-provider-azure \
+  --start-sha 1cf1ec4a1effd9340fe7370ab45b173a4979dc8f  \
+  --end-sha e843409f896981185ca31d6b4a4c939f27d975de
+  ```
+ - Manually format and categorize the release notes
 
 ## Promote image to prod repo
-
-- Images are built by the [post push images job](https://testgrid.k8s.io/sig-cluster-lifecycle-cluster-api-provider-azure#post-cluster-api-provider-azure-push-images)
-- Create a PR in https://github.com/kubernetes/k8s.io to add the image and tag
-  - Example PR: https://github.com/kubernetes/k8s.io/pull/1030/files
-- Location of image: https://console.cloud.google.com/gcr/images/k8s-staging-cluster-api-azure/GLOBAL/cluster-api-azure-controller?rImageListsize=30
+ Promote image
+ - Images are built by the [post push images job](https://testgrid.k8s.io/sig-cluster-lifecycle-cluster-api-provider-azure#post-cluster-api-provider-azure-push-images)
+ - Create a PR in https://github.com/kubernetes/k8s.io to add the image and tag
+   - Example PR: https://github.com/kubernetes/k8s.io/pull/1030/files
+ - Location of image: https://console.cloud.google.com/gcr/images/k8s-staging-cluster-api-azure/GLOBAL/cluster-api-azure-controller?rImageListsize=30
 
 ## Release in GitHub
-
-- Manually format and categorize the release notes
-- Publish release
-- [Announce][release-announcement] the release
+ Create the GitHub release in the UI
+ - Create a draft release in GitHub and associate it with the tag that was created
+ - Copy paste the release notes
+ - Upload [artifacts](#expected-artifacts) from the `out/` folder
+ - Publish release
+ - [Announce][release-announcement] the release
 
 ## Versioning
 
 cluster-api-provider-azure follows the [semantic versionining][semver] specification.
 
 Example versions:
-
 - Pre-release: `v0.1.1-alpha.1`
 - Minor release: `v0.1.0`
 - Patch release: `v0.1.1`
diff --git a/hack/install-go.sh b/hack/install-go.sh
new file mode 100644
index 000000000..d3a68603a
--- /dev/null
+++ b/hack/install-go.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "installing apt dependencies"
+sudo apt update && sudo apt install -y curl apt-transport-https
+
+echo "setting up work dir"
+WORKDIR="$(mktemp -d)"
+pushd "$WORKDIR"
+
+trap 'rm -rf ${WORKDIR}' EXIT
+
+GOLANG_VERSION="$(curl -sS https://golang.org/VERSION?m=text)"
+echo "Downloading ${GOLANG_VERSION}"
+curl -O "https://dl.google.com/go/${GOLANG_VERSION}.linux-amd64.tar.gz"
+
+echo "unpacking go"
+sudo mkdir -p /usr/local/go
+sudo chown -R "$(whoami):$(whoami)" /usr/local/go 
+tar -xvf "${GOLANG_VERSION}.linux-amd64.tar.gz" -C /usr/local
+
+echo "Successfully installed go"
diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml
index 570c782c4..c194b1afc 100644
--- a/templates/cluster-template-aad.yaml
+++ b/templates/cluster-template-aad.yaml
@@ -73,6 +73,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml
index ca2088242..14d4024bc 100644
--- a/templates/cluster-template-azure-bastion.yaml
+++ b/templates/cluster-template-azure-bastion.yaml
@@ -70,6 +70,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml
index 1e272b5b6..0d03569d9 100644
--- a/templates/cluster-template-ephemeral.yaml
+++ b/templates/cluster-template-ephemeral.yaml
@@ -68,6 +68,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-external-cloud-provider.yaml b/templates/cluster-template-external-cloud-provider.yaml
index 51335d140..28f17144f 100644
--- a/templates/cluster-template-external-cloud-provider.yaml
+++ b/templates/cluster-template-external-cloud-provider.yaml
@@ -69,6 +69,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml
index 39ca16e34..6cdf99444 100644
--- a/templates/cluster-template-ipv6.yaml
+++ b/templates/cluster-template-ipv6.yaml
@@ -89,6 +89,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       scheduler:
         extraArgs:
           bind-address: '::'
diff --git a/templates/cluster-template-machinepool-multiple-subnets.yaml b/templates/cluster-template-machinepool-multiple-subnets.yaml
index 41a65a70a..c1860a950 100644
--- a/templates/cluster-template-machinepool-multiple-subnets.yaml
+++ b/templates/cluster-template-machinepool-multiple-subnets.yaml
@@ -75,6 +75,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-machinepool-system-assigned-identity.yaml b/templates/cluster-template-machinepool-system-assigned-identity.yaml
index 713fe6307..d1ab4cfda 100644
--- a/templates/cluster-template-machinepool-system-assigned-identity.yaml
+++ b/templates/cluster-template-machinepool-system-assigned-identity.yaml
@@ -64,6 +64,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-machinepool-user-assigned-identity.yaml b/templates/cluster-template-machinepool-user-assigned-identity.yaml
index 720157195..0e7b492af 100644
--- a/templates/cluster-template-machinepool-user-assigned-identity.yaml
+++ b/templates/cluster-template-machinepool-user-assigned-identity.yaml
@@ -64,6 +64,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml
index 876069e40..f5592e508 100644
--- a/templates/cluster-template-machinepool-windows.yaml
+++ b/templates/cluster-template-machinepool-windows.yaml
@@ -69,6 +69,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml
index b2877869a..a564461fa 100644
--- a/templates/cluster-template-machinepool.yaml
+++ b/templates/cluster-template-machinepool.yaml
@@ -68,6 +68,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-nat-gateway.yaml b/templates/cluster-template-nat-gateway.yaml
index 3d576256f..79da52885 100644
--- a/templates/cluster-template-nat-gateway.yaml
+++ b/templates/cluster-template-nat-gateway.yaml
@@ -81,6 +81,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml
index 84fd0dfa9..408981d27 100644
--- a/templates/cluster-template-nvidia-gpu.yaml
+++ b/templates/cluster-template-nvidia-gpu.yaml
@@ -69,6 +69,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml
index ccb8fe734..69c634b19 100644
--- a/templates/cluster-template-private.yaml
+++ b/templates/cluster-template-private.yaml
@@ -77,6 +77,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-system-assigned-identity.yaml b/templates/cluster-template-system-assigned-identity.yaml
index 5b31f6892..7bdeb6a2a 100644
--- a/templates/cluster-template-system-assigned-identity.yaml
+++ b/templates/cluster-template-system-assigned-identity.yaml
@@ -64,6 +64,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-user-assigned-identity.yaml b/templates/cluster-template-user-assigned-identity.yaml
index a1e8bfcf9..85b374e2e 100644
--- a/templates/cluster-template-user-assigned-identity.yaml
+++ b/templates/cluster-template-user-assigned-identity.yaml
@@ -64,6 +64,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml
index 45fa9e813..b02080f5d 100644
--- a/templates/cluster-template-windows.yaml
+++ b/templates/cluster-template-windows.yaml
@@ -69,6 +69,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
index 3fe7e4e59..fff9af4a3 100644
--- a/templates/cluster-template.yaml
+++ b/templates/cluster-template.yaml
@@ -68,6 +68,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml
index d53df3468..b5e5a57f9 100644
--- a/templates/flavors/base/cluster-template.yaml
+++ b/templates/flavors/base/cluster-template.yaml
@@ -81,6 +81,8 @@ spec:
       etcd:
         local:
           dataDir: "/var/lib/etcddisk/etcd"
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     files:
       - contentFrom:
           secret:


Test patch:
diff --git a/api/v1alpha4/azurecluster_validation_test.go b/api/v1alpha4/azurecluster_validation_test.go
index fa77957c9..0ac8eb817 100644
--- a/api/v1alpha4/azurecluster_validation_test.go
+++ b/api/v1alpha4/azurecluster_validation_test.go
@@ -588,7 +588,7 @@ func TestValidateSubnetCIDR(t *testing.T) {
 				Type:     "FieldValueInvalid",
 				Field:    "subnets.cidrBlocks",
 				BadValue: "11.1.0.0/16",
-				Detail:   "subnet CIDR not in vnet address space: [10.0.0.0/8]",
+				Detail:   "subnet CIDR not in vnet CIDR range",
 			},
 		},
 		{
@@ -602,8 +602,14 @@ func TestValidateSubnetCIDR(t *testing.T) {
 		t.Run(testCase.name, func(t *testing.T) {
 			err := validateSubnetCIDR(testCase.subnetCidrBlocks, testCase.vnetCidrBlocks, field.NewPath("subnets.cidrBlocks"))
 			if testCase.wantErr {
-				// Searches for expected error in list of thrown errors
-				g.Expect(err).To(ContainElement(MatchError(testCase.expectedErr.Error())))
+				g.Expect(err).NotTo(HaveLen(0))
+				found := false
+				for _, actual := range err {
+					if actual.Error() == testCase.expectedErr.Error() {
+						found = true
+					}
+				}
+				g.Expect(found).To(BeTrue())
 			} else {
 				g.Expect(err).To(HaveLen(0))
 			}
diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml
index 22806f83f..52ed1db21 100644
--- a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml
+++ b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml
@@ -74,6 +74,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       kubernetesVersion: ci/${CI_VERSION}
     diskSetup:
       filesystems:
diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml
index e7482d57c..7404bf56c 100644
--- a/templates/test/ci/cluster-template-prow-ci-version.yaml
+++ b/templates/test/ci/cluster-template-prow-ci-version.yaml
@@ -73,6 +73,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       kubernetesVersion: ci/${CI_VERSION}
     diskSetup:
       filesystems:
diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml
index f5f14c33a..6e0d89d64 100644
--- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml
+++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml
@@ -84,6 +84,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml
index 837a36e44..4138dbc5d 100644
--- a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml
+++ b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml
@@ -74,6 +74,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml
index 5392986de..e0834c274 100644
--- a/templates/test/ci/cluster-template-prow-ipv6.yaml
+++ b/templates/test/ci/cluster-template-prow-ipv6.yaml
@@ -94,6 +94,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       scheduler:
         extraArgs:
           bind-address: '::'
diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml
index f1b5d450b..4174976d6 100644
--- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml
+++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml
@@ -73,6 +73,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       kubernetesVersion: ci/${CI_VERSION}
     diskSetup:
       filesystems:
diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml
index 7220a8faf..078fc989a 100644
--- a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml
+++ b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml
@@ -74,6 +74,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml
index 146ae8fef..7536bc5de 100644
--- a/templates/test/ci/cluster-template-prow-machine-pool.yaml
+++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml
@@ -73,6 +73,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml
index f380427ee..1026a35ea 100644
--- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml
+++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml
@@ -74,6 +74,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml
index 3d632da35..2efa78f0d 100644
--- a/templates/test/ci/cluster-template-prow-private.yaml
+++ b/templates/test/ci/cluster-template-prow-private.yaml
@@ -100,6 +100,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml
index e4f280e7a..5bc4db9ad 100644
--- a/templates/test/ci/cluster-template-prow-windows.yaml
+++ b/templates/test/ci/cluster-template-prow-windows.yaml
@@ -74,6 +74,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml
index a7cd8e310..d1694d76e 100644
--- a/templates/test/ci/cluster-template-prow.yaml
+++ b/templates/test/ci/cluster-template-prow.yaml
@@ -73,6 +73,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
     diskSetup:
       filesystems:
       - device: /dev/disk/azure/scsi1/lun0
diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml
index 83f4b06a7..e8f71856c 100644
--- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml
+++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml
@@ -75,6 +75,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       kubernetesVersion: ci/${CI_VERSION}
     diskSetup:
       filesystems:
diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml
index c043a81c2..73fa29a82 100644
--- a/templates/test/dev/cluster-template-custom-builds-windows.yaml
+++ b/templates/test/dev/cluster-template-custom-builds-windows.yaml
@@ -76,6 +76,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       kubernetesVersion: ci/${CI_VERSION}
     diskSetup:
       filesystems:
diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml
index 820ae98ae..dbc8a0d78 100644
--- a/templates/test/dev/cluster-template-custom-builds.yaml
+++ b/templates/test/dev/cluster-template-custom-builds.yaml
@@ -75,6 +75,8 @@ spec:
       etcd:
         local:
           dataDir: /var/lib/etcddisk/etcd
+          extraArgs:
+            quota-backend-bytes: "8589934592"
       kubernetesVersion: ci/${CI_VERSION}
     diskSetup:
       filesystems: