Repository: kubernetes/kops (2026-04-02T01:12:54Z)

Issue:
LB listener port for cilium-etcd's etcd cluster not created on GCE
/kind bug

**1. What `kops` version are you running? The command `kops version`, will display
 this information.**

Client version: 1.33.1 (git-v1.33.1)

**2. What Kubernetes version are you running? `kubectl version` will print the
 version if a cluster is running or provide the Kubernetes version specified as
 a `kops` flag.**

**3. What cloud provider are you using?**

GCE

**4. What commands did you run?  What is the simplest way to reproduce this issue?**

kops create cluster cluster.k8s.local \
  --topology private \
  --api-loadbalancer-type internal \
  --networking cilium-etcd \
  --bastion

**5. What happened after the commands executed?**

Cluster built, but cilium pod continually restarted as they couldn't reach APISERVER:4003.  Confirmed that APISERVER isn't configured to listen on port 4003.

**6. What did you expect to happen?**

**7. Please provide your cluster manifest. Execute
  `kops get --name my.example.com -o yaml` to display your cluster manifest.
  You may want to remove your cluster name and other sensitive information.**

```yaml
```

**8. Please run the commands with most verbose logging by adding the `-v 10` flag.
  Paste the logs into this report, or in a gist and provide the gist link here.**

**9. Anything else do we need to know?**



Discussion:
The Kubernetes project currently lacks enough contributors to adequately respond to all issues.

This bot triages un-triaged issues according to the following rules:
- After 90d of inactivity, `lifecycle/stale` is applied
- After 30d of inactivity since `lifecycle/stale` was applied, `lifecycle/rotten` is applied
- After 30d of inactivity since `lifecycle/rotten` was applied, the issue is closed

You can:
- Mark this issue as fresh with `/remove-lifecycle stale`
- Close this issue with `/close`
- Offer to help out with [Issue Triage][1]

Please send feedback to sig-contributor-experience at [kubernetes/community](https://github.com/kubernetes/community).

/lifecycle stale

[1]: https://www.kubernetes.dev/docs/guide/issue-triage/
/remove-lifecycle stale
The Kubernetes project currently lacks enough contributors to adequately respond to all issues.

This bot triages un-triaged issues according to the following rules:
- After 90d of inactivity, `lifecycle/stale` is applied
- After 30d of inactivity since `lifecycle/stale` was applied, `lifecycle/rotten` is applied
- After 30d of inactivity since `lifecycle/rotten` was applied, the issue is closed

You can:
- Mark this issue as fresh with `/remove-lifecycle stale`
- Close this issue with `/close`
- Offer to help out with [Issue Triage][1]

Please send feedback to sig-contributor-experience at [kubernetes/community](https://github.com/kubernetes/community).

/lifecycle stale

[1]: https://www.kubernetes.dev/docs/guide/issue-triage/
/remove-lifecycle stale



Patch:
diff --git a/pkg/model/gcemodel/api_loadbalancer.go b/pkg/model/gcemodel/api_loadbalancer.go
index d2c29112bbde7..f4c445d7569bf 100644
--- a/pkg/model/gcemodel/api_loadbalancer.go
+++ b/pkg/model/gcemodel/api_loadbalancer.go
@@ -22,6 +22,7 @@ import (
 
 	"golang.org/x/exp/slices"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/pkg/wellknownservices"
 	"k8s.io/kops/upup/pkg/fi"
@@ -127,6 +128,16 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
 				Allowed:      []string{"tcp:" + strconv.Itoa(wellknownports.KopsControllerPort)},
 			})
 		}
+
+		if model.UseCiliumEtcd(b.Cluster) {
+			b.AddFirewallRulesTasks(c, "cilium-etcd", &gcetasks.FirewallRule{
+				Lifecycle:    b.Lifecycle,
+				Network:      network,
+				SourceRanges: b.Cluster.Spec.API.Access,
+				TargetTags:   []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane)},
+				Allowed:      []string{"tcp:" + strconv.Itoa(wellknownports.EtcdCiliumClientPort)},
+			})
+		}
 	}
 	return nil
 
@@ -234,6 +245,24 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 
 			c.AddTask(fr)
 		}
+
+		if model.UseCiliumEtcd(b.Cluster) {
+			c.AddTask(&gcetasks.ForwardingRule{
+				Name:                s(b.NameForForwardingRule("cilium-etcd-" + sn.Name)),
+				Lifecycle:           b.Lifecycle,
+				BackendService:      bs,
+				Ports:               []string{strconv.Itoa(wellknownports.EtcdCiliumClientPort)},
+				IPAddress:           ipAddress,
+				IPProtocol:          "TCP",
+				LoadBalancingScheme: s("INTERNAL"),
+				Network:             network,
+				Subnetwork:          subnet,
+				Labels: map[string]string{
+					clusterLabel.Key: clusterLabel.Value,
+					"name":           "cilium-etcd-" + sn.Name,
+				},
+			})
+		}
 	}
 	return nil
 }


Test patch:
diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go
index 649df842a3da8..cb19d5b02bafd 100644
--- a/cmd/kops/integration_test.go
+++ b/cmd/kops/integration_test.go
@@ -397,6 +397,20 @@ func TestMinimalGCEInternalLoadBalancer(t *testing.T) {
 		runTestTerraformGCE(t)
 }
 
+// TestMinimalGCEInternalLoadBalancerCiliumEtcd runs tests on a minimal GCE configuration with an internal load balancer and cilium-etcd.
+func TestMinimalGCEInternalLoadBalancerCiliumEtcd(t *testing.T) {
+	newIntegrationTest("minimal-gce-ilb-cilium-etcd.example.com", "minimal_gce_ilb_cilium_etcd").
+		withCiliumEtcd().
+		withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium-master-us-test1-a").
+		withAddons(
+			ciliumAddon,
+			dnsControllerAddon,
+			gcpCCMAddon,
+			gcpPDCSIAddon,
+		).
+		runTestTerraformGCE(t)
+}
+
 // TestMinimalGCEPublicLoadBalancer runs tests on a minimal GCE configuration with a public load balancer.
 func TestMinimalGCEPublicLoadBalancer(t *testing.T) {
 	newIntegrationTest("minimal-gce-plb.example.com", "minimal_gce_plb").
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_cluster-completed.spec_content
new file mode 100644
index 0000000000000..5db07f0ce2e56
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_cluster-completed.spec_content
@@ -0,0 +1,255 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal-gce-ilb-cilium-etcd.example.com
+spec:
+  api:
+    loadBalancer:
+      subnets:
+      - name: us-test-1
+      type: Internal
+      useForInternalApi: true
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudConfig:
+    gceServiceAccount: default
+    gcpPDCSIDriver:
+      defaultStorageClassName: balanced-csi
+      enabled: true
+      version: v1.22.1
+    manageStorageClasses: true
+    multizone: true
+    nodeTags: minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node
+  cloudControllerManager:
+    allocateNodeCIDRs: true
+    cidrAllocatorType: CloudAllocator
+    clusterCIDR: 100.96.0.0/11
+    clusterName: minimal-gce-ilb-cilium-etcd-example-com
+    controllers:
+    - '*'
+    image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v35.0.0
+    leaderElection:
+      leaderElect: true
+  cloudProvider: gce
+  clusterDNSDomain: cluster.local
+  configBase: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com
+  containerd:
+    logLevel: info
+    runc:
+      version: 1.3.4
+    sandboxImage: registry.k8s.io/pause:3.10.1
+    version: 2.1.6
+  dnsZone: "1"
+  etcdClusters:
+  - backups:
+      backupStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/main
+    cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: main
+    version: 3.5.25
+  - backups:
+      backupStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/events
+    cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    memoryRequest: 100Mi
+    name: events
+    version: 3.5.25
+  - backups:
+      backupStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/cilium
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    manager:
+      backupRetentionDays: 90
+    name: cilium
+    version: 3.5.25
+  externalDns:
+    provider: dns-controller
+  iam:
+    legacy: false
+  keyStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/pki
+  kubeAPIServer:
+    allowPrivileged: true
+    anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
+    apiServerCount: 1
+    authorizationMode: Node,RBAC
+    bindAddress: 0.0.0.0
+    cloudProvider: external
+    enableAdmissionPlugins:
+    - DefaultStorageClass
+    - DefaultTolerationSeconds
+    - LimitRanger
+    - MutatingAdmissionWebhook
+    - NamespaceLifecycle
+    - NodeRestriction
+    - ResourceQuota
+    - RuntimeClass
+    - ServiceAccount
+    - ValidatingAdmissionPolicy
+    - ValidatingAdmissionWebhook
+    etcdServers:
+    - https://127.0.0.1:4001
+    etcdServersOverrides:
+    - /events#https://127.0.0.1:4002
+    image: registry.k8s.io/kube-apiserver:v1.32.0
+    kubeletPreferredAddressTypes:
+    - InternalIP
+    - Hostname
+    - ExternalIP
+    logLevel: 2
+    requestheaderAllowedNames:
+    - aggregator
+    requestheaderExtraHeaderPrefixes:
+    - X-Remote-Extra-
+    requestheaderGroupHeaders:
+    - X-Remote-Group
+    requestheaderUsernameHeaders:
+    - X-Remote-User
+    securePort: 443
+    serviceAccountIssuer: https://api.internal.minimal-gce-ilb-cilium-etcd.example.com
+    serviceAccountJWKSURI: https://api.internal.minimal-gce-ilb-cilium-etcd.example.com/openid/v1/jwks
+    serviceClusterIPRange: 100.64.0.0/13
+    storageBackend: etcd3
+  kubeControllerManager:
+    allocateNodeCIDRs: true
+    attachDetachReconcileSyncPeriod: 1m0s
+    cloudProvider: external
+    clusterCIDR: 100.96.0.0/11
+    clusterName: minimal-gce-ilb-cilium-etcd.example.com
+    configureCloudRoutes: false
+    image: registry.k8s.io/kube-controller-manager:v1.32.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+    useServiceAccountCredentials: true
+  kubeDNS:
+    cacheMaxConcurrent: 150
+    cacheMaxSize: 1000
+    cpuRequest: 100m
+    domain: cluster.local
+    memoryLimit: 170Mi
+    memoryRequest: 70Mi
+    nodeLocalDNS:
+      cpuRequest: 25m
+      enabled: false
+      image: registry.k8s.io/dns/k8s-dns-node-cache:1.26.0
+      memoryRequest: 5Mi
+    provider: CoreDNS
+    serverIP: 100.64.0.10
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    enabled: false
+    image: registry.k8s.io/kube-proxy:v1.32.0
+    logLevel: 2
+  kubeScheduler:
+    image: registry.k8s.io/kube-scheduler:v1.32.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+  kubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: external
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hairpinMode: promiscuous-bridge
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    registerSchedulable: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  - ::/0
+  kubernetesVersion: 1.32.0
+  masterKubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: external
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hairpinMode: promiscuous-bridge
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    registerSchedulable: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
+  masterPublicName: api.minimal-gce-ilb-cilium-etcd.example.com
+  networking:
+    cilium:
+      agentPrometheusPort: 9090
+      bpfCTGlobalAnyMax: 262144
+      bpfCTGlobalTCPMax: 524288
+      bpfLBAlgorithm: random
+      bpfLBMaglevTableSize: "16381"
+      bpfLBMapMax: 65536
+      bpfNATGlobalMax: 524288
+      bpfNeighGlobalMax: 524288
+      bpfPolicyMapMax: 16384
+      clusterName: default
+      cniExclusive: true
+      cpuRequest: 25m
+      disableCNPStatusUpdates: true
+      disableMasquerade: false
+      enableBPFMasquerade: false
+      enableEndpointHealthChecking: true
+      enableL7Proxy: true
+      enableLocalRedirectPolicy: false
+      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
+      etcdManaged: true
+      gatewayAPI:
+        enabled: false
+      hubble:
+        enabled: false
+      identityAllocationMode: crd
+      identityChangeGracePeriod: 5s
+      ingress:
+        enabled: false
+      ipam: kubernetes
+      memoryRequest: 128Mi
+      monitorAggregation: medium
+      sidecarIstioProxyImage: cilium/istio_proxy
+      toFqdnsDnsRejectResponseCode: refused
+      tunnel: vxlan
+      version: v1.18.6
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podCIDR: 100.96.0.0/11
+  project: testproject
+  secretStore: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/secrets
+  serviceClusterIPRange: 100.64.0.0/13
+  sshAccess:
+  - 0.0.0.0/0
+  - ::/0
+  subnets:
+  - cidr: 10.0.16.0/20
+    name: us-test1
+    region: us-test1
+    type: Private
+  topology:
+    dns:
+      type: Public
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-cilium_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-cilium_content
new file mode 100644
index 0000000000000..fb8835d7c40e7
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-cilium_content
@@ -0,0 +1,4 @@
+{
+  "memberCount": 1,
+  "etcdVersion": "3.5.25"
+}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-events_content
new file mode 100644
index 0000000000000..fb8835d7c40e7
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-events_content
@@ -0,0 +1,4 @@
+{
+  "memberCount": 1,
+  "etcdVersion": "3.5.25"
+}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-main_content
new file mode 100644
index 0000000000000..fb8835d7c40e7
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_etcd-cluster-spec-main_content
@@ -0,0 +1,4 @@
+{
+  "memberCount": 1,
+  "etcdVersion": "3.5.25"
+}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_kops-version.txt_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_kops-version.txt_content
new file mode 100644
index 0000000000000..86942cf6d89c9
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_kops-version.txt_content
@@ -0,0 +1,1 @@
+1.34.0-beta.1
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-cilium-master-us-test1-a_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-cilium-master-us-test1-a_content
new file mode 100644
index 0000000000000..e1cc65a9a8a31
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-cilium-master-us-test1-a_content
@@ -0,0 +1,166 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    k8s-app: etcd-manager-cilium
+  name: etcd-manager-cilium
+  namespace: kube-system
+spec:
+  containers:
+  - command:
+    - /bin/sh
+    - -c
+    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager
+      --backup-store=memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/cilium
+      --client-urls=https://api.internal.minimal-gce-ilb-cilium-etcd.example.com:4003
+      --cluster-name=etcd-cilium --containerized=true --dns-suffix=.internal.minimal-gce-ilb-cilium-etcd.example.com
+      --grpc-port=3991 --peer-urls=https://__name__:2382 --quarantine-client-urls=https://__name__:3992
+      --v=6 --volume-name-tag=k8s-io-etcd-cilium --volume-provider=gce --volume-tag=k8s-io-cluster-name=minimal-gce-ilb-cilium-etcd-example-com
+      --volume-tag=k8s-io-etcd-cilium --volume-tag=k8s-io-role-master=master > /tmp/pipe
+      2>&1
+    env:
+    - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
+      value: 90d
+    image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260227
+    name: etcd-manager
+    resources:
+      requests:
+        cpu: 200m
+        memory: 100Mi
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /rootfs
+      name: rootfs
+    - mountPath: /run
+      name: run
+    - mountPath: /etc/kubernetes/pki/etcd-manager
+      name: pki
+    - mountPath: /opt
+      name: opt
+    - mountPath: /var/log/etcd.log
+      name: varlogetcd
+  hostNetwork: true
+  hostPID: true
+  initContainers:
+  - args:
+    - --target-dir=/opt/kops-utils/
+    - --src=/ko-app/kops-utils-cp
+    command:
+    - /ko-app/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: kops-utils-cp
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.4.13
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.4.13
+    name: init-etcd-3-4-13
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.5.25
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.5.25
+    name: init-etcd-3-5-25
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.6.6
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.6.6
+    name: init-etcd-3-6-6
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.4.3
+    - --src=/opt/etcd-v3.4.13/etcd
+    - --src=/opt/etcd-v3.4.13/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-4-13
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.5.0
+    - --target-dir=/opt/etcd-v3.5.1
+    - --target-dir=/opt/etcd-v3.5.13
+    - --target-dir=/opt/etcd-v3.5.17
+    - --target-dir=/opt/etcd-v3.5.21
+    - --target-dir=/opt/etcd-v3.5.23
+    - --target-dir=/opt/etcd-v3.5.24
+    - --target-dir=/opt/etcd-v3.5.3
+    - --target-dir=/opt/etcd-v3.5.4
+    - --target-dir=/opt/etcd-v3.5.6
+    - --target-dir=/opt/etcd-v3.5.7
+    - --target-dir=/opt/etcd-v3.5.9
+    - --src=/opt/etcd-v3.5.25/etcd
+    - --src=/opt/etcd-v3.5.25/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-5-25
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.6.5
+    - --src=/opt/etcd-v3.6.6/etcd
+    - --src=/opt/etcd-v3.6.6/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-6-6
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  priorityClassName: system-cluster-critical
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+  volumes:
+  - hostPath:
+      path: /
+      type: Directory
+    name: rootfs
+  - hostPath:
+      path: /run
+      type: DirectoryOrCreate
+    name: run
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd-manager-cilium
+      type: DirectoryOrCreate
+    name: pki
+  - emptyDir: {}
+    name: opt
+  - hostPath:
+      path: /var/log/etcd-cilium.log
+      type: FileOrCreate
+    name: varlogetcd
+status: {}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-events-master-us-test1-a_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-events-master-us-test1-a_content
new file mode 100644
index 0000000000000..eddb92559adea
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-events-master-us-test1-a_content
@@ -0,0 +1,166 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    k8s-app: etcd-manager-events
+  name: etcd-manager-events
+  namespace: kube-system
+spec:
+  containers:
+  - command:
+    - /bin/sh
+    - -c
+    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager
+      --backup-store=memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/events
+      --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true
+      --dns-suffix=.internal.minimal-gce-ilb-cilium-etcd.example.com --grpc-port=3997
+      --peer-urls=https://__name__:2381 --quarantine-client-urls=https://__name__:3995
+      --v=6 --volume-name-tag=k8s-io-etcd-events --volume-provider=gce --volume-tag=k8s-io-cluster-name=minimal-gce-ilb-cilium-etcd-example-com
+      --volume-tag=k8s-io-etcd-events --volume-tag=k8s-io-role-master=master > /tmp/pipe
+      2>&1
+    env:
+    - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
+      value: 90d
+    image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260227
+    name: etcd-manager
+    resources:
+      requests:
+        cpu: 100m
+        memory: 100Mi
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /rootfs
+      name: rootfs
+    - mountPath: /run
+      name: run
+    - mountPath: /etc/kubernetes/pki/etcd-manager
+      name: pki
+    - mountPath: /opt
+      name: opt
+    - mountPath: /var/log/etcd.log
+      name: varlogetcd
+  hostNetwork: true
+  hostPID: true
+  initContainers:
+  - args:
+    - --target-dir=/opt/kops-utils/
+    - --src=/ko-app/kops-utils-cp
+    command:
+    - /ko-app/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: kops-utils-cp
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.4.13
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.4.13
+    name: init-etcd-3-4-13
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.5.25
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.5.25
+    name: init-etcd-3-5-25
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.6.6
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.6.6
+    name: init-etcd-3-6-6
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.4.3
+    - --src=/opt/etcd-v3.4.13/etcd
+    - --src=/opt/etcd-v3.4.13/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-4-13
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.5.0
+    - --target-dir=/opt/etcd-v3.5.1
+    - --target-dir=/opt/etcd-v3.5.13
+    - --target-dir=/opt/etcd-v3.5.17
+    - --target-dir=/opt/etcd-v3.5.21
+    - --target-dir=/opt/etcd-v3.5.23
+    - --target-dir=/opt/etcd-v3.5.24
+    - --target-dir=/opt/etcd-v3.5.3
+    - --target-dir=/opt/etcd-v3.5.4
+    - --target-dir=/opt/etcd-v3.5.6
+    - --target-dir=/opt/etcd-v3.5.7
+    - --target-dir=/opt/etcd-v3.5.9
+    - --src=/opt/etcd-v3.5.25/etcd
+    - --src=/opt/etcd-v3.5.25/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-5-25
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.6.5
+    - --src=/opt/etcd-v3.6.6/etcd
+    - --src=/opt/etcd-v3.6.6/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-6-6
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  priorityClassName: system-cluster-critical
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+  volumes:
+  - hostPath:
+      path: /
+      type: Directory
+    name: rootfs
+  - hostPath:
+      path: /run
+      type: DirectoryOrCreate
+    name: run
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd-manager-events
+      type: DirectoryOrCreate
+    name: pki
+  - emptyDir: {}
+    name: opt
+  - hostPath:
+      path: /var/log/etcd-events.log
+      type: FileOrCreate
+    name: varlogetcd
+status: {}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-main-master-us-test1-a_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-main-master-us-test1-a_content
new file mode 100644
index 0000000000000..84ec25175fca0
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-etcdmanager-main-master-us-test1-a_content
@@ -0,0 +1,166 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    k8s-app: etcd-manager-main
+  name: etcd-manager-main
+  namespace: kube-system
+spec:
+  containers:
+  - command:
+    - /bin/sh
+    - -c
+    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager
+      --backup-store=memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/main
+      --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true
+      --dns-suffix=.internal.minimal-gce-ilb-cilium-etcd.example.com --grpc-port=3996
+      --peer-urls=https://__name__:2380 --quarantine-client-urls=https://__name__:3994
+      --v=6 --volume-name-tag=k8s-io-etcd-main --volume-provider=gce --volume-tag=k8s-io-cluster-name=minimal-gce-ilb-cilium-etcd-example-com
+      --volume-tag=k8s-io-etcd-main --volume-tag=k8s-io-role-master=master > /tmp/pipe
+      2>&1
+    env:
+    - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
+      value: 90d
+    image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260227
+    name: etcd-manager
+    resources:
+      requests:
+        cpu: 200m
+        memory: 100Mi
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /rootfs
+      name: rootfs
+    - mountPath: /run
+      name: run
+    - mountPath: /etc/kubernetes/pki/etcd-manager
+      name: pki
+    - mountPath: /opt
+      name: opt
+    - mountPath: /var/log/etcd.log
+      name: varlogetcd
+  hostNetwork: true
+  hostPID: true
+  initContainers:
+  - args:
+    - --target-dir=/opt/kops-utils/
+    - --src=/ko-app/kops-utils-cp
+    command:
+    - /ko-app/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: kops-utils-cp
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.4.13
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.4.13
+    name: init-etcd-3-4-13
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.5.25
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.5.25
+    name: init-etcd-3-5-25
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --target-dir=/opt/etcd-v3.6.6
+    - --src=/usr/local/bin/etcd
+    - --src=/usr/local/bin/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/etcd:v3.6.6
+    name: init-etcd-3-6-6
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.4.3
+    - --src=/opt/etcd-v3.4.13/etcd
+    - --src=/opt/etcd-v3.4.13/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-4-13
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.5.0
+    - --target-dir=/opt/etcd-v3.5.1
+    - --target-dir=/opt/etcd-v3.5.13
+    - --target-dir=/opt/etcd-v3.5.17
+    - --target-dir=/opt/etcd-v3.5.21
+    - --target-dir=/opt/etcd-v3.5.23
+    - --target-dir=/opt/etcd-v3.5.24
+    - --target-dir=/opt/etcd-v3.5.3
+    - --target-dir=/opt/etcd-v3.5.4
+    - --target-dir=/opt/etcd-v3.5.6
+    - --target-dir=/opt/etcd-v3.5.7
+    - --target-dir=/opt/etcd-v3.5.9
+    - --src=/opt/etcd-v3.5.25/etcd
+    - --src=/opt/etcd-v3.5.25/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-5-25
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  - args:
+    - --symlink
+    - --target-dir=/opt/etcd-v3.6.5
+    - --src=/opt/etcd-v3.6.6/etcd
+    - --src=/opt/etcd-v3.6.6/etcdctl
+    command:
+    - /opt/kops-utils/kops-utils-cp
+    image: registry.k8s.io/kops/kops-utils-cp:1.35.0-beta.1
+    name: init-etcd-symlinks-3-6-6
+    resources: {}
+    volumeMounts:
+    - mountPath: /opt
+      name: opt
+  priorityClassName: system-cluster-critical
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
+  volumes:
+  - hostPath:
+      path: /
+      type: Directory
+    name: rootfs
+  - hostPath:
+      path: /run
+      type: DirectoryOrCreate
+    name: run
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd-manager-main
+      type: DirectoryOrCreate
+    name: pki
+  - emptyDir: {}
+    name: opt
+  - hostPath:
+      path: /var/log/etcd.log
+      type: FileOrCreate
+    name: varlogetcd
+status: {}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content
new file mode 100644
index 0000000000000..7867d3e53581a
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content
@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Pod
+metadata: {}
+spec:
+  containers:
+  - args:
+    - --ca-cert=/secrets/ca.crt
+    - --client-cert=/secrets/client.crt
+    - --client-key=/secrets/client.key
+    image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.34.0-beta.1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /.kube-apiserver-healthcheck/healthz
+        port: 3990
+      initialDelaySeconds: 5
+      timeoutSeconds: 5
+    name: healthcheck
+    resources: {}
+    securityContext:
+      runAsNonRoot: true
+      runAsUser: 10012
+    volumeMounts:
+    - mountPath: /secrets
+      name: healthcheck-secrets
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/kube-apiserver-healthcheck/secrets
+      type: Directory
+    name: healthcheck-secrets
+status: {}
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-bootstrap_content
new file mode 100644
index 0000000000000..5bf34f394b26c
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-bootstrap_content
@@ -0,0 +1,115 @@
+kind: Addons
+metadata:
+  name: bootstrap
+spec:
+  addons:
+  - id: k8s-1.16
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: 99889fc8356062a43edf7c268daebd7684cfaca146c891fbede6d89a8e42f02f
+    name: kops-controller.addons.k8s.io
+    needsRollingUpdate: control-plane
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.12
+    manifest: coredns.addons.k8s.io/k8s-1.12.yaml
+    manifestHash: c6ebdb7a6f37311e443a0dc608f1e85d46e1aa7932c423b7a8782ed4e1da6df2
+    name: coredns.addons.k8s.io
+    selector:
+      k8s-addon: coredns.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.9
+    manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
+    manifestHash: da91eb5cf9a29f1b03510007d6d54603aef2fc23a305abc9ba496c510dfd3bc7
+    name: kubelet-api.rbac.addons.k8s.io
+    selector:
+      k8s-addon: kubelet-api.rbac.addons.k8s.io
+    version: 9.99.0
+  - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
+    manifestHash: 686cc69e559a1c6f5e8b94e38de54a575a25c432ed5ceec565244b965fb5f07f
+    name: limit-range.addons.k8s.io
+    selector:
+      k8s-addon: limit-range.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.12
+    manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
+    manifestHash: 5c4088f2427f70a8ad8b6d574479b57f7764bd560a62aa18a1b1efad240e30a7
+    name: dns-controller.addons.k8s.io
+    selector:
+      k8s-addon: dns-controller.addons.k8s.io
+    version: 9.99.0
+  - id: v1.7.0
+    manifest: storage-gce.addons.k8s.io/v1.7.0.yaml
+    manifestHash: 4f73c7d683f04e61d60e90053a6db1fd82272dc0c13790390740f5e207181c77
+    name: storage-gce.addons.k8s.io
+    selector:
+      k8s-addon: storage-gce.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.23
+    manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml
+    manifestHash: 2bb04ab48e3fc035c4f2810723d91977860e261ec1feae2e38c5abe5bc0a8a9e
+    name: gcp-pd-csi-driver.addons.k8s.io
+    selector:
+      k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.23
+    manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
+    manifestHash: defbd944608641e77a855fa16ad1cf395ee32d2eeecf7ec08c2946e4ffb388a0
+    name: gcp-cloud-controller.addons.k8s.io
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+    selector:
+      k8s-addon: gcp-cloud-controller.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.16
+    manifest: networking.cilium.io/k8s-1.16-v1.15.yaml
+    manifestHash: a09fc4528f8138e4c953da73c41e6274c6634402622e42147f52f490b1dac8e0
+    name: networking.cilium.io
+    needsRollingUpdate: all
+    selector:
+      role.kubernetes.io/networking: "1"
+    version: 9.99.0
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content
new file mode 100644
index 0000000000000..02f679952641c
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content
@@ -0,0 +1,373 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: v1
+data:
+  Corefile: |-
+    .:53 {
+        errors
+        health {
+          lameduck 10s
+        }
+        ready
+        kubernetes cluster.local. in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+          max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    addonmanager.kubernetes.io/mode: EnsureExists
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: coredns
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  strategy:
+    rollingUpdate:
+      maxSurge: 10%
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - args:
+        - -conf
+        - /etc/coredns/Corefile
+        image: registry.k8s.io/coredns/coredns:v1.13.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 1
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+          periodSeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/coredns
+          name: config-volume
+          readOnly: true
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            k8s-app: kube-dns
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            k8s-app: kube-dns
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+      volumes:
+      - configMap:
+          name: coredns
+        name: config-volume
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: kube-dns
+  namespace: kube-system
+  resourceVersion: "0"
+spec:
+  clusterIP: 100.64.0.10
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+  - name: metrics
+    port: 9153
+    protocol: TCP
+  selector:
+    k8s-app: kube-dns
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: kube-dns
+  namespace: kube-system
+spec:
+  maxUnavailable: 50%
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - replicationcontrollers/scale
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - extensions
+  - apps
+  resources:
+  - deployments/scale
+  - replicasets/scale
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coredns-autoscaler
+subjects:
+- kind: ServiceAccount
+  name: coredns-autoscaler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: coredns-autoscaler
+    kubernetes.io/cluster-service: "true"
+  name: coredns-autoscaler
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: coredns-autoscaler
+  template:
+    metadata:
+      labels:
+        k8s-app: coredns-autoscaler
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - command:
+        - /cluster-proportional-autoscaler
+        - --namespace=kube-system
+        - --configmap=coredns-autoscaler
+        - --target=Deployment/coredns
+        - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
+        - --logtostderr=true
+        - --v=2
+        image: registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.9.0
+        name: autoscaler
+        resources:
+          requests:
+            cpu: 20m
+            memory: 10Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns-autoscaler
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content
new file mode 100644
index 0000000000000..ecf8cd0f34f7d
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content
@@ -0,0 +1,133 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: dns-controller.addons.k8s.io
+    k8s-app: dns-controller
+    version: v1.34.0-beta.1
+  name: dns-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: dns-controller
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        k8s-addon: dns-controller.addons.k8s.io
+        k8s-app: dns-controller
+        kops.k8s.io/managed-by: kops
+        version: v1.34.0-beta.1
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - args:
+        - --watch-ingress=false
+        - --dns=google-clouddns
+        - --zone=*/1
+        - --internal-ipv4
+        - --zone=*/*
+        - -v=2
+        command: null
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        image: registry.k8s.io/kops/dns-controller:1.34.0-beta.1
+        name: dns-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+        securityContext:
+          runAsNonRoot: true
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccount: dns-controller
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: dns-controller.addons.k8s.io
+  name: dns-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: dns-controller.addons.k8s.io
+  name: kops:dns-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - ingress
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: dns-controller.addons.k8s.io
+  name: kops:dns-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops:dns-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:dns-controller
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content
new file mode 100644
index 0000000000000..ecc95ad209bbe
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content
@@ -0,0 +1,549 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    component: cloud-controller-manager
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      component: cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        component: cloud-controller-manager
+        kops.k8s.io/managed-by: kops
+        tier: control-plane
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - args:
+        - --allocate-node-cidrs=true
+        - --cidr-allocator-type=CloudAllocator
+        - --cluster-cidr=100.96.0.0/11
+        - --cluster-name=minimal-gce-ilb-cilium-etcd-example-com
+        - --controllers=*
+        - --leader-elect=true
+        - --v=2
+        - --cloud-provider=gce
+        - --use-service-account-credentials=true
+        - --cloud-config=/etc/kubernetes/cloud.config
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v35.0.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 10258
+            scheme: HTTPS
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 15
+        name: cloud-controller-manager
+        resources:
+          requests:
+            cpu: 200m
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+          type: ""
+        name: cloudconfig
+  updateStrategy:
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - cloud-controller-manager
+  resources:
+  - leases
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+  - get
+  - update
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system::leader-locking-cloud-controller-manager
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - cloud-controller-manager
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - patch
+  - get
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system:controller:cloud-node-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - update
+  - delete
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - get
+  - list
+  - update
+  - delete
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - pods/status
+  verbs:
+  - list
+  - delete
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system::leader-locking-cloud-controller-manager
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system::leader-locking-cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system:controller:cloud-node-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:controller:cloud-node-controller
+subjects:
+- kind: ServiceAccount
+  name: cloud-node-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: system:controller:pvl-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  - persistentvolumes
+  verbs:
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: gce:cloud-provider
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - patch
+  - update
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: gce:cloud-provider
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - patch
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations:
+    kubernetes.io/deprecation: cloud-provider role is DEPRECATED in the concern of
+      potential collisions and will be removed in 1.16. Do not use this role.
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: cloud-provider
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - patch
+  - update
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    kubernetes.io/deprecation: cloud-provider clusterrole is DEPRECATED in the concern
+      of potential collisions and will be removed in 1.16. Do not use this role.
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: cloud-provider
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: gce:cloud-provider
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gce:cloud-provider
+subjects:
+- kind: ServiceAccount
+  name: cloud-provider
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io
+    addonmanager.kubernetes.io/mode: Reconcile
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-cloud-controller.addons.k8s.io
+  name: gce:cloud-provider
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gce:cloud-provider
+subjects:
+- kind: ServiceAccount
+  name: cloud-provider
+  namespace: kube-system
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content
new file mode 100644
index 0000000000000..8a568081f8cbe
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content
@@ -0,0 +1,868 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: standard-csi
+parameters:
+  type: pd-standard
+provisioner: pd.csi.storage.gke.io
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: balanced-csi
+parameters:
+  type: pd-balanced
+provisioner: pd.csi.storage.gke.io
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: balanced-storage
+parameters:
+  provisioned-iops-on-create: "5000"
+  provisioned-throughput-on-create: 250Mi
+  type: hyperdisk-balanced
+provisioner: pd.csi.storage.gke.io
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: ssd-csi
+parameters:
+  type: pd-ssd
+provisioner: pd.csi.storage.gke.io
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: gce-pd-csi-driver
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-node-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    k8s-app: gcp-compute-persistent-disk-csi-driver
+  name: csi-gce-pd-leaderelection-role
+  namespace: gce-pd-csi-driver
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-attacher-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-provisioner-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-resizer-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-snapshotter-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - delete
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents/status
+  verbs:
+  - update
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+    k8s-app: gcp-compute-persistent-disk-csi-driver
+  name: csi-gce-pd-controller-leaderelection-binding
+  namespace: gce-pd-csi-driver
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: csi-gce-pd-leaderelection-role
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-node-deploy
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller-attacher-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-attacher-role
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller-deploy
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-controller-deploy
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller-provisioner-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-provisioner-role
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller-snapshotter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-node-deploy
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-node-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-resizer-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-resizer-role
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: gce-pd-csi-driver
+
+---
+
+apiVersion: scheduling.k8s.io/v1
+description: This priority class should be used for the GCE PD CSI driver controller
+  deployment only.
+globalDefault: false
+kind: PriorityClass
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller
+value: 900000000
+
+---
+
+apiVersion: scheduling.k8s.io/v1
+description: This priority class should be used for the GCE PD CSI driver node deployment
+  only.
+globalDefault: false
+kind: PriorityClass
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-node
+value: 900001000
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-controller
+  namespace: gce-pd-csi-driver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: gcp-compute-persistent-disk-csi-driver
+  template:
+    metadata:
+      labels:
+        app: gcp-compute-persistent-disk-csi-driver
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      containers:
+      - args:
+        - --v=5
+        - --csi-address=/csi/csi.sock
+        - --feature-gates=Topology=true
+        - --http-endpoint=:22011
+        - --leader-election-namespace=$(PDCSI_NAMESPACE)
+        - --timeout=250s
+        - --extra-create-metadata
+        - --leader-election
+        - --default-fstype=ext4
+        - --controller-publish-readonly=true
+        env:
+        - name: PDCSI_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0
+        livenessProbe:
+          failureThreshold: 1
+          httpGet:
+            path: /healthz/leader-election
+            port: http-endpoint
+          initialDelaySeconds: 10
+          periodSeconds: 20
+          timeoutSeconds: 10
+        name: csi-provisioner
+        ports:
+        - containerPort: 22011
+          name: http-endpoint
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      - args:
+        - --v=5
+        - --csi-address=/csi/csi.sock
+        - --http-endpoint=:22012
+        - --leader-election
+        - --leader-election-namespace=$(PDCSI_NAMESPACE)
+        - --timeout=250s
+        env:
+        - name: PDCSI_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1
+        livenessProbe:
+          failureThreshold: 1
+          httpGet:
+            path: /healthz/leader-election
+            port: http-endpoint
+          initialDelaySeconds: 10
+          periodSeconds: 20
+          timeoutSeconds: 10
+        name: csi-attacher
+        ports:
+        - containerPort: 22012
+          name: http-endpoint
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      - args:
+        - --v=5
+        - --csi-address=/csi/csi.sock
+        - --http-endpoint=:22013
+        - --leader-election
+        - --leader-election-namespace=$(PDCSI_NAMESPACE)
+        - --handle-volume-inuse-error=false
+        env:
+        - name: PDCSI_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0
+        livenessProbe:
+          failureThreshold: 1
+          httpGet:
+            path: /healthz/leader-election
+            port: http-endpoint
+          initialDelaySeconds: 10
+          periodSeconds: 20
+          timeoutSeconds: 10
+        name: csi-resizer
+        ports:
+        - containerPort: 22013
+          name: http-endpoint
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      - args:
+        - --v=5
+        - --csi-address=/csi/csi.sock
+        - --metrics-address=:22014
+        - --leader-election
+        - --leader-election-namespace=$(PDCSI_NAMESPACE)
+        - --timeout=300s
+        - --retry-interval-max=60s
+        env:
+        - name: PDCSI_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1
+        name: csi-snapshotter
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      - args:
+        - --v=5
+        - --endpoint=unix:/csi/csi.sock
+        - --extra-labels=k8s-io-cluster-name=minimal-gce-ilb-cilium-etcd-example-com
+        - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme
+        - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml
+        - --enable-data-cache
+        - --run-node-service=false
+        env: []
+        image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1
+        name: gce-pd-driver
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: csi-gce-pd-controller
+      serviceAccountName: csi-gce-pd-controller-sa
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - emptyDir: {}
+        name: socket-dir
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: csi-gce-pd-node
+  namespace: gce-pd-csi-driver
+spec:
+  selector:
+    matchLabels:
+      app: gcp-compute-persistent-disk-csi-driver
+  template:
+    metadata:
+      labels:
+        app: gcp-compute-persistent-disk-csi-driver
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      containers:
+      - args:
+        - --v=5
+        - --csi-address=/csi/csi.sock
+        - --kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock
+        env:
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
+        name: csi-driver-registrar
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /registration
+          name: registration-dir
+      - args:
+        - --v=5
+        - --endpoint=unix:/csi/csi.sock
+        - --run-controller-service=false
+        - --enable-data-cache
+        image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1
+        name: gce-pd-driver
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+          name: kubelet-dir
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /dev
+          name: device-dir
+        - mountPath: /etc/udev
+          name: udev-rules-etc
+        - mountPath: /lib/udev
+          name: udev-rules-lib
+        - mountPath: /run/udev
+          name: udev-socket
+        - mountPath: /sys
+          name: sys
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: csi-gce-pd-node
+      serviceAccountName: csi-gce-pd-node-sa
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+        name: registration-dir
+      - hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+        name: kubelet-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/
+          type: DirectoryOrCreate
+        name: plugin-dir
+      - hostPath:
+          path: /dev
+          type: Directory
+        name: device-dir
+      - hostPath:
+          path: /etc/udev
+          type: Directory
+        name: udev-rules-etc
+      - hostPath:
+          path: /lib/udev
+          type: Directory
+        name: udev-rules-lib
+      - hostPath:
+          path: /run/udev
+          type: Directory
+        name: udev-socket
+      - hostPath:
+          path: /sys
+          type: Directory
+        name: sys
+
+---
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  labels:
+    addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: gcp-pd-csi-driver.addons.k8s.io
+  name: pd.csi.storage.gke.io
+spec:
+  attachRequired: true
+  podInfoOnMount: false
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content
new file mode 100644
index 0000000000000..5d959453d4839
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content
@@ -0,0 +1,219 @@
+apiVersion: v1
+data:
+  config.yaml: |
+    {"clusterName":"minimal-gce-ilb-cilium-etcd.example.com","cloud":"gce","configBase":"memfs://tests/minimal-gce-ilb-cilium-etcd.example.com","secretStore":"memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/secrets","server":{"Listen":":3988","provider":{"gce":{"projectID":"testproject","region":"us-test1","clusterName":"minimal-gce-ilb-cilium-etcd.example.com","MaxTimeSkew":300}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca","etcd-clients-ca-cilium"],"certNames":["kubelet","kubelet-server","etcd-client-cilium"]}}
+kind: ConfigMap
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.34.0-beta.1
+  name: kops-controller
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      annotations:
+        dns.alpha.kubernetes.io/internal: kops-controller.internal.minimal-gce-ilb-cilium-etcd.example.com
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        kops.k8s.io/managed-by: kops
+        version: v1.34.0-beta.1
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+              - key: kops.k8s.io/kops-controller-pki
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+              - key: kops.k8s.io/kops-controller-pki
+                operator: Exists
+      containers:
+      - args:
+        - --v=2
+        - --conf=/etc/kubernetes/kops-controller/config/config.yaml
+        command: null
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        - name: KOPS_RUN_TOO_NEW_VERSION
+          value: "1"
+        image: registry.k8s.io/kops/kops-controller:1.34.0-beta.1
+        name: kops-controller
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 10011
+        volumeMounts:
+        - mountPath: /etc/kubernetes/kops-controller/config/
+          name: kops-controller-config
+        - mountPath: /etc/kubernetes/kops-controller/pki/
+          name: kops-controller-pki
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccount: kops-controller
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      volumes:
+      - configMap:
+          name: kops-controller
+        name: kops-controller-config
+      - hostPath:
+          path: /etc/kubernetes/kops-controller/
+          type: Directory
+        name: kops-controller-pki
+  updateStrategy:
+    type: OnDelete
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+- apiGroups:
+  - ""
+  - coordination.k8s.io
+  resourceNames:
+  - kops-controller-leader
+  resources:
+  - configmaps
+  - leases
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+  - delete
+- apiGroups:
+  - ""
+  - coordination.k8s.io
+  resources:
+  - configmaps
+  - leases
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content
new file mode 100644
index 0000000000000..bb4d69ef0ef88
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content
@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: kubelet-api.rbac.addons.k8s.io
+  name: kops:system:kubelet-api-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kubelet-api-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: kubelet-api
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-limit-range.addons.k8s.io_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-limit-range.addons.k8s.io_content
new file mode 100644
index 0000000000000..c8939ddffd3ad
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-limit-range.addons.k8s.io_content
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: LimitRange
+metadata:
+  labels:
+    addon.kops.k8s.io/name: limit-range.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: limit-range.addons.k8s.io
+  name: limits
+  namespace: default
+spec:
+  limits:
+  - defaultRequest:
+      cpu: 100m
+    type: Container
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-networking.cilium.io-k8s-1.16_content
new file mode 100644
index 0000000000000..ee1f5e73caa50
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-networking.cilium.io-k8s-1.16_content
@@ -0,0 +1,1227 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: cilium-operator
+    app.kubernetes.io/part-of: cilium
+    io.cilium/app: operator
+    name: cilium-operator
+    role.kubernetes.io/networking: "1"
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      io.cilium/app: operator
+      name: cilium-operator
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
+  name: cilium
+  namespace: kube-system
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
+  name: cilium-operator
+  namespace: kube-system
+
+---
+
+apiVersion: v1
+data:
+  agent-not-ready-taint-key: node.cilium.io/agent-not-ready
+  auto-direct-node-routes: "false"
+  bpf-distributed-lru: "false"
+  bpf-events-drop-enabled: "true"
+  bpf-events-policy-verdict-enabled: "true"
+  bpf-events-trace-enabled: "true"
+  bpf-lb-acceleration: disabled
+  bpf-lb-algorithm-annotation: "false"
+  bpf-lb-external-clusterip: "false"
+  bpf-lb-map-max: "65536"
+  bpf-lb-mode-annotation: "false"
+  bpf-lb-sock: "false"
+  bpf-lb-source-range-all-types: "false"
+  bpf-map-dynamic-size-ratio: "0.0025"
+  bpf-policy-map-max: "16384"
+  bpf-policy-stats-map-max: "65536"
+  bpf-root: /sys/fs/bpf
+  cgroup-root: /run/cilium/cgroupv2
+  cilium-endpoint-gc-interval: 5m0s
+  cluster-name: default
+  cluster-pool-ipv4-mask-size: "24"
+  clustermesh-enable-endpoint-sync: "false"
+  clustermesh-enable-mcs-api: "false"
+  cni-exclusive: "true"
+  cni-log-file: /var/run/cilium/cilium-cni.log
+  custom-cni-conf: "false"
+  datapath-mode: veth
+  debug: "false"
+  debug-verbose: ""
+  default-lb-service-ipam: lbipam
+  direct-routing-skip-unreachable: "false"
+  disable-endpoint-crd: "false"
+  dnsproxy-enable-transparent-mode: "true"
+  dnsproxy-socket-linger-timeout: "10"
+  egress-gateway-reconciliation-trigger-interval: 1s
+  enable-auto-protect-node-port-range: "true"
+  enable-bpf-clock-probe: "false"
+  enable-bpf-masquerade: "false"
+  enable-endpoint-health-checking: "true"
+  enable-endpoint-lockdown-on-policy-overflow: "false"
+  enable-health-check-loadbalancer-ip: "false"
+  enable-health-check-nodeport: "true"
+  enable-health-checking: "true"
+  enable-internal-traffic-policy: "true"
+  enable-ipv4: "true"
+  enable-ipv4-big-tcp: "false"
+  enable-ipv4-masquerade: "true"
+  enable-ipv6: "false"
+  enable-ipv6-big-tcp: "false"
+  enable-ipv6-masquerade: "false"
+  enable-k8s-networkpolicy: "true"
+  enable-l2-neigh-discovery: "false"
+  enable-l7-proxy: "true"
+  enable-lb-ipam: "true"
+  enable-local-redirect-policy: "false"
+  enable-masquerade-to-route-source: "false"
+  enable-node-port: "false"
+  enable-node-selector-labels: "false"
+  enable-non-default-deny-policies: "true"
+  enable-policy: default
+  enable-sctp: "false"
+  enable-service-topology: "false"
+  enable-source-ip-verification: "true"
+  enable-svc-source-range-check: "true"
+  enable-tcx: "true"
+  enable-vtep: "false"
+  enable-well-known-identities: "false"
+  enable-xt-socket-fallback: "true"
+  envoy-access-log-buffer-size: "4096"
+  envoy-base-id: "0"
+  envoy-keep-cap-netbindservice: "false"
+  etcd-config: |-
+    ---
+    endpoints:
+      - https://api.internal.minimal-gce-ilb-cilium-etcd.example.com:4003
+    trusted-ca-file: '/var/lib/etcd-secrets/etcd-ca.crt'
+    key-file: '/var/lib/etcd-secrets/etcd-client-cilium.key'
+    cert-file: '/var/lib/etcd-secrets/etcd-client-cilium.crt'
+  external-envoy-proxy: "false"
+  health-check-icmp-failure-threshold: "3"
+  http-retry-count: "3"
+  http-stream-idle-timeout: "300"
+  identity-allocation-mode: crd
+  identity-change-grace-period: 5s
+  identity-gc-interval: 15m0s
+  identity-heartbeat-timeout: 30m0s
+  identity-management-mode: agent
+  install-no-conntrack-iptables-rules: "false"
+  ipam: kubernetes
+  ipam-cilium-node-update-rate: 15s
+  iptables-random-fully: "false"
+  k8s-require-ipv4-pod-cidr: "false"
+  k8s-require-ipv6-pod-cidr: "false"
+  kube-proxy-replacement: "false"
+  kvstore: etcd
+  kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}'
+  max-connected-clusters: "255"
+  mesh-auth-enabled: "true"
+  mesh-auth-gc-interval: 5m0s
+  mesh-auth-queue-size: "1024"
+  mesh-auth-rotated-identities-queue-size: "1024"
+  metrics-sampling-interval: 5m
+  monitor-aggregation: medium
+  monitor-aggregation-flags: all
+  monitor-aggregation-interval: 5s
+  nat-map-stats-entries: "32"
+  nat-map-stats-interval: 30s
+  node-port-bind-protection: "true"
+  nodeport-addresses: ""
+  nodes-gc-interval: 5m0s
+  operator-api-serve-addr: 127.0.0.1:9234
+  policy-cidr-match-mode: ""
+  policy-default-local-cluster: "false"
+  preallocate-bpf-maps: "false"
+  procfs: /host/proc
+  proxy-connect-timeout: "2"
+  proxy-idle-timeout-seconds: "60"
+  proxy-initial-fetch-timeout: "30"
+  proxy-max-concurrent-retries: "128"
+  proxy-max-connection-duration-seconds: "0"
+  proxy-max-requests-per-connection: "0"
+  proxy-xff-num-trusted-hops-egress: "0"
+  proxy-xff-num-trusted-hops-ingress: "0"
+  remove-cilium-node-taints: "true"
+  routing-mode: tunnel
+  service-no-backend-response: reject
+  set-cilium-is-up-condition: "true"
+  set-cilium-node-taints: "true"
+  synchronize-k8s-nodes: "true"
+  tofqdns-dns-reject-response-code: refused
+  tofqdns-enable-dns-compression: "true"
+  tofqdns-endpoint-max-ip-per-hostname: "1000"
+  tofqdns-idle-connection-grace-period: 0s
+  tofqdns-max-deferred-connection-deletes: "10000"
+  tofqdns-preallocate-identities: "true"
+  tofqdns-proxy-response-max-delay: 100ms
+  tunnel-protocol: vxlan
+  tunnel-source-port-range: 0-0
+  unmanaged-pod-watcher-interval: "15"
+  vtep-cidr: ""
+  vtep-endpoint: ""
+  vtep-mac: ""
+  vtep-mask: ""
+  write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
+kind: ConfigMap
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
+  name: cilium-config
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/part-of: cilium
+    role.kubernetes.io/networking: "1"
+  name: cilium
+rules:
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - services
+  - pods
+  - endpoints
+  - nodes
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - list
+  - watch
+  - get
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumloadbalancerippools
+  - ciliumbgppeeringpolicies
+  - ciliumbgpnodeconfigs
+  - ciliumbgpadvertisements
+  - ciliumbgppeerconfigs
+  - ciliumclusterwideenvoyconfigs
+  - ciliumclusterwidenetworkpolicies
+  - ciliumegressgatewaypolicies
+  - ciliumendpoints
+  - ciliumendpointslices
+  - ciliumenvoyconfigs
+  - ciliumidentities
+  - ciliumlocalredirectpolicies
+  - ciliumnetworkpolicies
+  - ciliumnodes
+  - ciliumnodeconfigs
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliumpodippools
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumidentities
+  - ciliumendpoints
+  - ciliumnodes
+  verbs:
+  - create
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumidentities
+  verbs:
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumendpoints
+  verbs:
+  - delete
+  - get
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnodes
+  - ciliumnodes/status
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumendpoints/status
+  - ciliumendpoints
+  - ciliuml2announcementpolicies/status
+  - ciliumbgpnodeconfigs/status
+  verbs:
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/part-of: cilium
+    role.kubernetes.io/networking: "1"
+  name: cilium-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - ""
+  resourceNames:
+  - cilium-config
+  resources:
+  - configmaps
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - delete
+  - patch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumclusterwidenetworkpolicies
+  verbs:
+  - create
+  - update
+  - deletecollection
+  - patch
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumendpoints
+  - ciliumidentities
+  verbs:
+  - delete
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumidentities
+  verbs:
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnodes
+  verbs:
+  - create
+  - update
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnodes/status
+  verbs:
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumendpointslices
+  - ciliumenvoyconfigs
+  - ciliumbgppeerconfigs
+  - ciliumbgpadvertisements
+  - ciliumbgpnodeconfigs
+  verbs:
+  - create
+  - update
+  - get
+  - list
+  - watch
+  - delete
+  - patch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumbgpclusterconfigs/status
+  - ciliumbgppeerconfigs/status
+  verbs:
+  - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resourceNames:
+  - ciliumloadbalancerippools.cilium.io
+  - ciliumbgppeeringpolicies.cilium.io
+  - ciliumbgpclusterconfigs.cilium.io
+  - ciliumbgppeerconfigs.cilium.io
+  - ciliumbgpadvertisements.cilium.io
+  - ciliumbgpnodeconfigs.cilium.io
+  - ciliumbgpnodeconfigoverrides.cilium.io
+  - ciliumclusterwideenvoyconfigs.cilium.io
+  - ciliumclusterwidenetworkpolicies.cilium.io
+  - ciliumegressgatewaypolicies.cilium.io
+  - ciliumendpoints.cilium.io
+  - ciliumendpointslices.cilium.io
+  - ciliumenvoyconfigs.cilium.io
+  - ciliumidentities.cilium.io
+  - ciliumlocalredirectpolicies.cilium.io
+  - ciliumnetworkpolicies.cilium.io
+  - ciliumnodes.cilium.io
+  - ciliumnodeconfigs.cilium.io
+  - ciliumcidrgroups.cilium.io
+  - ciliuml2announcementpolicies.cilium.io
+  - ciliumpodippools.cilium.io
+  - ciliumgatewayclassconfigs.cilium.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumloadbalancerippools
+  - ciliumpodippools
+  - ciliumbgppeeringpolicies
+  - ciliumbgpclusterconfigs
+  - ciliumbgpnodeconfigoverrides
+  - ciliumbgppeerconfigs
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumpodippools
+  verbs:
+  - create
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumloadbalancerippools/status
+  verbs:
+  - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/part-of: cilium
+    role.kubernetes.io/networking: "1"
+  name: cilium
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium
+subjects:
+- kind: ServiceAccount
+  name: cilium
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/part-of: cilium
+    role.kubernetes.io/networking: "1"
+  name: cilium-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-operator
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/part-of: cilium
+    role.kubernetes.io/networking: "1"
+  name: cilium-config-agent
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/part-of: cilium
+    role.kubernetes.io/networking: "1"
+  name: cilium-config-agent
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cilium-config-agent
+subjects:
+- kind: ServiceAccount
+  name: cilium
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: cilium-agent
+    app.kubernetes.io/part-of: cilium
+    k8s-app: cilium
+    kubernetes.io/cluster-service: "true"
+    role.kubernetes.io/networking: "1"
+  name: cilium
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium
+      kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      annotations:
+        container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined
+        container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined
+        container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined
+        container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined
+        kubectl.kubernetes.io/default-container: cilium-agent
+      labels:
+        app.kubernetes.io/name: cilium-agent
+        app.kubernetes.io/part-of: cilium
+        k8s-app: cilium
+        kops.k8s.io/managed-by: kops
+        kubernetes.io/cluster-service: "true"
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                k8s-app: cilium
+            topologyKey: kubernetes.io/hostname
+      automountServiceAccountToken: true
+      containers:
+      - args:
+        - --config-dir=/tmp/cilium/config-map
+        command:
+        - cilium-agent
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: CILIUM_CLUSTERMESH_CONFIG
+          value: /var/lib/cilium/clustermesh/
+        - name: GOMEMLIMIT
+          valueFrom:
+            resourceFieldRef:
+              divisor: "1"
+              resource: limits.memory
+        - name: KUBE_CLIENT_BACKOFF_BASE
+          value: "1"
+        - name: KUBE_CLIENT_BACKOFF_DURATION
+          value: "120"
+        - name: KUBERNETES_SERVICE_HOST
+          value: api.internal.minimal-gce-ilb-cilium-etcd.example.com
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        livenessProbe:
+          failureThreshold: 10
+          httpGet:
+            host: 127.0.0.1
+            httpHeaders:
+            - name: brief
+              value: "true"
+            - name: require-k8s-connectivity
+              value: "false"
+            path: /healthz
+            port: 9879
+            scheme: HTTP
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: cilium-agent
+        ports: null
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            host: 127.0.0.1
+            httpHeaders:
+            - name: brief
+              value: "true"
+            path: /healthz
+            port: 9879
+            scheme: HTTP
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 5
+        resources:
+          requests:
+            cpu: 25m
+            memory: 128Mi
+        securityContext:
+          capabilities:
+            add:
+            - CHOWN
+            - KILL
+            - NET_ADMIN
+            - NET_RAW
+            - IPC_LOCK
+            - SYS_MODULE
+            - SYS_ADMIN
+            - SYS_RESOURCE
+            - DAC_OVERRIDE
+            - FOWNER
+            - SETGID
+            - SETUID
+            drop:
+            - ALL
+          seLinuxOptions:
+            level: s0
+            type: spc_t
+        startupProbe:
+          failureThreshold: 300
+          httpGet:
+            host: 127.0.0.1
+            httpHeaders:
+            - name: brief
+              value: "true"
+            path: /healthz
+            port: 9879
+            scheme: HTTP
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          successThreshold: 1
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /host/proc/sys/net
+          name: host-proc-sys-net
+        - mountPath: /host/proc/sys/kernel
+          name: host-proc-sys-kernel
+        - mountPath: /sys/fs/bpf
+          mountPropagation: HostToContainer
+          name: bpf-maps
+        - mountPath: /run/cilium/cgroupv2
+          name: cilium-cgroup
+        - mountPath: /var/run/cilium
+          name: cilium-run
+        - mountPath: /var/run/cilium/netns
+          mountPropagation: HostToContainer
+          name: cilium-netns
+        - mountPath: /host/etc/cni/net.d
+          name: etc-cni-netd
+        - mountPath: /var/lib/etcd-config
+          name: etcd-config-path
+          readOnly: true
+        - mountPath: /var/lib/etcd-secrets
+          name: etcd-secrets
+          readOnly: true
+        - mountPath: /var/lib/cilium/clustermesh
+          name: clustermesh-secrets
+          readOnly: true
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+        - mountPath: /tmp
+          name: tmp
+      hostNetwork: true
+      initContainers:
+      - command:
+        - cilium-dbg
+        - build-config
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          value: api.internal.minimal-gce-ilb-cilium-etcd.example.com
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        name: config
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp
+      - command:
+        - sh
+        - -ec
+        - |
+          cp /usr/bin/cilium-mount /hostbin/cilium-mount;
+          nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT;
+          rm /hostbin/cilium-mount
+        env:
+        - name: CGROUP_ROOT
+          value: /run/cilium/cgroupv2
+        - name: BIN_PATH
+          value: /opt/cni/bin
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        name: mount-cgroup
+        securityContext:
+          capabilities:
+            add:
+            - SYS_ADMIN
+            - SYS_CHROOT
+            - SYS_PTRACE
+            drop:
+            - ALL
+          seLinuxOptions:
+            level: s0
+            type: spc_t
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /hostproc
+          name: hostproc
+        - mountPath: /hostbin
+          name: cni-path
+      - command:
+        - sh
+        - -ec
+        - |
+          cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
+          nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
+          rm /hostbin/cilium-sysctlfix
+        env:
+        - name: BIN_PATH
+          value: /opt/cni/bin
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        name: apply-sysctl-overwrites
+        securityContext:
+          capabilities:
+            add:
+            - SYS_ADMIN
+            - SYS_CHROOT
+            - SYS_PTRACE
+            drop:
+            - ALL
+          seLinuxOptions:
+            level: s0
+            type: spc_t
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /hostproc
+          name: hostproc
+        - mountPath: /hostbin
+          name: cni-path
+      - args:
+        - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf
+        command:
+        - /bin/bash
+        - -c
+        - --
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        name: mount-bpf-fs
+        securityContext:
+          privileged: true
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
+          name: bpf-maps
+      - command:
+        - /init-container.sh
+        env:
+        - name: CILIUM_ALL_STATE
+          valueFrom:
+            configMapKeyRef:
+              key: clean-cilium-state
+              name: cilium-config
+              optional: true
+        - name: CILIUM_BPF_STATE
+          valueFrom:
+            configMapKeyRef:
+              key: clean-cilium-bpf-state
+              name: cilium-config
+              optional: true
+        - name: WRITE_CNI_CONF_WHEN_READY
+          valueFrom:
+            configMapKeyRef:
+              key: write-cni-conf-when-ready
+              name: cilium-config
+              optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: api.internal.minimal-gce-ilb-cilium-etcd.example.com
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        name: clean-cilium-state
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - SYS_MODULE
+            - SYS_ADMIN
+            - SYS_RESOURCE
+            drop:
+            - ALL
+          seLinuxOptions:
+            level: s0
+            type: spc_t
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /sys/fs/bpf
+          name: bpf-maps
+        - mountPath: /run/cilium/cgroupv2
+          mountPropagation: HostToContainer
+          name: cilium-cgroup
+        - mountPath: /var/run/cilium
+          name: cilium-run
+      - command:
+        - /install-plugin.sh
+        image: quay.io/cilium/cilium:v1.18.6
+        imagePullPolicy: IfNotPresent
+        name: install-cni-binaries
+        resources:
+          requests:
+            cpu: 100m
+            memory: 10Mi
+        securityContext:
+          capabilities:
+            drop:
+            - ALL
+          seLinuxOptions:
+            level: s0
+            type: spc_t
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-path
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      securityContext:
+        seccompProfile:
+          type: Unconfined
+      serviceAccountName: cilium
+      terminationGracePeriodSeconds: 1
+      tolerations:
+      - operator: Exists
+      volumes:
+      - emptyDir: {}
+        name: tmp
+      - hostPath:
+          path: /var/run/cilium
+          type: DirectoryOrCreate
+        name: cilium-run
+      - hostPath:
+          path: /var/run/netns
+          type: DirectoryOrCreate
+        name: cilium-netns
+      - hostPath:
+          path: /sys/fs/bpf
+          type: DirectoryOrCreate
+        name: bpf-maps
+      - hostPath:
+          path: /proc
+          type: Directory
+        name: hostproc
+      - hostPath:
+          path: /run/cilium/cgroupv2
+          type: DirectoryOrCreate
+        name: cilium-cgroup
+      - hostPath:
+          path: /opt/cni/bin
+          type: DirectoryOrCreate
+        name: cni-path
+      - hostPath:
+          path: /etc/cni/net.d
+          type: DirectoryOrCreate
+        name: etc-cni-netd
+      - hostPath:
+          path: /lib/modules
+        name: lib-modules
+      - hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+        name: xtables-lock
+      - configMap:
+          defaultMode: 256
+          items:
+          - key: etcd-config
+            path: etcd.config
+          name: cilium-config
+        name: etcd-config-path
+      - hostPath:
+          path: /etc/kubernetes/pki/cilium
+          type: Directory
+        name: etcd-secrets
+      - name: clustermesh-secrets
+        projected:
+          defaultMode: 256
+          sources:
+          - secret:
+              name: cilium-clustermesh
+              optional: true
+          - secret:
+              items:
+              - key: tls.key
+                path: common-etcd-client.key
+              - key: tls.crt
+                path: common-etcd-client.crt
+              - key: ca.crt
+                path: common-etcd-client-ca.crt
+              name: clustermesh-apiserver-remote-cert
+              optional: true
+          - secret:
+              items:
+              - key: tls.key
+                path: local-etcd-client.key
+              - key: tls.crt
+                path: local-etcd-client.crt
+              - key: ca.crt
+                path: local-etcd-client-ca.crt
+              name: clustermesh-apiserver-local-cert
+              optional: true
+      - hostPath:
+          path: /proc/sys/net
+          type: Directory
+        name: host-proc-sys-net
+      - hostPath:
+          path: /proc/sys/kernel
+          type: Directory
+        name: host-proc-sys-kernel
+  updateStrategy:
+    type: OnDelete
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    addon.kops.k8s.io/name: networking.cilium.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: cilium-operator
+    app.kubernetes.io/part-of: cilium
+    io.cilium/app: operator
+    name: cilium-operator
+    role.kubernetes.io/networking: "1"
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      io.cilium/app: operator
+      name: cilium-operator
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 50%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: cilium-operator
+        app.kubernetes.io/part-of: cilium
+        io.cilium/app: operator
+        kops.k8s.io/managed-by: kops
+        name: cilium-operator
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                io.cilium/app: operator
+            topologyKey: kubernetes.io/hostname
+      automountServiceAccountToken: true
+      containers:
+      - args:
+        - --config-dir=/tmp/cilium/config-map
+        - --debug=$(CILIUM_DEBUG)
+        - --eni-tags=KubernetesCluster=minimal-gce-ilb-cilium-etcd.example.com
+        command:
+        - cilium-operator
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: CILIUM_DEBUG
+          valueFrom:
+            configMapKeyRef:
+              key: debug
+              name: cilium-config
+              optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: api.internal.minimal-gce-ilb-cilium-etcd.example.com
+        - name: KUBERNETES_SERVICE_PORT
+          value: "443"
+        image: quay.io/cilium/operator:v1.18.6
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 9234
+            scheme: HTTP
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: cilium-operator
+        readinessProbe:
+          failureThreshold: 5
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 9234
+            scheme: HTTP
+          initialDelaySeconds: 0
+          periodSeconds: 5
+          timeoutSeconds: 3
+        resources:
+          requests:
+            cpu: 25m
+            memory: 128Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /tmp/cilium/config-map
+          name: cilium-config-path
+          readOnly: true
+        - mountPath: /var/lib/etcd-config
+          name: etcd-config-path
+          readOnly: true
+        - mountPath: /var/lib/etcd-secrets
+          name: etcd-secrets
+          readOnly: true
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      restartPolicy: Always
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccountName: cilium-operator
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        operator: Exists
+      - key: node.cilium.io/agent-not-ready
+        operator: Exists
+      volumes:
+      - configMap:
+          name: cilium-config
+        name: cilium-config-path
+      - configMap:
+          defaultMode: 256
+          items:
+          - key: etcd-config
+            path: etcd.config
+          name: cilium-config
+        name: etcd-config-path
+      - hostPath:
+          path: /etc/kubernetes/pki/cilium
+          type: Directory
+        name: etcd-secrets
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content
new file mode 100644
index 0000000000000..b5b781d7554ae
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content
@@ -0,0 +1,13 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  labels:
+    addon.kops.k8s.io/name: storage-gce.addons.k8s.io
+    addonmanager.kubernetes.io/mode: EnsureExists
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: storage-gce.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: standard
+parameters:
+  type: pd-standard
+provisioner: kubernetes.io/gce-pd
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_nodeupconfig-master-us-test1-a_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_nodeupconfig-master-us-test1-a_content
new file mode 100644
index 0000000000000..0dc018380d780
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_nodeupconfig-master-us-test1-a_content
@@ -0,0 +1,396 @@
+APIServerConfig:
+  API:
+    loadBalancer:
+      useForInternalAPI: true
+    publicName: api.minimal-gce-ilb-cilium-etcd.example.com
+  ClusterDNSDomain: cluster.local
+  KubeAPIServer:
+    allowPrivileged: true
+    anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
+    apiServerCount: 1
+    authorizationMode: Node,RBAC
+    bindAddress: 0.0.0.0
+    cloudProvider: external
+    enableAdmissionPlugins:
+    - DefaultStorageClass
+    - DefaultTolerationSeconds
+    - LimitRanger
+    - MutatingAdmissionWebhook
+    - NamespaceLifecycle
+    - NodeRestriction
+    - ResourceQuota
+    - RuntimeClass
+    - ServiceAccount
+    - ValidatingAdmissionPolicy
+    - ValidatingAdmissionWebhook
+    etcdServers:
+    - https://127.0.0.1:4001
+    etcdServersOverrides:
+    - /events#https://127.0.0.1:4002
+    image: registry.k8s.io/kube-apiserver:v1.32.0
+    kubeletPreferredAddressTypes:
+    - InternalIP
+    - Hostname
+    - ExternalIP
+    logLevel: 2
+    requestheaderAllowedNames:
+    - aggregator
+    requestheaderExtraHeaderPrefixes:
+    - X-Remote-Extra-
+    requestheaderGroupHeaders:
+    - X-Remote-Group
+    requestheaderUsernameHeaders:
+    - X-Remote-User
+    securePort: 443
+    serviceAccountIssuer: https://api.internal.minimal-gce-ilb-cilium-etcd.example.com
+    serviceAccountJWKSURI: https://api.internal.minimal-gce-ilb-cilium-etcd.example.com/openid/v1/jwks
+    serviceClusterIPRange: 100.64.0.0/13
+    storageBackend: etcd3
+  ServiceAccountPublicKeys: |
+    -----BEGIN RSA PUBLIC KEY-----
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+    XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+    -----END RSA PUBLIC KEY-----
+    -----BEGIN RSA PUBLIC KEY-----
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+    Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+    -----END RSA PUBLIC KEY-----
+Assets:
+  amd64:
+  - 5ad4965598773d56a37a8e8429c3dc3d86b4c5c26d8417ab333ae345c053dae2@https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubelet
+  - 646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70@https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubectl
+  - eca252d94176f8e08084433d08cd478c28cba7b773b49d691f1bec0f1e94e7d1@https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/mounter
+  - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp
+  - 4793dc5c1f34ebf8402990d0050f3c294aa3c794cd5a4baa403c1cf10602326d@https://github.com/containerd/containerd/releases/download/v2.1.6/containerd-2.1.6-linux-amd64.tar.gz
+  - 5966ca40b6187b30e33bfc299c5f1fe72e8c1aa01cf3fefdadf391668f47f103@https://github.com/opencontainers/runc/releases/download/v1.3.4/runc.amd64
+  - 86189e1e8de9692eb02daf2f06db8495f687ce2c4ba09a6b64f135990dfb315d@https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/protokube-linux-amd64
+  - 0172d3c560aebe1eb4e8599f71c0d8fc68e4eca880add8031de41c8057ca8e3c@https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/channels-linux-amd64
+  arm64:
+  - bda9b2324c96693b38c41ecea051bab4c7c434be5683050b5e19025b50dbc0bf@https://dl.k8s.io/release/v1.32.0/bin/linux/arm64/kubelet
+  - ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896@https://dl.k8s.io/release/v1.32.0/bin/linux/arm64/kubectl
+  - ee06cd4a0e8428a3cced77f4f7db836138c589e8e4bf46f0c676f8ff4b54b942@https://dl.k8s.io/release/v1.32.0/bin/linux/arm64/mounter
+  - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp
+  - 88d6e32348c36628c8500a630c6dd4b3cb8c680b1d18dc8d1d19041f67757c6e@https://github.com/containerd/containerd/releases/download/v2.1.6/containerd-2.1.6-linux-arm64.tar.gz
+  - d6dcab36d1b6af1b72c7f0662e5fcf446a291271ba6006532b95c4144e19d428@https://github.com/opencontainers/runc/releases/download/v1.3.4/runc.arm64
+  - 25b57b0555fad42e5762246334681bf1c943794fcecdb680a79e482be5c08815@https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/protokube-linux-arm64
+  - 04470f8313796032fce85b974da4fc26420f36931e574fff6d117d21caf22770@https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/channels-linux-arm64
+CAs:
+  apiserver-aggregator-ca: |
+    -----BEGIN CERTIFICATE-----
+    MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe
+    BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX
+    DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It
+    Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM
+    x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB
+    o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
+    ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX
+    WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk
+    CzMeMdr4
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe
+    BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX
+    DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It
+    Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM
+    x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB
+    o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
+    ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN
+    QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW
+    HLtkTXH8
+    -----END CERTIFICATE-----
+  etcd-clients-ca: |
+    -----BEGIN CERTIFICATE-----
+    MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW
+    BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx
+    NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB
+    AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep
+    uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE
+    AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s
+    x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY
+    qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY=
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW
+    BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx
+    NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB
+    AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep
+    uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE
+    AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s
+    x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E
+    YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co=
+    -----END CERTIFICATE-----
+  etcd-clients-ca-cilium: |
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFotPsR9PsbCKkTJsMA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyMTUyWhcN
+    MzEwNjIxMjAyMTUyWjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+    XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW
+    3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EAR4UEW5ZK+NVtqm7s
+    HF/JbSYPd+BhcNaJVOv8JP+/CGfCOXOmxjpZICSYQqe6UjjjP7fbJy8FANTpKTuJ
+    UQC1kQ==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFotP940EXpD3N1D7MA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyNjU1WhcN
+    MzEwNjIxMjAyNjU1WjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+    XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW
+    3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EARXoKy6mExpD6tHFO
+    CN3ZGNZ5BsHl5W5y+gwUuVskgC7xt/bgTuXm5hz8TLgnG5kYtG4uxjFg4yCvtNg2
+    MQNfAQ==
+    -----END CERTIFICATE-----
+  etcd-manager-ca-cilium: |
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFo+bv6kG/ijs2GJsMA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1jaWxpdW0wHhcNMjEwNzA1MjAyMDM3WhcN
+    MzEwNzA1MjAyMDM3WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtY2lsaXVt
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAMHrFsj6jdcV2UZnTJmqNdbz7kQjh0NW
+    0PrIWcRAD6Y1q9/NvbndWF8jGay206KXJk1r/qHXyDuwHCKgZkfbnS0CAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFDKE
+    ITER3OCn4C7w9YVi2YdHDUkJMA0GCSqGSIb3DQEBCwUAA0EAo2zLlhHTpYlTM7dh
+    etdG+8zu6GpzoNs6caeYT1F7LCUp5CX8T05QVHZNSwTU41wFFu3nRa5Fr8/2nB+M
+    EcE5pA==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFo+b23aziPjha6o+MA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1jaWxpdW0wHhcNMjEwNzA1MjAyMjM3WhcN
+    MzEwNzA1MjAyMjM3WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtY2lsaXVt
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKObYoPZoxsjbLbCy1tA2JyHFKEPHg3X
+    gOPCmQLAYvnDOIxAewihwpdjjcuJP+xoz0vUA+fcJaBei/3lAFNV0MUCAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFOBa
+    mp4zlA4aPNrVCZgS+Ot9sG5BMA0GCSqGSIb3DQEBCwUAA0EABBJLTr+G+TxDLF3E
+    JyV/pgEM/QggrBJozK1bWCvxIUKsnZHiX6E/WVeDeT1QlM1HaxumLGMsKAAyxPV4
+    GY7LCw==
+    -----END CERTIFICATE-----
+  etcd-manager-ca-events: |
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN
+    MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H
+    g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6
+    CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O
+    sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs
+    GS/VUw==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN
+    MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL
+    DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW
+    LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE
+    hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV
+    cPfVNg==
+    -----END CERTIFICATE-----
+  etcd-manager-ca-main: |
+    -----BEGIN CERTIFICATE-----
+    MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb
+    BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx
+    MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN
+    BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm
+    ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO
+    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx
+    GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu
+    Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb
+    BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx
+    MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN
+    BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP
+    vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO
+    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP
+    DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9
+    t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd
+    -----END CERTIFICATE-----
+  etcd-peers-ca-cilium: |
+    -----BEGIN CERTIFICATE-----
+    MIIBfDCCASagAwIBAgIMFo+bv6kGnIBWECkZMA0GCSqGSIb3DQEBCwUAMB8xHTAb
+    BgNVBAMTFGV0Y2QtcGVlcnMtY2EtY2lsaXVtMB4XDTIxMDcwNTIwMjAzN1oXDTMx
+    MDcwNTIwMjAzN1owHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1jaWxpdW0wXDAN
+    BgkqhkiG9w0BAQEFAANLADBIAkEA2IAKqCL/e3GqQxXqRNIwDVlHoze6XhCIa2FS
+    5btM8qQ0eGermtEV3KEtxAlix62ZVHZOuAfSFAP02h+fkfnJaQIDAQABo0IwQDAO
+    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUfr/92gfR
+    qn/blYJEH3A38U51A8AwDQYJKoZIhvcNAQELBQADQQCC6qoc1PX3AXOtt+lqTtu0
+    oHrjU5/YXFbqDxEh/VdGYhqtpg3YuoHWAp3JDg1RVW1SRfUx30/375hoB5Nrw/5S
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBfDCCASagAwIBAgIMFo+b23acX0hZEkbkMA0GCSqGSIb3DQEBCwUAMB8xHTAb
+    BgNVBAMTFGV0Y2QtcGVlcnMtY2EtY2lsaXVtMB4XDTIxMDcwNTIwMjIzN1oXDTMx
+    MDcwNTIwMjIzN1owHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1jaWxpdW0wXDAN
+    BgkqhkiG9w0BAQEFAANLADBIAkEAw3T2pyEOgBPBKwofuILLokPxAFplVzdu540f
+    oREJ4iVqiroUlsz1G90mEwmqR+B7/0kt70ve9i5Z6E7Qz2nQaQIDAQABo0IwQDAO
+    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU0hyEvGir
+    2ucsJrojyZaDBIb8JLAwDQYJKoZIhvcNAQELBQADQQA9vQylgkvgROIMspzOlbZr
+    ZwsTAzp9J2ZxZL06AQ9iWzpvIw/H3oClV63q6zN2aHtpBTkhUOSX3Q4L/X/0MOkj
+    -----END CERTIFICATE-----
+  etcd-peers-ca-events: |
+    -----BEGIN CERTIFICATE-----
+    MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb
+    BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx
+    MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN
+    BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY
+    xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO
+    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O
+    Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB
+    DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb
+    BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx
+    MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN
+    BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW
+    03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO
+    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh
+    cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI
+    J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb
+    -----END CERTIFICATE-----
+  etcd-peers-ca-main: |
+    -----BEGIN CERTIFICATE-----
+    MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ
+    BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3
+    MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG
+    SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA
+    aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud
+    DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf
+    OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt
+    uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4=
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ
+    BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3
+    MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG
+    SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt
+    naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud
+    DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC
+    qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K
+    G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo=
+    -----END CERTIFICATE-----
+  kubernetes-ca: |
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
+    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
+    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
+    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
+    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
+    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
+    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
+    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
+    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
+    -----END CERTIFICATE-----
+ClusterName: minimal-gce-ilb-cilium-etcd.example.com
+ControlPlaneConfig:
+  KubeControllerManager:
+    allocateNodeCIDRs: true
+    attachDetachReconcileSyncPeriod: 1m0s
+    cloudProvider: external
+    clusterCIDR: 100.96.0.0/11
+    clusterName: minimal-gce-ilb-cilium-etcd.example.com
+    configureCloudRoutes: false
+    image: registry.k8s.io/kube-controller-manager:v1.32.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+    useServiceAccountCredentials: true
+  KubeScheduler:
+    image: registry.k8s.io/kube-scheduler:v1.32.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+DNSZone: "1"
+EtcdClusterNames:
+- main
+- events
+- cilium
+FileAssets:
+- content: |
+    apiVersion: kubescheduler.config.k8s.io/v1
+    clientConnection:
+      kubeconfig: /var/lib/kube-scheduler/kubeconfig
+    kind: KubeSchedulerConfiguration
+  path: /var/lib/kube-scheduler/config.yaml
+Hooks:
+- null
+- null
+KeypairIDs:
+  apiserver-aggregator-ca: "6980187172486667078076483355"
+  etcd-clients-ca: "6979622252718071085282986282"
+  etcd-clients-ca-cilium: "6977087239278090025924899436"
+  etcd-manager-ca-cilium: "6982282110587148051456483948"
+  etcd-manager-ca-events: "6982279354000777253151890266"
+  etcd-manager-ca-main: "6982279354000936168671127624"
+  etcd-peers-ca-cilium: "6982282110587040674744379673"
+  etcd-peers-ca-events: "6982279353999767935825892873"
+  etcd-peers-ca-main: "6982279353998887468930183660"
+  kubernetes-ca: "6982820025135291416230495506"
+  service-account: "2"
+KubeProxy: null
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: external
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hairpinMode: promiscuous-bridge
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  nodeLabels:
+    kops.k8s.io/kops-controller-pki: ""
+    node-role.kubernetes.io/control-plane: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
+  podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  registerSchedulable: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
+  taints:
+  - node-role.kubernetes.io/control-plane=:NoSchedule
+KubernetesVersion: 1.32.0
+Networking:
+  cilium: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  serviceClusterIPRange: 100.64.0.0/13
+UpdatePolicy: automatic
+UseCiliumEtcd: true
+channels:
+- memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/addons/bootstrap-channel.yaml
+configStore:
+  keypairs: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/pki
+  secrets: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/secrets
+containerdConfig:
+  logLevel: info
+  runc:
+    version: 1.3.4
+  sandboxImage: registry.k8s.io/pause:3.10.1
+  version: 2.1.6
+etcdManifests:
+- memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/etcd/main-master-us-test1-a.yaml
+- memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/etcd/events-master-us-test1-a.yaml
+- memfs://tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/etcd/cilium-master-us-test1-a.yaml
+multizone: true
+nodeTags: minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node
+staticManifests:
+- key: kube-apiserver-healthcheck
+  path: manifests/static/kube-apiserver-healthcheck.yaml
+usesLegacyGossip: false
+usesNoneDNS: false
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_nodeupconfig-nodes_content
new file mode 100644
index 0000000000000..ed39a9bfa4124
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/aws_s3_object_nodeupconfig-nodes_content
@@ -0,0 +1,83 @@
+Assets:
+  amd64:
+  - 5ad4965598773d56a37a8e8429c3dc3d86b4c5c26d8417ab333ae345c053dae2@https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubelet
+  - 646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70@https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubectl
+  - eca252d94176f8e08084433d08cd478c28cba7b773b49d691f1bec0f1e94e7d1@https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/mounter
+  - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp
+  - 4793dc5c1f34ebf8402990d0050f3c294aa3c794cd5a4baa403c1cf10602326d@https://github.com/containerd/containerd/releases/download/v2.1.6/containerd-2.1.6-linux-amd64.tar.gz
+  - 5966ca40b6187b30e33bfc299c5f1fe72e8c1aa01cf3fefdadf391668f47f103@https://github.com/opencontainers/runc/releases/download/v1.3.4/runc.amd64
+  arm64:
+  - bda9b2324c96693b38c41ecea051bab4c7c434be5683050b5e19025b50dbc0bf@https://dl.k8s.io/release/v1.32.0/bin/linux/arm64/kubelet
+  - ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896@https://dl.k8s.io/release/v1.32.0/bin/linux/arm64/kubectl
+  - ee06cd4a0e8428a3cced77f4f7db836138c589e8e4bf46f0c676f8ff4b54b942@https://dl.k8s.io/release/v1.32.0/bin/linux/arm64/mounter
+  - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp
+  - 88d6e32348c36628c8500a630c6dd4b3cb8c680b1d18dc8d1d19041f67757c6e@https://github.com/containerd/containerd/releases/download/v2.1.6/containerd-2.1.6-linux-arm64.tar.gz
+  - d6dcab36d1b6af1b72c7f0662e5fcf446a291271ba6006532b95c4144e19d428@https://github.com/opencontainers/runc/releases/download/v1.3.4/runc.arm64
+CAs:
+  etcd-clients-ca-cilium: |
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFotPsR9PsbCKkTJsMA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyMTUyWhcN
+    MzEwNjIxMjAyMTUyWjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+    XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW
+    3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EAR4UEW5ZK+NVtqm7s
+    HF/JbSYPd+BhcNaJVOv8JP+/CGfCOXOmxjpZICSYQqe6UjjjP7fbJy8FANTpKTuJ
+    UQC1kQ==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBgDCCASqgAwIBAgIMFotP940EXpD3N1D7MA0GCSqGSIb3DQEBCwUAMCExHzAd
+    BgNVBAMTFmV0Y2QtY2xpZW50cy1jYS1jaWxpdW0wHhcNMjEwNjIxMjAyNjU1WhcN
+    MzEwNjIxMjAyNjU1WjAhMR8wHQYDVQQDExZldGNkLWNsaWVudHMtY2EtY2lsaXVt
+    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+    XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAaNC
+    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW
+    3hR7ngBsk9aUOlEznWzH494EMA0GCSqGSIb3DQEBCwUAA0EARXoKy6mExpD6tHFO
+    CN3ZGNZ5BsHl5W5y+gwUuVskgC7xt/bgTuXm5hz8TLgnG5kYtG4uxjFg4yCvtNg2
+    MQNfAQ==
+    -----END CERTIFICATE-----
+ClusterName: minimal-gce-ilb-cilium-etcd.example.com
+Hooks:
+- null
+- null
+KeypairIDs:
+  etcd-clients-ca-cilium: "6977087239278090025924899436"
+  kubernetes-ca: "6982820025135291416230495506"
+KubeProxy: null
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: external
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hairpinMode: promiscuous-bridge
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  nodeLabels:
+    node-role.kubernetes.io/node: ""
+  podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  registerSchedulable: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
+KubernetesVersion: 1.32.0
+Networking:
+  cilium: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  serviceClusterIPRange: 100.64.0.0/13
+UpdatePolicy: automatic
+UseCiliumEtcd: true
+containerdConfig:
+  logLevel: info
+  runc:
+    version: 1.3.4
+  sandboxImage: registry.k8s.io/pause:3.10.1
+  version: 2.1.6
+multizone: true
+nodeTags: minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node
+usesLegacyGossip: false
+usesNoneDNS: false
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/google_compute_instance_template_master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/google_compute_instance_template_master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data
new file mode 100644
index 0000000000000..22005de08ed15
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/google_compute_instance_template_master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data
@@ -0,0 +1,133 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-amd64
+NODEUP_HASH_AMD64=c86e072f622b91546b7b3f3cb1a0f8a131e48b966ad018a0ac1520ceedf37725
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-arm64
+NODEUP_HASH_ARM64=64a9a9510538a449e85d05e13e3cd98b80377d68a673447c26821d40f00f0075
+
+
+
+
+
+sysctl -w net.core.rmem_max=16777216 || true
+sysctl -w net.core.wmem_max=16777216 || true
+sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
+sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, urls
+download-or-bust() {
+  echo "== Downloading $1 with hash $2 from $3 =="
+  local -r file="$1"
+  local -r hash="$2"
+  local -a urls
+  IFS=, read -r -a urls <<< "$3"
+
+  if [[ -f "${file}" ]]; then
+    if ! validate-hash "${file}" "${hash}"; then
+      rm -f "${file}"
+    else
+      return 0
+    fi
+  fi
+
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "== Downloading ${url} using ${cmd} =="
+        if ! (${cmd} "${url}"); then
+          echo "== Failed to download ${url} using ${cmd} =="
+          continue
+        fi
+        if ! validate-hash "${file}" "${hash}"; then
+          echo "== Failed to validate hash for ${url} =="
+          rm -f "${file}"
+        else
+          echo "== Downloaded ${url} with hash ${hash} =="
+          return 0
+        fi
+      done
+    done
+
+    echo "== All downloads failed; sleeping before retrying =="
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  cd ${INSTALL_DIR}/bin
+  download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
+
+  chmod +x nodeup
+
+  echo "== Running nodeup =="
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+CloudProvider: gce
+ClusterName: minimal-gce-ilb-cilium-etcd.example.com
+ConfigBase: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com
+InstanceGroupName: master-us-test1-a
+InstanceGroupRole: ControlPlane
+NodeupConfigHash: YnTo4Gmk4cvMAEVotthTRxKuuq3MSdIWZkfGHh/SOq0=
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/google_compute_instance_template_nodes-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/google_compute_instance_template_nodes-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data
new file mode 100644
index 0000000000000..baed393e36537
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/data/google_compute_instance_template_nodes-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data
@@ -0,0 +1,156 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-amd64
+NODEUP_HASH_AMD64=c86e072f622b91546b7b3f3cb1a0f8a131e48b966ad018a0ac1520ceedf37725
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-arm64
+NODEUP_HASH_ARM64=64a9a9510538a449e85d05e13e3cd98b80377d68a673447c26821d40f00f0075
+
+
+
+
+
+sysctl -w net.core.rmem_max=16777216 || true
+sysctl -w net.core.wmem_max=16777216 || true
+sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
+sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, urls
+download-or-bust() {
+  echo "== Downloading $1 with hash $2 from $3 =="
+  local -r file="$1"
+  local -r hash="$2"
+  local -a urls
+  IFS=, read -r -a urls <<< "$3"
+
+  if [[ -f "${file}" ]]; then
+    if ! validate-hash "${file}" "${hash}"; then
+      rm -f "${file}"
+    else
+      return 0
+    fi
+  fi
+
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "== Downloading ${url} using ${cmd} =="
+        if ! (${cmd} "${url}"); then
+          echo "== Failed to download ${url} using ${cmd} =="
+          continue
+        fi
+        if ! validate-hash "${file}" "${hash}"; then
+          echo "== Failed to validate hash for ${url} =="
+          rm -f "${file}"
+        else
+          echo "== Downloaded ${url} with hash ${hash} =="
+          return 0
+        fi
+      done
+    done
+
+    echo "== All downloads failed; sleeping before retrying =="
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  cd ${INSTALL_DIR}/bin
+  download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
+
+  chmod +x nodeup
+
+  echo "== Running nodeup =="
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+CloudProvider: gce
+ClusterName: minimal-gce-ilb-cilium-etcd.example.com
+ConfigServer:
+  CACertificates: |
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
+    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
+    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
+    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
+    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
+    -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
+    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
+    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
+    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
+    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
+    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
+    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
+    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
+    -----END CERTIFICATE-----
+  servers:
+  - https://kops-controller.internal.minimal-gce-ilb-cilium-etcd.example.com:3988/
+InstanceGroupName: nodes
+InstanceGroupRole: Node
+NodeupConfigHash: +sryUketA8co9qq2WFyu7yL6K6h8OqBGYWObyT1YCxo=
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/id_rsa.pub b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/id_rsa.pub
new file mode 100755
index 0000000000000..81cb0127830e7
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/id_rsa.pub
@@ -0,0 +1,1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/in-v1alpha2.yaml
new file mode 100644
index 0000000000000..1868fb0fff8dc
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/in-v1alpha2.yaml
@@ -0,0 +1,103 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal-gce-ilb-cilium-etcd.example.com
+spec:
+  api:
+    loadBalancer:
+      type: Internal
+      useForInternalApi: true
+      subnets:
+        - name: us-test-1
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudProvider: gce
+  configBase: memfs://tests/minimal-gce-ilb-cilium-etcd.example.com
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    memoryRequest: 100Mi
+    name: main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    memoryRequest: 100Mi
+    name: events
+  - etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    name: cilium
+  cloudConfig:
+    gceServiceAccount: default
+  iam:
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubeProxy:
+    enabled: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  - ::/0
+  kubernetesVersion: v1.32.0
+  masterPublicName: api.minimal-gce-ilb-cilium-etcd.example.com
+  networking:
+    cilium:
+      etcdManaged: true
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  - ::/0
+  subnets:
+  - name: us-test1
+    region: us-test1
+    type: Private
+  topology:
+    dns:
+      type: Public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-gce-ilb-cilium-etcd.example.com
+  name: master-us-test1-a
+spec:
+  image: ubuntu-os-cloud/ubuntu-2004-focal-v20221018
+  machineType: e2-medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-gce-ilb-cilium-etcd.example.com
+  name: nodes
+spec:
+  image: ubuntu-os-cloud/ubuntu-2004-focal-v20221018
+  machineType: e2-medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+  - us-test1-b
diff --git a/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/kubernetes.tf
new file mode 100644
index 0000000000000..2ac2a46fae14a
--- /dev/null
+++ b/tests/integration/update_cluster/minimal_gce_ilb_cilium_etcd/kubernetes.tf
@@ -0,0 +1,734 @@
+locals {
+  cluster_name = "minimal-gce-ilb-cilium-etcd.example.com"
+  project      = "testproject"
+  region       = "us-test1"
+}
+
+output "cluster_name" {
+  value = "minimal-gce-ilb-cilium-etcd.example.com"
+}
+
+output "project" {
+  value = "testproject"
+}
+
+output "region" {
+  value = "us-test1"
+}
+
+provider "google" {
+  project = "testproject"
+  region  = "us-test1"
+}
+
+provider "aws" {
+  alias  = "files"
+  region = "us-test-1"
+}
+
+resource "aws_s3_object" "cluster-completed-spec" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_cluster-completed.spec_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/cluster-completed.spec"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "etcd-cluster-spec-cilium" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-cilium_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/cilium/control/etcd-cluster-spec"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "etcd-cluster-spec-events" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-events_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/events/control/etcd-cluster-spec"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "etcd-cluster-spec-main" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-main_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/backups/etcd/main/control/etcd-cluster-spec"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "kops-version-txt" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_kops-version.txt_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/kops-version.txt"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "manifests-etcdmanager-cilium-master-us-test1-a" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-cilium-master-us-test1-a_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/etcd/cilium-master-us-test1-a.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test1-a" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test1-a_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/etcd/events-master-us-test1-a.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test1-a" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test1-a_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/etcd/main-master-us-test1-a.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/manifests/static/kube-apiserver-healthcheck.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-bootstrap" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-bootstrap_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/bootstrap-channel.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-coredns-addons-k8s-io-k8s-1-12" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-gcp-cloud-controller-addons-k8s-io-k8s-1-23" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-gcp-pd-csi-driver-addons-k8s-io-k8s-1-23" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-limit-range-addons-k8s-io" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-limit-range.addons.k8s.io_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/limit-range.addons.k8s.io/v1.5.0.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-networking-cilium-io-k8s-1-16" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-networking.cilium.io-k8s-1.16_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/networking.cilium.io/k8s-1.16-v1.15.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-gce-ilb-cilium-etcd-example-com-addons-storage-gce-addons-k8s-io-v1-7-0" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal-gce-ilb-cilium-etcd.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/addons/storage-gce.addons.k8s.io/v1.7.0.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "nodeupconfig-master-us-test1-a" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_nodeupconfig-master-us-test1-a_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/igconfig/control-plane/master-us-test1-a/nodeupconfig.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "nodeupconfig-nodes" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes_content")
+  key                    = "tests/minimal-gce-ilb-cilium-etcd.example.com/igconfig/node/nodes/nodeupconfig.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "google_compute_address" "api-us-test1-minimal-gce-ilb-cilium-etcd-example-com" {
+  address_type = "INTERNAL"
+  name         = "api-us-test1-minimal-gce-ilb-cilium-etcd-example-com"
+  purpose      = "SHARED_LOADBALANCER_VIP"
+  subnetwork   = google_compute_subnetwork.us-test1-minimal-gce-ilb-cilium-etcd-example-com.name
+}
+
+resource "google_compute_disk" "a-etcd-cilium-minimal-gce-ilb-cilium-etcd-example-com" {
+  labels = {
+    "k8s-io-cluster-name" = "minimal-gce-ilb-cilium-etcd-example-com"
+    "k8s-io-etcd-cilium"  = "a-2fa"
+    "k8s-io-role-master"  = "master"
+  }
+  name = "a-etcd-cilium-minimal-gce-ilb-cilium-etcd-example-com"
+  size = 20
+  type = "pd-ssd"
+  zone = "us-test1-a"
+}
+
+resource "google_compute_disk" "a-etcd-events-minimal-gce-ilb-cilium-etcd-example-com" {
+  labels = {
+    "k8s-io-cluster-name" = "minimal-gce-ilb-cilium-etcd-example-com"
+    "k8s-io-etcd-events"  = "a-2fa"
+    "k8s-io-role-master"  = "master"
+  }
+  name = "a-etcd-events-minimal-gce-ilb-cilium-etcd-example-com"
+  size = 20
+  type = "pd-ssd"
+  zone = "us-test1-a"
+}
+
+resource "google_compute_disk" "a-etcd-main-minimal-gce-ilb-cilium-etcd-example-com" {
+  labels = {
+    "k8s-io-cluster-name" = "minimal-gce-ilb-cilium-etcd-example-com"
+    "k8s-io-etcd-main"    = "a-2fa"
+    "k8s-io-role-master"  = "master"
+  }
+  name = "a-etcd-main-minimal-gce-ilb-cilium-etcd-example-com"
+  size = 20
+  type = "pd-ssd"
+  zone = "us-test1-a"
+}
+
+resource "google_compute_firewall" "cilium-etcd-ipv6-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["4003"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "cilium-etcd-ipv6-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["::/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane"]
+}
+
+resource "google_compute_firewall" "cilium-etcd-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["4003"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "cilium-etcd-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane"]
+}
+
+resource "google_compute_firewall" "https-api-ipv6-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["443"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "https-api-ipv6-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["::/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane"]
+}
+
+resource "google_compute_firewall" "https-api-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["443"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "https-api-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane"]
+}
+
+resource "google_compute_firewall" "lb-health-checks-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "lb-health-checks-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["35.191.0.0/16", "130.211.0.0/22", "209.85.204.0/22", "209.85.152.0/22"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane"]
+}
+
+resource "google_compute_firewall" "master-to-master-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    protocol = "tcp"
+  }
+  allow {
+    protocol = "udp"
+  }
+  allow {
+    protocol = "icmp"
+  }
+  allow {
+    protocol = "esp"
+  }
+  allow {
+    protocol = "ah"
+  }
+  allow {
+    protocol = "sctp"
+  }
+  disabled    = false
+  name        = "master-to-master-minimal-gce-ilb-cilium-etcd-example-com"
+  network     = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_tags = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+  target_tags = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "master-to-node-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    protocol = "tcp"
+  }
+  allow {
+    protocol = "udp"
+  }
+  allow {
+    protocol = "icmp"
+  }
+  allow {
+    protocol = "esp"
+  }
+  allow {
+    protocol = "ah"
+  }
+  allow {
+    protocol = "sctp"
+  }
+  disabled    = false
+  name        = "master-to-node-minimal-gce-ilb-cilium-etcd-example-com"
+  network     = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_tags = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+  target_tags = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "node-to-master-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["443"]
+    protocol = "tcp"
+  }
+  allow {
+    ports    = ["10250"]
+    protocol = "tcp"
+  }
+  allow {
+    ports    = ["3988"]
+    protocol = "tcp"
+  }
+  allow {
+    ports    = ["8472"]
+    protocol = "udp"
+  }
+  allow {
+    ports    = ["4003"]
+    protocol = "tcp"
+  }
+  disabled    = false
+  name        = "node-to-master-minimal-gce-ilb-cilium-etcd-example-com"
+  network     = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_tags = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+  target_tags = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "node-to-node-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    protocol = "tcp"
+  }
+  allow {
+    protocol = "udp"
+  }
+  allow {
+    protocol = "icmp"
+  }
+  allow {
+    protocol = "esp"
+  }
+  allow {
+    protocol = "ah"
+  }
+  allow {
+    protocol = "sctp"
+  }
+  disabled    = false
+  name        = "node-to-node-minimal-gce-ilb-cilium-etcd-example-com"
+  network     = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_tags = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+  target_tags = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "nodeport-external-to-node-ipv6-minimal-gce-ilb-cilium-et-eqdptu" {
+  allow {
+    ports    = ["30000-32767"]
+    protocol = "tcp"
+  }
+  allow {
+    ports    = ["30000-32767"]
+    protocol = "udp"
+  }
+  disabled      = true
+  name          = "nodeport-external-to-node-ipv6-minimal-gce-ilb-cilium-et-eqdptu"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["::/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "nodeport-external-to-node-minimal-gce-ilb-cilium-etcd-ex-eqdptu" {
+  allow {
+    ports    = ["30000-32767"]
+    protocol = "tcp"
+  }
+  allow {
+    ports    = ["30000-32767"]
+    protocol = "udp"
+  }
+  disabled      = true
+  name          = "nodeport-external-to-node-minimal-gce-ilb-cilium-etcd-ex-eqdptu"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "ssh-external-to-master-ipv6-minimal-gce-ilb-cilium-etcd--eqdptu" {
+  allow {
+    ports    = ["22"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "ssh-external-to-master-ipv6-minimal-gce-ilb-cilium-etcd--eqdptu"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["::/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "ssh-external-to-master-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["22"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "ssh-external-to-master-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "ssh-external-to-node-ipv6-minimal-gce-ilb-cilium-etcd-ex-eqdptu" {
+  allow {
+    ports    = ["22"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "ssh-external-to-node-ipv6-minimal-gce-ilb-cilium-etcd-ex-eqdptu"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["::/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-ilb-cilium-etcd-example-com" {
+  allow {
+    ports    = ["22"]
+    protocol = "tcp"
+  }
+  disabled      = false
+  name          = "ssh-external-to-node-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-ilb-cilium-etcd-example-com" {
+  backend_service = google_compute_region_backend_service.api-minimal-gce-ilb-cilium-etcd-example-com.id
+  ip_address      = google_compute_address.api-us-test1-minimal-gce-ilb-cilium-etcd-example-com.address
+  ip_protocol     = "TCP"
+  labels = {
+    "k8s-io-cluster-name" = "minimal-gce-ilb-cilium-etcd-example-com"
+    "name"                = "api-us-test1"
+  }
+  load_balancing_scheme = "INTERNAL"
+  name                  = "api-us-test1-minimal-gce-ilb-cilium-etcd-example-com"
+  network               = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  ports                 = ["443"]
+  subnetwork            = google_compute_subnetwork.us-test1-minimal-gce-ilb-cilium-etcd-example-com.name
+}
+
+resource "google_compute_forwarding_rule" "cilium-etcd-us-test1-minimal-gce-ilb-cilium-etcd-example-com" {
+  backend_service = google_compute_region_backend_service.api-minimal-gce-ilb-cilium-etcd-example-com.id
+  ip_address      = google_compute_address.api-us-test1-minimal-gce-ilb-cilium-etcd-example-com.address
+  ip_protocol     = "TCP"
+  labels = {
+    "k8s-io-cluster-name" = "minimal-gce-ilb-cilium-etcd-example-com"
+    "name"                = "cilium-etcd-us-test1"
+  }
+  load_balancing_scheme = "INTERNAL"
+  name                  = "cilium-etcd-us-test1-minimal-gce-ilb-cilium-etcd-example-com"
+  network               = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  ports                 = ["4003"]
+  subnetwork            = google_compute_subnetwork.us-test1-minimal-gce-ilb-cilium-etcd-example-com.name
+}
+
+resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com" {
+  base_instance_name = "master-us-test1-a"
+  lifecycle {
+    ignore_changes = [target_size]
+  }
+  list_managed_instances_results = "PAGINATED"
+  name                           = "a-master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com"
+  target_size                    = 1
+  update_policy {
+    minimal_action = "REPLACE"
+    type           = "OPPORTUNISTIC"
+  }
+  version {
+    instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com.self_link
+  }
+  zone = "us-test1-a"
+}
+
+resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-ilb-cilium-etcd-example-com" {
+  base_instance_name = "nodes"
+  lifecycle {
+    ignore_changes = [target_size]
+  }
+  list_managed_instances_results = "PAGINATED"
+  name                           = "a-nodes-minimal-gce-ilb-cilium-etcd-example-com"
+  target_size                    = 1
+  update_policy {
+    minimal_action = "REPLACE"
+    type           = "OPPORTUNISTIC"
+  }
+  version {
+    instance_template = google_compute_instance_template.nodes-minimal-gce-ilb-cilium-etcd-example-com.self_link
+  }
+  zone = "us-test1-a"
+}
+
+resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-ilb-cilium-etcd-example-com" {
+  base_instance_name = "nodes"
+  lifecycle {
+    ignore_changes = [target_size]
+  }
+  list_managed_instances_results = "PAGINATED"
+  name                           = "b-nodes-minimal-gce-ilb-cilium-etcd-example-com"
+  target_size                    = 1
+  update_policy {
+    minimal_action = "REPLACE"
+    type           = "OPPORTUNISTIC"
+  }
+  version {
+    instance_template = google_compute_instance_template.nodes-minimal-gce-ilb-cilium-etcd-example-com.self_link
+  }
+  zone = "us-test1-b"
+}
+
+resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete            = true
+    boot                   = true
+    device_name            = "persistent-disks-0"
+    disk_name              = ""
+    disk_size_gb           = 64
+    disk_type              = "pd-standard"
+    interface              = ""
+    mode                   = "READ_WRITE"
+    provisioned_iops       = 0
+    provisioned_throughput = 0
+    source                 = ""
+    source_image           = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221018"
+    type                   = "PERSISTENT"
+  }
+  labels = {
+    "k8s-io-cluster-name"       = "minimal-gce-ilb-cilium-etcd-example-com"
+    "k8s-io-instance-group"     = "master-us-test1-a"
+    "k8s-io-role-control-plane" = "control-plane"
+    "k8s-io-role-master"        = "master"
+  }
+  lifecycle {
+    create_before_destroy = true
+  }
+  machine_type = "e2-medium"
+  metadata = {
+    "cluster-name"                    = "minimal-gce-ilb-cilium-etcd.example.com"
+    "kops-k8s-io-instance-group-name" = "master-us-test1-a"
+    "ssh-keys"                        = "admin: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ=="
+    "user-data"                       = file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data")
+  }
+  name_prefix = "master-us-test1-a-minimal-ukspr5-"
+  network_interface {
+    network    = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+    stack_type = "IPV4_ONLY"
+    subnetwork = google_compute_subnetwork.us-test1-minimal-gce-ilb-cilium-etcd-example-com.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+    provisioning_model  = "STANDARD"
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
+  }
+  tags = ["minimal-gce-ilb-cilium-etcd-ex-eqdptu-k8s-io-role-control-plane", "minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_instance_template" "nodes-minimal-gce-ilb-cilium-etcd-example-com" {
+  can_ip_forward = true
+  disk {
+    auto_delete            = true
+    boot                   = true
+    device_name            = "persistent-disks-0"
+    disk_name              = ""
+    disk_size_gb           = 128
+    disk_type              = "pd-standard"
+    interface              = ""
+    mode                   = "READ_WRITE"
+    provisioned_iops       = 0
+    provisioned_throughput = 0
+    source                 = ""
+    source_image           = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221018"
+    type                   = "PERSISTENT"
+  }
+  labels = {
+    "k8s-io-cluster-name"   = "minimal-gce-ilb-cilium-etcd-example-com"
+    "k8s-io-instance-group" = "nodes"
+    "k8s-io-role-node"      = "node"
+  }
+  lifecycle {
+    create_before_destroy = true
+  }
+  machine_type = "e2-medium"
+  metadata = {
+    "cluster-name"                    = "minimal-gce-ilb-cilium-etcd.example.com"
+    "kops-k8s-io-instance-group-name" = "nodes"
+    "kube-env"                        = "AUTOSCALER_ENV_VARS: os_distribution=ubuntu;arch=amd64;os=linux"
+    "ssh-keys"                        = "admin: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ=="
+    "user-data"                       = file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-ilb-cilium-etcd-example-com_metadata_user-data")
+  }
+  name_prefix = "nodes-minimal-gce-ilb-cil-vfh7jq-"
+  network_interface {
+    network    = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+    stack_type = "IPV4_ONLY"
+    subnetwork = google_compute_subnetwork.us-test1-minimal-gce-ilb-cilium-etcd-example-com.name
+  }
+  scheduling {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+    provisioning_model  = "STANDARD"
+  }
+  service_account {
+    email  = "default"
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_only"]
+  }
+  tags = ["minimal-gce-ilb-cilium-etcd-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_network" "minimal-gce-ilb-cilium-etcd-example-com" {
+  auto_create_subnetworks = false
+  name                    = "minimal-gce-ilb-cilium-etcd-example-com"
+}
+
+resource "google_compute_region_backend_service" "api-minimal-gce-ilb-cilium-etcd-example-com" {
+  backend {
+    balancing_mode = "CONNECTION"
+    group          = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-ilb-cilium-etcd-example-com.instance_group
+  }
+  health_checks         = [google_compute_region_health_check.api-minimal-gce-ilb-cilium-etcd-example-com.id]
+  load_balancing_scheme = "INTERNAL"
+  name                  = "api-minimal-gce-ilb-cilium-etcd-example-com"
+  protocol              = "TCP"
+}
+
+resource "google_compute_region_health_check" "api-minimal-gce-ilb-cilium-etcd-example-com" {
+  name = "api-minimal-gce-ilb-cilium-etcd-example-com"
+  tcp_health_check {
+    port = 443
+  }
+}
+
+resource "google_compute_router" "nat-minimal-gce-ilb-cilium-etcd-example-com" {
+  name    = "nat-minimal-gce-ilb-cilium-etcd-example-com"
+  network = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+}
+
+resource "google_compute_router_nat" "nat-minimal-gce-ilb-cilium-etcd-example-com" {
+  name                               = "nat-minimal-gce-ilb-cilium-etcd-example-com"
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  region                             = "us-test1"
+  router                             = google_compute_router.nat-minimal-gce-ilb-cilium-etcd-example-com.name
+  source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
+  subnetwork {
+    name                    = google_compute_subnetwork.us-test1-minimal-gce-ilb-cilium-etcd-example-com.name
+    source_ip_ranges_to_nat = ["ALL_IP_RANGES"]
+  }
+}
+
+resource "google_compute_subnetwork" "us-test1-minimal-gce-ilb-cilium-etcd-example-com" {
+  ip_cidr_range = "10.0.16.0/20"
+  name          = "us-test1-minimal-gce-ilb-cilium-etcd-example-com"
+  network       = google_compute_network.minimal-gce-ilb-cilium-etcd-example-com.name
+  region        = "us-test1"
+  stack_type    = "IPV4_ONLY"
+}
+
+terraform {
+  required_version = ">= 0.15.0"
+  required_providers {
+    aws = {
+      "configuration_aliases" = [aws.files]
+      "source"                = "hashicorp/aws"
+      "version"               = ">= 5.0.0"
+    }
+    google = {
+      "source"  = "hashicorp/google"
+      "version" = ">= 5.11.0"
+    }
+  }
+}