Repository: kubernetes/kops (go)

Issue:
containerd/runc assets are still downloaded if skipInstall is true
/kind bug

**1. What `kops` version are you running? The command `kops version`, will display
 this information.**
1.26.4

**2. What Kubernetes version are you running? `kubectl version` will print the
 version if a cluster is running or provide the Kubernetes version specified as
 a `kops` flag.**
1.26.6

**3. What cloud provider are you using?**
AWS/Spotinst

**4. What commands did you run?  What is the simplest way to reproduce this issue?**
Have a cluster with `.spec.containerd.skipInstall = true`.

**5. What happened after the commands executed?**
The node logs indicate that containerd and runc assets are downloaded and extracted. They are never used.

**6. What did you expect to happen?**
For this to not happen

Pull request:
Don't download container runtime assets when skipping the installation
For now, this only works when `SkipInstall=true` is set at cluster level.

Closes #15558

Patch:
diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go
index f756d3cdfb..8216212e7d 100644
--- a/upup/pkg/fi/cloudup/apply_cluster.go
+++ b/upup/pkg/fi/cloudup/apply_cluster.go
@@ -72,7 +72,6 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
 	"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
 	"k8s.io/kops/util/pkg/architectures"
-	"k8s.io/kops/util/pkg/hashing"
 	"k8s.io/kops/util/pkg/mirrors"
 	"k8s.io/kops/util/pkg/vfs"
 )
@@ -1072,37 +1071,49 @@ func (c *ApplyClusterCmd) addFileAssets(assetBuilder *assets.AssetBuilder) error
 			c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
 		}
 
-		cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder, arch)
-		if err != nil {
-			return err
+		{
+			cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder, arch)
+			if err != nil {
+				return err
+			}
+			c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(cniAsset, cniAssetHash))
 		}
-		c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(cniAsset, cniAssetHash))
 
-		var containerRuntimeAssetUrl *url.URL
-		var containerRuntimeAssetHash *hashing.Hash
 		switch c.Cluster.Spec.ContainerRuntime {
 		case "docker":
-			containerRuntimeAssetUrl, containerRuntimeAssetHash, err = findDockerAsset(c.Cluster, assetBuilder, arch)
+			if c.Cluster.Spec.Docker != nil && c.Cluster.Spec.Docker.SkipInstall {
+				break
+			}
+
+			dockerAssetUrl, dockerAssetHash, err := findDockerAsset(c.Cluster, assetBuilder, arch)
+			if err != nil {
+				return err
+			}
+			if dockerAssetUrl != nil && dockerAssetHash != nil {
+				c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(dockerAssetUrl, dockerAssetHash))
+			}
 		case "containerd":
-			containerRuntimeAssetUrl, containerRuntimeAssetHash, err = findContainerdAsset(c.Cluster, assetBuilder, arch)
-		default:
-			err = fmt.Errorf("unknown container runtime: %q", c.Cluster.Spec.ContainerRuntime)
-		}
-		if err != nil {
-			return err
-		}
-		c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(containerRuntimeAssetUrl, containerRuntimeAssetHash))
+			if c.Cluster.Spec.Containerd != nil && c.Cluster.Spec.Containerd.SkipInstall {
+				break
+			}
 
-		if c.Cluster.Spec.ContainerRuntime == "containerd" {
-			var runcAssetUrl *url.URL
-			var runcAssetHash *hashing.Hash
-			runcAssetUrl, runcAssetHash, err = findRuncAsset(c.Cluster, assetBuilder, arch)
+			containerdAssetUrl, containerdAssetHash, err := findContainerdAsset(c.Cluster, assetBuilder, arch)
+			if err != nil {
+				return err
+			}
+			if containerdAssetUrl != nil && containerdAssetHash != nil {
+				c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(containerdAssetUrl, containerdAssetHash))
+			}
+
+			runcAssetUrl, runcAssetHash, err := findRuncAsset(c.Cluster, assetBuilder, arch)
 			if err != nil {
 				return err
 			}
 			if runcAssetUrl != nil && runcAssetHash != nil {
 				c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(runcAssetUrl, runcAssetHash))
 			}
+		default:
+			return fmt.Errorf("unknown container runtime: %q", c.Cluster.Spec.ContainerRuntime)
 		}
 
 		asset, err := NodeUpAsset(assetBuilder, arch)


Test patch:
diff --git a/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_master-us-test-1a.masters.containerd.example.com_user_data b/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_master-us-test-1a.masters.containerd.example.com_user_data
index 851b608719..ee552605d0 100644
--- a/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_master-us-test-1a.masters.containerd.example.com_user_data
+++ b/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_master-us-test-1a.masters.containerd.example.com_user_data
@@ -252,7 +252,7 @@ ClusterName: containerd.example.com
 ConfigBase: memfs://clusters.example.com/containerd.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: iqQyu3HS9EWX0wah8wCb5yoTDYOcqdYWZCud1dPBQP0=
+NodeupConfigHash: M8eJSK74gm1DVgZdbXfSfieaCHiMS1swoLeVi9ob6mo=
 
 __EOF_KUBE_ENV
 
diff --git a/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_nodes.containerd.example.com_user_data b/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_nodes.containerd.example.com_user_data
index 36e563f083..d782054897 100644
--- a/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_nodes.containerd.example.com_user_data
+++ b/tests/integration/update_cluster/containerd-custom/data/aws_launch_template_nodes.containerd.example.com_user_data
@@ -177,7 +177,7 @@ ConfigServer:
   - https://kops-controller.internal.containerd.example.com:3988/
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: Os7bU/JHurv4KhpI5nVsiMKCIJsiWdr5kqZKhfvvmdY=
+NodeupConfigHash: 5x427DhSZT7lT8vph43JBYFpa3xob6ddiGOrZtPUMxc=
 
 __EOF_KUBE_ENV
 
diff --git a/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_cluster-completed.spec_content
index eb3d784c20..bb2041cbeb 100644
--- a/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_cluster-completed.spec_content
@@ -39,6 +39,7 @@ spec:
       - https://registry-1.docker.io
     runc:
       version: 1.1.5
+    skipInstall: true
     version: 1.6.20
   dnsZone: Z1AFAKE1ZON3YO
   docker:
diff --git a/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
index b92ff1f65a..83c5b138ae 100644
--- a/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
+++ b/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
@@ -58,16 +58,12 @@ Assets:
   - b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5@https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubelet
   - b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae@https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl
   - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
-  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
   - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
   arm64:
   - fb033c1d079cac8babb04a25abecbc6cc1a2afb53f56ef1d73f8dc3b15b3c09e@https://dl.k8s.io/release/v1.26.0/bin/linux/arm64/kubelet
   - 79b14e4ddada9e81d2989f36a89faa9e56f8abe6e0246e7bdc305c93c3731ea4@https://dl.k8s.io/release/v1.26.0/bin/linux/arm64/kubectl
   - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
-  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
 CAs:
@@ -293,6 +289,7 @@ containerdConfig:
     - https://registry-1.docker.io
   runc:
     version: 1.1.5
+  skipInstall: true
   version: 1.6.20
 docker:
   skipInstall: true
diff --git a/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-nodes_content
index 40153fb6a7..0361d34f3d 100644
--- a/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-nodes_content
+++ b/tests/integration/update_cluster/containerd-custom/data/aws_s3_object_nodeupconfig-nodes_content
@@ -3,14 +3,10 @@ Assets:
   - b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5@https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubelet
   - b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae@https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl
   - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
-  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   arm64:
   - fb033c1d079cac8babb04a25abecbc6cc1a2afb53f56ef1d73f8dc3b15b3c09e@https://dl.k8s.io/release/v1.26.0/bin/linux/arm64/kubelet
   - 79b14e4ddada9e81d2989f36a89faa9e56f8abe6e0246e7bdc305c93c3731ea4@https://dl.k8s.io/release/v1.26.0/bin/linux/arm64/kubectl
   - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
-  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
 CAs: {}
 ClusterName: containerd.example.com
 ContainerRuntime: containerd
@@ -65,6 +61,7 @@ containerdConfig:
     - https://registry-1.docker.io
   runc:
     version: 1.1.5
+  skipInstall: true
   version: 1.6.20
 docker:
   skipInstall: true
diff --git a/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml b/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml
index bb26b84ccd..d8752231a9 100644
--- a/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml
+++ b/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml
@@ -11,6 +11,7 @@ spec:
   configBase: memfs://clusters.example.com/containerd.example.com
   containerRuntime: containerd
   containerd:
+    skipInstall: true
     registryMirrors:
       docker.io:
       - https://registry-1.docker.io