Repository: kubernetes/kops (go)

Issue:
kops 1.25, updating clusters does not work if ig has taints
/kind bug

**1. What `kops` version are you running? The command `kops version`, will display
 this information.**

commit https://github.com/kubernetes/kops/commit/ce73a5b830d625122e12fa8bd40bfeeaae348581 from release-1.25

**2. What Kubernetes version are you running? `kubectl version` will print the
 version if a cluster is running or provide the Kubernetes version specified as
 a `kops` flag.**

1.25.1

**3. What cloud provider are you using?**

openstack

**4. What commands did you run?  What is the simplest way to reproduce this issue?**

```
kops edit cluster --set=spec.kubernetesVersion=1.25.1
kops update cluster --yes
kops rolling-update cluster --yes
```

**5. What happened after the commands executed?**

the ig spec before command executed

```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2021-02-10T08:29:37Z"
  generation: 9
  labels:
    kops.k8s.io/cluster: sre-ci.k8s.local
  name: e2etest
spec:
  cloudLabels:
    kopsGroupName: e2etest
  image: ubuntu-2004-200622-devops
  machineType: m1.large
  manager: CloudGroup
  maxSize: 2
  minSize: 2
  nodeLabels:
    kops.k8s.io/instancegroup: e2etest
  role: Node
  subnets:
  - espsti
  - esptnl
  - helpa
  taints:
  - e2etest:NoSchedule
```

after

```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2021-02-10T08:29:37Z"
  generation: 10
  labels:
    kops.k8s.io/cluster: sre-ci.k8s.local
  name: e2etest
spec:
  cloudLabels:
    kopsGroupName: e2etest
  image: ubuntu-2004-010922-devops
  kubelet:
    anonymousAuth: false
    authenticationTokenWebhook: true
    authorizationMode: Webhook
    evictionHard: memory.available<5%,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
    evictionMinimumReclaim: nodefs.available=1Gi,imagefs.available=2Gi
    nodeLabels:
      kops.k8s.io/instancegroup: e2etest
      node-role.kubernetes.io/node: ""
    systemReserved:
      memory: 512Mi
    taints:
    - e2etest:NoSchedule
  machineType: m1.large
  manager: CloudGroup
  maxSize: 2
  minSize: 2
  nodeLabels:
    kops.k8s.io/instancegroup: e2etest
  role: Node
  subnets:
  - espsti
  - esptnl
  - helpa
  taints:
  - e2etest:NoSchedule
```

However, because ig now contains taints twice it leads to situation that this does not work. When e2etest instance starts again the result is that its not coming online.

The error from machine log:

```
Sep 17 04:37:15 e2etest-8jyglf kubelet[101939]: E0917 04:37:15.627003  101939 run.go:74] "command failed" err="failed to parse kubelet flag: invalid argument \"e2etest:NoSchedule,e2etest:NoSchedule\" for \"--register-with-taints\" flag: duplicated taints with the same key and effect: {e2etest  NoSchedule <nil>}"
```




Pull request description:
Prevent kops edit cluster from writing the populated IG spec to state store
Fixes #14285

Patch:
diff --git a/cmd/kops/edit_instancegroup.go b/cmd/kops/edit_instancegroup.go
index 47cf77d2b4..7ebb99a004 100644
--- a/cmd/kops/edit_instancegroup.go
+++ b/cmd/kops/edit_instancegroup.go
@@ -157,7 +157,7 @@ func RunEditInstanceGroup(ctx context.Context, f *util.Factory, out io.Writer, o
 			return err
 		}
 
-		failure, err := updateInstanceGroup(ctx, clientset, channel, cluster, oldGroup, newGroup)
+		failure, err := updateInstanceGroup(ctx, clientset, channel, cluster, newGroup)
 		if err != nil {
 			return err
 		}
@@ -263,7 +263,7 @@ func RunEditInstanceGroup(ctx context.Context, f *util.Factory, out io.Writer, o
 			continue
 		}
 
-		failure, err := updateInstanceGroup(ctx, clientset, channel, cluster, oldGroup, newGroup)
+		failure, err := updateInstanceGroup(ctx, clientset, channel, cluster, newGroup)
 		if err != nil {
 			return preservedFile(err, file, out)
 		}
@@ -280,7 +280,7 @@ func RunEditInstanceGroup(ctx context.Context, f *util.Factory, out io.Writer, o
 	}
 }
 
-func updateInstanceGroup(ctx context.Context, clientset simple.Clientset, channel *api.Channel, cluster *api.Cluster, oldGroup, newGroup *api.InstanceGroup) (string, error) {
+func updateInstanceGroup(ctx context.Context, clientset simple.Clientset, channel *api.Channel, cluster *api.Cluster, newGroup *api.InstanceGroup) (string, error) {
 	cloud, err := cloudup.BuildCloud(cluster)
 	if err != nil {
 		return "", err
@@ -310,6 +310,6 @@ func updateInstanceGroup(ctx context.Context, clientset simple.Clientset, channe
 	}
 
 	// Note we perform as much validation as we can, before writing a bad config
-	_, err = clientset.InstanceGroupsFor(cluster).Update(ctx, fullGroup, metav1.UpdateOptions{})
+	_, err = clientset.InstanceGroupsFor(cluster).Update(ctx, newGroup, metav1.UpdateOptions{})
 	return "", err
 }


Test patch:
diff --git a/cmd/kops/edit_instancegroup_test.go b/cmd/kops/edit_instancegroup_test.go
new file mode 100644
index 0000000000..4d3d135a30
--- /dev/null
+++ b/cmd/kops/edit_instancegroup_test.go
@@ -0,0 +1,91 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/kopscodecs"
+	"k8s.io/kops/pkg/testutils"
+	"k8s.io/kops/pkg/testutils/golden"
+)
+
+func TestEditInstanceGroup(t *testing.T) {
+	t.Setenv("SKIP_REGION_CHECK", "1")
+	var stdout bytes.Buffer
+
+	clusterName := "test.k8s.io"
+
+	cluster := testutils.BuildMinimalCluster(clusterName)
+	nodes := testutils.BuildMinimalNodeInstanceGroup("nodes", "subnet-us-test-1a")
+	nodes.Spec.Image = "ami-xyz"
+	nodes.Spec.Taints = []string{"e2etest:NoSchedule"}
+
+	testutils.NewIntegrationTestHarness(t).SetupMockAWS()
+
+	ctx := context.Background()
+
+	factoryOptions := &util.FactoryOptions{}
+	factoryOptions.RegistryPath = "memfs://tests"
+
+	factory := util.NewFactory(factoryOptions)
+	clientSet, err := factory.KopsClient()
+	if err != nil {
+		t.Fatalf("could not create clientset: %v", err)
+	}
+
+	cluster, err = clientSet.CreateCluster(ctx, cluster)
+	if err != nil {
+		t.Fatalf("could not create cluster: %v", err)
+	}
+	_, err = clientSet.InstanceGroupsFor(cluster).Create(ctx, &nodes, v1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("could not create instance group: %v", err)
+	}
+
+	{
+		editOptions := &EditInstanceGroupOptions{
+			ClusterName: clusterName,
+			GroupName:   "nodes",
+			Sets:        []string{"spec.maxSize=10"},
+		}
+		err := RunEditInstanceGroup(ctx, factory, &stdout, editOptions)
+		if err != nil {
+			t.Fatalf("could not edit instance group: %v", err)
+		}
+	}
+
+	storedIG, err := clientSet.InstanceGroupsFor(cluster).Get(ctx, "nodes", v1.GetOptions{})
+	if err != nil {
+		t.Fatalf("could not get instance group: %v", err)
+	}
+	storedIG.CreationTimestamp = MagicTimestamp
+	actualYAMLBytes, err := kopscodecs.ToVersionedYamlWithVersion(storedIG, schema.GroupVersion{Group: "kops.k8s.io", Version: "v1alpha2"})
+	if err != nil {
+		t.Fatalf("unexpected error serializing Addon: %v", err)
+	}
+
+	actualYAML := strings.TrimSpace(string(actualYAMLBytes))
+
+	golden.AssertMatchesFile(t, actualYAML, "test/edit_instance_group.yaml")
+}
diff --git a/cmd/kops/test/edit_instance_group.yaml b/cmd/kops/test/edit_instance_group.yaml
new file mode 100644
index 0000000000..7179495cff
--- /dev/null
+++ b/cmd/kops/test/edit_instance_group.yaml
@@ -0,0 +1,16 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  generation: 1
+  labels:
+    kops.k8s.io/cluster: test.k8s.io
+  name: nodes
+spec:
+  image: ami-xyz
+  maxSize: 10
+  role: Node
+  subnets:
+  - subnet-us-test-1a
+  taints:
+  - e2etest:NoSchedule
diff --git a/pkg/model/awsmodel/autoscalinggroup_test.go b/pkg/model/awsmodel/autoscalinggroup_test.go
index 33085da0d7..ff7ccd3b4e 100644
--- a/pkg/model/awsmodel/autoscalinggroup_test.go
+++ b/pkg/model/awsmodel/autoscalinggroup_test.go
@@ -50,7 +50,7 @@ func buildNodeInstanceGroup(subnets ...string) *kops.InstanceGroup {
 // Tests that RootVolumeOptimization flag gets added to the awstasks
 func TestRootVolumeOptimizationFlag(t *testing.T) {
 	cluster := buildMinimalCluster()
-	ig := buildNodeInstanceGroup("subnet-us-mock-1a")
+	ig := buildNodeInstanceGroup("subnet-us-test-1a")
 	ig.Spec.RootVolumeOptimization = fi.Bool(true)
 
 	k := [][]byte{}
diff --git a/pkg/testutils/cluster.go b/pkg/testutils/cluster.go
index e780468f69..c11f74353b 100644
--- a/pkg/testutils/cluster.go
+++ b/pkg/testutils/cluster.go
@@ -30,7 +30,7 @@ func BuildMinimalCluster(clusterName string) *kops.Cluster {
 	c.ObjectMeta.Name = clusterName
 	c.Spec.KubernetesVersion = "1.14.6"
 	c.Spec.Subnets = []kops.ClusterSubnetSpec{
-		{Name: "subnet-us-mock-1a", Zone: "us-mock-1a", CIDR: "172.20.1.0/24", Type: kops.SubnetTypePrivate},
+		{Name: "subnet-us-test-1a", Zone: "us-test-1a", CIDR: "172.20.1.0/24", Type: kops.SubnetTypePrivate},
 	}
 
 	c.Spec.ContainerRuntime = "containerd"
@@ -54,9 +54,9 @@ func BuildMinimalCluster(clusterName string) *kops.Cluster {
 
 	c.Spec.NetworkCIDR = "172.20.0.0/16"
 	c.Spec.Subnets = []kops.ClusterSubnetSpec{
-		{Name: "subnet-us-mock-1a", Zone: "us-mock-1a", CIDR: "172.20.1.0/24", Type: kops.SubnetTypePublic},
-		{Name: "subnet-us-mock-1b", Zone: "us-mock-1b", CIDR: "172.20.2.0/24", Type: kops.SubnetTypePublic},
-		{Name: "subnet-us-mock-1c", Zone: "us-mock-1c", CIDR: "172.20.3.0/24", Type: kops.SubnetTypePublic},
+		{Name: "subnet-us-test-1a", Zone: "us-test-1a", CIDR: "172.20.1.0/24", Type: kops.SubnetTypePublic},
+		{Name: "subnet-us-test-1b", Zone: "us-test-1b", CIDR: "172.20.2.0/24", Type: kops.SubnetTypePublic},
+		{Name: "subnet-us-test-1c", Zone: "us-test-1c", CIDR: "172.20.3.0/24", Type: kops.SubnetTypePublic},
 	}
 
 	c.Spec.NonMasqueradeCIDR = "100.64.0.0/10"
diff --git a/upup/pkg/fi/cloudup/awsup/aws_utils_test.go b/upup/pkg/fi/cloudup/awsup/aws_utils_test.go
index 13e4529e1a..0348423e2f 100644
--- a/upup/pkg/fi/cloudup/awsup/aws_utils_test.go
+++ b/upup/pkg/fi/cloudup/awsup/aws_utils_test.go
@@ -28,13 +28,13 @@ import (
 func TestValidateRegion(t *testing.T) {
 	allRegions = []*ec2.Region{
 		{
-			RegionName: aws.String("us-mock-1"),
+			RegionName: aws.String("us-test-1"),
 		},
 		{
-			RegionName: aws.String("us-mock-2"),
+			RegionName: aws.String("us-test-2"),
 		},
 	}
-	for _, region := range []string{"us-mock-1", "us-mock-2"} {
+	for _, region := range []string{"us-test-1", "us-test-2"} {
 		err := ValidateRegion(region)
 		if err != nil {
 			t.Fatalf("unexpected error validating region %q: %v", region, err)
diff --git a/upup/pkg/fi/cloudup/deepvalidate_test.go b/upup/pkg/fi/cloudup/deepvalidate_test.go
index 866474c435..0d7148e8e7 100644
--- a/upup/pkg/fi/cloudup/deepvalidate_test.go
+++ b/upup/pkg/fi/cloudup/deepvalidate_test.go
@@ -42,14 +42,14 @@ func TestDeepValidate_OK(t *testing.T) {
 func TestDeepValidate_NoNodeZones(t *testing.T) {
 	c := buildDefaultCluster(t)
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1a"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1a"))
 	expectErrorFromDeepValidate(t, c, groups, "must configure at least one Node InstanceGroup")
 }
 
 func TestDeepValidate_NoMasterZones(t *testing.T) {
 	c := buildDefaultCluster(t)
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1a"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1a"))
 	expectErrorFromDeepValidate(t, c, groups, "must configure at least one Master InstanceGroup")
 }
 
@@ -57,11 +57,11 @@ func TestDeepValidate_BadZone(t *testing.T) {
 	t.Skipf("Zone validation not checked by DeepValidate")
 	c := buildDefaultCluster(t)
 	c.Spec.Subnets = []kopsapi.ClusterSubnetSpec{
-		{Name: "subnet-badzone", Zone: "us-mock-1z", CIDR: "172.20.1.0/24"},
+		{Name: "subnet-badzone", Zone: "us-test-1z", CIDR: "172.20.1.0/24"},
 	}
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1z"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1z"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1z"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1z"))
 	expectErrorFromDeepValidate(t, c, groups, "Zone is not a recognized AZ")
 }
 
@@ -69,12 +69,12 @@ func TestDeepValidate_MixedRegion(t *testing.T) {
 	t.Skipf("Region validation not checked by DeepValidate")
 	c := buildDefaultCluster(t)
 	c.Spec.Subnets = []kopsapi.ClusterSubnetSpec{
-		{Name: "mock1a", Zone: "us-mock-1a", CIDR: "172.20.1.0/24"},
+		{Name: "test1a", Zone: "us-test-1a", CIDR: "172.20.1.0/24"},
 		{Name: "west1b", Zone: "us-west-1b", CIDR: "172.20.2.0/24"},
 	}
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1a"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1a", "subnet-us-west-1b"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1a"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1a", "subnet-us-west-1b"))
 
 	expectErrorFromDeepValidate(t, c, groups, "Clusters cannot span multiple regions")
 }
@@ -83,11 +83,11 @@ func TestDeepValidate_RegionAsZone(t *testing.T) {
 	t.Skipf("Region validation not checked by DeepValidate")
 	c := buildDefaultCluster(t)
 	c.Spec.Subnets = []kopsapi.ClusterSubnetSpec{
-		{Name: "mock1", Zone: "us-mock-1", CIDR: "172.20.1.0/24"},
+		{Name: "test1", Zone: "us-test-1", CIDR: "172.20.1.0/24"},
 	}
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1"))
 
 	expectErrorFromDeepValidate(t, c, groups, "Region is not a recognized EC2 region: \"us-east-\" (check you have specified valid zones?)")
 }
@@ -95,17 +95,17 @@ func TestDeepValidate_RegionAsZone(t *testing.T) {
 func TestDeepValidate_NotIncludedZone(t *testing.T) {
 	c := buildDefaultCluster(t)
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1d"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1d"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1d"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1d"))
 
-	expectErrorFromDeepValidate(t, c, groups, "spec.subnets[0]: Not found: \"subnet-us-mock-1d\"")
+	expectErrorFromDeepValidate(t, c, groups, "spec.subnets[0]: Not found: \"subnet-us-test-1d\"")
 }
 
 func TestDeepValidate_DuplicateZones(t *testing.T) {
 	c := buildDefaultCluster(t)
 	c.Spec.Subnets = []kopsapi.ClusterSubnetSpec{
-		{Name: "dup1", Zone: "us-mock-1a", CIDR: "172.20.1.0/24"},
-		{Name: "dup1", Zone: "us-mock-1a", CIDR: "172.20.2.0/24"},
+		{Name: "dup1", Zone: "us-test-1a", CIDR: "172.20.1.0/24"},
+		{Name: "dup1", Zone: "us-test-1a", CIDR: "172.20.2.0/24"},
 	}
 	var groups []*kopsapi.InstanceGroup
 	groups = append(groups, buildMinimalMasterInstanceGroup("dup1"))
@@ -116,16 +116,16 @@ func TestDeepValidate_DuplicateZones(t *testing.T) {
 func TestDeepValidate_ExtraMasterZone(t *testing.T) {
 	c := buildDefaultCluster(t)
 	c.Spec.Subnets = []kopsapi.ClusterSubnetSpec{
-		{Name: "mock1a", Zone: "us-mock-1a", CIDR: "172.20.1.0/24", Type: kopsapi.SubnetTypePublic},
-		{Name: "mock1b", Zone: "us-mock-1b", CIDR: "172.20.2.0/24", Type: kopsapi.SubnetTypePublic},
+		{Name: "test1a", Zone: "us-test-1a", CIDR: "172.20.1.0/24", Type: kopsapi.SubnetTypePublic},
+		{Name: "test1b", Zone: "us-test-1b", CIDR: "172.20.2.0/24", Type: kopsapi.SubnetTypePublic},
 	}
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1a"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1b"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1c"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1a", "subnet-us-mock-1b"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1a"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1b"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1c"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1a", "subnet-us-test-1b"))
 
-	expectErrorFromDeepValidate(t, c, groups, "spec.subnets[0]: Not found: \"subnet-us-mock-1a\"")
+	expectErrorFromDeepValidate(t, c, groups, "spec.subnets[0]: Not found: \"subnet-us-test-1a\"")
 }
 
 func TestDeepValidate_EvenEtcdClusterSize(t *testing.T) {
@@ -134,18 +134,18 @@ func TestDeepValidate_EvenEtcdClusterSize(t *testing.T) {
 		{
 			Name: "main",
 			Members: []kopsapi.EtcdMemberSpec{
-				{Name: "us-mock-1a", InstanceGroup: fi.String("us-mock-1a")},
-				{Name: "us-mock-1b", InstanceGroup: fi.String("us-mock-1b")},
+				{Name: "us-test-1a", InstanceGroup: fi.String("us-test-1a")},
+				{Name: "us-test-1b", InstanceGroup: fi.String("us-test-1b")},
 			},
 		},
 	}
 
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1a"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1b"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1c"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1d"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1a"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1a"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1b"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1c"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1d"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1a"))
 
 	expectErrorFromDeepValidate(t, c, groups, "Should be an odd number of master-zones for quorum. Use --zones and --master-zones to declare node zones and master zones separately")
 }
@@ -156,21 +156,21 @@ func TestDeepValidate_MissingEtcdMember(t *testing.T) {
 		{
 			Name: "main",
 			Members: []kopsapi.EtcdMemberSpec{
-				{Name: "us-mock-1a", InstanceGroup: fi.String("us-mock-1a")},
-				{Name: "us-mock-1b", InstanceGroup: fi.String("us-mock-1b")},
-				{Name: "us-mock-1c", InstanceGroup: fi.String("us-mock-1c")},
+				{Name: "us-test-1a", InstanceGroup: fi.String("us-test-1a")},
+				{Name: "us-test-1b", InstanceGroup: fi.String("us-test-1b")},
+				{Name: "us-test-1c", InstanceGroup: fi.String("us-test-1c")},
 			},
 		},
 	}
 
 	var groups []*kopsapi.InstanceGroup
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1a"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1b"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1c"))
-	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-mock-1d"))
-	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-mock-1a"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1a"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1b"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1c"))
+	groups = append(groups, buildMinimalMasterInstanceGroup("subnet-us-test-1d"))
+	groups = append(groups, buildMinimalNodeInstanceGroup("subnet-us-test-1a"))
 
-	expectErrorFromDeepValidate(t, c, groups, "spec.metadata.name: Forbidden: InstanceGroup \"master-subnet-us-mock-1a\" with role Master must have a member in etcd cluster \"main\"")
+	expectErrorFromDeepValidate(t, c, groups, "spec.metadata.name: Forbidden: InstanceGroup \"master-subnet-us-test-1a\" with role Master must have a member in etcd cluster \"main\"")
 }
 
 func expectErrorFromDeepValidate(t *testing.T, c *kopsapi.Cluster, groups []*kopsapi.InstanceGroup, message string) {
diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec_test.go b/upup/pkg/fi/cloudup/populate_cluster_spec_test.go
index bb49a6b8e9..63ff170c26 100644
--- a/upup/pkg/fi/cloudup/populate_cluster_spec_test.go
+++ b/upup/pkg/fi/cloudup/populate_cluster_spec_test.go
@@ -34,7 +34,7 @@ import (
 )
 
 func buildMinimalCluster() (*awsup.MockAWSCloud, *kopsapi.Cluster) {
-	cloud := awsup.InstallMockAWSCloud(MockAWSRegion, "abcd")
+	cloud := awsup.InstallMockAWSCloud(testAWSRegion, "abcd")
 
 	c := testutils.BuildMinimalCluster("testcluster.test.com")
 
@@ -199,9 +199,9 @@ func TestPopulateCluster_Custom_CIDR(t *testing.T) {
 	cloud, c := buildMinimalCluster()
 	c.Spec.NetworkCIDR = "172.20.2.0/24"
 	c.Spec.Subnets = []kopsapi.ClusterSubnetSpec{
-		{Name: "subnet-us-mock-1a", Zone: "us-mock-1a", CIDR: "172.20.2.0/27", Type: kopsapi.SubnetTypePublic},
-		{Name: "subnet-us-mock-1b", Zone: "us-mock-1b", CIDR: "172.20.2.32/27", Type: kopsapi.SubnetTypePublic},
-		{Name: "subnet-us-mock-1c", Zone: "us-mock-1c", CIDR: "172.20.2.64/27", Type: kopsapi.SubnetTypePublic},
+		{Name: "subnet-us-test-1a", Zone: "us-test-1a", CIDR: "172.20.2.0/27", Type: kopsapi.SubnetTypePublic},
+		{Name: "subnet-us-test-1b", Zone: "us-test-1b", CIDR: "172.20.2.32/27", Type: kopsapi.SubnetTypePublic},
+		{Name: "subnet-us-test-1c", Zone: "us-test-1c", CIDR: "172.20.2.64/27", Type: kopsapi.SubnetTypePublic},
 	}
 
 	err := PerformAssignments(c, cloud)
diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go
index a713b19dd9..64270e4395 100644
--- a/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go
+++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec_test.go
@@ -70,6 +70,7 @@ func TestPopulateInstanceGroup_Role_Required(t *testing.T) {
 	expectErrorFromPopulateInstanceGroup(t, cluster, g, channel, "spec.role")
 }
 
+// TestPopulateInstanceGroup_AddTaintsCollision ensures we handle IGs with a user configured taint that kOps also adds by default
 func TestPopulateInstanceGroup_AddTaintsCollision(t *testing.T) {
 	_, cluster := buildMinimalCluster()
 	input := buildMinimalNodeInstanceGroup()
@@ -87,11 +88,41 @@ func TestPopulateInstanceGroup_AddTaintsCollision(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error from PopulateInstanceGroupSpec: %v", err)
 	}
-	if len(output.Spec.Taints) != 1 {
+	if len(output.Spec.Kubelet.Taints) != 1 {
 		t.Errorf("Expected only 1 taint, got %d", len(output.Spec.Taints))
 	}
 }
 
+// TestPopulateInstanceGroup_AddTaintsCollision2 ensures we handle taints that are configured in multiple parts of the spec and multiple resources.
+// This one also adds a second taint that we should see in the final result
+func TestPopulateInstanceGroup_AddTaintsCollision3(t *testing.T) {
+	taint := "e2etest:NoSchedule"
+	taint2 := "e2etest:NoExecute"
+	_, cluster := buildMinimalCluster()
+	cluster.Spec.Kubelet = &kopsapi.KubeletConfigSpec{
+		Taints: []string{taint, taint2},
+	}
+	input := buildMinimalNodeInstanceGroup()
+	input.Spec.Taints = []string{taint}
+	input.Spec.Kubelet = &kopsapi.KubeletConfigSpec{
+		Taints: []string{taint},
+	}
+
+	channel := &kopsapi.Channel{}
+
+	cloud, err := BuildCloud(cluster)
+	if err != nil {
+		t.Fatalf("error from BuildCloud: %v", err)
+	}
+	output, err := PopulateInstanceGroupSpec(cluster, input, cloud, channel)
+	if err != nil {
+		t.Fatalf("error from PopulateInstanceGroupSpec: %v", err)
+	}
+	if len(output.Spec.Kubelet.Taints) != 2 {
+		t.Errorf("Expected only 2 taints, got %d", len(output.Spec.Kubelet.Taints))
+	}
+}
+
 func TestPopulateInstanceGroup_AddTaints(t *testing.T) {
 	_, cluster := buildMinimalCluster()
 	input := buildMinimalNodeInstanceGroup()
diff --git a/upup/pkg/fi/cloudup/validation_test.go b/upup/pkg/fi/cloudup/validation_test.go
index 52a288f7de..a442ec78f7 100644
--- a/upup/pkg/fi/cloudup/validation_test.go
+++ b/upup/pkg/fi/cloudup/validation_test.go
@@ -27,7 +27,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-const MockAWSRegion = "us-mock-1"
+const testAWSRegion = "us-test-1"
 
 func buildDefaultCluster(t *testing.T) *api.Cluster {
 	cloud, c := buildMinimalCluster()
@@ -46,8 +46,8 @@ func buildDefaultCluster(t *testing.T) *api.Cluster {
 	//c.Cluster = &api.Cluster{}
 	//c.Cluster.Name = "testcluster.mydomain.com"
 
-	//c.InstanceGroups = append(c.InstanceGroups, buildNodeInstanceGroup("us-mock-1a"))
-	//c.InstanceGroups = append(c.InstanceGroups, buildMasterInstanceGroup("us-mock-1a"))
+	//c.InstanceGroups = append(c.InstanceGroups, buildNodeInstanceGroup("us-test-1a"))
+	//c.InstanceGroups = append(c.InstanceGroups, buildMasterInstanceGroup("us-test-1a"))
 	//c.SSHPublicKey = path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub")
 	//
 	//c.Cluster.Spec.Kubelet = &api.KubeletConfig{}