Repository: kubernetes/kops (go)

Issue:
Horizontal Pod Autoscaler Label clash
**1. What `kops` version are you running? The command `kops version`, will display
 this information.**
`Version 1.20.0-alpha.2`
I'm working on adding features under https://github.com/kubernetes/kops/issues/10706 however I think this bug is independant

**2. What Kubernetes version are you running? `kubectl version` will print the
 version if a cluster is running or provide the Kubernetes version specified as
 a `kops` flag.**
`Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-13T13:23:52Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"}
`
I do not believe the client version is applicable as the error is occurring as I am performing `kops edit cluster`

**3. What cloud provider are you using?**
AWS

**4. What commands did you run?  What is the simplest way to reproduce this issue?**

`kops create cluster` your normal options + `--cloud-labels "k8s.io/cluster-autoscaler/${KUBE_NAME}=owned"`
Edit the cluster and replace the Classic Loadbalancer with a Network Loadbalancer.

In my case
```
#!/bin/bash
K8S_ZONES="eu-west-1b,eu-west-1c"
K8S_VPC_ID="vpc-0XXXXXXX1"
K8S_CLUSTER_NAME="common-dev-euw1-cluster"
K8S_SUBNETS='subnet-01aXXXXXb0c,subnet-0418XXXXXXX1f7'
K8S_UTILITY_SUBNETS="subnet-04dXXXXX417,subnet-0a4XXXXX0b"
K8S_BASTION="ssh-rsa   REMOVED=="
K8S_VPC_CIDR="172.19.192.0/20"


./kops create cluster --state "s3://${S3_BUCKET}" \
            --cloud aws \
            --vpc ${K8S_VPC_ID} \
            --subnets ${K8S_SUBNETS} \
            --zones ${K8S_ZONES} \
            --utility-subnets ${K8S_UTILITY_SUBNETS} \
            --topology private \
            --master-count 1 \
            --master-size t3a.large \
            --node-count 2 \
            --node-size r5.xlarge \
            --networking calico \
            --ssh-public-key="${ENV}_key.pub" \
            --bastion=false \
            --dns-zone ${KUBE_NAME} \
            --encrypt-etcd-storage \
            --cloud-labels "HostType=k8s,TargetEnvironment=${ENV},k8s.io/cluster-autoscaler/${KUBE_NAME}=owned" \
            --admin-access ${CORP_IP_CIDR} \
            --admin-access ${K8S_VPC_CIDR} \
            --admin-access ${NAT_GW_1}/32 \
            --admin-access $(NAT_GW_2)/32 \
            --dry-run \
            -o yaml ${KUBE_NAME} > cluster-${KUBE_NAME}.yaml
```
I'm editing the yaml to add additional instance groups, set the auth etc

Followed by
```
kops --state "s3://${S3_BUCKET}" create -f ${KUBE_NAME}.config.yaml
```

After the cluster was up I edited the cluster to change the loadbalancer

**5. What happened after the commands executed?**
I received the error message
```
spec.cloudLabels.k8s.io/cluster-autoscaler/test.example.com: Forbidden: "k8s.io/" is a reserved label prefix and cannot be used as a custom label
```

**6. What did you expect to happen?**
The cluster edit to not complain. i'd then need to apply the changes with `kops update cluster --yes`

**7. Please provide your cluster manifest. Execute
  `kops get --name my.example.com -o yaml` to display your cluster manifest.
  You may want to remove your cluster name and other sensitive information.**

```yaml
```

**8. Please run the commands with most verbose logging by adding the `-v 10` flag.
  Paste the logs into this report, or in a gist and provide the gist link here.**

**9. Anything else do we need to know?**


Pull request:
Further improve cloudLabel validation
We were too strict on some labels that blocked common use cases such as CAS. At the same time we allowed IG-level cloudLabels that could result in broken clusters.

Fixes #10862 

Patch:
diff --git a/pkg/apis/kops/validation/cluster.go b/pkg/apis/kops/validation/cluster.go
index 4918758c5f..d24137b316 100644
--- a/pkg/apis/kops/validation/cluster.go
+++ b/pkg/apis/kops/validation/cluster.go
@@ -127,6 +127,10 @@ func validateEtcdMemberUpdate(fp *field.Path, obj kops.EtcdMemberSpec, status *k
 
 func validateClusterCloudLabels(cluster *kops.Cluster, fldPath *field.Path) (allErrs field.ErrorList) {
 	labels := cluster.Spec.CloudLabels
+	return validateCloudLabels(labels, fldPath)
+}
+
+func validateCloudLabels(labels map[string]string, fldPath *field.Path) (allErrs field.ErrorList) {
 	if labels == nil {
 		return allErrs
 	}
@@ -142,11 +146,10 @@ func validateClusterCloudLabels(cluster *kops.Cluster, fldPath *field.Path) (all
 			allErrs = append(allErrs, field.Forbidden(fldPath.Child(reservedKey), fmt.Sprintf("%q is a reserved label and cannot be used as a custom label", reservedKey)))
 		}
 	}
-
 	reservedPrefixes := []string{
+		"kubernetes.io/cluster/",
+		"k8s.io/role/",
 		"kops.k8s.io/",
-		"k8s.io/",
-		"kubernetes.io/",
 	}
 
 	for _, reservedPrefix := range reservedPrefixes {
diff --git a/pkg/apis/kops/validation/instancegroup.go b/pkg/apis/kops/validation/instancegroup.go
index 92bf19e1f0..858e4a9202 100644
--- a/pkg/apis/kops/validation/instancegroup.go
+++ b/pkg/apis/kops/validation/instancegroup.go
@@ -281,11 +281,21 @@ func validateIGCloudLabels(ig *kops.InstanceGroup, fldPath *field.Path) (allErrs
 		return allErrs
 	}
 
+	genericLabels := make(map[string]string)
+
 	for key, value := range labels {
-		if key == aws.CloudTagInstanceGroupName && value != ig.ObjectMeta.Name {
-			allErrs = append(allErrs, field.Invalid(fldPath.Child(aws.CloudTagInstanceGroupName), key, "Node label may only contain a single slash"))
+		if key == aws.CloudTagInstanceGroupName {
+
+			if value != ig.ObjectMeta.Name {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child(aws.CloudTagInstanceGroupName), key, "Node label may only contain a single slash"))
+			}
+		} else {
+			genericLabels[key] = value
 		}
 	}
+
+	allErrs = append(allErrs, validateCloudLabels(genericLabels, fldPath)...)
+
 	return allErrs
 }
 


Test patch:
diff --git a/pkg/apis/kops/validation/instancegroup_test.go b/pkg/apis/kops/validation/instancegroup_test.go
index f9f3a84089..a5a2909fef 100644
--- a/pkg/apis/kops/validation/instancegroup_test.go
+++ b/pkg/apis/kops/validation/instancegroup_test.go
@@ -270,6 +270,44 @@ func TestValidNodeLabels(t *testing.T) {
 	}
 }
 
+func TestValidateIGCloudLabels(t *testing.T) {
+
+	grid := []struct {
+		label    string
+		expected []string
+	}{
+
+		{
+			label: "k8s.io/cluster-autoscaler/test.example.com",
+		},
+		{
+			label:    "KubernetesCluster",
+			expected: []string{"Forbidden::spec.cloudLabels.KubernetesCluster"},
+		},
+		{
+			label: "MyBillingLabel",
+		},
+		{
+			label: "subdomain.domain.tld/foo/bar",
+		},
+	}
+
+	for _, g := range grid {
+		ig := &kops.InstanceGroup{
+			ObjectMeta: v1.ObjectMeta{
+				Name: "some-ig",
+			},
+			Spec: kops.InstanceGroupSpec{
+				Role:        "Node",
+				CloudLabels: make(map[string]string),
+			},
+		}
+		ig.Spec.CloudLabels[g.label] = "placeholder"
+		errs := ValidateInstanceGroup(ig, nil)
+		testErrors(t, g.label, errs, g.expected)
+	}
+}
+
 func TestIGCloudLabelIsIGName(t *testing.T) {
 
 	grid := []struct {