Repository: kubernetes-sigs/cluster-api-provider-aws
Pull request #3677 (2022-08-19 08:08:06)

Fix the update logic for aws-node daemonset environment properties
<!-- Thanks for this PR! If this is your first PR please read the [contributing guide](../CONTRIBUTING.md) -->
<!-- If this PR is still work-in-progress and is being open for visibility please prefix the title with `WIP:` -->

**What type of PR is this?**

/kind bug

**What this PR does / why we need it**:

We apply environment property changes to aws-node. These changes are only applied to the loop variable, but the actual update doesn't update them correctly.

When I change the values, the update doesn't do anything.

Before:

- create node
```
Environment:
      ADDITIONAL_ENI_TAGS:                    {}
      AWS_VPC_CNI_NODE_PORT_SUPPORT:          true
      AWS_VPC_ENI_MTU:                        9001
      AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:     false
      AWS_VPC_K8S_CNI_EXTERNALSNAT:           false
      AWS_VPC_K8S_CNI_LOGLEVEL:               DEBUG
      AWS_VPC_K8S_CNI_LOG_FILE:               /host/var/log/aws-routed-eni/ipamd.log
      AWS_VPC_K8S_CNI_RANDOMIZESNAT:          prng
      AWS_VPC_K8S_CNI_VETHPREFIX:             eni
      AWS_VPC_K8S_PLUGIN_LOG_FILE:            /var/log/aws-routed-eni/plugin.log
      AWS_VPC_K8S_PLUGIN_LOG_LEVEL:           DEBUG
      DISABLE_INTROSPECTION:                  false
      DISABLE_METRICS:                        false
      DISABLE_NETWORK_RESOURCE_PROVISIONING:  false
      ENABLE_IPv4:                            true
      ENABLE_IPv6:                            false
      ENABLE_POD_ENI:                         false
      ENABLE_PREFIX_DELEGATION:               false
      MY_NODE_NAME:                            (v1:spec.nodeName)
      WARM_ENI_TARGET:                        1
      WARM_PREFIX_TARGET:                     1
      AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG:     true
```
- change value
```
Environment:
      ADDITIONAL_ENI_TAGS:                    {}
      AWS_VPC_CNI_NODE_PORT_SUPPORT:          true
      AWS_VPC_ENI_MTU:                        9001
      AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:     false
      AWS_VPC_K8S_CNI_EXTERNALSNAT:           false
      AWS_VPC_K8S_CNI_LOGLEVEL:               DEBUG
      AWS_VPC_K8S_CNI_LOG_FILE:               /host/var/log/aws-routed-eni/ipamd.log
      AWS_VPC_K8S_CNI_RANDOMIZESNAT:          prng
      AWS_VPC_K8S_CNI_VETHPREFIX:             eni
      AWS_VPC_K8S_PLUGIN_LOG_FILE:            /var/log/aws-routed-eni/plugin.log
      AWS_VPC_K8S_PLUGIN_LOG_LEVEL:           DEBUG
      DISABLE_INTROSPECTION:                  false
      DISABLE_METRICS:                        false
      DISABLE_NETWORK_RESOURCE_PROVISIONING:  false
      ENABLE_IPv4:                            true
      ENABLE_IPv6:                            false
      ENABLE_POD_ENI:                         false
      ENABLE_PREFIX_DELEGATION:               false
      MY_NODE_NAME:                            (v1:spec.nodeName)
      WARM_ENI_TARGET:                        1
      WARM_PREFIX_TARGET:                     1
      AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG:     true
```
- observe that nothing changes

With this modification:

- create aws-node without hardcoded new values:
```
Environment:
      ADDITIONAL_ENI_TAGS:                    {}
      AWS_VPC_CNI_NODE_PORT_SUPPORT:          true
      AWS_VPC_ENI_MTU:                        9001
      AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:     false
      AWS_VPC_K8S_CNI_EXTERNALSNAT:           false
      AWS_VPC_K8S_CNI_LOGLEVEL:               DEBUG
      AWS_VPC_K8S_CNI_LOG_FILE:               /host/var/log/aws-routed-eni/ipamd.log
      AWS_VPC_K8S_CNI_RANDOMIZESNAT:          prng
      AWS_VPC_K8S_CNI_VETHPREFIX:             eni
      AWS_VPC_K8S_PLUGIN_LOG_FILE:            /var/log/aws-routed-eni/plugin.log
      AWS_VPC_K8S_PLUGIN_LOG_LEVEL:           DEBUG
      DISABLE_INTROSPECTION:                  false
      DISABLE_METRICS:                        false
      DISABLE_NETWORK_RESOURCE_PROVISIONING:  false
      ENABLE_IPv4:                            true
      ENABLE_IPv6:                            false
      ENABLE_POD_ENI:                         false
      ENABLE_PREFIX_DELEGATION:               false
      MY_NODE_NAME:                            (v1:spec.nodeName)
      WARM_ENI_TARGET:                        1
      WARM_PREFIX_TARGET:                     1
```
- then change the code to include new values:
```
Environment:
      ADDITIONAL_ENI_TAGS:                    {}
      AWS_VPC_CNI_NODE_PORT_SUPPORT:          true
      AWS_VPC_ENI_MTU:                        9001
      AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:     false
      AWS_VPC_K8S_CNI_EXTERNALSNAT:           false
      AWS_VPC_K8S_CNI_LOGLEVEL:               DEBUG
      AWS_VPC_K8S_CNI_LOG_FILE:               /host/var/log/aws-routed-eni/ipamd.log
      AWS_VPC_K8S_CNI_RANDOMIZESNAT:          prng
      AWS_VPC_K8S_CNI_VETHPREFIX:             eni
      AWS_VPC_K8S_PLUGIN_LOG_FILE:            /var/log/aws-routed-eni/plugin.log
      AWS_VPC_K8S_PLUGIN_LOG_LEVEL:           DEBUG
      DISABLE_INTROSPECTION:                  false
      DISABLE_METRICS:                        false
      DISABLE_NETWORK_RESOURCE_PROVISIONING:  false
      ENABLE_IPv4:                            true
      ENABLE_IPv6:                            false
      ENABLE_POD_ENI:                         false
      ENABLE_PREFIX_DELEGATION:               false
      MY_NODE_NAME:                            (v1:spec.nodeName)
      WARM_ENI_TARGET:                        1
      WARM_PREFIX_TARGET:                     1
      AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG:     true
      ENI_CONFIG_LABEL_DEF:                   failure-domain.beta.kubernetes.io/zone
      SOME_NAME:                              some-vale
```

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Checklist**:
<!-- Put an "X" character inside the brackets of each completed task. Some may be optional depending on the PR in which case these can be deleted -->

- [ ] squashed commits
- [ ] includes documentation
- [ ] adds unit tests
- [ ] adds or updates e2e tests


k8s-ci-robot: @Skarlso: This issue is currently awaiting triage.

If CAPA/CAPI contributors determines this is a relevant issue, they will accept it by applying the `triage/accepted` label and provide further guidance.

The `triage/accepted` label can be added by org members by writing `/triage accepted` in a comment.


<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository.
</details>
k8s-ci-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by:
**Once this PR has been reviewed and has the lgtm label**, please assign richardcase for approval by writing `/assign @richardcase` in a comment. For more information see:[The Kubernetes Code Review Process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process).

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fcluster-api-provider-aws).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/main/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["richardcase"]} -->
Skarlso: /ok-to-test
Skarlso: Unfortunately, since this is happening inside the kube update function once the value is passed over, the unit test happily tells you that everything is fine.
Skarlso: Ah! I fixed this already in ONE of the places I see. :D 

```go
		for i := range ds.Spec.Template.Spec.Containers {
			container := &ds.Spec.Template.Spec.Containers[i]
			if container.Name == "aws-node" {
				container.Env = s.filterEnv(container.Env)
				container.Env, needsUpdate = s.applyUserProvidedEnvironmentProperties(container.Env)
			}
		}
```

But didn't update this bit at the very end after SecondaryCidrBlock is defined. :)
Skarlso: Added unit test. :)
k8s-ci-robot: @Skarlso: The following test **failed**, say `/retest` to rerun all failed tests or `/retest-required` to rerun all mandatory failed tests:

Test name | Commit | Details | Required | Rerun command
--- | --- | --- | --- | ---
pull-cluster-api-provider-aws-test | dd15ef7d5495c550e598a05197379839035bbc8a | [link](https://prow.k8s.io/view/gs/kubernetes-jenkins/pr-logs/pull/kubernetes-sigs_cluster-api-provider-aws/3677/pull-cluster-api-provider-aws-test/1560545090265419776) | true | `/test pull-cluster-api-provider-aws-test`

[Full PR test history](https://prow.k8s.io/pr-history?org=kubernetes-sigs&repo=cluster-api-provider-aws&pr=3677). [Your PR dashboard](https://prow.k8s.io/pr?query=is%3Apr%20state%3Aopen%20author%3ASkarlso). Please help us cut down on flakes by [linking to](https://git.k8s.io/community/contributors/devel/sig-testing/flaky-tests.md#filing-issues-for-flaky-tests) an [open issue](https://github.com/kubernetes-sigs/cluster-api-provider-aws/issues?q=is:issue+is:open) when you hit one in your PR.

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->

Patch:
diff --git a/pkg/cloud/services/awsnode/cni.go b/pkg/cloud/services/awsnode/cni.go
index dc62487ad..9153a325f 100644
--- a/pkg/cloud/services/awsnode/cni.go
+++ b/pkg/cloud/services/awsnode/cni.go
@@ -159,9 +159,9 @@ func (s *Service) ReconcileCNI(ctx context.Context) error {
 	}
 
 	s.scope.Info("updating containers", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace())
-	for _, container := range ds.Spec.Template.Spec.Containers {
-		if container.Name == "aws-node" {
-			container.Env = append(s.filterEnv(container.Env),
+	for i := range ds.Spec.Template.Spec.Containers {
+		if ds.Spec.Template.Spec.Containers[i].Name == "aws-node" {
+			ds.Spec.Template.Spec.Containers[i].Env = append(s.filterEnv(ds.Spec.Template.Spec.Containers[i].Env),
 				corev1.EnvVar{
 					Name:  "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG",
 					Value: "true",


Test patch:
diff --git a/pkg/cloud/services/awsnode/cni_test.go b/pkg/cloud/services/awsnode/cni_test.go
index 20f7add36..7c418af1b 100644
--- a/pkg/cloud/services/awsnode/cni_test.go
+++ b/pkg/cloud/services/awsnode/cni_test.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/api/apps/v1"
@@ -27,6 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
 	ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope"
 )
@@ -206,6 +208,92 @@ func TestReconcileCniVpcCniValues(t *testing.T) {
 	}
 }
 
+func TestReconcileCniVpcCniValuesWithSecondaryCidrBlock(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+	g := NewWithT(t)
+	daemonSet := &v1.DaemonSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "DaemonSet",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "aws-node",
+			Namespace: "kube-system",
+		},
+		Spec: v1.DaemonSetSpec{
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "aws-node",
+							Env: []corev1.EnvVar{
+								{
+									Name:  "NAME1",
+									Value: "OVERWRITE",
+								},
+								{
+									Name:  "NAME3",
+									Value: "VALUE3",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	values := ekscontrolplanev1.VpcCni{
+		Env: []corev1.EnvVar{
+			{
+				Name:  "NAME1",
+				Value: "VALUE1",
+			},
+		},
+	}
+	mockClient := &cachingClient{
+		getValue: daemonSet,
+	}
+	m := &mockScope{
+		client:             mockClient,
+		cni:                values,
+		secondaryCidrBlock: aws.String("100.0.0.1/20"),
+		securityGroups: map[infrav1.SecurityGroupRole]infrav1.SecurityGroup{
+			"node": {
+				ID:   "sg-1234",
+				Name: "node",
+			},
+		},
+		subnets: []infrav1.SubnetSpec{},
+	}
+	s := NewService(m)
+
+	err := s.ReconcileCNI(context.Background())
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(mockClient.updateChain).NotTo(BeEmpty())
+	ds, ok := mockClient.updateChain[0].(*v1.DaemonSet)
+	g.Expect(ok).To(BeTrue())
+	g.Expect(ds.Spec.Template.Spec.Containers).NotTo(BeEmpty())
+	g.Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf([]corev1.EnvVar{
+		{
+			Name:  "NAME1",
+			Value: "VALUE1",
+		},
+		{
+			Name:  "NAME3",
+			Value: "VALUE3",
+		},
+		{
+			Name:  "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG",
+			Value: "true",
+		},
+		{
+			Name:  "ENI_CONFIG_LABEL_DEF",
+			Value: "failure-domain.beta.kubernetes.io/zone",
+		},
+	}))
+}
+
 type cachingClient struct {
 	client.Client
 	getValue    client.Object
@@ -225,10 +313,17 @@ func (c *cachingClient) Update(ctx context.Context, obj client.Object, opts ...c
 	return nil
 }
 
+func (c *cachingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
+	return nil
+}
+
 type mockScope struct {
 	scope.AWSNodeScope
-	client client.Client
-	cni    ekscontrolplanev1.VpcCni
+	client             client.Client
+	cni                ekscontrolplanev1.VpcCni
+	secondaryCidrBlock *string
+	securityGroups     map[infrav1.SecurityGroupRole]infrav1.SecurityGroup
+	subnets            infrav1.Subnets
 }
 
 func (s *mockScope) RemoteClient() (client.Client, error) {
@@ -256,5 +351,13 @@ func (s *mockScope) DisableVPCCNI() bool {
 }
 
 func (s *mockScope) SecondaryCidrBlock() *string {
-	return nil
+	return s.secondaryCidrBlock
+}
+
+func (s *mockScope) SecurityGroups() map[infrav1.SecurityGroupRole]infrav1.SecurityGroup {
+	return s.securityGroups
+}
+
+func (s *mockScope) Subnets() infrav1.Subnets {
+	return s.subnets
 }