Repository: kubernetes/kops (go)

Issue:
Support Cilium Operator pod annotations
/kind feature

**1. Describe IN DETAIL the feature/behavior/change you would like to see.**

Kops `1.23` introduced the possibility to add some custom annotation on every Cilium agent pods using the `cilium.agentPodAnnotations` key in the cluster definition (https://github.com/kubernetes/kops/pull/12414). 

Annotations are pretty useful when you need third-party tool to add additional behavior for a k8s resource.
Lots of auto-discovery tools are based on this annotations.

I would expect the same feature but for the Cilium operator pods which is installed along with the Cilium agents on any K8S cluster backed by Cilium CNI.

**2. Feel free to provide a design supporting your feature request.**

I would propose to mimic the PR #12414 and introduce the `cilium.operatorPodAnnotations` key. This specific key will then support adding annotation on the Cilium operator pod.


Pull request:
Support Cilium Operator Pod Annotations
<!--
Thanks for contributing to kubernetes/kops!  Here are some tips for you:
1. If this is your first time, read our contributor guidelines:

 https://git.k8s.io/kops/CONTRIBUTING.md

2. Also, you'll probably want to checkout our development documentation:

https://git.k8s.io/kops/docs/development

3. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests

4. Finally, make sure all verifications and tests pass by running:
```
 make pr
```
-->

**What this PR does / why we need it**: Add support for customizing annotations on the Cilium Operator Deployment / Pod

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
fixes #15310

**Special notes for your reviewer**:


Patch:
diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
index ae875b9ac8..af4677ffb3 100644
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -5060,6 +5060,12 @@ spec:
                       nodeInitBootstrapFile:
                         description: NodeInitBootstrapFile is unused.
                         type: string
+                      operatorPodAnnotations:
+                        additionalProperties:
+                          type: string
+                        description: 'OperatorPodAnnotations makes possible to add
+                          additional annotations to cilium operator. Default: none'
+                        type: object
                       pprof:
                         description: Pprof is unused.
                         type: boolean
diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go
index fd139c7806..0f202ae918 100644
--- a/pkg/apis/kops/networking.go
+++ b/pkg/apis/kops/networking.go
@@ -391,6 +391,9 @@ type CiliumNetworkingSpec struct {
 	// AgentPodAnnotations makes possible to add additional annotations to cilium agent.
 	// Default: none
 	AgentPodAnnotations map[string]string `json:"agentPodAnnotations,omitempty"`
+	// OperatorPodAnnotations makes possible to add additional annotations to cilium operator.
+	// Default: none
+	OperatorPodAnnotations map[string]string `json:"operatorPodAnnotations,omitempty"`
 	// Tunnel specifies the Cilium tunnelling mode. Possible values are "vxlan", "geneve", or "disabled".
 	// Default: vxlan
 	Tunnel string `json:"tunnel,omitempty"`
diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go
index 829e24f470..cea7b43bda 100644
--- a/pkg/apis/kops/v1alpha2/networking.go
+++ b/pkg/apis/kops/v1alpha2/networking.go
@@ -464,6 +464,9 @@ type CiliumNetworkingSpec struct {
 	// AgentPodAnnotations makes possible to add additional annotations to the cilium agent.
 	// Default: none
 	AgentPodAnnotations map[string]string `json:"agentPodAnnotations,omitempty"`
+	// OperatorPodAnnotations makes possible to add additional annotations to cilium operator.
+	// Default: none
+	OperatorPodAnnotations map[string]string `json:"operatorPodAnnotations,omitempty"`
 	// Pprof is unused.
 	// +k8s:conversion-gen=false
 	Pprof bool `json:"pprof,omitempty"`
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
index c4a3e21e9b..214094ff4f 100644
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@@ -1963,6 +1963,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.Masquerade = in.Masquerade
 	// INFO: in.Nat46Range opted out of conversion generation
 	out.AgentPodAnnotations = in.AgentPodAnnotations
+	out.OperatorPodAnnotations = in.OperatorPodAnnotations
 	// INFO: in.Pprof opted out of conversion generation
 	// INFO: in.PrefilterDevice opted out of conversion generation
 	// INFO: in.PrometheusServeAddr opted out of conversion generation
@@ -2037,6 +2038,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.IdentityChangeGracePeriod = in.IdentityChangeGracePeriod
 	out.Masquerade = in.Masquerade
 	out.AgentPodAnnotations = in.AgentPodAnnotations
+	out.OperatorPodAnnotations = in.OperatorPodAnnotations
 	out.Tunnel = in.Tunnel
 	out.MonitorAggregation = in.MonitorAggregation
 	out.BPFCTGlobalTCPMax = in.BPFCTGlobalTCPMax
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
index 0edd912e25..c30eb4d064 100644
--- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
@@ -604,6 +604,13 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.OperatorPodAnnotations != nil {
+		in, out := &in.OperatorPodAnnotations, &out.OperatorPodAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.InstallIptablesRules != nil {
 		in, out := &in.InstallIptablesRules, &out.InstallIptablesRules
 		*out = new(bool)
diff --git a/pkg/apis/kops/v1alpha3/networking.go b/pkg/apis/kops/v1alpha3/networking.go
index 15ab8ef1f6..c85a13a819 100644
--- a/pkg/apis/kops/v1alpha3/networking.go
+++ b/pkg/apis/kops/v1alpha3/networking.go
@@ -354,6 +354,9 @@ type CiliumNetworkingSpec struct {
 	// AgentPodAnnotations makes possible to add additional annotations to the cilium agent.
 	// Default: none
 	AgentPodAnnotations map[string]string `json:"agentPodAnnotations,omitempty"`
+	// OperatorPodAnnotations makes possible to add additional annotations to cilium operator.
+	// Default: none
+	OperatorPodAnnotations map[string]string `json:"operatorPodAnnotations,omitempty"`
 	// Tunnel specifies the Cilium tunnelling mode. Possible values are "vxlan", "geneve", or "disabled".
 	// Default: vxlan
 	Tunnel string `json:"tunnel,omitempty"`
diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
index 4f405102d5..bc39c5d291 100644
--- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
@@ -2099,6 +2099,7 @@ func autoConvert_v1alpha3_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.IdentityChangeGracePeriod = in.IdentityChangeGracePeriod
 	out.Masquerade = in.Masquerade
 	out.AgentPodAnnotations = in.AgentPodAnnotations
+	out.OperatorPodAnnotations = in.OperatorPodAnnotations
 	out.Tunnel = in.Tunnel
 	out.MonitorAggregation = in.MonitorAggregation
 	out.BPFCTGlobalTCPMax = in.BPFCTGlobalTCPMax
@@ -2162,6 +2163,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha3_CiliumNetworkingSpec(in *
 	out.IdentityChangeGracePeriod = in.IdentityChangeGracePeriod
 	out.Masquerade = in.Masquerade
 	out.AgentPodAnnotations = in.AgentPodAnnotations
+	out.OperatorPodAnnotations = in.OperatorPodAnnotations
 	out.Tunnel = in.Tunnel
 	out.MonitorAggregation = in.MonitorAggregation
 	out.BPFCTGlobalTCPMax = in.BPFCTGlobalTCPMax
diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
index e2f2296127..d756bdaee2 100644
--- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
@@ -626,6 +626,13 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.OperatorPodAnnotations != nil {
+		in, out := &in.OperatorPodAnnotations, &out.OperatorPodAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.InstallIptablesRules != nil {
 		in, out := &in.InstallIptablesRules, &out.InstallIptablesRules
 		*out = new(bool)
diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go
index 228209fe97..53cfcd23ab 100644
--- a/pkg/apis/kops/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/zz_generated.deepcopy.go
@@ -707,6 +707,13 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.OperatorPodAnnotations != nil {
+		in, out := &in.OperatorPodAnnotations, &out.OperatorPodAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.InstallIptablesRules != nil {
 		in, out := &in.InstallIptablesRules, &out.InstallIptablesRules
 		*out = new(bool)
diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template
index 0938022fc4..9ef77e333d 100644
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template
@@ -964,6 +964,10 @@ spec:
     type: RollingUpdate
   template:
     metadata:
+      annotations:
+        {{- range $key, $value := .OperatorPodAnnotations }}
+        {{ $key }}: "{{ $value }}"
+        {{- end }}
       labels:
         io.cilium/app: operator
         name: cilium-operator


Test patch:
diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content
index 88fa7fceb1..a0e1a0d875 100644
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content
@@ -230,6 +230,10 @@ spec:
       ipam: kubernetes
       memoryRequest: 128Mi
       monitorAggregation: medium
+      operatorPodAnnotations:
+        test1: "true"
+        test2: "123"
+        test3: cilium-operator
       sidecarIstioProxyImage: cilium/istio_proxy
       toFqdnsDnsRejectResponseCode: refused
       tunnel: vxlan
diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
index 4492d7925d..9ad53c8c4e 100644
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 2a1ee49e7de7f6109240c11f32555b269d41ac87416a0203f2cbac9fda60283a
+    manifestHash: d1db96e7bf2e42c0e9514182f66fc48ca5eca29063c103f5e1b73c770f750c3a
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:
diff --git a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
index 1b675ed5e5..a6370f28f2 100644
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
@@ -648,6 +648,10 @@ spec:
     type: RollingUpdate
   template:
     metadata:
+      annotations:
+        test1: "true"
+        test2: "123"
+        test3: cilium-operator
       creationTimestamp: null
       labels:
         io.cilium/app: operator
diff --git a/tests/integration/update_cluster/privatecilium/in-v1alpha2.yaml b/tests/integration/update_cluster/privatecilium/in-v1alpha2.yaml
index f3d1b5dac9..52681f71cc 100644
--- a/tests/integration/update_cluster/privatecilium/in-v1alpha2.yaml
+++ b/tests/integration/update_cluster/privatecilium/in-v1alpha2.yaml
@@ -30,6 +30,10 @@ spec:
         test1: "true"
         test2: "123"
         test3: awesome
+      operatorPodAnnotations:
+        test1: "true"
+        test2: "123"
+        test3: cilium-operator
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0