Repository: fluxcd/flagger
Pull request #1049 (2021-11-10 14:52:07)

Prevent Flux from overriding Flagger managed objects
Annotate the routing objects managed by Flagger (Kubernetes services, virtual services, ingresses, etc) to prevent Flux from overriding them. This change requires [Flux 0.22.0](https://github.com/fluxcd/flux2/releases/tag/v0.22.0) or newer.

This allows Flux users to add Flagger Canaries to their repos without having to delete any existing Kubernetes Services. Users that deploy apps with HelmReleases they must remove the Kubernetes Services from their charts, because Flux, when used with Helm, is not capable of ignoring changes in-cluster.

codecov-commenter: # [Codecov](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd) Report
> Merging [#1049](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd) (9a973f6) into [main](https://codecov.io/gh/fluxcd/flagger/commit/9c7db58d8783a5ec7b5dbf453dd25f017b3a6101?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd) (9c7db58) will **increase** coverage by `0.00%`.
> The diff coverage is `100.00%`.

> :exclamation: Current head 9a973f6 differs from pull request most recent head 322285e. Consider uploading reports for the commit 322285e to get more accurate results
[![Impacted file tree graph](https://codecov.io/gh/fluxcd/flagger/pull/1049/graphs/tree.svg?width=650&height=150&src=pr&token=o7A3i1tCXM&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd)](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd)

```diff
@@           Coverage Diff           @@
##             main    #1049   +/-   ##
=======================================
  Coverage   57.15%   57.15%           
=======================================
  Files          76       76           
  Lines        6138     6139    +1     
=======================================
+ Hits         3508     3509    +1     
  Misses       2103     2103           
  Partials      527      527           
```


| [Impacted Files](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd) | Coverage Δ | |
|---|---|---|
| [pkg/router/util.go](https://codecov.io/gh/fluxcd/flagger/pull/1049/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd#diff-cGtnL3JvdXRlci91dGlsLmdv) | `100.00% <100.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd). Last update [9c7db58...322285e](https://codecov.io/gh/fluxcd/flagger/pull/1049?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=fluxcd).


Patch:
diff --git a/pkg/router/kubernetes_default.go b/pkg/router/kubernetes_default.go
index ad0b63a5..118d5ab0 100644
--- a/pkg/router/kubernetes_default.go
+++ b/pkg/router/kubernetes_default.go
@@ -150,7 +150,7 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam
 				Name:        name,
 				Namespace:   canary.Namespace,
 				Labels:      metadata.Labels,
-				Annotations: metadata.Annotations,
+				Annotations: filterMetadata(metadata.Annotations),
 				OwnerReferences: []metav1.OwnerReference{
 					*metav1.NewControllerRef(canary, schema.GroupVersionKind{
 						Group:   flaggerv1.SchemeGroupVersion.Group,
@@ -217,6 +217,10 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam
 		}
 
 		if updateService {
+			if svcClone.ObjectMeta.Annotations == nil {
+				svcClone.ObjectMeta.Annotations = make(map[string]string)
+			}
+			svcClone.ObjectMeta.Annotations = filterMetadata(svcClone.ObjectMeta.Annotations)
 			_, err = c.kubeClient.CoreV1().Services(canary.Namespace).Update(context.TODO(), svcClone, metav1.UpdateOptions{})
 			if err != nil {
 				return fmt.Errorf("service %s update error: %w", name, err)
diff --git a/pkg/router/util.go b/pkg/router/util.go
index 48e0d1ae..5f389fb0 100644
--- a/pkg/router/util.go
+++ b/pkg/router/util.go
@@ -4,7 +4,11 @@ import (
 	"strings"
 )
 
-const toolkitMarker = "toolkit.fluxcd.io"
+const (
+	toolkitMarker         = "toolkit.fluxcd.io"
+	toolkitReconcileKey   = "kustomize.toolkit.fluxcd.io/reconcile"
+	toolkitReconcileValue = "disabled"
+)
 
 func includeLabelsByPrefix(labels map[string]string, includeLabelPrefixes []string) map[string]string {
 	filteredLabels := make(map[string]string)
@@ -26,10 +30,14 @@ func includeLabelsByPrefix(labels map[string]string, includeLabelPrefixes []stri
 func filterMetadata(meta map[string]string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range meta {
+		// remove Flux ownership
 		if strings.Contains(k, toolkitMarker) {
 			continue
 		}
 		res[k] = v
 	}
+
+	// prevent Flux from overriding Flagger managed objects
+	res[toolkitReconcileKey] = toolkitReconcileValue
 	return res
 }


Test patch:
diff --git a/pkg/router/kubernetes_default_test.go b/pkg/router/kubernetes_default_test.go
index 4121199c..a1888341 100644
--- a/pkg/router/kubernetes_default_test.go
+++ b/pkg/router/kubernetes_default_test.go
@@ -391,7 +391,7 @@ func TestServiceRouter_InitializeMetadata(t *testing.T) {
 
 	primarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
 	require.NoError(t, err)
-	assert.Equal(t, 0, len(primarySvc.Annotations))
+	assert.Equal(t, 1, len(primarySvc.Annotations))
 	assert.Equal(t, "podinfo-primary", primarySvc.Labels["app"])
 }
 
@@ -423,12 +423,12 @@ func TestServiceRouter_ReconcileMetadata(t *testing.T) {
 
 	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
 	require.NoError(t, err)
-	assert.Equal(t, 0, len(canarySvc.Annotations))
+	assert.Equal(t, 1, len(canarySvc.Annotations))
 	assert.Equal(t, "podinfo-canary", canarySvc.Labels["app"])
 
 	primarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
 	require.NoError(t, err)
-	assert.Equal(t, 0, len(primarySvc.Annotations))
+	assert.Equal(t, 1, len(primarySvc.Annotations))
 	assert.Equal(t, "podinfo-primary", primarySvc.Labels["app"])
 
 	mocks.canary.Spec.Service.Apex = &flaggerv1.CustomMetadata{
diff --git a/pkg/router/traefik_test.go b/pkg/router/traefik_test.go
index 69d9a40d..9e172cfc 100644
--- a/pkg/router/traefik_test.go
+++ b/pkg/router/traefik_test.go
@@ -52,7 +52,7 @@ func TestTraefikRouter_Reconcile(t *testing.T) {
 	assert.Equal(t, uint(100), services[0].Weight)
 
 	assert.Equal(t, ts.ObjectMeta.Labels, mocks.canary.Spec.Service.Apex.Labels)
-	assert.Equal(t, ts.ObjectMeta.Annotations, mocks.canary.Spec.Service.Apex.Annotations)
+	assert.Equal(t, ts.ObjectMeta.Annotations, filterMetadata(mocks.canary.Spec.Service.Apex.Annotations))
 
 	for _, tt := range []struct {
 		name        string
diff --git a/test/kubernetes/test-deployment.sh b/test/kubernetes/test-deployment.sh
index 836e44e7..d4220af3 100755
--- a/test/kubernetes/test-deployment.sh
+++ b/test/kubernetes/test-deployment.sh
@@ -8,6 +8,21 @@ set -o errexit
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: podinfo-svc
+  namespace: test
+spec:
+  type: ClusterIP
+  selector:
+    app: podinfo
+  ports:
+    - name: http
+      port: 9898
+      protocol: TCP
+      targetPort: http
+---
 apiVersion: flagger.app/v1beta1
 kind: Canary
 metadata:
@@ -72,6 +87,12 @@ until ${ok}; do
     fi
 done
 
+passed=$(kubectl -n test get svc/podinfo-svc -oyaml 2>&1 | { grep 'kustomize.toolkit.fluxcd.io/reconcile' || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 toolkit annotation test failed'
+  kubectl -n test get svc/podinfo-svc -oyaml
+  exit 1
+fi
 passed=$(kubectl -n test get deploy/podinfo-primary -oyaml 2>&1 | { grep test-label-prefix || true; })
 if [ -z "$passed" ]; then
   echo -e '\u2716 primary copy labels by prefix test failed'