Repository: fluxcd/kustomize-controller

Issue:
Kustomization reconciliation fails on multiple $patch: delete uses in a single strategic merge patch
# Environment

* Flux v2.4.0 (kustomize-controller v1.4.0)
* Kubernetes v1.29.3 (seemingly irrelevant)

# Description

If multiple `$patch: delete` strategic merge patches are provided in a single `spec.patches.patch` in a `kustomize.toolkit.fluxcd.io/v1.Kustomization` resource, kustomize-controller fails to reconcile with the following status:

```
flux-system   kustomization.kustomize.toolkit.fluxcd.io/debug         3m20s   False     kustomize build failed: recovered from kustomize build panic: runtime error: invalid memory address or nil pointer dereference
```

kustomize-controller log reports the following:

```json
{"level":"error","ts":"2024-12-09T18:41:35.677Z","msg":"Reconciliation failed after 108.007784ms, next try in 1h0m0s","controller":"kustomization","controllerGroup":"kustomize.toolkit.fluxcd.io","controllerKind":"Kustomization","Kustomization":{"name":"debug","namespace":"flux-system"},"namespace":"flux-system","name":"debug","reconcileID":"5eef40e9-8bf0-4a65-a178-eae10b04f877","revision":"main@sha1:4258282a038ab9506e1b37b3a44cc5ab4d38075e","error":"kustomize build failed: recovered from kustomize build panic: runtime error: invalid memory address or nil pointer dereference"}
```

The Kustomization resource needs to look like that:

```yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: whatever
spec:
  patches:
    - patch: |-
        $patch: delete
        apiVersion: v1
        kind: Whatever
        metadata:
          name: whatever1
        ---
        $patch: delete
        apiVersion: v1
        kind: Whatever
        metadata:
          name: whatever2
```

# Reproduction steps

0. Create a fresh Kubernetes cluster and install Flux v2.4.0 in whatever way convenient.
1. Create a Kustomization with the following files:

```
% cat > kustomization.yaml
resources:
  - configmaps.yml
% cat > configmaps.yml
apiVersion: v1
kind: Namespace
metadata:
  name: debug
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: debug1
  namespace: debug
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: debug2
  namespace: debug
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: debug3
  namespace: debug
```

2. Setup Flux to reconcile this Kustomization while deleting two of the ConfigMaps:

```yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: debug
  namespace: flux-system
spec:
  interval: 1h
  sourceRef:
    kind: GitRepository
    name: flux-system
  path: ./debug
  prune: true
  patches:
    - patch: |-
        $patch: delete
        apiVersion: v1
        kind: ConfigMap
        metadata:
          name: debug1
          namespace: debug
        ---
        $patch: delete
        apiVersion: v1
        kind: ConfigMap
        metadata:
          name: debug2
          namespace: debug
```

3. Wait for Flux to reconcile and observe the status above.

# Additional details

Apparently closely related to https://github.com/kubernetes-sigs/kustomize/issues/5552? The current Flux-specific workaround is simply to split every removal patch into its separate `- patch:`.

Discussion:
@stefanprodan: Flux uses Kustomize Go SDK, if this doesn't work with the kustomize CLI then it can't work with Flux.@cappyzawa: I've added a test case for this issue in #1466. The test currently fails due to the panic described in this issue, but it will serve as a regression test once the upstream fix in kubernetes-sigs/kustomize#5552 is included in a future kustomize release.

This ensures we can verify the fix works correctly when the upstream dependency is updated.@cappyzawa: FYI: kustomize v5.7.0 has been released! :tada:
https://github.com/kubernetes-sigs/kustomize/releases/tag/kustomize/v5.7.0

It includes the following fix:
> #5859: fix: Don't panic on multiple $patch: delete strategic merge patches in a single patch file@matheuscscp: @cappyzawa We need dedicated PRs to upgrade kustomize in these places:

https://github.com/search?q=org%3Afluxcd%20Pin%20kustomize%20to%20v5.6.0&type=code@cappyzawa: If no one is assigned to this yet, I’ll submit individual PRs starting tomorrow.

Solution patch:
diff --git a/go.mod b/go.mod
index c50caa0..f77bf73 100644
--- a/go.mod
+++ b/go.mod
@@ -47,13 +47,13 @@ require (
 	k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e
 	sigs.k8s.io/controller-runtime v0.21.0
 	sigs.k8s.io/kustomize/api v0.19.0
-	sigs.k8s.io/yaml v1.4.0
+	sigs.k8s.io/yaml v1.5.0
 )
 
-// Pin kustomize to v5.6.0
+// Pin kustomize to v5.7.0
 replace (
-	sigs.k8s.io/kustomize/api => sigs.k8s.io/kustomize/api v0.19.0
-	sigs.k8s.io/kustomize/kyaml => sigs.k8s.io/kustomize/kyaml v0.19.0
+	sigs.k8s.io/kustomize/api => sigs.k8s.io/kustomize/api v0.20.0
+	sigs.k8s.io/kustomize/kyaml => sigs.k8s.io/kustomize/kyaml v0.20.0
 )
 
 // Fix CVE-2022-28948
@@ -106,6 +106,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/carapace-sh/carapace-shlex v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.3 // indirect
@@ -226,6 +227,8 @@ require (
 	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.3 // indirect
 	golang.org/x/crypto v0.38.0 // indirect
 	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
 	golang.org/x/sync v0.14.0 // indirect
@@ -252,7 +255,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
 	k8s.io/kubectl v0.33.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.19.0 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.20.0 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 0d8947f..ed6056d 100644
--- a/go.sum
+++ b/go.sum
@@ -119,6 +119,8 @@ github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdn
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/carapace-sh/carapace-shlex v1.0.1 h1:ww0JCgWpOVuqWG7k3724pJ18Lq8gh5pHQs9j3ojUs1c=
+github.com/carapace-sh/carapace-shlex v1.0.1/go.mod h1:lJ4ZsdxytE0wHJ8Ta9S7Qq0XpjgjU0mdfCqiI2FHx7M=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -492,6 +494,10 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE=
+go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -595,14 +601,15 @@ sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytI
 sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
-sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ=
-sigs.k8s.io/kustomize/api v0.19.0/go.mod h1:/BbwnivGVcBh1r+8m3tH1VNxJmHSk1PzP5fkP6lbL1o=
-sigs.k8s.io/kustomize/kyaml v0.19.0 h1:RFge5qsO1uHhwJsu3ipV7RNolC7Uozc0jUBC/61XSlA=
-sigs.k8s.io/kustomize/kyaml v0.19.0/go.mod h1:FeKD5jEOH+FbZPpqUghBP8mrLjJ3+zD3/rf9NNu1cwY=
+sigs.k8s.io/kustomize/api v0.20.0 h1:xPLqcobHI0bThyRUteO+nCV8G4d1Rlo5HafO57VRcas=
+sigs.k8s.io/kustomize/api v0.20.0/go.mod h1:F6CfaV27oevRCMJgehLqyX81dlUnRX/Fc13Uo7+OSo4=
+sigs.k8s.io/kustomize/kyaml v0.20.0 h1:tT8KMKi4R3hCJ1+9HDdek2VoXpkerP92ZfF6fDgGw14=
+sigs.k8s.io/kustomize/kyaml v0.20.0/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po=
 sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
 sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
 sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
 sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI=
 sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
-sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+sigs.k8s.io/yaml v1.5.0 h1:M10b2U7aEUY6hRtU870n2VTPgR5RZiL/I6Lcc2F4NUQ=
+sigs.k8s.io/yaml v1.5.0/go.mod h1:wZs27Rbxoai4C0f8/9urLZtZtF3avA3gKvGyPdDqTO4=



Test patch:
diff --git a/internal/controller/kustomization_patch_delete_test.go b/internal/controller/kustomization_patch_delete_test.go
new file mode 100644
index 0000000..9b54fa5
--- /dev/null
+++ b/internal/controller/kustomization_patch_delete_test.go
@@ -0,0 +1,243 @@
+/*
+Copyright 2025 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/fluxcd/pkg/apis/kustomize"
+	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/testserver"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1"
+)
+
+// TestKustomizationReconciler_MultiplePatchDelete tests the handling of multiple
+// $patch: delete directives in strategic merge patches.
+// This test ensures that the controller properly handles scenarios where multiple
+// resources are deleted using a single patch specification.
+func TestKustomizationReconciler_MultiplePatchDelete(t *testing.T) {
+	g := NewWithT(t)
+	id := "multi-patch-delete-" + randStringRunes(5)
+	revision := "v1.0.0"
+
+	err := createNamespace(id)
+	g.Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+	err = createKubeConfigSecret(id)
+	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
+
+	// Create test files with multiple ConfigMaps
+	manifests := func(name string, data string) []testserver.File {
+		return []testserver.File{
+			{
+				Name: "configmaps.yaml",
+				Body: `---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm1
+  namespace: ` + name + `
+data:
+  key: ` + data + `1
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm2
+  namespace: ` + name + `
+data:
+  key: ` + data + `2
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm3
+  namespace: ` + name + `
+data:
+  key: ` + data + `3
+`,
+			},
+		}
+	}
+
+	artifact, err := testServer.ArtifactFromFiles(manifests(id, randStringRunes(5)))
+	g.Expect(err).NotTo(HaveOccurred())
+
+	repositoryName := types.NamespacedName{
+		Name:      randStringRunes(5),
+		Namespace: id,
+	}
+
+	err = applyGitRepository(repositoryName, artifact, revision)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	kustomizationKey := types.NamespacedName{
+		Name:      "patch-delete-" + randStringRunes(5),
+		Namespace: id,
+	}
+
+	t.Run("multiple patch delete in single patch should work", func(t *testing.T) {
+		// This test verifies that multiple $patch: delete directives in a single patch work correctly
+		// Ref: https://github.com/fluxcd/kustomize-controller/issues/1306
+		kustomization := &kustomizev1.Kustomization{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kustomizationKey.Name,
+				Namespace: kustomizationKey.Namespace,
+			},
+			Spec: kustomizev1.KustomizationSpec{
+				Interval: metav1.Duration{Duration: reconciliationInterval},
+				Path:     "./",
+				KubeConfig: &meta.KubeConfigReference{
+					SecretRef: meta.SecretKeyReference{
+						Name: "kubeconfig",
+					},
+				},
+				SourceRef: kustomizev1.CrossNamespaceSourceReference{
+					Name:      repositoryName.Name,
+					Namespace: repositoryName.Namespace,
+					Kind:      sourcev1.GitRepositoryKind,
+				},
+				Prune: true,
+				Patches: []kustomize.Patch{
+					{
+						// Multiple $patch: delete in a single patch
+						Patch: `$patch: delete
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm1
+  namespace: ` + id + `
+---
+$patch: delete
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm2
+  namespace: ` + id + ``,
+					},
+				},
+			},
+		}
+
+		g.Expect(k8sClient.Create(context.Background(), kustomization)).To(Succeed())
+
+		// Wait for reconciliation and check that it succeeds without panic
+		g.Eventually(func() bool {
+			var obj kustomizev1.Kustomization
+			_ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), &obj)
+			return obj.Status.LastAppliedRevision == revision
+		}, timeout, time.Second).Should(BeTrue())
+
+		// Verify that only cm3 ConfigMap exists (cm1 and cm2 should be deleted)
+		var cm corev1.ConfigMap
+		err := k8sClient.Get(context.Background(), client.ObjectKey{Name: "cm1", Namespace: id}, &cm)
+		g.Expect(err).To(HaveOccurred(), "cm1 should have been deleted")
+
+		err = k8sClient.Get(context.Background(), client.ObjectKey{Name: "cm2", Namespace: id}, &cm)
+		g.Expect(err).To(HaveOccurred(), "cm2 should have been deleted")
+
+		err = k8sClient.Get(context.Background(), client.ObjectKey{Name: "cm3", Namespace: id}, &cm)
+		g.Expect(err).NotTo(HaveOccurred(), "cm3 should still exist")
+
+		// Cleanup
+		g.Expect(k8sClient.Delete(context.Background(), kustomization)).To(Succeed())
+		g.Eventually(func() bool {
+			err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), kustomization)
+			return apierrors.IsNotFound(err)
+		}, timeout, time.Second).Should(BeTrue())
+	})
+
+	t.Run("multiple patch delete in separate patches should work", func(t *testing.T) {
+		// This test verifies that separate patches (which was previously a workaround) still work correctly
+		kustomizationSeparate := &kustomizev1.Kustomization{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kustomizationKey.Name + "-separate",
+				Namespace: kustomizationKey.Namespace,
+			},
+			Spec: kustomizev1.KustomizationSpec{
+				Interval: metav1.Duration{Duration: reconciliationInterval},
+				Path:     "./",
+				KubeConfig: &meta.KubeConfigReference{
+					SecretRef: meta.SecretKeyReference{
+						Name: "kubeconfig",
+					},
+				},
+				SourceRef: kustomizev1.CrossNamespaceSourceReference{
+					Name:      repositoryName.Name,
+					Namespace: repositoryName.Namespace,
+					Kind:      sourcev1.GitRepositoryKind,
+				},
+				Prune: true,
+				Patches: []kustomize.Patch{
+					{
+						Patch: `$patch: delete
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm1
+  namespace: ` + id + ``,
+					},
+					{
+						Patch: `$patch: delete
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cm2
+  namespace: ` + id + ``,
+					},
+				},
+			},
+		}
+
+		g.Expect(k8sClient.Create(context.Background(), kustomizationSeparate)).To(Succeed())
+
+		// Wait for successful reconciliation
+		g.Eventually(func() bool {
+			var obj kustomizev1.Kustomization
+			_ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomizationSeparate), &obj)
+			return obj.Status.LastAppliedRevision == revision
+		}, timeout, time.Second).Should(BeTrue())
+
+		// Verify that only cm3 ConfigMap exists
+		var cm corev1.ConfigMap
+		err := k8sClient.Get(context.Background(), client.ObjectKey{Name: "cm1", Namespace: id}, &cm)
+		g.Expect(err).To(HaveOccurred(), "cm1 should have been deleted")
+
+		err = k8sClient.Get(context.Background(), client.ObjectKey{Name: "cm2", Namespace: id}, &cm)
+		g.Expect(err).To(HaveOccurred(), "cm2 should have been deleted")
+
+		err = k8sClient.Get(context.Background(), client.ObjectKey{Name: "cm3", Namespace: id}, &cm)
+		g.Expect(err).NotTo(HaveOccurred(), "cm3 should still exist")
+
+		// Cleanup
+		g.Expect(k8sClient.Delete(context.Background(), kustomizationSeparate)).To(Succeed())
+		g.Eventually(func() bool {
+			err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomizationSeparate), kustomizationSeparate)
+			return apierrors.IsNotFound(err)
+		}, timeout, time.Second).Should(BeTrue())
+	})
+}