Repository: cloudnative-pg/cloudnative-pg

Issue:
[Bug]: unable to backup when we are more than 1 container in pod
### Is there an existing issue already for this bug?

- [x] I have searched for an existing issue, and could not find anything. I believe this is a new bug.

### I have read the troubleshooting guide

- [x] I have read the troubleshooting guide and I think this is a new bug.

### I am running a supported version of CloudNativePG

- [x] I have read the troubleshooting guide and I think this is a new bug.

### Contact Details


### Version

1.27 (latest patch)

### What version of Kubernetes are you using?

1.34

### What is your Kubernetes environment?

Self-managed: kind (evaluation)

### How did you install the operator?

YAML manifest

### What happened?

When deploying a PostgreSQL cluster via CNPG and Istio is enabled on the namespace, Istio automatically adds a sidecar to the pod or controller when it performs the backup, comparing the image of the first container with the image defined in the CNPG cluster. The problem is that if the first container is not PostgreSQL, the backup remains stuck in ‘Active’ status and this error message appears in the controller logs:

`{‘level’:‘debug’,“ts”:" 2025-10-26T07:49:29.003680028Z‘,’msg‘:’Instance not having expected image, discarded as target for backup‘,’controller‘:’backup‘,’controllerGroup‘:’postgresql.cnpg.io‘,’controllerKind‘:“Backup”,’ Backup‘:{’name‘:’postgresql-20251026084856‘,’namespace‘:’databases‘},’namespace‘:’databases‘,“name”:’postgresql-20251026084856‘,’ reconcileID‘:’f4dd8a41-2755-4086-89f8-e1daa10bb2bc‘,“caller”:’internal/controller/backup_controller.go:692"}`


### Cluster resource

```shell
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: postgresql
  namespace: databases
spec:
  monitoring:
    enablePodMonitor: true
  instances: 3
  imageCatalogRef:
    apiGroup: postgresql.cnpg.io
    kind: ClusterImageCatalog
    name: postgresql-minimal-trixie
    major: 18
  resources:
    limits:
      memory: 600Mi
    requests:
      cpu: 200m
      memory: 600Mi
  plugins:
    - name: barman-cloud.cloudnative-pg.io
      isWALArchiver: true
      parameters:
        barmanObjectName: minio-store
  storage:
    storageClass: longhorn
    size: 4Gi
  walStorage:
    storageClass: longhorn
    size: 4Gi
  backup:
    volumeSnapshot:
       className: longhorn
  bootstrap:
    initdb:
      database: bitnami_keycloak
      owner: bn_keycloak
      secret:
        name: postgresql-app
 managed:
    roles:
    - name: admin
      ensure: present
      comment: admin user
      login: true
      superuser: true
      passwordSecret:
        name: postgresql-admin
    - name: adhesion
      ensure: present
      comment: adhesion user
      login: true
      superuser: false
      passwordSecret:
        name: postgresql-adhesion
    - name: bn_keycloak
      ensure: present
      comment: bn_keycloak user
      login: true
      superuser: false
      passwordSecret:
        name: postgresql-app
    - name: communication
      ensure: present
      comment: communication user
      login: true
      superuser: false
      passwordSecret:
        name: postgresql-communication
    - name: finance
      ensure: present
      comment: finance user
      login: true
      superuser: false
      passwordSecret:
        name: postgresql-finance
    - name: sonar
      ensure: present
      comment: sonar user
      login: true
      superuser: false
      passwordSecret:
        name: postgresql-sonar
```

### Relevant log output

```shell
{"level":"debug","ts":"2025-10-26T07:49:28.615298496Z","msg":"reconciling object `databases/postgresql-20251026084856`","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:116"}
{"level":"debug","ts":"2025-10-26T07:49:28.637618Z","msg":"Found cluster for backup","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:164","cluster":"postgresql"}
{"level":"debug","ts":"2025-10-26T07:49:29.003337907Z","msg":"Instance not having expected image, discarded as target for backup","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:692"}
{"level":"debug","ts":"2025-10-26T07:49:29.003383699Z","msg":"Instance not having expected image, discarded as target for backup","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:692"}
{"level":"debug","ts":"2025-10-26T07:49:29.003396823Z","msg":"Instance not having expected image, discarded as target for backup","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:692"}
{"level":"debug","ts":"2025-10-26T07:49:29.003407323Z","msg":"No ready instances found as target for backup, defaulting to primary","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:738"}
{"level":"debug","ts":"2025-10-26T07:49:29.003680028Z","msg":"Instance not having expected image, discarded as target for backup","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","caller":"internal/controller/backup_controller.go:692"}
{"level":"info","ts":"2025-10-26T07:49:29.003763735Z","msg":"Couldn't find target pod, will retry in 30 seconds","controller":"backup","controllerGroup":"postgresql.cnpg.io","controllerKind":"Backup","Backup":{"name":"postgresql-20251026084856","namespace":"databases"},"namespace":"databases","name":"postgresql-20251026084856","reconcileID":"f4dd8a41-2755-4086-89f8-e1daa10bb2bc","target":"postgresql-2"}
{"level":"info","ts":"2025-10-26T07:49:29.003988607Z","logger":"events","msg":"Couldn't find target pod postgresql-2, will retry in 30 seconds","type":"Warning","object":{"kind":"Backup","namespace":"databases","name":"postgresql-20251026084856","uid":"a22c1a8e-aaa9-46ec-9a96-aadefdafa1c1","apiVersion":"postgresql.cnpg.io/v1","resourceVersion":"1077019247"},"reason":"FindingPod"}
```

### Code of Conduct

- [x] I agree to follow this project's Code of Conduct

Discussion:
@dosubot: <!-- Greeting -->
Hi @Joda89! I'm [Dosu](https://go.dosu.dev/dosubot) and I’m helping the cloudnative-pg team.

<!-- Answer -->
This is a known limitation in CloudNativePG: the backup controller checks only the first container in each pod to see if its image matches the expected PostgreSQL image. When Istio injects a sidecar, it often becomes the first container, causing the backup controller to discard the pod as a valid backup target and leaving backups stuck in 'Active' status. This logic is hardcoded in the backup controller and does not support multi-container pods or search for the PostgreSQL container elsewhere in the list.

There are currently no documented workarounds or fixes for this in the official documentation or recent releases. However, some users running CNPG with Istio have found that creating headless services for each PostgreSQL pod can help with other Istio-related communication issues, though this does not solve the backup image check problem itself. You may also want to experiment with pod template ordering, but Istio's injection is not always controllable.

For now, running CNPG backups in environments with injected sidecars like Istio is not supported due to this container image check logic. If you need to run backups, you will need to disable Istio injection for your PostgreSQL pods or wait for a future update that addresses multi-container support in the backup controller. For more details, see the relevant [code logic](https://github.com/cloudnative-pg/cloudnative-pg/blob/2cac8d6ff066b20cdc2e6ff1fd58a109cb08dec3/internal/controller/backup_controller.go) and [community discussion](https://github.com/cloudnative-pg/cloudnative-pg/discussions/1280).


<!-- Dosu Comment Footer -->
*To reply, just mention [@dosu](https://go.dosu.dev/dosubot).*

---
<sup>How did I do? [Good](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=great_response) | [Irrelevant](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=irrelevant_answer) | [Incorrect](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=incorrect_sources) | [Verbose](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=too_verbose) | [Hallucination](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=hallucination) | [Report 🐛](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=bug_report) | [Other](https://app.dosu.dev/response-feedback/4059be96-caca-453a-a747-1b813eb0cb9e?feedback_type=other)</sup>&nbsp;&nbsp;[![Chat with Dosu](https://dosu.dev/dosu-chat-badge.svg)](https://app.dosu.dev/89b818f3-8b5e-449b-bfb3-64a9360d9665/ask?utm_source=github)&nbsp;[![Join Discord](https://img.shields.io/badge/join-5865F2?logo=discord&logoColor=white&label=)](https://go.dosu.dev/discord-bot)&nbsp;[![Share on X](https://img.shields.io/badge/X-share-black)](https://twitter.com/intent/tweet?text=%40dosu_ai%20helped%20me%20solve%20this%20issue!&url=https%3A//github.com/cloudnative-pg/cloudnative-pg/issues/8966)@Joda89: Hello [@dosu](https://go.dosu.dev/dosubot),
I have already created the PR that allows you to retrieve the Postgres container.
Can you check my PR ?

Kind regards 

Solution patch:
diff --git a/internal/controller/backup_controller.go b/internal/controller/backup_controller.go
index 5ad58093..0960fef2 100644
--- a/internal/controller/backup_controller.go
+++ b/internal/controller/backup_controller.go
@@ -466,8 +466,15 @@ func (r *BackupReconciler) isValidBackupRunning(
 		return false, fmt.Errorf("unknown.spec.target received: %s", backup.Spec.Target)
 	}
 
+	pgContainerStatus, err := getPostgresContainerStatus(&pod)
+	if err != nil {
+		contextLogger.Warning("Cannot get postgres container status, assuming container restarted",
+			"error", err)
+		return false, nil
+	}
+
 	containerIsNotRestarted := utils.PodHasContainerStatuses(pod) &&
-		backup.Status.InstanceID.ContainerID == pod.Status.ContainerStatuses[0].ContainerID
+		backup.Status.InstanceID.ContainerID == pgContainerStatus.ContainerID
 	isPodActive := utils.IsPodActive(pod)
 	if isCorrectPodElected && containerIsNotRestarted && isPodActive {
 		contextLogger.Info("Backup is already running on",
@@ -539,9 +546,14 @@ func (r *BackupReconciler) reconcileSnapshotBackup(
 	}
 
 	if len(backup.Status.Phase) == 0 || backup.Status.Phase == apiv1.BackupPhasePending {
+		pgContainerStatus, err := getPostgresContainerStatus(targetPod)
+		if err != nil {
+			return nil, fmt.Errorf("cannot get postgres container status: %w", err)
+		}
+
 		backup.Status.SetAsStarted(
 			targetPod.Name,
-			targetPod.Status.ContainerStatuses[0].ContainerID,
+			pgContainerStatus.ContainerID,
 			apiv1.BackupMethodVolumeSnapshot,
 		)
 		// given that we use only kubernetes resources we can use the backup name as ID
@@ -688,7 +700,13 @@ func (r *BackupReconciler) getBackupTargetPod(ctx context.Context,
 			return false
 		}
 
-		if pod.Spec.Containers[0].Image != cluster.Status.Image {
+		pgContainer, err := getPostgresContainer(pod)
+		if err != nil {
+			contextLogger.Warning("Instance has no postgres container, discarded as target for backup")
+			return false
+		}
+
+		if pgContainer.Image != cluster.Status.Image {
 			contextLogger.Debug("Instance not having expected image, discarded as target for backup")
 			return false
 		}
@@ -752,6 +770,26 @@ func (r *BackupReconciler) getBackupTargetPod(ctx context.Context,
 	return &pod, nil
 }
 
+// getPostgresContainerStatus returns the container status for the postgres container in a pod
+func getPostgresContainerStatus(pod *corev1.Pod) (*corev1.ContainerStatus, error) {
+	for i := range pod.Status.ContainerStatuses {
+		if pod.Status.ContainerStatuses[i].Name == specs.PostgresContainerName {
+			return &pod.Status.ContainerStatuses[i], nil
+		}
+	}
+	return nil, fmt.Errorf("postgres container status not found in pod %s", pod.Name)
+}
+
+// getPostgresContainer returns the postgres container spec from a pod
+func getPostgresContainer(pod *corev1.Pod) (*corev1.Container, error) {
+	for i := range pod.Spec.Containers {
+		if pod.Spec.Containers[i].Name == specs.PostgresContainerName {
+			return &pod.Spec.Containers[i], nil
+		}
+	}
+	return nil, fmt.Errorf("postgres container not found in pod %s", pod.Name)
+}
+
 // startInstanceManagerBackup request a backup in a Pod and marks the backup started
 // or failed if needed
 func startInstanceManagerBackup(
@@ -761,9 +799,14 @@ func startInstanceManagerBackup(
 	pod *corev1.Pod,
 	cluster *apiv1.Cluster,
 ) error {
+	pgContainerStatus, err := getPostgresContainerStatus(pod)
+	if err != nil {
+		return fmt.Errorf("cannot get postgres container status: %w", err)
+	}
+
 	// This backup has been started
 	status := backup.GetStatus()
-	status.SetAsStarted(pod.Name, pod.Status.ContainerStatuses[0].ContainerID, backup.Spec.Method)
+	status.SetAsStarted(pod.Name, pgContainerStatus.ContainerID, backup.Spec.Method)
 
 	if err := postgres.PatchBackupStatusAndRetry(ctx, client, backup); err != nil {
 		return err
@@ -771,10 +814,10 @@ func startInstanceManagerBackup(
 	config := ctrl.GetConfigOrDie()
 	clientInterface := kubernetes.NewForConfigOrDie(config)
 
-	var err error
 	var stdout, stderr string
 	err = retry.OnError(retry.DefaultBackoff, func(error) bool { return true }, func() error {
-		stdout, stderr, err = utils.ExecCommand(
+		var execErr error
+		stdout, stderr, execErr = utils.ExecCommand(
 			ctx,
 			clientInterface,
 			config,
@@ -785,7 +828,7 @@ func startInstanceManagerBackup(
 			"backup",
 			backup.GetName(),
 		)
-		return err
+		return execErr
 	})
 	if err != nil {
 		log.FromContext(ctx).Error(err, "executing backup", "stdout", stdout, "stderr", stderr)



Test patch:
diff --git a/internal/controller/backup_controller_test.go b/internal/controller/backup_controller_test.go
index ba570c31..81de25a4 100644
--- a/internal/controller/backup_controller_test.go
+++ b/internal/controller/backup_controller_test.go
@@ -107,6 +107,7 @@ var _ = Describe("backup_controller barmanObjectStore unit tests", func() {
 				Phase: corev1.PodRunning,
 				ContainerStatuses: []corev1.ContainerStatus{
 					{
+						Name:        "postgres",
 						ContainerID: containerID,
 					},
 				},