Repository: cloudnative-pg/cloudnative-pg

Issue:
[Bug]: Pods not coming back up after adding barman cloud native plugin and killing pods
### Is there an existing issue already for this bug?

- [x] I have searched for an existing issue, and could not find anything. I believe this is a new bug.

### I have read the troubleshooting guide

- [x] I have read the troubleshooting guide and I think this is a new bug.

### I am running a supported version of CloudNativePG

- [x] I have read the troubleshooting guide and I think this is a new bug.

### Contact Details

sv7n@pm.me

### Version

1.26 (latest patch)

### What version of Kubernetes are you using?

1.33

### What is your Kubernetes environment?

Self-managed: k3s

### How did you install the operator?

Helm

### What happened?

I have installed barman cloud plugin to move to new way of doing backups and have adjusted the yaml to have `.spec.plugins` section and adjusted `.spec.externalClusters` accordingly, after that I tried to do backup and it didn't work for me is is likely me doing something wrong (or the plugin is not working as intended but I suspect I am the problem here). This alone sounds like a plugin related issue and not cnpg operator issue, until i killed the pods.

I decided to kill the pods and see if that helps but the pods are not coming back up when running 
`cnpg status -n freshrss psql-freshrss` I see

```
❯ cnpg status -n freshrss psql-freshrss
Cluster Summary
Name                 freshrss/psql-freshrss
PostgreSQL Image:    ghcr.io/cloudnative-pg/postgresql:17
Primary instance:    psql-freshrss-11
Primary start time:  2025-06-15 17:36:06 +0000 UTC (uptime 160h35m35s)
Status:              Cluster in healthy state 
Instances:           3
Ready instances:     0
Size:                container not found

Continuous Backup status
Not configured

Physical backups
Primary instance not found

Streaming Replication status
Primary instance not found

Instances status
Name  Current LSN  Replication role  Status  QoS  Manager Version  Node
----  -----------  ----------------  ------  ---  ---------------  ----

Plugins status
Name                            Version  Status  Reported Operator Capabilities
----                            -------  ------  ------------------------------
barman-cloud.cloudnative-pg.io  0.5.0    N/A     Reconciler Hooks, Lifecycle Service
```
which straight away looks like a contradictory statement, how can there be 0 ready instances out of 3 and yet `Cluster in healthy state` that can't possibly be correct. I would argue that no plugin should have such affect and as such this is likely an operator issue.

### Cluster resource

```shell
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  annotations:
    argocd.argoproj.io/tracking-id: freshrss:postgresql.cnpg.io/Cluster:freshrss/psql-freshrss
    cnpg.io/skipEmptyWalArchiveCheck: enabled
    kubectl.kubernetes.io/last-applied-configuration: ...
  creationTimestamp: "2024-07-26T16:48:09Z"
  generation: 31
  labels:
    app: freshrss
    name: psql-freshrss
    subtype: psql
    type: database
  name: psql-freshrss
  namespace: freshrss
  resourceVersion: "1964759239"
  uid: 37235ce2-45c8-481b-bdd1-34291d8682f2
spec:
  affinity:
    enablePodAntiAffinity: true
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - preference:
          matchExpressions:
          - key: power-usage
            operator: In
            values:
            - high
        weight: 1
      - preference:
          matchExpressions:
          - key: power-usage
            operator: In
            values:
            - medium
        weight: 30
      - preference:
          matchExpressions:
          - key: power-usage
            operator: In
            values:
            - low
        weight: 90
    podAntiAffinityType: required
    topologyKey: kubernetes.io/hostname
  bootstrap:
    initdb:
      database: app
      encoding: UTF8
      localeCType: C
      localeCollate: C
      owner: app
  enablePDB: true
  enableSuperuserAccess: false
  externalClusters:
  - name: rzr
    plugin:
      enabled: true
      isWALArchiver: true
      name: barman-cloud.cloudnative-pg.io
      parameters:
        barmanObjectName: rzr-object-store
        serverName: freshrss-freshrss
  - name: opiz2
    plugin:
      enabled: true
      isWALArchiver: true
      name: barman-cloud.cloudnative-pg.io
      parameters:
        barmanObjectName: opiz2-object-store
        serverName: freshrss-freshrss
  failoverDelay: 0
  imageName: ghcr.io/cloudnative-pg/postgresql:17
  instances: 3
  logLevel: info
  maxSyncReplicas: 0
  minSyncReplicas: 0
  monitoring:
    customQueriesConfigMap:
    - key: queries
      name: cnpg-default-monitoring
    disableDefaultQueries: false
    enablePodMonitor: false
  plugins:
  - enabled: true
    isWALArchiver: true
    name: barman-cloud.cloudnative-pg.io
    parameters:
      barmanObjectName: rzr-object-store
      serverName: freshrss-freshrss
  postgresGID: 26
  postgresUID: 26
  postgresql:
    parameters:
      archive_mode: "on"
      archive_timeout: 5min
      auto_explain.log_analyze: "true"
      auto_explain.log_min_duration: 10s
      dynamic_shared_memory_type: posix
      full_page_writes: "on"
      log_destination: csvlog
      log_directory: /controller/log
      log_filename: postgres
      log_rotation_age: "0"
      log_rotation_size: "0"
      log_truncate_on_rotation: "false"
      logging_collector: "on"
      max_parallel_workers: "32"
      max_replication_slots: "32"
      max_worker_processes: "32"
      shared_memory_type: mmap
      shared_preload_libraries: ""
      ssl_max_protocol_version: TLSv1.3
      ssl_min_protocol_version: TLSv1.3
      wal_keep_size: 2GB
      wal_level: logical
      wal_log_hints: "on"
      wal_receiver_timeout: 5s
      wal_sender_timeout: 5s
    shared_preload_libraries: []
    syncReplicaElectionConstraint:
      enabled: false
    synchronous:
      dataDurability: required
      method: any
      number: 1
  primaryUpdateMethod: restart
  primaryUpdateStrategy: unsupervised
  replicationSlots:
    highAvailability:
      enabled: true
      slotPrefix: _cnpg_
    synchronizeReplicas:
      enabled: true
    updateInterval: 30
  resources:
    limits:
      cpu: "4"
      memory: 6Gi
    requests:
      cpu: 150m
      memory: 150Mi
  smartShutdownTimeout: 180
  startDelay: 3600
  stopDelay: 1800
  storage:
    resizeInUseVolumes: false
    size: 20Gi
    storageClass: local-path
  switchoverDelay: 3600
  walStorage:
    resizeInUseVolumes: false
    size: 20Gi
    storageClass: local-path
status:
  availableArchitectures:
  - goArch: amd64
    hash: 024c5162e1dee336b21121d781397b347d04939a857d9d8a47664d1796e8b7c9
  - goArch: arm64
    hash: 63f3a3fbe19b8fe1c26a0e516ecb965c3412e390729a02706825399f585d9298
  certificates:
    clientCASecret: psql-freshrss-ca
    expirations:
      psql-freshrss-ca: 2025-07-01 02:23:56 +0000 UTC
      psql-freshrss-replication: 2025-07-01 02:23:56 +0000 UTC
      psql-freshrss-server: 2025-07-31 03:35:00 +0000 UTC
    replicationTLSSecret: psql-freshrss-replication
    serverAltDNSNames:
    - psql-freshrss-rw
    - psql-freshrss-rw.freshrss
    - psql-freshrss-rw.freshrss.svc
    - psql-freshrss-rw.freshrss.svc.cluster.local
    - psql-freshrss-r
    - psql-freshrss-r.freshrss
    - psql-freshrss-r.freshrss.svc
    - psql-freshrss-r.freshrss.svc.cluster.local
    - psql-freshrss-ro
    - psql-freshrss-ro.freshrss
    - psql-freshrss-ro.freshrss.svc
    - psql-freshrss-ro.freshrss.svc.cluster.local
    serverCASecret: psql-freshrss-ca
    serverTLSSecret: psql-freshrss-server
  cloudNativePGCommitHash: 1535f3c17
  cloudNativePGOperatorHash: 024c5162e1dee336b21121d781397b347d04939a857d9d8a47664d1796e8b7c9
  conditions:
  - lastTransitionTime: "2025-06-15T17:58:33Z"
    message: Cluster is Ready
    reason: ClusterIsReady
    status: "True"
    type: Ready
  - lastTransitionTime: "2025-06-22T09:49:04Z"
    message: 'wal archive plugin is not available: barman-cloud.cloudnative-pg.io'
    reason: ContinuousArchivingFailing
    status: "False"
    type: ContinuousArchiving
  - lastTransitionTime: "2025-06-22T09:57:53Z"
    message: 'requested plugin is not available: barman-cloud.cloudnative-pg.io'
    reason: LastBackupFailed
    status: "False"
    type: LastBackupSucceeded
  configMapResourceVersion:
    metrics:
      cnpg-default-monitoring: "1932118216"
  currentPrimary: psql-freshrss-11
  currentPrimaryTimestamp: "2025-06-15T17:36:05.503623Z"
  danglingPVC:
  - psql-freshrss-10
  - psql-freshrss-10-wal
  - psql-freshrss-11
  - psql-freshrss-11-wal
  - psql-freshrss-13
  - psql-freshrss-13-wal
  firstRecoverabilityPoint: "2025-05-27T06:24:22Z"
  firstRecoverabilityPointByMethod:
    barmanObjectStore: "2025-05-27T06:24:22Z"
  image: ghcr.io/cloudnative-pg/postgresql:17
  instanceNames:
  - psql-freshrss-10
  - psql-freshrss-11
  - psql-freshrss-13
  instances: 3
  instancesReportedState:
    psql-freshrss-10:
      ip: 10.0.4.252
      isPrimary: false
      timeLineID: 6
    psql-freshrss-11:
      ip: 10.0.6.173
      isPrimary: true
      timeLineID: 6
    psql-freshrss-13:
      ip: 10.0.5.114
      isPrimary: false
      timeLineID: 6
  lastFailedBackup: "2025-06-22T10:02:57Z"
  lastSuccessfulBackup: "2025-06-22T03:40:13Z"
  lastSuccessfulBackupByMethod:
    barmanObjectStore: "2025-06-22T03:40:13Z"
  latestGeneratedNode: 13
  managedRolesStatus: {}
  pgDataImageInfo:
    image: ghcr.io/cloudnative-pg/postgresql:17
    majorVersion: 17
  phase: Cluster in healthy state
  pluginStatus:
  - capabilities:
    - TYPE_RECONCILER_HOOKS
    - TYPE_LIFECYCLE_SERVICE
    name: barman-cloud.cloudnative-pg.io
    version: 0.5.0
  poolerIntegrations:
    pgBouncerIntegration: {}
  pvcCount: 6
  readService: psql-freshrss-r
  secretsResourceVersion:
    applicationSecretVersion: "1935460851"
    clientCaSecretVersion: "1878297696"
    replicationSecretVersion: "1878297697"
    serverCaSecretVersion: "1878297696"
    serverSecretVersion: "1906696179"
  switchReplicaClusterStatus: {}
  targetPrimary: psql-freshrss-11
  targetPrimaryTimestamp: "2025-06-15T17:35:41.412276Z"
  timelineID: 6
  topology:
    successfullyExtracted: true
  writeService: psql-freshrss-rw
```

### Relevant log output

```shell
No relevant log output, the operator is quiet happy with the state as if nothing is wrong


{"level":"info","ts":"2025-06-22T10:19:00.974984556Z","msg":"Pre-reconcile hook stopped the reconciliation loop","controller":"cluster","controllerGroup":"postgresql.cnpg.io","controllerKind":"Cluster","Cluster":{"name":"psql-freshrss","namespace":"freshrss"},"namespace":"freshrss","name":"psql-freshrss","reconcileID":"9f09482b-b757-45ce-bb35-9c9df318b05a","hookResult":{"result":{"Requeue":true,"RequeueAfter":0},"err":null,"stopReconciliation":true,"identifier":"barman-cloud.cloudnative-pg.io"}}


barman plugin is not so happy with

{"level":"info","ts":"2025-06-22T10:19:00.97479926Z","msg":"barman object configuration not found, requeuing","name":"psql-freshrss","namespace":"freshrss","name":"rzr-object-store","namespace":"freshrss"}

but I don't think happiness of a plugin should be causing such behaviour no matter what.
```

### Code of Conduct

- [x] I agree to follow this project's Code of Conduct

Discussion:
@dosubot: <!-- Greeting -->
Hi @atropos112! I'm [Dosu](https://go.dosu.dev/dosubot) and I’m helping the cloudnative-pg team.

<!-- Answer -->
Your cluster is stuck because the barman-cloud plugin is blocking the reconciliation loop. The log message `Pre-reconcile hook stopped the reconciliation loop` means the operator is deferring all progress until the plugin is happy. In your case, the plugin reports `barman object configuration not found, requeuing`, which usually means the referenced ObjectStore resource (like `rzr-object-store`) is missing or misconfigured. When this happens, the operator doesn't update pod or instance status, so you get contradictory output like "0 ready instances" but "Cluster in healthy state"—the status is stale because the operator can't proceed past the plugin check. This is a known behavior with the new plugin system: plugin errors can block all reconciliation and leave the cluster in a confusing state [docs](https://cloudnative-pg.io/plugin-barman-cloud/docs/usage).

To fix this, double-check that every ObjectStore referenced in `.spec.plugins` and `.spec.externalClusters` exists and is correctly configured. The plugin expects these resources to be present and valid; otherwise, it will keep blocking reconciliation [docs](https://cloudnative-pg.io/plugin-barman-cloud/docs/usage). Also, make sure cert-manager is installed and working, as it's required for plugin communication [docs](https://cloudnative-pg.io/plugin-barman-cloud/docs/intro).

If you want the cluster to report "Continuous Backup status: Configured", you also need a `.spec.backup` section—even an empty one—since the plugin checks for both WAL archiving and scheduled backups [issue](https://github.com/cloudnative-pg/cloudnative-pg/issues/7658).

Once the plugin is happy (all ObjectStores are present and valid), the operator should resume reconciliation, pods should come back up, and the status should update correctly. If you still see issues after verifying the above, try increasing operator log verbosity or temporarily disabling the plugin to confirm the root cause.

Let me know if you need help reviewing your ObjectStore resources or plugin configuration.


<!-- Dosu Comment Footer -->
*To reply, just mention [@dosu](https://go.dosu.dev/dosubot).*

---
<sup>How did I do? [Good](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=great_response) | [Irrelevant](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=irrelevant_answer) | [Incorrect](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=incorrect_sources) | [Verbose](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=too_verbose) | [Hallucination](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=hallucination) | [Report 🐛](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=bug_report) | [Other](https://app.dosu.dev/response-feedback/ca81d8ce-e09a-4d78-a8c7-9620682039f2?feedback_type=other)</sup>&nbsp;&nbsp;[![Join Discord](https://img.shields.io/badge/join-5865F2?logo=discord&logoColor=white&label=)](https://go.dosu.dev/discord-bot)&nbsp;[![Share on X](https://img.shields.io/badge/X-share-black)](https://twitter.com/intent/tweet?text=%40dosu_ai%20helped%20me%20solve%20this%20issue!&url=https%3A//github.com/cloudnative-pg/cloudnative-pg/issues/7867)@atropos112: I can see how cnpg operator choosing to not reconcile until a plugin is happy has its merits.

What I am concerned of is the operator claiming everything is fine while waiting for the plugin to respond. 
It should still be responsible for seeing the cluster, seeing that pods are all down and reflecting that in the status accordingly. Having status `Cluster in healthy state` when it in-fact isn't in a healthy state is worse than not providing status at all.

Maybe its worth having "Waiting for plugin" or if it truly doesn't know then it should say "Unknown" deferring usage of `Cluster in healthy state` at all cost unless it is certain.

P.S. I like this bot, its very helpful!

Solution patch:
diff --git a/api/v1/cluster_types.go b/api/v1/cluster_types.go
index 0fd9cd3c..987a813d 100644
--- a/api/v1/cluster_types.go
+++ b/api/v1/cluster_types.go
@@ -640,6 +640,9 @@ const (
 	// loaded still
 	PhaseUnknownPlugin = "Cluster cannot proceed to reconciliation due to an unknown plugin being required"
 
+	// PhaseFailurePlugin is triggered when the cluster cannot proceed to reconciliation due to an interaction failure
+	PhaseFailurePlugin = "Cluster cannot proceed to reconciliation due to an error while interacting with plugins"
+
 	// PhaseImageCatalogError is triggered when the cluster cannot select the image to
 	// apply because of an invalid or incomplete catalog
 	PhaseImageCatalogError = "Cluster has incomplete or invalid image catalog"
diff --git a/internal/cnpi/plugin/client/backup.go b/internal/cnpi/plugin/client/backup.go
index 5178aa3a..61034623 100644
--- a/internal/cnpi/plugin/client/backup.go
+++ b/internal/cnpi/plugin/client/backup.go
@@ -22,7 +22,6 @@ package client
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"slices"
 	"time"
@@ -33,20 +32,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var (
-	// ErrPluginNotLoaded is raised when the plugin that should manage the backup
-	// have not been loaded inside the cluster
-	ErrPluginNotLoaded = errors.New("plugin not loaded")
-
-	// ErrPluginNotSupportBackup is raised when the plugin that should manage the backup
-	// doesn't support the Backup service
-	ErrPluginNotSupportBackup = errors.New("plugin does not support Backup service")
-
-	// ErrPluginNotSupportBackupEndpoint is raised when the plugin that should manage the backup
-	// doesn't support the Backup RPC endpoint
-	ErrPluginNotSupportBackupEndpoint = errors.New("plugin does not support the Backup RPC call")
-)
-
 // BackupResponse is the status of a newly created backup. This is used as a return
 // type for the Backup RPC Call
 type BackupResponse struct {
@@ -100,6 +85,17 @@ func (data *data) Backup(
 	backupObject client.Object,
 	pluginName string,
 	parameters map[string]string,
+) (*BackupResponse, error) {
+	b, err := data.innerBackup(ctx, cluster, backupObject, pluginName, parameters)
+	return b, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerBackup(
+	ctx context.Context,
+	cluster client.Object,
+	backupObject client.Object,
+	pluginName string,
+	parameters map[string]string,
 ) (*BackupResponse, error) {
 	contextLogger := log.FromContext(ctx)
 
diff --git a/internal/cnpi/plugin/client/cluster.go b/internal/cnpi/plugin/client/cluster.go
index 0742dba8..6429cb8b 100644
--- a/internal/cnpi/plugin/client/cluster.go
+++ b/internal/cnpi/plugin/client/cluster.go
@@ -22,7 +22,6 @@ package client
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"slices"
 
@@ -34,6 +33,11 @@ import (
 )
 
 func (data *data) MutateCluster(ctx context.Context, object client.Object, mutatedObject client.Object) error {
+	err := data.innerMutateCluster(ctx, object, mutatedObject)
+	return wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerMutateCluster(ctx context.Context, object client.Object, mutatedObject client.Object) error {
 	contextLogger := log.FromContext(ctx)
 
 	serializedObject, err := json.Marshal(object)
@@ -96,11 +100,16 @@ func (data *data) MutateCluster(ctx context.Context, object client.Object, mutat
 }
 
 var (
-	errInvalidJSON        = errors.New("invalid json")
-	errSetStatusInCluster = errors.New("SetStatusInCluster invocation failed")
+	errInvalidJSON        = newPluginError("invalid json")
+	errSetStatusInCluster = newPluginError("SetStatusInCluster invocation failed")
 )
 
 func (data *data) SetStatusInCluster(ctx context.Context, cluster client.Object) (map[string]string, error) {
+	m, err := data.innerSetStatusInCluster(ctx, cluster)
+	return m, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerSetStatusInCluster(ctx context.Context, cluster client.Object) (map[string]string, error) {
 	contextLogger := log.FromContext(ctx)
 	serializedObject, err := json.Marshal(cluster)
 	if err != nil {
@@ -150,6 +159,14 @@ func (data *data) SetStatusInCluster(ctx context.Context, cluster client.Object)
 func (data *data) ValidateClusterCreate(
 	ctx context.Context,
 	object client.Object,
+) (field.ErrorList, error) {
+	result, err := data.innerValidateClusterCreate(ctx, object)
+	return result, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerValidateClusterCreate(
+	ctx context.Context,
+	object client.Object,
 ) (field.ErrorList, error) {
 	contextLogger := log.FromContext(ctx)
 
@@ -192,6 +209,15 @@ func (data *data) ValidateClusterUpdate(
 	ctx context.Context,
 	oldObject client.Object,
 	newObject client.Object,
+) (field.ErrorList, error) {
+	result, err := data.innerValidateClusterUpdate(ctx, oldObject, newObject)
+	return result, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerValidateClusterUpdate(
+	ctx context.Context,
+	oldObject client.Object,
+	newObject client.Object,
 ) (field.ErrorList, error) {
 	contextLogger := log.FromContext(ctx)
 
diff --git a/internal/cnpi/plugin/client/create.go b/internal/cnpi/plugin/client/create.go
index a17c3a1c..fc08a0f7 100644
--- a/internal/cnpi/plugin/client/create.go
+++ b/internal/cnpi/plugin/client/create.go
@@ -31,6 +31,11 @@ import (
 
 // NewClient creates a new CNPI client
 func NewClient(ctx context.Context, enabledPlugin *stringset.Data) (Client, error) {
+	cli, err := innerNewClient(ctx, enabledPlugin)
+	return cli, wrapAsPluginErrorIfNeeded(err)
+}
+
+func innerNewClient(ctx context.Context, enabledPlugin *stringset.Data) (Client, error) {
 	contextLogger := log.FromContext(ctx)
 	plugins := repository.New()
 
diff --git a/internal/cnpi/plugin/client/errors.go b/internal/cnpi/plugin/client/errors.go
new file mode 100644
index 00000000..ad99ed2b
--- /dev/null
+++ b/internal/cnpi/plugin/client/errors.go
@@ -0,0 +1,73 @@
+/*
+Copyright © contributors to CloudNativePG, established as
+CloudNativePG a Series of LF Projects, LLC.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package client
+
+import "errors"
+
+var (
+	// ErrPluginNotLoaded is raised when the plugin that should manage the backup
+	// have not been loaded inside the cluster
+	ErrPluginNotLoaded = newPluginError("plugin not loaded")
+
+	// ErrPluginNotSupportBackup is raised when the plugin that should manage the backup
+	// doesn't support the Backup service
+	ErrPluginNotSupportBackup = newPluginError("plugin does not support Backup service")
+
+	// ErrPluginNotSupportBackupEndpoint is raised when the plugin that should manage the backup
+	// doesn't support the Backup RPC endpoint
+	ErrPluginNotSupportBackupEndpoint = newPluginError("plugin does not support the Backup RPC call")
+)
+
+type pluginError struct {
+	innerErr error
+}
+
+func (e *pluginError) Error() string {
+	return e.innerErr.Error()
+}
+
+func (e *pluginError) Unwrap() error {
+	return e.innerErr
+}
+
+func newPluginError(msg string) error {
+	return &pluginError{innerErr: errors.New(msg)}
+}
+
+// ContainsPluginError checks if the provided error chain contains a plugin error.
+func ContainsPluginError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	var pluginErr *pluginError
+	return errors.As(err, &pluginErr)
+}
+
+func wrapAsPluginErrorIfNeeded(err error) error {
+	if err == nil {
+		return nil
+	}
+	if ContainsPluginError(err) {
+		return err
+	}
+
+	return &pluginError{innerErr: err}
+}
diff --git a/internal/cnpi/plugin/client/lifecycle.go b/internal/cnpi/plugin/client/lifecycle.go
index 98e3a0dd..1ba289e8 100644
--- a/internal/cnpi/plugin/client/lifecycle.go
+++ b/internal/cnpi/plugin/client/lifecycle.go
@@ -49,6 +49,16 @@ func (data *data) LifecycleHook(
 	operationType plugin.OperationVerb,
 	cluster client.Object,
 	object client.Object,
+) (client.Object, error) {
+	obj, err := data.innerLifecycleHook(ctx, operationType, cluster, object)
+	return obj, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerLifecycleHook(
+	ctx context.Context,
+	operationType plugin.OperationVerb,
+	cluster client.Object,
+	object client.Object,
 ) (client.Object, error) {
 	contextLogger := log.FromContext(ctx).WithName("lifecycle_hook")
 
diff --git a/internal/cnpi/plugin/client/postgres.go b/internal/cnpi/plugin/client/postgres.go
index a3259a66..f25d2b66 100644
--- a/internal/cnpi/plugin/client/postgres.go
+++ b/internal/cnpi/plugin/client/postgres.go
@@ -34,6 +34,16 @@ func (data *data) EnrichConfiguration(
 	cluster client.Object,
 	config map[string]string,
 	operationType postgresClient.OperationType_Type,
+) (map[string]string, error) {
+	m, err := data.innerEnrichConfiguration(ctx, cluster, config, operationType)
+	return m, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerEnrichConfiguration(
+	ctx context.Context,
+	cluster client.Object,
+	config map[string]string,
+	operationType postgresClient.OperationType_Type,
 ) (map[string]string, error) {
 	tempConfig := config
 
diff --git a/internal/cnpi/plugin/client/reconciler.go b/internal/cnpi/plugin/client/reconciler.go
index b234ef48..cc0eded1 100644
--- a/internal/cnpi/plugin/client/reconciler.go
+++ b/internal/cnpi/plugin/client/reconciler.go
@@ -61,7 +61,7 @@ func newReconcilerRequeueResult(identifier string, after int64) ReconcilerHookRe
 
 // newReconcilerErrorResult creates a new result from an error
 func newReconcilerErrorResult(identifier string, err error) ReconcilerHookResult {
-	return ReconcilerHookResult{Err: err, StopReconciliation: true, Identifier: identifier}
+	return ReconcilerHookResult{Err: wrapAsPluginErrorIfNeeded(err), StopReconciliation: true, Identifier: identifier}
 }
 
 func (data *data) PreReconcile(ctx context.Context, cluster client.Object, object client.Object) ReconcilerHookResult {
diff --git a/internal/cnpi/plugin/client/restore_job.go b/internal/cnpi/plugin/client/restore_job.go
index 86fc0751..2ea986d1 100644
--- a/internal/cnpi/plugin/client/restore_job.go
+++ b/internal/cnpi/plugin/client/restore_job.go
@@ -40,6 +40,14 @@ type gvkEnsurer interface {
 func (data *data) Restore(
 	ctx context.Context,
 	cluster gvkEnsurer,
+) (*restore.RestoreResponse, error) {
+	r, err := data.innerRestore(ctx, cluster)
+	return r, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerRestore(
+	ctx context.Context,
+	cluster gvkEnsurer,
 ) (*restore.RestoreResponse, error) {
 	cluster.EnsureGVKIsPresent()
 
diff --git a/internal/cnpi/plugin/client/wal.go b/internal/cnpi/plugin/client/wal.go
index e9d36b55..7e86ebd5 100644
--- a/internal/cnpi/plugin/client/wal.go
+++ b/internal/cnpi/plugin/client/wal.go
@@ -35,6 +35,14 @@ func (data *data) ArchiveWAL(
 	ctx context.Context,
 	cluster client.Object,
 	sourceFileName string,
+) error {
+	return wrapAsPluginErrorIfNeeded(data.innerArchiveWAL(ctx, cluster, sourceFileName))
+}
+
+func (data *data) innerArchiveWAL(
+	ctx context.Context,
+	cluster client.Object,
+	sourceFileName string,
 ) error {
 	contextLogger := log.FromContext(ctx)
 
@@ -79,6 +87,16 @@ func (data *data) RestoreWAL(
 	cluster client.Object,
 	sourceWALName string,
 	destinationFileName string,
+) (bool, error) {
+	b, err := data.innerRestoreWAL(ctx, cluster, sourceWALName, destinationFileName)
+	return b, wrapAsPluginErrorIfNeeded(err)
+}
+
+func (data *data) innerRestoreWAL(
+	ctx context.Context,
+	cluster client.Object,
+	sourceWALName string,
+	destinationFileName string,
 ) (bool, error) {
 	var errorCollector error
 
diff --git a/internal/controller/cluster_controller.go b/internal/controller/cluster_controller.go
index 9cd46f57..08556196 100644
--- a/internal/controller/cluster_controller.go
+++ b/internal/controller/cluster_controller.go
@@ -207,6 +207,15 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 				)
 		}
 
+		if regErr := r.RegisterPhase(
+			ctx,
+			cluster,
+			apiv1.PhaseFailurePlugin,
+			fmt.Sprintf("Error while discovering plugins: %s", err.Error()),
+		); regErr != nil {
+			contextLogger.Error(regErr, "unable to register phase", "outerErr", err.Error())
+		}
+
 		contextLogger.Error(err, "Error loading plugins, retrying")
 		return ctrl.Result{}, err
 	}
@@ -224,6 +233,21 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	if errors.Is(err, utils.ErrTerminateLoop) {
 		return ctrl.Result{}, nil
 	}
+
+	// This code assumes that we always end the reconciliation loop if we encounter an error.
+	// In case that the assumption is false this code could overwrite an error phase.
+	if cnpgiClient.ContainsPluginError(err) {
+		if regErr := r.RegisterPhase(
+			ctx,
+			cluster,
+			apiv1.PhaseFailurePlugin,
+			fmt.Sprintf("Encountered an error while interacting with plugins: %s", err.Error()),
+		); regErr != nil {
+			contextLogger.Error(regErr, "unable to register phase", "outerErr", err.Error())
+		}
+		return ctrl.Result{RequeueAfter: 15 * time.Second}, nil
+	}
+
 	if err != nil {
 		return ctrl.Result{}, err
 	}



Test patch:
diff --git a/internal/cnpi/plugin/client/errors_test.go b/internal/cnpi/plugin/client/errors_test.go
new file mode 100644
index 00000000..9c68a3da
--- /dev/null
+++ b/internal/cnpi/plugin/client/errors_test.go
@@ -0,0 +1,122 @@
+/*
+Copyright © contributors to CloudNativePG, established as
+CloudNativePG a Series of LF Projects, LLC.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package client
+
+import (
+	"errors"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("wrapAsPluginErrorIfNeeded", func() {
+	It("should return nil when err is nil", func() {
+		result := wrapAsPluginErrorIfNeeded(nil)
+		Expect(result).ToNot(HaveOccurred())
+	})
+
+	It("should return the same error when it already contains a plugin error", func() {
+		originalErr := newPluginError("original plugin error")
+		wrappedErr := fmt.Errorf("wrapped: %w", originalErr)
+
+		result := wrapAsPluginErrorIfNeeded(wrappedErr)
+		Expect(result).To(Equal(wrappedErr))
+		Expect(ContainsPluginError(result)).To(BeTrue())
+	})
+
+	It("should wrap a non-plugin error as a plugin error", func() {
+		originalErr := errors.New("some regular error")
+
+		result := wrapAsPluginErrorIfNeeded(originalErr)
+		Expect(result).ToNot(Equal(originalErr))
+		Expect(ContainsPluginError(result)).To(BeTrue())
+		Expect(result.Error()).To(Equal(originalErr.Error()))
+	})
+
+	It("should wrap a nested non-plugin error as a plugin error", func() {
+		originalErr := errors.New("base error")
+		wrappedErr := fmt.Errorf("context: %w", originalErr)
+
+		result := wrapAsPluginErrorIfNeeded(wrappedErr)
+		Expect(result).ToNot(Equal(wrappedErr))
+		Expect(ContainsPluginError(result)).To(BeTrue())
+		Expect(result.Error()).To(Equal(wrappedErr.Error()))
+		Expect(errors.Is(result, originalErr)).To(BeTrue())
+	})
+
+	It("should not double-wrap an already wrapped plugin error", func() {
+		originalErr := errors.New("base error")
+		pluginErr := &pluginError{innerErr: originalErr}
+		wrappedPluginErr := fmt.Errorf("additional context: %w", pluginErr)
+
+		result := wrapAsPluginErrorIfNeeded(wrappedPluginErr)
+		Expect(result).To(Equal(wrappedPluginErr))
+		Expect(ContainsPluginError(result)).To(BeTrue())
+	})
+})
+
+var _ = Describe("ContainsPluginError", func() {
+	It("should return false when err is nil", func() {
+		result := ContainsPluginError(nil)
+		Expect(result).To(BeFalse())
+	})
+
+	It("should return true when error is a direct plugin error", func() {
+		pluginErr := newPluginError("test plugin error")
+
+		result := ContainsPluginError(pluginErr)
+		Expect(result).To(BeTrue())
+	})
+
+	It("should return true when plugin error is wrapped with fmt.Errorf", func() {
+		pluginErr := newPluginError("original plugin error")
+		wrappedErr := fmt.Errorf("context: %w", pluginErr)
+
+		result := ContainsPluginError(wrappedErr)
+		Expect(result).To(BeTrue())
+	})
+
+	It("should return true when plugin error is deeply nested", func() {
+		pluginErr := newPluginError("base plugin error")
+		wrappedOnce := fmt.Errorf("level 1: %w", pluginErr)
+		wrappedTwice := fmt.Errorf("level 2: %w", wrappedOnce)
+
+		result := ContainsPluginError(wrappedTwice)
+		Expect(result).To(BeTrue())
+	})
+
+	It("should return false when error chain contains no plugin error", func() {
+		baseErr := errors.New("base error")
+		wrappedErr := fmt.Errorf("wrapped: %w", baseErr)
+
+		result := ContainsPluginError(wrappedErr)
+		Expect(result).To(BeFalse())
+	})
+
+	It("should return true when error chain contains plugin error mixed with other errors", func() {
+		baseErr := errors.New("base error")
+		pluginErr := &pluginError{innerErr: baseErr}
+		wrappedErr := fmt.Errorf("additional context: %w", pluginErr)
+
+		result := ContainsPluginError(wrappedErr)
+		Expect(result).To(BeTrue())
+	})
+})
diff --git a/internal/cnpi/plugin/client/postgres_test.go b/internal/cnpi/plugin/client/postgres_test.go
index 80f426cd..187320fc 100644
--- a/internal/cnpi/plugin/client/postgres_test.go
+++ b/internal/cnpi/plugin/client/postgres_test.go
@@ -21,7 +21,6 @@ package client
 
 import (
 	"context"
-	"errors"
 
 	"github.com/cloudnative-pg/cnpg-i/pkg/postgres"
 	"google.golang.org/grpc"
@@ -119,7 +118,7 @@ var _ = Describe("EnrichConfiguration", func() {
 	})
 
 	It("should return error when plugin returns error", func(ctx SpecContext) {
-		expectedErr := errors.New("plugin error")
+		expectedErr := newPluginError("plugin error")
 
 		postgresClient := &fakePostgresClient{
 			enrichConfigError: expectedErr,