Repository: mariadb-operator/mariadb-operator

Issue:
[Bug] MariaDB custom liveness exec is ignored in 25.10.1
25.10.1 has brought a regression to liveness/readiness probes for replication setup. Now, the replica container is unconditionally restarted when SQL or IO thread is not running. But this makes it impossible to investigate and manually fix the issue if I can't or don't want to rely on the new automatic remediation feature.

I thought that I could revert to regular liveness exec `select 1` probes as it seemed to be allowed per https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/api_reference.md#probe, but it turned out that exec field is ignored, and I can only change thresholds like was pointed out here https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/configuration.md#probes

I believe we should allow setting regular exec probes. WDYT?

Discussion:
@mmontes11: We could do it for liveness probe, for readiness probe using select 1 as readiness probe will lead to data loss and stale reads, the readiness condition of the Pod is took into account during updates and failover.

With your select 1 liveness probe and disabling the automated recovery, you could fix the Pods yourself.

Would you like to open a PR for liveness?@hedgieinsocks: Sure, liveness is good enough. I'll have a look at the builder and raise a pr today or tomorrow.

Solution patch:
diff --git a/api/v1alpha1/mariadb_replication_types.go b/api/v1alpha1/mariadb_replication_types.go
index c79384be..f7d7615c 100644
--- a/api/v1alpha1/mariadb_replication_types.go
+++ b/api/v1alpha1/mariadb_replication_types.go
@@ -284,6 +284,11 @@ type ReplicationSpec struct {
 	// +optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
 	Agent Agent `json:"agent,omitempty"`
+	// StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
+	// It is disabled by default
+	// +optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	StandaloneProbes *bool `json:"standaloneProbes,omitempty"`
 }
 
 // IsGtidStrictModeEnabled determines whether GTID strict mode is enabled.
@@ -319,6 +324,9 @@ func (r *Replication) SetDefaults(mdb *MariaDB, env *environment.OperatorEnv) er
 	if r.SemiSyncEnabled == nil {
 		r.SemiSyncEnabled = ptr.To(true)
 	}
+	if r.StandaloneProbes == nil {
+		r.StandaloneProbes = ptr.To(false)
+	}
 
 	if reflect.ValueOf(r.InitContainer).IsZero() {
 		r.InitContainer = InitContainer{
@@ -533,3 +541,11 @@ type ReplicationStatus struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=status
 	ReplicaToRecover *string `json:"replicaToRecover,omitempty"`
 }
+
+// UseStandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
+func (m *MariaDB) UseStandaloneProbes() bool {
+	replication := ptr.Deref(m.Spec.Replication, Replication{})
+	standaloneProbes := ptr.Deref(replication.StandaloneProbes, false)
+
+	return standaloneProbes
+}
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index c9f302f3..6e3318fe 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -4028,6 +4028,11 @@ func (in *ReplicationSpec) DeepCopyInto(out *ReplicationSpec) {
 	}
 	in.InitContainer.DeepCopyInto(&out.InitContainer)
 	in.Agent.DeepCopyInto(&out.Agent)
+	if in.StandaloneProbes != nil {
+		in, out := &in.StandaloneProbes, &out.StandaloneProbes
+		*out = new(bool)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationSpec.
diff --git a/config/crd/bases/k8s.mariadb.com_mariadbs.yaml b/config/crd/bases/k8s.mariadb.com_mariadbs.yaml
index f2b251c5..4eaf78f5 100644
--- a/config/crd/bases/k8s.mariadb.com_mariadbs.yaml
+++ b/config/crd/bases/k8s.mariadb.com_mariadbs.yaml
@@ -6106,6 +6106,11 @@ spec:
                     - AfterSync
                     - AfterCommit
                     type: string
+                  standaloneProbes:
+                    description: |-
+                      StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
+                      It is disabled by default
+                    type: boolean
                   syncBinlog:
                     description: |-
                       SyncBinlog indicates after how many events the binary log is synchronized to the disk.
diff --git a/deploy/charts/mariadb-operator-crds/templates/crds.yaml b/deploy/charts/mariadb-operator-crds/templates/crds.yaml
index f0859400..3c0f410f 100644
--- a/deploy/charts/mariadb-operator-crds/templates/crds.yaml
+++ b/deploy/charts/mariadb-operator-crds/templates/crds.yaml
@@ -8220,6 +8220,11 @@ spec:
                     - AfterSync
                     - AfterCommit
                     type: string
+                  standaloneProbes:
+                    description: |-
+                      StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
+                      It is disabled by default
+                    type: boolean
                   syncBinlog:
                     description: |-
                       SyncBinlog indicates after how many events the binary log is synchronized to the disk.
diff --git a/deploy/crds/crds.yaml b/deploy/crds/crds.yaml
index f0859400..3c0f410f 100644
--- a/deploy/crds/crds.yaml
+++ b/deploy/crds/crds.yaml
@@ -8220,6 +8220,11 @@ spec:
                     - AfterSync
                     - AfterCommit
                     type: string
+                  standaloneProbes:
+                    description: |-
+                      StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.
+                      It is disabled by default
+                    type: boolean
                   syncBinlog:
                     description: |-
                       SyncBinlog indicates after how many events the binary log is synchronized to the disk.
diff --git a/docs/api_reference.md b/docs/api_reference.md
index 10b52754..3c926239 100644
--- a/docs/api_reference.md
+++ b/docs/api_reference.md
@@ -2341,6 +2341,7 @@ _Appears in:_
 | `syncBinlog` _integer_ | SyncBinlog indicates after how many events the binary log is synchronized to the disk.<br />See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-and-binary-log-system-variables#sync_binlog |  |  |
 | `initContainer` _[InitContainer](#initcontainer)_ | InitContainer is an init container that runs in the MariaDB Pod and co-operates with mariadb-operator. |  |  |
 | `agent` _[Agent](#agent)_ | Agent is a sidecar agent that runs in the MariaDB Pod and co-operates with mariadb-operator. |  |  |
+| `standaloneProbes` _boolean_ | StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.<br />It is disabled by default |  |  |
 | `enabled` _boolean_ | Enabled is a flag to enable replication. |  |  |
 
 
@@ -2368,6 +2369,7 @@ _Appears in:_
 | `syncBinlog` _integer_ | SyncBinlog indicates after how many events the binary log is synchronized to the disk.<br />See: https://mariadb.com/docs/server/ha-and-performance/standard-replication/replication-and-binary-log-system-variables#sync_binlog |  |  |
 | `initContainer` _[InitContainer](#initcontainer)_ | InitContainer is an init container that runs in the MariaDB Pod and co-operates with mariadb-operator. |  |  |
 | `agent` _[Agent](#agent)_ | Agent is a sidecar agent that runs in the MariaDB Pod and co-operates with mariadb-operator. |  |  |
+| `standaloneProbes` _boolean_ | StandaloneProbes indicates whether to use the default non-HA startup and liveness probes.<br />It is disabled by default |  |  |
 
 
 #### ResourceRequirements
diff --git a/pkg/builder/container_builder.go b/pkg/builder/container_builder.go
index f5913fb3..9f2be32f 100644
--- a/pkg/builder/container_builder.go
+++ b/pkg/builder/container_builder.go
@@ -705,14 +705,14 @@ func mariadbPorts(mariadb *mariadbv1alpha1.MariaDB) []corev1.ContainerPort {
 }
 
 func mariadbLivenessProbe(mariadb *mariadbv1alpha1.MariaDB) (*corev1.Probe, error) {
-	if mariadb.IsHAEnabled() {
+	if mariadb.IsHAEnabled() && !mariadb.UseStandaloneProbes() {
 		return mariadbAgentProbe(mariadb, "/liveness", mariadb.Spec.LivenessProbe)
 	}
 	return mariadbProbe(mariadb, mariadb.Spec.LivenessProbe), nil
 }
 
 func mariadbStartupProbe(mariadb *mariadbv1alpha1.MariaDB) (*corev1.Probe, error) {
-	if mariadb.IsHAEnabled() {
+	if mariadb.IsHAEnabled() && !mariadb.UseStandaloneProbes() {
 		return mariadbAgentProbe(mariadb, "/liveness", mariadb.Spec.StartupProbe)
 	}
 	return mariadbProbe(mariadb, mariadb.Spec.StartupProbe), nil



Test patch:
diff --git a/api/v1alpha1/mariadb_types_test.go b/api/v1alpha1/mariadb_types_test.go
index ccde0c89..f80963d4 100644
--- a/api/v1alpha1/mariadb_types_test.go
+++ b/api/v1alpha1/mariadb_types_test.go
@@ -1358,6 +1358,7 @@ var _ = Describe("MariaDB types", func() {
 								SemiSyncWaitPoint:  nil,
 								SyncBinlog:         nil,
 								SemiSyncAckTimeout: nil,
+								StandaloneProbes:   ptr.To(false),
 								InitContainer: InitContainer{
 									Image:             env.MariadbOperatorImage,
 									ContainerTemplate: ContainerTemplate{},
@@ -1445,6 +1446,7 @@ var _ = Describe("MariaDB types", func() {
 								SemiSyncEnabled:    ptr.To(true),
 								SemiSyncWaitPoint:  nil,
 								SemiSyncAckTimeout: nil,
+								StandaloneProbes:   ptr.To(false),
 								InitContainer: InitContainer{
 									Image:             env.MariadbOperatorImage,
 									ContainerTemplate: ContainerTemplate{},
diff --git a/pkg/builder/container_builder_test.go b/pkg/builder/container_builder_test.go
index b8acd4db..3745f174 100644
--- a/pkg/builder/container_builder_test.go
+++ b/pkg/builder/container_builder_test.go
@@ -208,6 +208,33 @@ func TestMariadbStartupProbe(t *testing.T) {
 				PeriodSeconds:       10,
 			},
 		},
+		{
+			name: "MariaDB replication with standalone probe",
+			mariadb: &mariadbv1alpha1.MariaDB{
+				Spec: mariadbv1alpha1.MariaDBSpec{
+					Replication: &mariadbv1alpha1.Replication{
+						Enabled: true,
+						ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+							StandaloneProbes: ptr.To(true),
+						},
+					},
+				},
+			},
+			wantProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{
+							"bash",
+							"-c",
+							"mariadb -u root -p\"${MARIADB_ROOT_PASSWORD}\" -e \"SELECT 1;\"",
+						},
+					},
+				},
+				InitialDelaySeconds: 20,
+				TimeoutSeconds:      5,
+				PeriodSeconds:       10,
+			},
+		},
 		{
 			name: "MariaDB Galera",
 			mariadb: &mariadbv1alpha1.MariaDB{
@@ -510,6 +537,33 @@ func TestMariadbLivenessProbe(t *testing.T) {
 				PeriodSeconds:       10,
 			},
 		},
+		{
+			name: "MariaDB replication with standalone probe",
+			mariadb: &mariadbv1alpha1.MariaDB{
+				Spec: mariadbv1alpha1.MariaDBSpec{
+					Replication: &mariadbv1alpha1.Replication{
+						Enabled: true,
+						ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+							StandaloneProbes: ptr.To(true),
+						},
+					},
+				},
+			},
+			wantProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					Exec: &corev1.ExecAction{
+						Command: []string{
+							"bash",
+							"-c",
+							"mariadb -u root -p\"${MARIADB_ROOT_PASSWORD}\" -e \"SELECT 1;\"",
+						},
+					},
+				},
+				InitialDelaySeconds: 20,
+				TimeoutSeconds:      5,
+				PeriodSeconds:       10,
+			},
+		},
 		{
 			name: "MariaDB Galera",
 			mariadb: &mariadbv1alpha1.MariaDB{
@@ -810,6 +864,33 @@ func TestMariadbReadinessProbe(t *testing.T) {
 				PeriodSeconds:       10,
 			},
 		},
+		{
+			name: "MariaDB replication with ignored standalone probe",
+			mariadb: &mariadbv1alpha1.MariaDB{
+				Spec: mariadbv1alpha1.MariaDBSpec{
+					Replication: &mariadbv1alpha1.Replication{
+						Enabled: true,
+						ReplicationSpec: mariadbv1alpha1.ReplicationSpec{
+							StandaloneProbes: ptr.To(true),
+							Agent: mariadbv1alpha1.Agent{
+								ProbePort: 5566,
+							},
+						},
+					},
+				},
+			},
+			wantProbe: &corev1.Probe{
+				ProbeHandler: corev1.ProbeHandler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path: "/readiness",
+						Port: intstr.FromInt(5566),
+					},
+				},
+				InitialDelaySeconds: 20,
+				TimeoutSeconds:      5,
+				PeriodSeconds:       10,
+			},
+		},
 		{
 			name: "MariaDB Galera",
 			mariadb: &mariadbv1alpha1.MariaDB{