Repository: percona/percona-xtradb-cluster-operator
Pull request #2018: K8SPXC-1609: Fix needlessly updating binlog collector deployment
Author: login: egegunes
Labels: name: size/M
description: 30-99 lines
Created: 2025-04-02T14:17:14Z | Merged: 2025-04-04T08:18:03Z

[![K8SPXC-1609](https://badgen.net/badge/JIRA/K8SPXC-1609/green)](https://jira.percona.com/browse/K8SPXC-1609) [<img width="16" alt="Powered by Pull Request Badge" src="https://user-images.githubusercontent.com/1393946/111216524-d2bb8e00-85d4-11eb-821b-ed4c00989c02.png">](https://pullrequestbadge.com/?utm_medium=github&utm_source=percona&utm_campaign=badge_info)<!-- PR-BADGE: PLEASE DO NOT REMOVE THIS COMMENT -->

**CHANGE DESCRIPTION**
---
**Problem:**
Binlog collector deployment was constantly updated by operator because of `deployment.kubernetes.io/revision` annotation added to Deployments automatically by Kubernetes.

**Solution:**
This PR introduces:
* Ignore `deployment.kubernetes.io/revision` annotation in deployment while checking if metadata is changed.
* Add `VERBOSE` log level and print object diff if it's enabled.

**CHECKLIST**
---
**Jira**
- [ ] Is the Jira ticket created and referenced properly?
- [ ] Does the Jira ticket have the proper statuses for documentation (`Needs Doc`) and QA (`Needs QA`)?
- [ ] Does the Jira ticket link to the proper milestone (Fix Version field)?

**Tests**
- [ ] Is an E2E test/test case added for the new feature/change?
- [ ] Are unit tests added where appropriate?
- [ ] Are OpenShift compare files changed for E2E tests (`compare/*-oc.yml`)?

**Config/Logging/Testability**
- [ ] Are all needed new/changed options added to default YAML files?
- [ ] Are all needed new/changed options added to the [Helm Chart](https://github.com/percona/percona-helm-charts)?
- [ ] Did we add proper logging messages for operator actions?
- [ ] Did we ensure compatibility with the previous version or cluster upgrade process?
- [ ] Does the change support oldest and newest supported PXC version?
- [ ] Does the change support oldest and newest supported Kubernetes version?

[K8SPXC-1609]: https://perconadev.atlassian.net/browse/K8SPXC-1609?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ


Diff:
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
index c43a8a60a1..04f465575e 100644
--- a/cmd/manager/main.go
+++ b/cmd/manager/main.go
@@ -217,7 +217,7 @@ func getLogLevel(log logr.Logger) zapcore.LevelEnabler {
 	}
 
 	switch strings.ToUpper(l) {
-	case "DEBUG":
+	case "VERBOSE", "DEBUG":
 		return zapcore.DebugLevel
 	case "INFO":
 		return zapcore.InfoLevel
diff --git a/pkg/controller/pxc/controller.go b/pkg/controller/pxc/controller.go
index 1799e2d2f2..060b15e1b4 100644
--- a/pkg/controller/pxc/controller.go
+++ b/pkg/controller/pxc/controller.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"reflect"
 	"strings"
 	"sync"
@@ -11,6 +12,7 @@ import (
 	"time"
 
 	cm "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
+	"github.com/google/go-cmp/cmp"
 	"github.com/pkg/errors"
 	"github.com/robfig/cron/v3"
 	appsv1 "k8s.io/api/apps/v1"
@@ -40,6 +42,7 @@ import (
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/config"
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset"
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/backup"
+	"github.com/percona/percona-xtradb-cluster-operator/pkg/util"
 	"github.com/percona/percona-xtradb-cluster-operator/version"
 )
 
@@ -1132,9 +1135,14 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(ctx context.Context, cr *
 		return r.client.Create(ctx, obj)
 	}
 
-	if oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash ||
-		!isObjectMetaEqual(obj, oldObject) {
+	switch obj.(type) {
+	case *appsv1.Deployment:
+		objAnnotations := oldObject.GetAnnotations()
+		delete(objAnnotations, "deployment.kubernetes.io/revision")
+		oldObject.SetAnnotations(objAnnotations)
+	}
 
+	if oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash || !isObjectMetaEqual(obj, oldObject) {
 		switch object := obj.(type) {
 		case *corev1.Service:
 			object.Spec.ClusterIP = oldObject.(*corev1.Service).Spec.ClusterIP
@@ -1145,7 +1153,15 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(ctx context.Context, cr *
 			obj.SetResourceVersion(oldObject.GetResourceVersion())
 		}
 
-		log.V(1).Info("Updating object", "object", obj.GetName(), "kind", obj.GetObjectKind())
+		log.V(1).Info("Updating object",
+			"object", obj.GetName(),
+			"kind", obj.GetObjectKind(),
+			"hashChanged", oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash,
+			"metaChanged", !isObjectMetaEqual(obj, oldObject),
+		)
+		if util.IsLogLevelVerbose() && !util.IsLogStructured() {
+			fmt.Println(cmp.Diff(oldObject, obj))
+		}
 
 		return r.client.Update(ctx, obj)
 	}
diff --git a/pkg/util/log.go b/pkg/util/log.go
new file mode 100644
index 0000000000..10e67c031e
--- /dev/null
+++ b/pkg/util/log.go
@@ -0,0 +1,30 @@
+package util
+
+import (
+	"os"
+	"strconv"
+	"strings"
+)
+
+func IsLogLevelVerbose() bool {
+	l, found := os.LookupEnv("LOG_LEVEL")
+	if !found {
+		return false
+	}
+
+	return strings.ToUpper(l) == "VERBOSE"
+}
+
+func IsLogStructured() bool {
+	s, found := os.LookupEnv("LOG_STRUCTURED")
+	if !found {
+		return false
+	}
+
+	useJson, err := strconv.ParseBool(s)
+	if err != nil {
+		return false
+	}
+
+	return useJson
+}


Reviews:
user: login: pooknull
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T09:43:19Z
user: login: gkech
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T09:48:26Z
user: login: gkech
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T09:49:23Z
user: login: gkech
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T12:43:46Z
user: login: hors
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-03T14:08:51Z
user: login: pooknull
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T14:16:09Z

Review comments:
diff_hunk: @@ -1145,7 +1153,15 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(ctx context.Context, cr *
 			obj.SetResourceVersion(oldObject.GetResourceVersion())
 		}
 
-		log.V(1).Info("Updating object", "object", obj.GetName(), "kind", obj.GetObjectKind())
+		log.V(1).Info("Updating object",
+			"object", obj.GetName(),
+			"kind", obj.GetObjectKind(),
+			"hashChanged", oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash,
+			"metaChanged", !isObjectMetaEqual(obj, oldObject),
+		)
+		if util.IsLogLevelVerbose() {
+			fmt.Println(cmp.Diff(oldObject, obj))
path: pkg/controller/pxc/controller.go
user: login: pooknull
body: I think we should use a `log` here. This can break parsing of logs when json logging format is used.
created_at: 2025-04-03T09:43:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1145,7 +1153,15 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(ctx context.Context, cr *
 			obj.SetResourceVersion(oldObject.GetResourceVersion())
 		}
 
-		log.V(1).Info("Updating object", "object", obj.GetName(), "kind", obj.GetObjectKind())
+		log.V(1).Info("Updating object",
+			"object", obj.GetName(),
+			"kind", obj.GetObjectKind(),
+			"hashChanged", oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash,
+			"metaChanged", !isObjectMetaEqual(obj, oldObject),
+		)
+		if util.IsLogLevelVerbose() {
+			fmt.Println(cmp.Diff(oldObject, obj))
path: pkg/controller/pxc/controller.go
user: login: gkech
body: On top of @pooknull comment, I was wondering why we are not using `log.V(2)` which is essentially verbose level 2. 
created_at: 2025-04-03T09:48:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1145,7 +1153,15 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(ctx context.Context, cr *
 			obj.SetResourceVersion(oldObject.GetResourceVersion())
 		}
 
-		log.V(1).Info("Updating object", "object", obj.GetName(), "kind", obj.GetObjectKind())
+		log.V(1).Info("Updating object",
+			"object", obj.GetName(),
+			"kind", obj.GetObjectKind(),
+			"hashChanged", oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash,
+			"metaChanged", !isObjectMetaEqual(obj, oldObject),
+		)
+		if util.IsLogLevelVerbose() {
path: pkg/controller/pxc/controller.go
user: login: gkech
body: e.g. `if log.V(2).Enabled(){...`
created_at: 2025-04-03T09:49:23Z
author_association: CONTRIBUTOR

PR comments:
user: login: JNKPercona
created_at: 2025-04-04T07:59:46Z
body: | Test name | Status |
| ------------- | ------------- |
| affinity-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/affinity-8-0.log) |
| auto-tuning-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/auto-tuning-8-0.log) |
| cross-site-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/cross-site-8-0.log) |
| custom-users-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/custom-users-8-0.log) |
| demand-backup-cloud-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/demand-backup-cloud-8-0.log) |
| demand-backup-encrypted-with-tls-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/demand-backup-encrypted-with-tls-8-0.log) |
| demand-backup-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/demand-backup-8-0.log) |
| demand-backup-flow-control-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/demand-backup-flow-control-8-0.log) |
| demand-backup-parallel-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/demand-backup-parallel-8-0.log) |
| haproxy-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/haproxy-5-7.log) |
| haproxy-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/haproxy-8-0.log) |
| init-deploy-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/init-deploy-5-7.log) |
| init-deploy-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/init-deploy-8-0.log) |
| limits-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/limits-8-0.log) |
| monitoring-2-0-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/monitoring-2-0-8-0.log) |
| one-pod-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/one-pod-5-7.log) |
| one-pod-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/one-pod-8-0.log) |
| pitr-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/pitr-8-0.log) |
| pitr-gap-errors-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/pitr-gap-errors-8-0.log) |
| proxy-protocol-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/proxy-protocol-8-0.log) |
| proxysql-sidecar-res-limits-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/proxysql-sidecar-res-limits-8-0.log) |
| pvc-resize-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/pvc-resize-5-7.log) |
| pvc-resize-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/pvc-resize-8-0.log) |
| recreate-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/recreate-8-0.log) |
| restore-to-encrypted-cluster-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/restore-to-encrypted-cluster-8-0.log) |
| scaling-proxysql-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/scaling-proxysql-8-0.log) |
| scaling-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/scaling-8-0.log) |
| scheduled-backup-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/scheduled-backup-5-7.log) |
| scheduled-backup-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/scheduled-backup-8-0.log) |
| security-context-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/security-context-8-0.log) |
| smart-update1-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/smart-update1-8-0.log) |
| smart-update2-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/smart-update2-8-0.log) |
| storage-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/storage-8-0.log) |
| tls-issue-cert-manager-ref-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/tls-issue-cert-manager-ref-8-0.log) |
| tls-issue-cert-manager-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/tls-issue-cert-manager-8-0.log) |
| tls-issue-self-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/tls-issue-self-8-0.log) |
| upgrade-consistency-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/upgrade-consistency-8-0.log) |
| upgrade-haproxy-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/upgrade-haproxy-5-7.log) |
| upgrade-haproxy-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/upgrade-haproxy-8-0.log) |
| upgrade-proxysql-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/upgrade-proxysql-5-7.log) |
| upgrade-proxysql-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/upgrade-proxysql-8-0.log) |
| users-5-7 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/users-5-7.log) |
| users-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/users-8-0.log) |
| validation-hook-8-0 | [passed](https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator/PR-2018/810c4c52/validation-hook-8-0.log) |
| We run 44 out of 44|

commit: https://github.com/percona/percona-xtradb-cluster-operator/pull/2018/commits/810c4c5238e15be3fb0a5a6eac3aa82e23936190
image: `perconalab/percona-xtradb-cluster-operator:PR-2018-810c4c52`

author_association: COLLABORATOR