Repository: openshift/library-go
Pull request #1938: OCPBUGS-35102: certrotation: don't update metadata in RefreshOnlyWhenExpired mode
Author: login: vrutkovs
Labels: name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: Indicates that a PR is ready to be merged.
name: jira/valid-bug
description: Indicates that a referenced Jira bug is valid for the branch this PR is targeting.
name: jira/valid-reference
description: Indicates that this PR references a valid Jira ticket of any type.
Created: 2025-02-28T10:42:44Z | Merged: 2025-03-13T11:49:11Z

There are two copies of this controller running in the cluster: one in the operator with RefreshOnlyWhenExpired=false and one leader in kube-apiserver sidecar with RefreshOnlyWhenExpired=true. This method allows us to generate certs in operator before kube-apiserver pods are created - and regenerate expired certs after the cluster was shutdown.

However, regardless of the mode, this controller applies metadata changes.

So if an update introduces a change in some annotations these controller copies are racing, as there are different versions of the controller running - an updated one in the operator and outdated in kube-apiserver sidecar.

This change ensures that controllers in
RefreshOnlyWhenExpired=true mode don't update metadata. The primary goal is to have certificate contents fresh enough to run the control plane - and the operator will take care of the secondary things like secret/configmap metadata.


Diff:
diff --git a/pkg/operator/certrotation/cabundle.go b/pkg/operator/certrotation/cabundle.go
index 1cb4685b1f..9940490ed9 100644
--- a/pkg/operator/certrotation/cabundle.go
+++ b/pkg/operator/certrotation/cabundle.go
@@ -30,6 +30,10 @@ type CABundleConfigMap struct {
 	Namespace string
 	// Name is the name of the ConfigMap to maintain.
 	Name string
+	// RefreshOnlyWhenExpired set to true means to ignore 80% of validity and the Refresh duration for rotation,
+	// but only rotate when the certificate expires. This is useful for auto-recovery when we want to enforce
+	// rotation on expiration only, but not interfere with the ordinary rotation controller.
+	RefreshOnlyWhenExpired bool
 	// Owner is an optional reference to add to the secret that this rotator creates.
 	Owner *metav1.OwnerReference
 	// AdditionalAnnotations is a collection of annotations set for the secret
@@ -62,12 +66,15 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
 		creationRequired = true
 	}
 
-	needsOwnerUpdate := false
-	if c.Owner != nil {
-		needsOwnerUpdate = ensureOwnerReference(&caBundleConfigMap.ObjectMeta, c.Owner)
+	// run Update if metadata needs changing unless running in RefreshOnlyWhenExpired mode
+	if !c.RefreshOnlyWhenExpired {
+		needsOwnerUpdate := false
+		if c.Owner != nil {
+			needsOwnerUpdate = ensureOwnerReference(&caBundleConfigMap.ObjectMeta, c.Owner)
+		}
+		needsMetadataUpdate := c.AdditionalAnnotations.EnsureTLSMetadataUpdate(&caBundleConfigMap.ObjectMeta)
+		updateRequired = needsOwnerUpdate || needsMetadataUpdate
 	}
-	needsMetadataUpdate := c.AdditionalAnnotations.EnsureTLSMetadataUpdate(&caBundleConfigMap.ObjectMeta)
-	updateRequired = needsOwnerUpdate || needsMetadataUpdate
 
 	updatedCerts, err := manageCABundleConfigMap(caBundleConfigMap, signingCertKeyPair.Config.Certs[0])
 	if err != nil {
diff --git a/pkg/operator/certrotation/cabundle_test.go b/pkg/operator/certrotation/cabundle_test.go
index 76ea08311e..cf15bafff1 100644
--- a/pkg/operator/certrotation/cabundle_test.go
+++ b/pkg/operator/certrotation/cabundle_test.go
@@ -7,13 +7,13 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"errors"
-	clocktesting "k8s.io/utils/clock/testing"
 	"math/big"
-	"os"
 	"strings"
 	"testing"
 	"time"
 
+	clocktesting "k8s.io/utils/clock/testing"
+
 	"k8s.io/client-go/util/cert"
 
 	"github.com/davecgh/go-spew/spew"
@@ -30,11 +30,17 @@ import (
 )
 
 func TestEnsureConfigMapCABundle(t *testing.T) {
+	certs, err := newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	tests := []struct {
 		name string
 
-		initialConfigMapFn func() *corev1.ConfigMap
-		caFn               func() (*crypto.CA, error)
+		initialConfigMapFn     func() *corev1.ConfigMap
+		caFn                   func() (*crypto.CA, error)
+		RefreshOnlyWhenExpired bool
 
 		verifyActions func(t *testing.T, client *kubefake.Clientset)
 		expectedError string
@@ -44,7 +50,8 @@ func TestEnsureConfigMapCABundle(t *testing.T) {
 			caFn: func() (*crypto.CA, error) {
 				return newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
 			},
-			initialConfigMapFn: func() *corev1.ConfigMap { return nil },
+			initialConfigMapFn:     func() *corev1.ConfigMap { return nil },
+			RefreshOnlyWhenExpired: false,
 			verifyActions: func(t *testing.T, client *kubefake.Clientset) {
 				actions := client.Actions()
 				if len(actions) != 1 {
@@ -84,6 +91,7 @@ func TestEnsureConfigMapCABundle(t *testing.T) {
 				caBundleConfigMap.Data["ca-bundle.crt"] = string(caBytes)
 				return caBundleConfigMap
 			},
+			RefreshOnlyWhenExpired: false,
 			verifyActions: func(t *testing.T, client *kubefake.Clientset) {
 				actions := client.Actions()
 				if len(actions) != 1 {
@@ -115,6 +123,7 @@ func TestEnsureConfigMapCABundle(t *testing.T) {
 			caFn: func() (*crypto.CA, error) {
 				return newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
 			},
+			RefreshOnlyWhenExpired: false,
 			initialConfigMapFn: func() *corev1.ConfigMap {
 				caBundleConfigMap := &corev1.ConfigMap{
 					ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "trust-bundle", ResourceVersion: "10"},
@@ -158,24 +167,17 @@ func TestEnsureConfigMapCABundle(t *testing.T) {
 			},
 		},
 		{
-			name: "update remove duplicate",
+			name: "no update when RefreshOnlyWhenExpired set",
 			caFn: func() (*crypto.CA, error) {
-				return newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
+				return certs, nil
 			},
+			RefreshOnlyWhenExpired: true,
 			initialConfigMapFn: func() *corev1.ConfigMap {
 				caBundleConfigMap := &corev1.ConfigMap{
 					ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "trust-bundle", ResourceVersion: "10"},
 					Data:       map[string]string{},
 				}
-				certBytes, err := os.ReadFile("./testfiles/tls-expired.crt")
-				if err != nil {
-					t.Fatal(err)
-				}
-				certs, err := cert.ParseCertsPEM(certBytes)
-				if err != nil {
-					t.Fatal(err)
-				}
-				caBytes, err := crypto.EncodeCertificates(certs...)
+				caBytes, err := crypto.EncodeCertificates(certs.Config.Certs[0])
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -184,28 +186,9 @@ func TestEnsureConfigMapCABundle(t *testing.T) {
 			},
 			verifyActions: func(t *testing.T, client *kubefake.Clientset) {
 				actions := client.Actions()
-				if len(actions) != 1 {
+				if len(actions) != 0 {
 					t.Fatal(spew.Sdump(actions))
 				}
-
-				if !actions[0].Matches("update", "configmaps") {
-					t.Error(actions[0])
-				}
-
-				actual := actions[0].(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap)
-				if len(actual.Data["ca-bundle.crt"]) == 0 {
-					t.Error(actual.Data)
-				}
-				if certType, _ := CertificateTypeFromObject(actual); certType != CertificateTypeCABundle {
-					t.Errorf("expected certificate type 'ca-bundle', got: %v", certType)
-				}
-				result, err := cert.ParseCertsPEM([]byte(actual.Data["ca-bundle.crt"]))
-				if err != nil {
-					t.Fatal(err)
-				}
-				if len(result) != 1 {
-					t.Error(len(result))
-				}
 			},
 		},
 	}
diff --git a/pkg/operator/certrotation/signer.go b/pkg/operator/certrotation/signer.go
index 2eb761bbb3..6f4941de45 100644
--- a/pkg/operator/certrotation/signer.go
+++ b/pkg/operator/certrotation/signer.go
@@ -78,10 +78,12 @@ func (c RotatedSigningCASecret) EnsureSigningCertKeyPair(ctx context.Context) (*
 		creationRequired = true
 	}
 
-	// run Update if metadata needs changing
-	needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
-	needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
-	updateRequired = needsMetadataUpdate || needsTypeChange
+	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired mode
+	if !c.RefreshOnlyWhenExpired {
+		needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
+		needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
+		updateRequired = needsMetadataUpdate || needsTypeChange
+	}
 
 	// run Update if signer content needs changing
 	signerUpdated := false
diff --git a/pkg/operator/certrotation/signer_test.go b/pkg/operator/certrotation/signer_test.go
index d965ce3a95..bdcbf49bec 100644
--- a/pkg/operator/certrotation/signer_test.go
+++ b/pkg/operator/certrotation/signer_test.go
@@ -24,13 +24,15 @@ func TestEnsureSigningCertKeyPair(t *testing.T) {
 	tests := []struct {
 		name string
 
-		initialSecret *corev1.Secret
+		initialSecret          *corev1.Secret
+		RefreshOnlyWhenExpired bool
 
 		verifyActions func(t *testing.T, client *kubefake.Clientset, controllerUpdatedSecret bool)
 		expectedError string
 	}{
 		{
-			name: "initial create",
+			name:                   "initial create",
+			RefreshOnlyWhenExpired: false,
 			verifyActions: func(t *testing.T, client *kubefake.Clientset, controllerUpdatedSecret bool) {
 				t.Helper()
 				actions := client.Actions()
@@ -78,6 +80,7 @@ func TestEnsureSigningCertKeyPair(t *testing.T) {
 				Type:       corev1.SecretTypeTLS,
 				Data:       map[string][]byte{"tls.crt": {}, "tls.key": {}},
 			},
+			RefreshOnlyWhenExpired: false,
 			verifyActions: func(t *testing.T, client *kubefake.Clientset, controllerUpdatedSecret bool) {
 				t.Helper()
 				actions := client.Actions()
@@ -131,6 +134,32 @@ func TestEnsureSigningCertKeyPair(t *testing.T) {
 				Type: corev1.SecretTypeTLS,
 				Data: map[string][]byte{"tls.crt": {}, "tls.key": {}},
 			},
+			RefreshOnlyWhenExpired: false,
+			verifyActions: func(t *testing.T, client *kubefake.Clientset, controllerUpdatedSecret bool) {
+				t.Helper()
+				actions := client.Actions()
+				if len(actions) != 0 {
+					t.Fatal(spew.Sdump(actions))
+				}
+			},
+			expectedError: "certFile missing", // this means we tried to read the cert from the existing secret.  If we created one, we fail in the client check
+		},
+		{
+			name: "update with RefreshOnlyWhenExpired set",
+			initialSecret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace:       "ns",
+					Name:            "signer",
+					ResourceVersion: "10",
+					Annotations: map[string]string{
+						"auth.openshift.io/certificate-not-after":  "2108-09-08T22:47:31-07:00",
+						"auth.openshift.io/certificate-not-before": "2108-09-08T20:47:31-07:00",
+					},
+				},
+				Type: corev1.SecretTypeTLS,
+				Data: map[string][]byte{"tls.crt": {}, "tls.key": {}},
+			},
+			RefreshOnlyWhenExpired: true,
 			verifyActions: func(t *testing.T, client *kubefake.Clientset, controllerUpdatedSecret bool) {
 				t.Helper()
 				actions := client.Actions()
@@ -166,6 +195,7 @@ func TestEnsureSigningCertKeyPair(t *testing.T) {
 				Owner: &metav1.OwnerReference{
 					Name: "operator",
 				},
+				RefreshOnlyWhenExpired: test.RefreshOnlyWhenExpired,
 			}
 
 			_, updated, err := c.EnsureSigningCertKeyPair(context.TODO())
diff --git a/pkg/operator/certrotation/target.go b/pkg/operator/certrotation/target.go
index b68aea1633..deb5dfa335 100644
--- a/pkg/operator/certrotation/target.go
+++ b/pkg/operator/certrotation/target.go
@@ -112,9 +112,12 @@ func (c RotatedSelfSignedCertKeySecret) EnsureTargetCertKeyPair(ctx context.Cont
 		creationRequired = true
 	}
 
-	needsMetadataUpdate := ensureMetadataUpdate(targetCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
-	needsTypeChange := ensureSecretTLSTypeSet(targetCertKeyPairSecret)
-	updateRequired = needsMetadataUpdate || needsTypeChange
+	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired mode
+	if !c.RefreshOnlyWhenExpired {
+		needsMetadataUpdate := ensureMetadataUpdate(targetCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
+		needsTypeChange := ensureSecretTLSTypeSet(targetCertKeyPairSecret)
+		updateRequired = needsMetadataUpdate || needsTypeChange
+	}
 
 	if reason := c.CertCreator.NeedNewTargetCertKeyPair(targetCertKeyPairSecret, signingCertKeyPair, caBundleCerts, c.Refresh, c.RefreshOnlyWhenExpired, creationRequired); len(reason) > 0 {
 		c.EventRecorder.Eventf("TargetUpdateRequired", "%q in %q requires a new target cert/key pair: %v", c.Name, c.Namespace, reason)
diff --git a/pkg/operator/certrotation/target_test.go b/pkg/operator/certrotation/target_test.go
index c3ed58a8cd..9fff60f983 100644
--- a/pkg/operator/certrotation/target_test.go
+++ b/pkg/operator/certrotation/target_test.go
@@ -137,8 +137,9 @@ func TestEnsureTargetCertKeyPair(t *testing.T) {
 	tests := []struct {
 		name string
 
-		initialSecretFn func() *corev1.Secret
-		caFn            func() (*crypto.CA, error)
+		initialSecretFn        func() *corev1.Secret
+		caFn                   func() (*crypto.CA, error)
+		RefreshOnlyWhenExpired bool
 
 		verifyActions func(t *testing.T, client *kubefake.Clientset)
 		expectedError string
@@ -148,7 +149,8 @@ func TestEnsureTargetCertKeyPair(t *testing.T) {
 			caFn: func() (*crypto.CA, error) {
 				return newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
 			},
-			initialSecretFn: func() *corev1.Secret { return nil },
+			RefreshOnlyWhenExpired: false,
+			initialSecretFn:        func() *corev1.Secret { return nil },
 			verifyActions: func(t *testing.T, client *kubefake.Clientset) {
 				actions := client.Actions()
 				if len(actions) != 1 {
@@ -185,6 +187,7 @@ func TestEnsureTargetCertKeyPair(t *testing.T) {
 			caFn: func() (*crypto.CA, error) {
 				return newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
 			},
+			RefreshOnlyWhenExpired: false,
 			initialSecretFn: func() *corev1.Secret {
 				caBundleSecret := &corev1.Secret{
 					ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "target-secret", ResourceVersion: "10"},
@@ -228,6 +231,37 @@ func TestEnsureTargetCertKeyPair(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "no update when RefreshOnlyWhenExpired set",
+			caFn: func() (*crypto.CA, error) {
+				return newTestCACertificate(pkix.Name{CommonName: "signer-tests"}, int64(1), metav1.Duration{Duration: time.Hour * 24 * 60}, time.Now)
+			},
+			RefreshOnlyWhenExpired: true,
+			initialSecretFn: func() *corev1.Secret {
+				caBundleSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace:       "ns",
+						Name:            "target-secret",
+						ResourceVersion: "10",
+						Annotations: map[string]string{
+							"auth.openshift.io/certificate-not-after":  "2108-09-08T22:47:31-07:00",
+							"auth.openshift.io/certificate-not-before": "2108-09-08T20:47:31-07:00",
+							"auth.openshift.io/certificate-issuer":     "signer-tests",
+							"auth.openshift.io/certificate-hostnames":  "foo,bar",
+						},
+					},
+					Data: map[string][]byte{},
+					Type: corev1.SecretTypeTLS,
+				}
+				return caBundleSecret
+			},
+			verifyActions: func(t *testing.T, client *kubefake.Clientset) {
+				actions := client.Actions()
+				if len(actions) != 0 {
+					t.Fatal(spew.Sdump(actions))
+				}
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -258,6 +292,7 @@ func TestEnsureTargetCertKeyPair(t *testing.T) {
 				Owner: &metav1.OwnerReference{
 					Name: "operator",
 				},
+				RefreshOnlyWhenExpired: test.RefreshOnlyWhenExpired,
 			}
 
 			newCA, err := test.caFn()


Reviews:
user: login: benluddy
body: This makes sense as a clear backportable mitigation for annotation flapping. The description of OCPBUGS-35102 mentions some other approaches that might be better, but not backportable. Are we planning more work after the flapping issue is mitigated in 4.18?
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T15:29:33Z
user: login: vrutkovs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T15:36:08Z
user: login: vrutkovs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T15:37:28Z
user: login: vrutkovs
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T16:41:06Z

Review comments:
diff_hunk: @@ -78,10 +78,12 @@ func (c RotatedSigningCASecret) EnsureSigningCertKeyPair(ctx context.Context) (*
 		creationRequired = true
 	}
 
-	// run Update if metadata needs changing
-	needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
-	needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
-	updateRequired = needsMetadataUpdate || needsTypeChange
+	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired malformed
path: pkg/operator/certrotation/signer.go
user: login: benluddy
body: ```suggestion
	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired mode
```
created_at: 2025-03-05T15:08:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -112,9 +112,12 @@ func (c RotatedSelfSignedCertKeySecret) EnsureTargetCertKeyPair(ctx context.Cont
 		creationRequired = true
 	}
 
-	needsMetadataUpdate := ensureMetadataUpdate(targetCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
-	needsTypeChange := ensureSecretTLSTypeSet(targetCertKeyPairSecret)
-	updateRequired = needsMetadataUpdate || needsTypeChange
+	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired malformed
path: pkg/operator/certrotation/target.go
user: login: benluddy
body: ```suggestion
	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired mode
```
created_at: 2025-03-05T15:08:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -62,12 +66,15 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
 		creationRequired = true
 	}
 
-	needsOwnerUpdate := false
-	if c.Owner != nil {
-		needsOwnerUpdate = ensureOwnerReference(&caBundleConfigMap.ObjectMeta, c.Owner)
+	// run Update if metadata needs changing unless running in RefreshOnlyWhenExpired mode
path: pkg/operator/certrotation/cabundle.go
user: login: benluddy
body: Will there be unit tests covering the three special cases?
created_at: 2025-03-05T15:21:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,10 +78,12 @@ func (c RotatedSigningCASecret) EnsureSigningCertKeyPair(ctx context.Context) (*
 		creationRequired = true
 	}
 
-	// run Update if metadata needs changing
-	needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
-	needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
-	updateRequired = needsMetadataUpdate || needsTypeChange
+	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired malformed
+	if !c.RefreshOnlyWhenExpired {
+		needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
+		needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
path: pkg/operator/certrotation/signer.go
user: login: benluddy
body: I guess type changes aren't involved in the observed flapping, but in practice it won't matter and is simpler to exclude?
created_at: 2025-03-05T15:29:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -62,12 +66,15 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
 		creationRequired = true
 	}
 
-	needsOwnerUpdate := false
-	if c.Owner != nil {
-		needsOwnerUpdate = ensureOwnerReference(&caBundleConfigMap.ObjectMeta, c.Owner)
+	// run Update if metadata needs changing unless running in RefreshOnlyWhenExpired mode
path: pkg/operator/certrotation/cabundle.go
user: login: vrutkovs
body: Good idea, I'll add some
created_at: 2025-03-05T15:36:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,10 +78,12 @@ func (c RotatedSigningCASecret) EnsureSigningCertKeyPair(ctx context.Context) (*
 		creationRequired = true
 	}
 
-	// run Update if metadata needs changing
-	needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
-	needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
-	updateRequired = needsMetadataUpdate || needsTypeChange
+	// run Update if metadata needs changing unless we're in RefreshOnlyWhenExpired malformed
+	if !c.RefreshOnlyWhenExpired {
+		needsMetadataUpdate := ensureMetadataUpdate(signingCertKeyPairSecret, c.Owner, c.AdditionalAnnotations)
+		needsTypeChange := ensureSecretTLSTypeSet(signingCertKeyPairSecret)
path: pkg/operator/certrotation/signer.go
user: login: vrutkovs
body: Yes, I can't think of a way of type change issue coming back, but we want to guard against entire `updateRequire` changing anyway
created_at: 2025-03-05T15:37:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -62,12 +66,15 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
 		creationRequired = true
 	}
 
-	needsOwnerUpdate := false
-	if c.Owner != nil {
-		needsOwnerUpdate = ensureOwnerReference(&caBundleConfigMap.ObjectMeta, c.Owner)
+	// run Update if metadata needs changing unless running in RefreshOnlyWhenExpired mode
path: pkg/operator/certrotation/cabundle.go
user: login: vrutkovs
body: Done
created_at: 2025-03-05T16:41:06Z
author_association: CONTRIBUTOR

PR comments:
user: login: openshift-ci-robot
created_at: 2025-02-28T10:42:49Z
body: @vrutkovs: This pull request references [Jira Issue OCPBUGS-35102](https://issues.redhat.com//browse/OCPBUGS-35102), which is invalid:
 - expected the bug to target either version "4.19.*" or "openshift-4.19.*", but it targets "4.18.z" instead

Comment <code>/jira refresh</code> to re-evaluate validity if changes to the Jira bug are made, or edit the title of this pull request to link to a different bug.

The bug has been updated to refer to the pull request using the external bug tracker.

<details>

In response to [this](https://github.com/openshift/library-go/pull/1938):

>There are two copies of this controller running in the cluster: one in the operator with RefreshOnlyWhenExpired=false and one leader in kube-apiserver sidecar with RefreshOnlyWhenExpired=true. This method allows us to generate certs in operator before kube-apiserver pods are created - and regenerate expired certs after the cluster was shutdown.
>
>However, regardless of the mode, this controller applies metadata changes.
>
>So if an update introduces a change in some annotations these controller copies are racing, as there are different versions of the controller running - an updated one in the operator and outdated in kube-apiserver sidecar.
>
>This change ensures that controllers in
>RefreshOnlyWhenExpired=true mode don't update metadata. The primary goal is to have certificate contents fresh enough to run the control plane - and the operator will take care of the secondary things like secret/configmap metadata.


Instructions for interacting with me using PR comments are available [here](https://prow.ci.openshift.org/command-help?repo=openshift%2Flibrary-go).  If you have questions or suggestions related to my behavior, please file an issue against the [openshift-eng/jira-lifecycle-plugin](https://github.com/openshift-eng/jira-lifecycle-plugin/issues/new) repository.
</details>
author_association: NONE
user: login: vrutkovs
created_at: 2025-02-28T10:56:54Z
body: /cherrypick release-4.18
author_association: CONTRIBUTOR
user: login: openshift-cherrypick-robot
created_at: 2025-02-28T10:56:57Z
body: @vrutkovs: once the present PR merges, I will cherry-pick it on top of `release-4.18` in a new PR and assign it to you.

<details>

In response to [this](https://github.com/openshift/library-go/pull/1938#issuecomment-2690345967):

>/cherrypick release-4.18


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE
user: login: vrutkovs
created_at: 2025-03-05T15:35:42Z
body: >The description of OCPBUGS-35102 mentions some other approaches that might be better, but not backportable

Yes, another idea is to have a proper leader election - but previously we've seen it unreliable, in 1.29 in case of emergency it did give back leader status, so this solution is now left as last resort option
author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-03-05T17:05:57Z
body: @vrutkovs: all tests **passed!**


[Full PR test history](https://prow.ci.openshift.org/pr-history?org=openshift&repo=library-go&pr=1938). [Your PR dashboard](https://prow.ci.openshift.org/pr?query=is:pr+state:open+author:vrutkovs).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
author_association: CONTRIBUTOR
user: login: vrutkovs
created_at: 2025-03-11T09:50:18Z
body: /jira refresh
author_association: CONTRIBUTOR
user: login: openshift-ci-robot
created_at: 2025-03-11T09:50:23Z
body: @vrutkovs: This pull request references [Jira Issue OCPBUGS-35102](https://issues.redhat.com//browse/OCPBUGS-35102), which is valid.

<details><summary>3 validation(s) were run on this bug</summary>

* bug is open, matching expected state (open)
* bug target version (4.19.0) matches configured target version for branch (4.19.0)
* bug is in the state POST, which is one of the valid states (NEW, ASSIGNED, POST)</details>

<details>

In response to [this](https://github.com/openshift/library-go/pull/1938#issuecomment-2713414150):

>/jira refresh


Instructions for interacting with me using PR comments are available [here](https://prow.ci.openshift.org/command-help?repo=openshift%2Flibrary-go).  If you have questions or suggestions related to my behavior, please file an issue against the [openshift-eng/jira-lifecycle-plugin](https://github.com/openshift-eng/jira-lifecycle-plugin/issues/new) repository.
</details>
author_association: NONE
user: login: hasbro17
created_at: 2025-03-12T17:50:59Z
body: /lgtm
author_association: NONE
user: login: jsafrane
created_at: 2025-03-13T11:45:00Z
body: /approve
author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-03-13T11:45:16Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/openshift/library-go/pull/1938#issuecomment-2718667284" title="LGTM">hasbro17</a>*, *<a href="https://github.com/openshift/library-go/pull/1938#issuecomment-2720977701" title="Approved">jsafrane</a>*, *<a href="https://github.com/openshift/library-go/pull/1938#" title="Author self-approved">vrutkovs</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=openshift%2Flibrary-go).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[pkg/operator/OWNERS](https://github.com/openshift/library-go/blob/master/pkg/operator/OWNERS)~~ [jsafrane]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: openshift-ci-robot
created_at: 2025-03-13T11:49:16Z
body: @vrutkovs: [Jira Issue OCPBUGS-35102](https://issues.redhat.com//browse/OCPBUGS-35102): Some pull requests linked via external trackers have merged:
 * [openshift/library-go#1724](https://github.com/openshift/library-go/pull/1724)
 * [openshift/origin#28639](https://github.com/openshift/origin/pull/28639)
 * [openshift/release#53573](https://github.com/openshift/release/pull/53573)
 * [openshift/library-go#1772](https://github.com/openshift/library-go/pull/1772)

The following pull requests linked via external trackers have not merged:
 * [openshift/library-go#1939](https://github.com/openshift/library-go/pull/1939) is open

These pull request must merge or be unlinked from the Jira bug in order for it to move to the next state. Once unlinked, request a bug refresh with <code>/jira refresh</code>.

[Jira Issue OCPBUGS-35102](https://issues.redhat.com//browse/OCPBUGS-35102) has not been moved to the MODIFIED state.

<details>

In response to [this](https://github.com/openshift/library-go/pull/1938):

>There are two copies of this controller running in the cluster: one in the operator with RefreshOnlyWhenExpired=false and one leader in kube-apiserver sidecar with RefreshOnlyWhenExpired=true. This method allows us to generate certs in operator before kube-apiserver pods are created - and regenerate expired certs after the cluster was shutdown.
>
>However, regardless of the mode, this controller applies metadata changes.
>
>So if an update introduces a change in some annotations these controller copies are racing, as there are different versions of the controller running - an updated one in the operator and outdated in kube-apiserver sidecar.
>
>This change ensures that controllers in
>RefreshOnlyWhenExpired=true mode don't update metadata. The primary goal is to have certificate contents fresh enough to run the control plane - and the operator will take care of the secondary things like secret/configmap metadata.


Instructions for interacting with me using PR comments are available [here](https://prow.ci.openshift.org/command-help?repo=openshift%2Flibrary-go).  If you have questions or suggestions related to my behavior, please file an issue against the [openshift-eng/jira-lifecycle-plugin](https://github.com/openshift-eng/jira-lifecycle-plugin/issues/new) repository.
</details>
author_association: NONE
user: login: openshift-cherrypick-robot
created_at: 2025-03-13T11:50:09Z
body: @vrutkovs: new pull request created: #1945

<details>

In response to [this](https://github.com/openshift/library-go/pull/1938#issuecomment-2690345967):

>/cherrypick release-4.18


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE