Repository: Azure/ARO-RP
Pull request #2854 (2023-04-19 00:26:58)

Rewrite some of the cert/key encoding and decoding
### Which issue this PR addresses:
Splitting out some of #2842 that is unrelated

### What this PR does / why we need it:

Rewrites some of the PEM encoding/decoding which will be used when the installer dependencies are removed

### Test plan for issue:
Unit tests, should mostly be a no-op

### Is there any documentation that needs to be updated for this PR?
N/A



Patch:
diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml
index ed3460ee0..0a16ae7c4 100644
--- a/.github/workflows/yamllint.yml
+++ b/.github/workflows/yamllint.yml
@@ -1,42 +1,17 @@
 ---
 # MegaLinter GitHub Action configuration file
-# More info at https://megalinter.github.io
+# More info at https://oxsecurity.github.io/megalinter
 name: mega-linter-yaml
 
 on:
-  # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main
-  push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
   pull_request:
 
-env: # Comment env block if you do not want to apply fixes
-  APPLY_FIXES: none # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
-
 jobs:
   build:
     name: mega-linter-yaml
     runs-on: ubuntu-latest
     steps:
-      # Git Checkout
       - name: Checkout Code
-        uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
-          fetch-depth: 0
-
-      # MegaLinter
+        uses: actions/checkout@v3
       - name: yamllint
-        id: ml
-        # You can override MegaLinter flavor used to have faster performances
-        # More info at https://megalinter.github.io/flavors/
-        uses: megalinter/megalinter/flavors/ci_light@v5
-        env:
-          # All available variables are described in documentation
-          # https://megalinter.github.io/configuration/
-          VALIDATE_ALL_CODEBASE: true
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY
-          # DISABLE: COPYPASTE,SPELL # Uncomment to disable copy-paste and spell checks
-          ENABLE_LINTERS: YAML_YAMLLINT
-            #          YAML_YAMLLINT_CONFIG_FILE: .yaml-lint.yml
-            #          YAML_YAMLLINT_RULES_PATH: .github/linters
-          PRINT_ALPACA: false
+        uses: oxsecurity/megalinter/flavors/ci_light@v6
diff --git a/.gitignore b/.gitignore
index a32a9d817..cb0449513 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,4 @@ portal/v2/.vscode/
 /hack/hive-config/crds
 /hack/hive-config/hive-deployment.yaml
 cmd/aro/__debug_bin
+megalinter-reports/
diff --git a/.mega-linter.yml b/.mega-linter.yml
index 7b8c31a77..10a0a8303 100644
--- a/.mega-linter.yml
+++ b/.mega-linter.yml
@@ -1,14 +1,15 @@
 # Configuration file for MegaLinter
-# See all available variables at https://megalinter.github.io/configuration/ and in linters documentation
+# See all available variables at https://oxsecurity.github.io/megalinter/configuration/ and in linters documentation
 
 APPLY_FIXES: none
-# ENABLE:  # If you use ENABLE variable, all other languages/formats/tooling formats will be disabled by default
 ENABLE_LINTERS:
-  - YAML_YAMLLINT # If you use ENABLE_LINTERS variable, all other linters will be disabled by default
+  - YAML_YAMLLINT
 EXCLUDED_DIRECTORIES:
-  - vendor
+  - "vendor"
+  - "node_modules"
 YAML_YAMLLINT_CONFIG_FILE: .yaml-lint.yml
 LINTER_RULES_PATH: .
 PRINT_ALPACA: false
 GITHUB_STATUS_REPORTER: false
 GITHUB_COMMENT_REPORTER: false
+VALIDATE_ALL_CODEBASE: true
diff --git a/Dockerfile.guardrails b/Dockerfile.guardrails
new file mode 100644
index 000000000..341678ff2
--- /dev/null
+++ b/Dockerfile.guardrails
@@ -0,0 +1,33 @@
+ARG REGISTRY
+FROM ${REGISTRY}/ubi8/go-toolset:1.18.10-1 AS builder
+ARG GATEKEEPER_VERSION
+ENV DOWNLOAD_URL=https://github.com/open-policy-agent/gatekeeper/archive/v${GATEKEEPER_VERSION}.tar.gz
+
+ARG BUILDKIT_SBOM_SCAN_STAGE=true
+
+ENV GO111MODULE=on \
+    GOOS=linux \
+    GOPATH=/go/ \
+    GOARCH=amd64 \
+    CGO_ENABLED=0
+
+WORKDIR ${GOPATH}/src/github.com/open-policy-agent/gatekeeper
+USER root
+RUN curl -Lq $DOWNLOAD_URL | tar -xz --strip-components=1
+
+RUN go build -mod vendor -a -ldflags "-X github.com/open-policy-agent/gatekeeper/pkg/version.Version=latest" -o manager
+
+#### Runtime container
+FROM ${REGISTRY}/ubi8/ubi-minimal:latest
+
+ENV USER_UID=1001 \
+    USER_NAME=guardrails-operator
+
+WORKDIR /
+RUN microdnf update && microdnf clean all
+COPY --from=builder /go/src/github.com/open-policy-agent/gatekeeper/manager .
+ENTRYPOINT ["/manager"]
+
+USER ${USER_UID}
+LABEL io.openshift.managed.name="guardrails-operator" \
+      io.openshift.managed.description="Operator to enforce guardrails policies for Openshift version 4 clusters"
diff --git a/Makefile b/Makefile
index e8307416b..5ae2f229f 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,8 @@ FLUENTBIT_VERSION = 1.9.10
 FLUENTBIT_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/fluentbit:$(FLUENTBIT_VERSION)-cm$(MARINER_VERSION)
 AUTOREST_VERSION = 3.6.3
 AUTOREST_IMAGE = quay.io/openshift-on-azure/autorest:${AUTOREST_VERSION}
+GATEKEEPER_VERSION = 3.10.0
+GATEKEEPER_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/guardrails:$(GATEKEEPER_VERSION)
 
 ifneq ($(shell uname -s),Darwin)
     export CGO_CFLAGS=-Dgpgme_off_t=off_t
@@ -102,6 +104,9 @@ image-proxy:
 	docker pull $(REGISTRY)/ubi8/ubi-minimal
 	docker build --platform=linux/amd64 --no-cache -f Dockerfile.proxy -t $(REGISTRY)/proxy:latest --build-arg REGISTRY=$(REGISTRY) .
 
+image-guardrails:
+	docker build --platform=linux/amd64 --network=host --build-arg GATEKEEPER_VERSION=$(GATEKEEPER_VERSION) --build-arg REGISTRY=$(REGISTRY) -f Dockerfile.guardrails -t $(GATEKEEPER_IMAGE) .
+
 publish-image-aro: image-aro
 	docker push $(ARO_IMAGE)
 ifeq ("${RP_IMAGE_ACR}-$(BRANCH)","arointsvc-master")
@@ -125,6 +130,9 @@ publish-image-fluentbit: image-fluentbit
 publish-image-proxy: image-proxy
 	docker push ${RP_IMAGE_ACR}.azurecr.io/proxy:latest
 
+publish-image-guardrails: image-guardrails
+	docker push $(GATEKEEPER_IMAGE)
+
 image-e2e:
 	docker build --platform=linux/amd64 --network=host --no-cache -f Dockerfile.aro-e2e -t $(ARO_IMAGE) --build-arg REGISTRY=$(REGISTRY) .
 
diff --git a/pkg/api/validate/vm.go b/pkg/api/validate/vm.go
index 72ecf7907..dd4a9af53 100644
--- a/pkg/api/validate/vm.go
+++ b/pkg/api/validate/vm.go
@@ -78,7 +78,7 @@ var supportedMasterVmSizes = map[api.VMSize]api.VMSizeStruct{
 	api.VMSizeStandardE32asV4: api.VMSizeStandardE32asV4Struct,
 	api.VMSizeStandardE48asV4: api.VMSizeStandardE48asV4Struct,
 	api.VMSizeStandardE64asV4: api.VMSizeStandardE64asV4Struct,
-	api.VMSizeStandardE96asV4: api.VMSizeStandardE64asV4Struct,
+	api.VMSizeStandardE96asV4: api.VMSizeStandardE96asV4Struct,
 
 	api.VMSizeStandardE8asV5:  api.VMSizeStandardE8asV5Struct,
 	api.VMSizeStandardE16asV5: api.VMSizeStandardE16asV5Struct,
@@ -168,7 +168,7 @@ var supportedWorkerVmSizes = map[api.VMSize]api.VMSizeStruct{
 	api.VMSizeStandardE32asV4: api.VMSizeStandardE32asV4Struct,
 	api.VMSizeStandardE48asV4: api.VMSizeStandardE48asV4Struct,
 	api.VMSizeStandardE64asV4: api.VMSizeStandardE64asV4Struct,
-	api.VMSizeStandardE96asV4: api.VMSizeStandardE64asV4Struct,
+	api.VMSizeStandardE96asV4: api.VMSizeStandardE96asV4Struct,
 
 	api.VMSizeStandardE8asV5:  api.VMSizeStandardE8asV5Struct,
 	api.VMSizeStandardE16asV5: api.VMSizeStandardE16asV5Struct,
diff --git a/pkg/bootstraplogging/bootstraplogging.go b/pkg/bootstraplogging/bootstraplogging.go
index 115aa8ec4..f55e166b2 100644
--- a/pkg/bootstraplogging/bootstraplogging.go
+++ b/pkg/bootstraplogging/bootstraplogging.go
@@ -9,7 +9,7 @@ import (
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/env"
-	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
+	utilpem "github.com/Azure/ARO-RP/pkg/util/pem"
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
@@ -23,12 +23,12 @@ func GetConfig(env env.Interface, oc *api.OpenShiftCluster) (*bootstraplogging.C
 
 	key, cert := env.ClusterGenevaLoggingSecret()
 
-	gcsKeyBytes, err := utiltls.PrivateKeyAsBytes(key)
+	gcsKeyBytes, err := utilpem.Encode(key)
 	if err != nil {
 		return nil, err
 	}
 
-	gcsCertBytes, err := utiltls.CertAsBytes(cert)
+	gcsCertBytes, err := utilpem.Encode(cert)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index e645ba1a1..9fa67d844 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -36,11 +36,11 @@ import (
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging"
 	"github.com/Azure/ARO-RP/pkg/util/dynamichelper"
+	utilpem "github.com/Azure/ARO-RP/pkg/util/pem"
 	"github.com/Azure/ARO-RP/pkg/util/pullsecret"
 	"github.com/Azure/ARO-RP/pkg/util/ready"
 	"github.com/Azure/ARO-RP/pkg/util/restconfig"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
-	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
 )
 
 //go:embed staticresources
@@ -172,12 +172,12 @@ func (o *operator) resources() ([]kruntime.Object, error) {
 
 	// then dynamic resources
 	key, cert := o.env.ClusterGenevaLoggingSecret()
-	gcsKeyBytes, err := utiltls.PrivateKeyAsBytes(key)
+	gcsKeyBytes, err := utilpem.Encode(key)
 	if err != nil {
 		return nil, err
 	}
 
-	gcsCertBytes, err := utiltls.CertAsBytes(cert)
+	gcsCertBytes, err := utilpem.Encode(cert)
 	if err != nil {
 		return nil, err
 	}
@@ -367,11 +367,11 @@ func (o *operator) CreateOrUpdate(ctx context.Context) error {
 
 func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
 	key, cert := o.env.ClusterGenevaLoggingSecret()
-	gcsKeyBytes, err := utiltls.PrivateKeyAsBytes(key)
+	gcsKeyBytes, err := utilpem.Encode(key)
 	if err != nil {
 		return err
 	}
-	gcsCertBytes, err := utiltls.CertAsBytes(cert)
+	gcsCertBytes, err := utilpem.Encode(cert)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/util/pem/pem.go b/pkg/util/pem/decode.go
similarity index 51%
rename from pkg/util/pem/pem.go
rename to pkg/util/pem/decode.go
index dc9f9be92..e55df12e1 100644
--- a/pkg/util/pem/pem.go
+++ b/pkg/util/pem/decode.go
@@ -7,9 +7,34 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
+	"errors"
 	"fmt"
 )
 
+func parsePrivateKey(block *pem.Block) (*rsa.PrivateKey, error) {
+	var key *rsa.PrivateKey
+	// try PKCS1
+	key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+	if err == nil {
+		// the key is PKCS1, return it
+		return key, nil
+	}
+
+	// if it's not PKCS1, try PKCS8
+	k, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, errors.New("private key is not PKCS#1 or PKCS#8")
+	}
+
+	var ok bool
+	key, ok = k.(*rsa.PrivateKey)
+	if !ok {
+		return nil, fmt.Errorf("unimplemented private key type %T in PKCS#8 wrapping", k)
+	}
+
+	return key, nil
+}
+
 func Parse(b []byte) (key *rsa.PrivateKey, certs []*x509.Certificate, err error) {
 	for {
 		var block *pem.Block
@@ -19,23 +44,11 @@ func Parse(b []byte) (key *rsa.PrivateKey, certs []*x509.Certificate, err error)
 		}
 
 		switch block.Type {
-		case "RSA PRIVATE KEY":
-			key, err = x509.ParsePKCS1PrivateKey(block.Bytes)
+		case "RSA PRIVATE KEY", "PRIVATE KEY":
+			key, err = parsePrivateKey(block)
 			if err != nil {
 				return nil, nil, err
 			}
-
-		case "PRIVATE KEY":
-			k, err := x509.ParsePKCS8PrivateKey(block.Bytes)
-			if err != nil {
-				return nil, nil, err
-			}
-			var ok bool
-			key, ok = k.(*rsa.PrivateKey)
-			if !ok {
-				return nil, nil, fmt.Errorf("unimplemented private key type %T in PKCS#8 wrapping", k)
-			}
-
 		case "CERTIFICATE":
 			c, err := x509.ParseCertificate(block.Bytes)
 			if err != nil {
diff --git a/pkg/util/pem/encode.go b/pkg/util/pem/encode.go
new file mode 100644
index 000000000..356a5a4ae
--- /dev/null
+++ b/pkg/util/pem/encode.go
@@ -0,0 +1,45 @@
+package pem
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+)
+
+type Encodable interface {
+	*x509.Certificate | *x509.CertificateRequest | *rsa.PublicKey | *rsa.PrivateKey
+}
+
+func Encode[V Encodable](inputs ...V) (r []byte, err error) {
+	for _, i := range inputs {
+		pemType := "UNKNOWN"
+		var rawBytes []byte
+
+		switch t := any(i).(type) {
+		case *x509.Certificate:
+			pemType = "CERTIFICATE"
+			rawBytes = t.Raw
+		case *x509.CertificateRequest:
+			pemType = "CERTIFICATE REQUEST"
+			rawBytes = t.Raw
+		case *rsa.PrivateKey:
+			pemType = "RSA PRIVATE KEY"
+			rawBytes = x509.MarshalPKCS1PrivateKey(t)
+		case *rsa.PublicKey:
+			pemType = "RSA PUBLIC KEY"
+			rawBytes, err = x509.MarshalPKIXPublicKey(t)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("unable to identify %v", t)
+		}
+
+		r = append(r, pem.EncodeToMemory(&pem.Block{Type: pemType, Bytes: rawBytes})...)
+	}
+	return
+}
diff --git a/pkg/util/tls/marshal.go b/pkg/util/tls/marshal.go
deleted file mode 100644
index 36d8bff22..000000000
--- a/pkg/util/tls/marshal.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package tls
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"bytes"
-	"crypto/rsa"
-	"crypto/x509"
-	"encoding/pem"
-	"fmt"
-)
-
-func CertAsBytes(certs ...*x509.Certificate) (b []byte, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			b, err = nil, fmt.Errorf("CertAsBytes: %v", r)
-		}
-	}()
-
-	buf := &bytes.Buffer{}
-	for _, cert := range certs {
-		err = pem.Encode(buf, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return buf.Bytes(), nil
-}
-
-func PrivateKeyAsBytes(key *rsa.PrivateKey) (b []byte, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			b, err = nil, fmt.Errorf("PrivateKeyAsBytes: %v", r)
-		}
-	}()
-
-	buf := &bytes.Buffer{}
-
-	err = pem.Encode(buf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	if err != nil {
-		return nil, err
-	}
-
-	return buf.Bytes(), nil
-}


Test patch:
diff --git a/pkg/env/certificateRefresher_test.go b/pkg/env/certificateRefresher_test.go
index a9ca10cd5..aeb1706e2 100644
--- a/pkg/env/certificateRefresher_test.go
+++ b/pkg/env/certificateRefresher_test.go
@@ -117,33 +117,6 @@ PeihXgcD511I3eMq7A8r7+xlNFEFYaleYwmoYSOt0cBGtoZttXDYacR/xvEkVaxO
 1LQ=
 -----END CERTIFICATE-----`
 
-// equalPrivKey is added manually, it has been added to go1.15, whilst ARO uses go1.14
-// TODO: Remove once ARO is on go1.15
-func equalPrivKey(one, other *rsa.PrivateKey) bool {
-	if one == nil && other == nil {
-		return true
-	}
-
-	pubEqual := one.PublicKey.N.Cmp(other.PublicKey.N) == 0 && one.PublicKey.E == other.PublicKey.E
-	expEqual := one.D.Cmp(other.D) == 0
-	if !pubEqual || !expEqual {
-		return false
-	}
-
-	primesEqual := len(one.Primes) == len(other.Primes)
-	if !primesEqual {
-		return false
-	}
-
-	for i := range one.Primes {
-		if one.Primes[i].Cmp(other.Primes[i]) != 0 {
-			return false
-		}
-	}
-
-	return true
-}
-
 func TestRefreshingCertificate(t *testing.T) {
 	mockController := gomock.NewController(t)
 
@@ -311,7 +284,7 @@ func TestRefreshingCertificate(t *testing.T) {
 			tick(cancel)
 
 			testkey, testCerts := refreshing.GetCertificates()
-			if !equalPrivKey(testkey, test.wantKey) {
+			if !testkey.Equal(test.wantKey) {
 				t.Error("returned private key does not match")
 			}
 
diff --git a/pkg/portal/kubeconfig/proxy_test.go b/pkg/portal/kubeconfig/proxy_test.go
index 0c150dea3..1b22e6220 100644
--- a/pkg/portal/kubeconfig/proxy_test.go
+++ b/pkg/portal/kubeconfig/proxy_test.go
@@ -26,6 +26,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/azureclient"
 	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
 	mock_proxy "github.com/Azure/ARO-RP/pkg/util/mocks/proxy"
+	utilpem "github.com/Azure/ARO-RP/pkg/util/pem"
 	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
 	testdatabase "github.com/Azure/ARO-RP/test/database"
 	"github.com/Azure/ARO-RP/test/util/listener"
@@ -73,17 +74,17 @@ func testKubeconfig(cacerts []*x509.Certificate, clientkey *rsa.PrivateKey, clie
 	}
 
 	var err error
-	kc.AuthInfos[0].AuthInfo.ClientKeyData, err = utiltls.PrivateKeyAsBytes(clientkey)
+	kc.AuthInfos[0].AuthInfo.ClientKeyData, err = utilpem.Encode(clientkey)
 	if err != nil {
 		return nil, err
 	}
 
-	kc.AuthInfos[0].AuthInfo.ClientCertificateData, err = utiltls.CertAsBytes(clientcerts[0])
+	kc.AuthInfos[0].AuthInfo.ClientCertificateData, err = utilpem.Encode(clientcerts[0])
 	if err != nil {
 		return nil, err
 	}
 
-	kc.Clusters[0].Cluster.CertificateAuthorityData, err = utiltls.CertAsBytes(cacerts[0])
+	kc.Clusters[0].Cluster.CertificateAuthorityData, err = utilpem.Encode(cacerts[0])
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/portal/prometheus/proxy_test.go b/pkg/portal/prometheus/proxy_test.go
index 72223a71e..487d4a477 100644
--- a/pkg/portal/prometheus/proxy_test.go
+++ b/pkg/portal/prometheus/proxy_test.go
@@ -32,6 +32,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
 	"github.com/Azure/ARO-RP/pkg/portal/util/responsewriter"
 	mock_proxy "github.com/Azure/ARO-RP/pkg/util/mocks/proxy"
+	utilpem "github.com/Azure/ARO-RP/pkg/util/pem"
 	"github.com/Azure/ARO-RP/pkg/util/portforward"
 	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
 	testdatabase "github.com/Azure/ARO-RP/test/database"
@@ -150,17 +151,17 @@ func testKubeconfig(cacerts []*x509.Certificate, clientkey *rsa.PrivateKey, clie
 	}
 
 	var err error
-	kc.AuthInfos[0].AuthInfo.ClientKeyData, err = utiltls.PrivateKeyAsBytes(clientkey)
+	kc.AuthInfos[0].AuthInfo.ClientKeyData, err = utilpem.Encode(clientkey)
 	if err != nil {
 		return nil, err
 	}
 
-	kc.AuthInfos[0].AuthInfo.ClientCertificateData, err = utiltls.CertAsBytes(clientcerts[0])
+	kc.AuthInfos[0].AuthInfo.ClientCertificateData, err = utilpem.Encode(clientcerts[0])
 	if err != nil {
 		return nil, err
 	}
 
-	kc.Clusters[0].Cluster.CertificateAuthorityData, err = utiltls.CertAsBytes(cacerts[0])
+	kc.Clusters[0].Cluster.CertificateAuthorityData, err = utilpem.Encode(cacerts[0])
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/util/clientauthorizer/subject_name_and_issuer_test.go b/pkg/util/clientauthorizer/subject_name_and_issuer_test.go
index 19804385b..1cda51498 100644
--- a/pkg/util/clientauthorizer/subject_name_and_issuer_test.go
+++ b/pkg/util/clientauthorizer/subject_name_and_issuer_test.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	utilpem "github.com/Azure/ARO-RP/pkg/util/pem"
 	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
 )
 
@@ -129,7 +130,7 @@ func TestSubjectNameAndIssuer(t *testing.T) {
 						return nil, errors.New("noop")
 					}
 
-					return utiltls.CertAsBytes(validCaCerts...)
+					return utilpem.Encode(validCaCerts...)
 				},
 			}
 			err := authorizer.readCABundle(caBundlePath)
@@ -171,7 +172,7 @@ func TestSubjectNameAndIssuerReadCABundle(t *testing.T) {
 		{
 			name: "valid ca cert",
 			readFile: func(string) ([]byte, error) {
-				return utiltls.CertAsBytes(validCaCerts...)
+				return utilpem.Encode(validCaCerts...)
 			},
 			want: true,
 		},
diff --git a/pkg/util/pem/pem_test.go b/pkg/util/pem/pem_test.go
new file mode 100644
index 000000000..7d19541a8
--- /dev/null
+++ b/pkg/util/pem/pem_test.go
@@ -0,0 +1,48 @@
+package pem
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"bytes"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
+)
+
+var _ = Describe("PEM", func() {
+	validCaKey, validCaCerts, err := utiltls.GenerateKeyAndCertificate("validca", nil, nil, true, false)
+	Expect(err).To(BeNil())
+
+	Describe("encoding keys", func() {
+		It("succeeds", func() {
+			keyOut, err := Encode(validCaKey)
+			Expect(err).To(BeNil())
+			Expect(keyOut).To(ContainSubstring("BEGIN RSA PRIVATE KEY"))
+		})
+	})
+
+	Describe("encoding single certificate", func() {
+		It("succeeds", func() {
+			certsOut, err := Encode(validCaCerts...)
+			Expect(err).To(BeNil())
+			Expect(certsOut).To(ContainSubstring("BEGIN CERTIFICATE"))
+		})
+	})
+
+	Describe("encoding multiple certificates", func() {
+		It("succeeds", func() {
+			certsOut, err := Encode(validCaCerts[0], validCaCerts[0])
+			Expect(err).To(BeNil())
+			Expect(bytes.Count(certsOut, []byte("BEGIN CERTIFICATE"))).To(Equal(2))
+		})
+	})
+})
+
+func TestPEM(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "PEM Suite")
+}
diff --git a/test/e2e/operator.go b/test/e2e/operator.go
index 5ed8f2e34..1a1e4467e 100644
--- a/test/e2e/operator.go
+++ b/test/e2e/operator.go
@@ -285,20 +285,14 @@ var _ = Describe("ARO Operator - Conditions", func() {
 	})
 })
 
-func timestampIsUpdated(annotations map[string]string) bool {
+func subnetReconciliationAnnotationExists(annotations map[string]string) bool {
 	if annotations == nil {
 		return false
 	}
 
 	timestamp := annotations[subnetController.AnnotationTimestamp]
-	t, err := time.Parse(time.RFC1123, timestamp)
-	if err != nil {
-		return false
-	}
-
-	// 11 seconds here since we have set 10 seconds as the default polling
-	// interval for the ginkgo's Eventually block. (e2e/setup.go)
-	return t.Add(time.Second * 11).After(time.Now())
+	_, err := time.Parse(time.RFC1123, timestamp)
+	return err == nil
 }
 
 var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
@@ -379,7 +373,7 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() {
 
 				co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{})
 				g.Expect(err).NotTo(HaveOccurred())
-				g.Expect(co.Annotations).To(Satisfy(timestampIsUpdated))
+				g.Expect(co.Annotations).To(Satisfy(subnetReconciliationAnnotationExists))
 			}).WithContext(ctx).Should(Succeed())
 		}
 	})