Repository: kubeflow/katib (language: python)

Issue:
Migrate KatibCertGenerator to OPA CertController
/kind feature

**Describe the solution you'd like**
[A clear and concise description of what you want to happen.]
Today, we maintain the [katib-cert-generator](https://github.com/kubeflow/katib/tree/7df05c23a55363ba7cbbd7b9859943562afd71f4/pkg/certgenerator/v1beta1) to generate certifications for the webhook.
But, it doesn't have some useful features like a certification rotation mechanism.

So, we should migrate the katib-cert-generator to [open-policy-agent/cert-controller](https://github.com/open-policy-agent/cert-controller).

This training-operator cert generation mechanism would be a good example to introduce the opa/cert-controller to the katib: https://github.com/kubeflow/training-operator/blob/e85dbf9e010421813ea03bf0f82a4801f4587989/pkg/cert/cert.go

**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]

---

<!-- Don't delete this message to encourage users to support your issue! -->

Love this feature? Give it a 👍 We prioritize the features with the most 👍



Discussion / hints:

Fix patch:
diff --git a/go.mod b/go.mod
index 11ef406e143..e908c993958 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
 	github.com/lib/pq v1.10.6
 	github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a
 	github.com/onsi/gomega v1.30.0
+	github.com/open-policy-agent/cert-controller v0.10.1
 	github.com/prometheus/client_golang v1.18.0
 	github.com/shirou/gopsutil/v3 v3.22.5
 	github.com/spf13/viper v1.9.0
@@ -123,6 +124,7 @@ require (
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
diff --git a/go.sum b/go.sum
index 5d756af16fc..f0f3ee30e17 100644
--- a/go.sum
+++ b/go.sum
@@ -540,6 +540,10 @@ github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY
 github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw=
 github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4=
+github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ=
+github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8=
@@ -687,6 +691,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -1248,6 +1254,8 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
+k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs=
+k8s.io/kube-aggregator v0.28.1/go.mod h1:JaLizMe+AECSpO2OmrWVsvnG0V3dX1RpW+Wq/QHbu18=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
diff --git a/manifests/v1beta1/components/controller/rbac.yaml b/manifests/v1beta1/components/controller/rbac.yaml
index 3118ea7dedf..e93fbd031cf 100644
--- a/manifests/v1beta1/components/controller/rbac.yaml
+++ b/manifests/v1beta1/components/controller/rbac.yaml
@@ -58,6 +58,7 @@ rules:
       - "list"
       - "watch"
       - "patch"
+      - "update"
   - apiGroups:
       - apps
     resources:
@@ -127,6 +128,7 @@ rules:
       - "watch"
       - "list"
       - "patch"
+      - "update"
 ---
 apiVersion: v1
 kind: ServiceAccount
diff --git a/pkg/certgenerator/v1beta1/certificate.go b/pkg/certgenerator/v1beta1/certificate.go
deleted file mode 100644
index 19d07615f67..00000000000
--- a/pkg/certgenerator/v1beta1/certificate.go
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-Copyright 2022 The Kubeflow Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package certgenerator
-
-import (
-	"bytes"
-	"crypto/rsa"
-	"crypto/x509"
-	"encoding/pem"
-)
-
-// certificates contains all certificates for katib-webhook-cert.
-type certificates struct {
-	certPem []byte
-	keyPem  []byte
-	cert    *x509.Certificate
-	key     *rsa.PrivateKey
-}
-
-// encode creates PEM key and convert DER to CRT.
-func encode(rawKey *rsa.PrivateKey, der []byte) (*certificates, error) {
-	keyPem := &bytes.Buffer{}
-	if err := pem.Encode(keyPem, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rawKey)}); err != nil {
-		return nil, err
-	}
-
-	certPem := &bytes.Buffer{}
-	if err := pem.Encode(certPem, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil {
-		return nil, err
-	}
-
-	cert, err := x509.ParseCertificate(der)
-	if err != nil {
-		return nil, err
-	}
-
-	return &certificates{
-		certPem: certPem.Bytes(),
-		keyPem:  keyPem.Bytes(),
-		cert:    cert,
-		key:     rawKey,
-	}, nil
-}
diff --git a/pkg/certgenerator/v1beta1/const.go b/pkg/certgenerator/v1beta1/const.go
deleted file mode 100644
index 09aac6f6669..00000000000
--- a/pkg/certgenerator/v1beta1/const.go
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
-Copyright 2022 The Kubeflow Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package certgenerator
-
-const (
-	Webhook           = "katib.kubeflow.org"
-	serverKeyName     = "tls.key"
-	serverCertName    = "tls.crt"
-	ssaFieldOwnerName = "cert-generator"
-)
diff --git a/pkg/certgenerator/v1beta1/generator.go b/pkg/certgenerator/v1beta1/generator.go
index 0784a625ca7..3a61bf7ac45 100644
--- a/pkg/certgenerator/v1beta1/generator.go
+++ b/pkg/certgenerator/v1beta1/generator.go
@@ -17,276 +17,41 @@ limitations under the License.
 package certgenerator
 
 import (
-	"bytes"
-	"context"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"errors"
 	"fmt"
-	"math/big"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
 
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	cert "github.com/open-policy-agent/cert-controller/pkg/rotator"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	configv1beta1 "github.com/kubeflow/katib/pkg/apis/config/v1beta1"
 	"github.com/kubeflow/katib/pkg/controller.v1beta1/consts"
 )
 
-var (
-	log = logf.Log.WithName("cert-generator")
+const Webhook = "katib.kubeflow.org"
 
-	errServiceNotFound      = errors.New("unable to locate controller service")
-	errCertCheckFail        = errors.New("failed to check if certs already exist")
-	errCreateCertFail       = errors.New("failed to create certs")
-	errCreateCertSecretFail = errors.New("failed to create secret embedded certs")
-	errInjectCertError      = errors.New("failed to inject certs into WebhookConfigurations")
-)
-
-// CertGenerator is the manager to generate certs.
-type CertGenerator struct {
-	namespace          string
-	webhookServiceName string
-	webhookSecretName  string
-	fullServiceDomain  string
-	kubeClient         client.Client
-	certsReady         chan struct{}
-
-	certs *certificates
-}
-
-var _ manager.Runnable = &CertGenerator{}
-var _ manager.LeaderElectionRunnable = &CertGenerator{}
-
-func (c *CertGenerator) Start(ctx context.Context) error {
-	if err := c.generate(ctx); err != nil {
-		return err
-	}
-	log.Info("Waiting for certs to get ready.")
-	if err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{
-		Duration: time.Second,
-		Factor:   2,
-		Jitter:   1,
-		Steps:    10,
-		Cap:      time.Minute * 5,
-	}, ensureCertMounted(time.Now())); err != nil {
-		return err
-	}
-	// Sending an empty data to a certsReady means it starts to register controllers to the manager.
-	c.certsReady <- struct{}{}
-	return nil
-}
-
-// ensureCertMounted ensures that the generated certs are mounted inside the container.
-func ensureCertMounted(start time.Time) func(context.Context) (bool, error) {
-	return func(ctx context.Context) (bool, error) {
-		now := time.Now()
-		outputLog := false
-		if now.Sub(start) >= 15*time.Second {
-			start = now
-			outputLog = true
-		}
-
-		certFile := filepath.Join(consts.CertDir, serverCertName)
-		if _, err := os.Stat(certFile); err != nil {
-			if outputLog {
-				log.Info("Public key file doesn't exist in the container yet.", "publicKeyFile", certFile)
-			}
-			return false, nil
-		}
-		keyFile := filepath.Join(consts.CertDir, serverKeyName)
-		if _, err := os.Stat(keyFile); err != nil {
-			if outputLog {
-				log.Info("Private key file doesn't exist in the container yet.", "privateKeyFile", keyFile)
-			}
-			return false, nil
-		}
-		log.Info("Succeeded to be mounted certs inside the container.")
-		return true, nil
-	}
-}
-
-func (c *CertGenerator) NeedLeaderElection() bool {
-	return false
-}
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update
+// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update
+// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update
 
 // AddToManager adds the cert-generator to the manager.
-func AddToManager(mgr manager.Manager, config configv1beta1.CertGeneratorConfig, certsReady chan struct{}) error {
-	return mgr.Add(&CertGenerator{
-		namespace:          consts.DefaultKatibNamespace,
-		webhookServiceName: config.WebhookServiceName,
-		webhookSecretName:  config.WebhookSecretName,
-		fullServiceDomain: strings.Join([]string{
-			config.WebhookServiceName,
-			consts.DefaultKatibNamespace,
-			"svc",
-		}, "."),
-		kubeClient: mgr.GetClient(),
-		certsReady: certsReady,
-	})
-}
-
-// generate generates certificates for the admission webhooks.
-func (c *CertGenerator) generate(ctx context.Context) error {
-	controllerService := &corev1.Service{}
-	if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: c.webhookServiceName, Namespace: c.namespace}, controllerService); err != nil {
-		return fmt.Errorf("%w: %v", errServiceNotFound, err)
-	}
-
-	certExist, err := c.isCertExist(ctx)
-	if err != nil {
-		return fmt.Errorf("%w: %v", errCertCheckFail, err)
-	}
-	if !certExist {
-		if err = c.createCert(); err != nil {
-			return fmt.Errorf("%w: %v", errCreateCertFail, err)
-		}
-		if err = c.updateCertSecret(ctx); err != nil {
-			return fmt.Errorf("%w: %v", errCreateCertSecretFail, err)
-		}
-	}
-	if err = c.injectCert(ctx); err != nil {
-		return fmt.Errorf("%w: %v", errInjectCertError, err)
-	}
-	return nil
-}
-
-// isCertExist checks if a secret embedded certs already exists.
-// For example, it will return true if the katib-controller is created with enabled leader-election
-// since another controller pod will create the secret.
-func (c *CertGenerator) isCertExist(ctx context.Context) (bool, error) {
-	secret := &corev1.Secret{}
-	if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: c.webhookSecretName, Namespace: c.namespace}, secret); err != nil {
-		return false, err
-	}
-	key := secret.Data[serverKeyName]
-	cert := secret.Data[serverCertName]
-	if len(key) != 0 && len(cert) != 0 {
-		c.certs = &certificates{
-			keyPem:  key,
-			certPem: cert,
-		}
-		return true, nil
-	}
-	return false, nil
-}
-
-// createCert creates the self-signed certificate and private key.
-func (c *CertGenerator) createCert() error {
-	now := time.Now()
-	template := &x509.Certificate{
-		SerialNumber: big.NewInt(0),
-		Subject: pkix.Name{
-			CommonName: c.fullServiceDomain,
+func AddToManager(mgr manager.Manager, cfg configv1beta1.CertGeneratorConfig, certsReady chan struct{}) error {
+	return cert.AddRotator(mgr, &cert.CertRotator{
+		SecretKey: types.NamespacedName{
+			Namespace: consts.DefaultKatibNamespace,
+			Name:      cfg.WebhookSecretName,
 		},
-		DNSNames: []string{
-			c.fullServiceDomain,
+		CertDir:        consts.CertDir,
+		CAName:         "katib-ca",
+		CAOrganization: "katib",
+		DNSName:        fmt.Sprintf("%s.%s.svc", cfg.WebhookServiceName, consts.DefaultKatibNamespace),
+		IsReady:        certsReady,
+		Webhooks: []cert.WebhookInfo{
+			{Name: Webhook, Type: cert.Validating},
+			{Name: Webhook, Type: cert.Mutating},
 		},
-		NotBefore:   now,
-		NotAfter:    now.Add(24 * time.Hour * 365 * 10),
-		KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
-		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-	}
-
-	log.Info("Generating self-signed public certificate and private key.")
-	rawKey, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return err
-	}
-
-	der, err := x509.CreateCertificate(rand.Reader, template, template, rawKey.Public(), rawKey)
-	if err != nil {
-		return err
-	}
-	if c.certs, err = encode(rawKey, der); err != nil {
-		return err
-	}
-	return nil
-}
-
-// updateCertSecret updates Secret embedded tls.key and tls.crt.
-func (c *CertGenerator) updateCertSecret(ctx context.Context) error {
-	secret := &corev1.Secret{}
-	if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: c.webhookSecretName, Namespace: c.namespace}, secret); err != nil {
-		return err
-	}
-	newSecret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			UID:       secret.UID,
-			Name:      secret.Name,
-			Namespace: secret.Namespace,
-		},
-		TypeMeta: secret.TypeMeta,
-	}
-	newSecret.Data = map[string][]byte{
-		serverKeyName:  c.certs.keyPem,
-		serverCertName: c.certs.certPem,
-	}
-	return c.kubeClient.Patch(ctx, newSecret, client.Apply, client.FieldOwner(ssaFieldOwnerName), client.ForceOwnership)
-}
-
-// injectCert applies patch to ValidatingWebhookConfiguration and MutatingWebhookConfiguration.
-func (c *CertGenerator) injectCert(ctx context.Context) error {
-	vWebhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{}
-	if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: Webhook}, vWebhookConfig); err != nil {
-		return err
-	}
-	if !bytes.Equal(vWebhookConfig.Webhooks[0].ClientConfig.CABundle, c.certs.certPem) {
-		newVWebhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{
-			ObjectMeta: metav1.ObjectMeta{
-				UID:        vWebhookConfig.UID,
-				Name:       vWebhookConfig.Name,
-				Namespace:  vWebhookConfig.Namespace,
-				Generation: vWebhookConfig.Generation,
-			},
-			TypeMeta: vWebhookConfig.TypeMeta,
-		}
-		newVWebhookConfig.Webhooks = vWebhookConfig.Webhooks
-		newVWebhookConfig.Webhooks[0].ClientConfig.CABundle = c.certs.certPem
-
-		log.Info("Trying to patch ValidatingWebhookConfiguration adding the caBundle.")
-		err := c.kubeClient.Patch(ctx, newVWebhookConfig, client.Apply, client.FieldOwner(ssaFieldOwnerName), client.ForceOwnership)
-		if err != nil {
-			log.Error(err, "Unable to patch ValidatingWebhookConfiguration", "ValidatingWebhookConfiguration", Webhook)
-			return err
-		}
-	}
-
-	mWebhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{}
-	if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: Webhook}, mWebhookConfig); err != nil {
-		return err
-	}
-	if !bytes.Equal(mWebhookConfig.Webhooks[0].ClientConfig.CABundle, c.certs.certPem) ||
-		!bytes.Equal(mWebhookConfig.Webhooks[1].ClientConfig.CABundle, c.certs.certPem) {
-		newMWebhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{
-			ObjectMeta: metav1.ObjectMeta{
-				UID:        mWebhookConfig.UID,
-				Name:       mWebhookConfig.Name,
-				Namespace:  mWebhookConfig.Namespace,
-				Generation: mWebhookConfig.Generation,
-			},
-			TypeMeta: mWebhookConfig.TypeMeta,
-		}
-		newMWebhookConfig.Webhooks = mWebhookConfig.Webhooks
-		newMWebhookConfig.Webhooks[0].ClientConfig.CABundle = c.certs.certPem
-		newMWebhookConfig.Webhooks[1].ClientConfig.CABundle = c.certs.certPem
-
-		log.Info("Trying to patch MutatingWebhookConfiguration adding the caBundle.")
-		err := c.kubeClient.Patch(ctx, newMWebhookConfig, client.Apply, client.FieldOwner(ssaFieldOwnerName), client.ForceOwnership)
-		if err != nil {
-			log.Error(err, "Unable to patch MutatingWebhookConfiguration", "MutatingWebhookConfiguration", Webhook)
-			return err
-		}
-	}
-	return nil
+		FieldOwner: "cert-generator",
+		// When the Katib is running in the leader election mode,
+		// we expect webhook server will run in primary and secondary instance
+		RequireLeaderElection: false,
+	})
 }


Test patch:
diff --git a/pkg/certgenerator/v1beta1/generator_test.go b/pkg/certgenerator/v1beta1/generator_test.go
deleted file mode 100644
index 05dc5600749..00000000000
--- a/pkg/certgenerator/v1beta1/generator_test.go
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
-Copyright 2022 The Kubeflow Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package certgenerator
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	admissionregistration "k8s.io/api/admissionregistration/v1"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
-
-	configv1beta1 "github.com/kubeflow/katib/pkg/apis/config/v1beta1"
-	"github.com/kubeflow/katib/pkg/controller.v1beta1/consts"
-)
-
-func TestGenerate(t *testing.T) {
-	const testNamespace = "test"
-
-	emptyVWebhookConfig := &admissionregistration.ValidatingWebhookConfiguration{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: admissionregistration.SchemeGroupVersion.String(),
-			Kind:       "ValidatingWebhookConfiguration",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: Webhook,
-		},
-		Webhooks: []admissionregistration.ValidatingWebhook{
-			{
-				Name:         strings.Join([]string{"validator.experiment", Webhook}, "."),
-				ClientConfig: admissionregistration.WebhookClientConfig{},
-			},
-		},
-	}
-	emptyMWebhookConfig := &admissionregistration.MutatingWebhookConfiguration{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: admissionregistration.SchemeGroupVersion.String(),
-			Kind:       "MutatingWebhookConfiguration",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: Webhook,
-		},
-		Webhooks: []admissionregistration.MutatingWebhook{
-			{
-				Name:         strings.Join([]string{"defaulter.experiment", Webhook}, "."),
-				ClientConfig: admissionregistration.WebhookClientConfig{},
-			},
-			{
-				Name:         strings.Join([]string{"mutator.pod", Webhook}, "."),
-				ClientConfig: admissionregistration.WebhookClientConfig{},
-			},
-		},
-	}
-	webhookSecret := &corev1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: corev1.SchemeGroupVersion.String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "katib-test-secret",
-			Namespace: testNamespace,
-		},
-	}
-	controllerService := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: corev1.SchemeGroupVersion.String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      configv1beta1.DefaultWebhookServiceName,
-			Namespace: testNamespace,
-		},
-	}
-
-	tests := map[string]struct {
-		objects   []client.Object
-		opts      *CertGenerator
-		wantError error
-	}{
-		"Generate successfully": {
-			opts: &CertGenerator{
-				namespace:          testNamespace,
-				webhookServiceName: "katib-controller",
-				webhookSecretName:  "katib-test-secret",
-			},
-			objects: []client.Object{
-				emptyVWebhookConfig,
-				emptyMWebhookConfig,
-				controllerService,
-				webhookSecret,
-			},
-		},
-		"There is not ValidatingWebhookConfiguration": {
-			opts: &CertGenerator{
-				namespace:          testNamespace,
-				webhookServiceName: "katib-controller",
-				webhookSecretName:  "katib-test-secret",
-			},
-			objects: []client.Object{
-				emptyMWebhookConfig,
-				controllerService,
-				webhookSecret,
-			},
-			wantError: errInjectCertError,
-		},
-		"There is not MutatingWebhookConfiguration": {
-			opts: &CertGenerator{
-				namespace:          testNamespace,
-				webhookServiceName: "katib-controller",
-				webhookSecretName:  "katib-test-secret",
-			},
-			objects: []client.Object{
-				emptyVWebhookConfig,
-				controllerService,
-				webhookSecret,
-			},
-			wantError: errInjectCertError,
-		},
-		"There is not Service katib-controller": {
-			opts: &CertGenerator{
-				namespace:          testNamespace,
-				webhookServiceName: "katib-controller",
-				webhookSecretName:  "katib-test-secret",
-			},
-			objects: []client.Object{
-				emptyVWebhookConfig,
-				emptyMWebhookConfig,
-				webhookSecret,
-			},
-			wantError: errServiceNotFound,
-		},
-		"There is not Secret katib-webhook-cert": {
-			opts: &CertGenerator{
-				namespace:          testNamespace,
-				webhookServiceName: "katib-controller",
-				webhookSecretName:  "katib-test-secret",
-			},
-			objects: []client.Object{
-				emptyVWebhookConfig,
-				emptyMWebhookConfig,
-				controllerService,
-			},
-			wantError: errCertCheckFail,
-		},
-	}
-	for name, tc := range tests {
-		t.Run(name, func(t *testing.T) {
-			kc := buildFakeClient(tc.objects)
-			tc.opts.kubeClient = kc
-			err := tc.opts.generate(context.Background())
-			if diff := cmp.Diff(tc.wantError, err, cmpopts.EquateErrors()); len(diff) != 0 {
-				t.Errorf("Unexpected error from generate() (-want,+got):\n%s", diff)
-			}
-
-			if tc.wantError == nil {
-				secret := &corev1.Secret{}
-				if err = kc.Get(context.Background(), client.ObjectKey{Name: tc.opts.webhookSecretName, Namespace: testNamespace}, secret); err != nil {
-					t.Fatalf("Failed to get a webhookSecret: %v", err)
-				}
-				if len(secret.Data[serverKeyName]) == 0 {
-					t.Errorf("Unexpected tls.key embedded in secret: %v", secret.Data)
-				}
-				if len(secret.Data[serverCertName]) == 0 {
-					t.Errorf("Unexpected tls.crt embedded in secret: %v", secret.Data)
-				}
-
-				vConfig := &admissionregistration.ValidatingWebhookConfiguration{}
-				if err = kc.Get(context.Background(), client.ObjectKey{Name: Webhook}, vConfig); err != nil {
-					t.Fatalf("Failed to get a ValidatingWebhookConfiguration: %v", err)
-				}
-				if len(vConfig.Webhooks[0].ClientConfig.CABundle) == 0 {
-					t.Errorf("Unexpected tls.crt embedded in ValidatingWebhookConfiguration: %v", vConfig.Webhooks)
-				}
-
-				mConfig := &admissionregistration.MutatingWebhookConfiguration{}
-				if err = kc.Get(context.Background(), client.ObjectKey{Name: Webhook}, mConfig); err != nil {
-					t.Fatalf("Failed to get a MutatingWebhookConfiguration: %v", err)
-				}
-				if len(mConfig.Webhooks[0].ClientConfig.CABundle) == 0 || len(mConfig.Webhooks[1].ClientConfig.CABundle) == 0 {
-					t.Errorf("Unexpected tls.crt embedded in MutatingWebhookConfiguration: %v", mConfig.Webhooks)
-				}
-			}
-		})
-	}
-}
-
-func buildFakeClient(kubeResources []client.Object) client.Client {
-	fakeClientBuilder := fake.NewClientBuilder().
-		WithScheme(scheme.Scheme).
-		WithInterceptorFuncs(interceptor.Funcs{Patch: ssaAsStrategicMergePatchFunc})
-	if len(kubeResources) > 0 {
-		fakeClientBuilder.WithObjects(kubeResources...)
-	}
-	return fakeClientBuilder.Build()
-}
-
-type ssaPatchAsStrategicMerge struct {
-	client.Patch
-}
-
-func (*ssaPatchAsStrategicMerge) Type() types.PatchType {
-	return types.StrategicMergePatchType
-}
-
-func wrapSSAPatch(patch client.Patch) client.Patch {
-	if patch.Type() == types.ApplyPatchType {
-		return &ssaPatchAsStrategicMerge{Patch: patch}
-	}
-	return patch
-}
-
-// ssaAsStrategicMergePatchFunc returns the patch interceptor.
-// TODO (tenzen-y): Once the fake client supports server-side apply, we should remove these interceptor.
-// REF: https://github.com/kubernetes/kubernetes/issues/115598
-func ssaAsStrategicMergePatchFunc(
-	ctx context.Context,
-	cli client.WithWatch,
-	obj client.Object,
-	patch client.Patch,
-	opts ...client.PatchOption,
-) error {
-	return cli.Patch(ctx, obj, wrapSSAPatch(patch), opts...)
-}
-
-func TestEnsureCertMounted(t *testing.T) {
-	tests := map[string]struct {
-		keyExist  bool
-		certExist bool
-		wantExist bool
-	}{
-		"key and cert exist": {
-			keyExist:  true,
-			certExist: true,
-			wantExist: true,
-		},
-		"key doesn't exist": {
-			keyExist:  false,
-			certExist: true,
-			wantExist: false,
-		},
-		"cert doesn't exist": {
-			keyExist:  true,
-			certExist: false,
-			wantExist: false,
-		},
-		"all files doesn't exist": {
-			keyExist:  false,
-			certExist: false,
-			wantExist: false,
-		},
-	}
-	for name, tc := range tests {
-		t.Run(name, func(t *testing.T) {
-			if tc.keyExist || tc.certExist {
-				if err := os.MkdirAll(consts.CertDir, 0760); err != nil {
-					t.Fatalf("Failed to set up directory: %v", err)
-				}
-				defer func() {
-					if err := os.RemoveAll(consts.CertDir); err != nil {
-						t.Fatalf("Failed to clean up directory: %v", err)
-					}
-				}()
-			}
-			if tc.keyExist {
-				if _, err := os.Create(filepath.Join(consts.CertDir, serverKeyName)); err != nil {
-					t.Fatalf("Failed to create tls.key: %v", err)
-				}
-			}
-			if tc.certExist {
-				if _, err := os.Create(filepath.Join(consts.CertDir, serverCertName)); err != nil {
-					t.Fatalf("Failed to create tls.crt: %v", err)
-				}
-			}
-			ensureFunc := ensureCertMounted(time.Now())
-			got, _ := ensureFunc(context.Background())
-			if tc.wantExist != got {
-				t.Errorf("Unexpected value from ensureCertMounted: \n(want: %v, got: %v)\n", tc.wantExist, got)
-			}
-		})
-	}
-}