Repository: sigstore/fulcio
Pull request #1258 (2023-07-04 21:34:12)

add HTTP and GRPC health check endpoints
#### Summary
This adds both HTTP and gRPC health check endpoints to Fulcio.

HTTP: `/healthz` should return `HTTP 200 OK`
gRPC: implements [gRPC Health Checking Protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md)

This also adds coverage for this via K8S local testing using readiness and liveness probes in cluster.

#### Release Note
* HTTP and gRPC health check endpoints are now exposed from Fulcio

#### Documentation

- [ ] update docs.sigstore.dev to denote this feature


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/sigstore/fulcio/pull/1258?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore) Report
> Merging [#1258](https://app.codecov.io/gh/sigstore/fulcio/pull/1258?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore) (54a5481) into [main](https://app.codecov.io/gh/sigstore/fulcio/commit/a5b774db0cff2c91ef9ac3618fd54ec947a07ead?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore) (a5b774d) will **increase** coverage by `0.20%`.
> The diff coverage is `52.38%`.

```diff
@@            Coverage Diff             @@
##             main    #1258      +/-   ##
==========================================
+ Coverage   56.07%   56.27%   +0.20%     
==========================================
  Files          50       50              
  Lines        2921     2939      +18     
==========================================
+ Hits         1638     1654      +16     
- Misses       1138     1139       +1     
- Partials      145      146       +1     
```


| [Impacted Files](https://app.codecov.io/gh/sigstore/fulcio/pull/1258?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore) | Coverage Δ | |
|---|---|---|
| [pkg/server/grpc\_server.go](https://app.codecov.io/gh/sigstore/fulcio/pull/1258?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore#diff-cGtnL3NlcnZlci9ncnBjX3NlcnZlci5nbw==) | `50.55% <20.00%> (-1.15%)` | :arrow_down: |
| [cmd/app/http.go](https://app.codecov.io/gh/sigstore/fulcio/pull/1258?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore#diff-Y21kL2FwcC9odHRwLmdv) | `52.05% <50.00%> (-1.92%)` | :arrow_down: |
| [cmd/app/grpc.go](https://app.codecov.io/gh/sigstore/fulcio/pull/1258?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore#diff-Y21kL2FwcC9ncnBjLmdv) | `47.82% <100.00%> (+1.88%)` | :arrow_up: |

... and [2 files with indirect coverage changes](https://app.codecov.io/gh/sigstore/fulcio/pull/1258/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sigstore)



Patch:
diff --git a/.github/workflows/protoc-dependabot-hack.yml b/.github/workflows/protoc-dependabot-hack.yml
index 372b5de..57b57be 100644
--- a/.github/workflows/protoc-dependabot-hack.yml
+++ b/.github/workflows/protoc-dependabot-hack.yml
@@ -16,4 +16,4 @@ jobs:
 
       # update the version in these places manually when Dependabot proposes a change to it here:
       # 1. the version in main.yml used to install protoc
-      - uses: protocolbuffers/protobuf@v23.3
+      - uses: protocolbuffers/protobuf@v23.4
diff --git a/.github/workflows/verify-k8s.yml b/.github/workflows/verify-k8s.yml
index fb14137..3469a45 100644
--- a/.github/workflows/verify-k8s.yml
+++ b/.github/workflows/verify-k8s.yml
@@ -73,9 +73,9 @@ jobs:
       - uses: ko-build/setup-ko@ace48d793556083a76f1e3e6068850c1f4a369aa # v0.6
 
       - name: Setup Cluster
-        uses: chainguard-dev/actions/setup-kind@7d1eb557f464d97e5fe5177807a9226141eb9308 # main
+        uses: chainguard-dev/actions/setup-kind@f5a6616ce43b6ffabeddb87480a13721fffb3588 # main
         with:
-          k8s-version: v1.22.x
+          k8s-version: 1.24.x
           registry-authority: ${{ env.REGISTRY_NAME }}:${{ env.REGISTRY_PORT }}
 
       - name: Generate temporary CA files
@@ -121,6 +121,7 @@ jobs:
             server.yaml: |-
               host: 0.0.0.0
               port: 5555
+              grpc-port: 5554
               ca: fileca
               fileca-cert: /etc/fulcio-secret/cert.pem
               fileca-key: /etc/fulcio-secret/key.pem
diff --git a/Dockerfile b/Dockerfile
index a156dac..0d055bf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20.5@sha256:344193a70dc3588452ea39b4a1e465a8d3c91f788ae053f7ee168cebf18e0a50 AS builder
+FROM golang:1.20.5@sha256:fd9306e1c664bd49a11d4a4a04e41303430e069e437d137876e9290a555e06fb AS builder
 ENV APP_ROOT=/opt/app-root
 ENV GOPATH=$APP_ROOT
 
@@ -28,7 +28,7 @@ RUN go build -o server main.go
 RUN CGO_ENABLED=1 go build -gcflags "all=-N -l" -o server_debug main.go
 
 # Multi-Stage production build
-FROM golang:1.20.5@sha256:344193a70dc3588452ea39b4a1e465a8d3c91f788ae053f7ee168cebf18e0a50 as deploy
+FROM golang:1.20.5@sha256:fd9306e1c664bd49a11d4a4a04e41303430e069e437d137876e9290a555e06fb as deploy
 
 # Retrieve the binary from the previous stage
 COPY --from=builder /opt/app-root/src/server /usr/local/bin/fulcio-server
diff --git a/Dockerfile.ctfe_init b/Dockerfile.ctfe_init
index 36c53e1..f45c68a 100644
--- a/Dockerfile.ctfe_init
+++ b/Dockerfile.ctfe_init
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20.5@sha256:344193a70dc3588452ea39b4a1e465a8d3c91f788ae053f7ee168cebf18e0a50 AS builder
+FROM golang:1.20.5@sha256:fd9306e1c664bd49a11d4a4a04e41303430e069e437d137876e9290a555e06fb AS builder
 
 WORKDIR /root/
 
diff --git a/cmd/app/grpc.go b/cmd/app/grpc.go
index 51587f2..3aaf283 100644
--- a/cmd/app/grpc.go
+++ b/cmd/app/grpc.go
@@ -22,7 +22,9 @@ import (
 	"net"
 	"os"
 	"runtime"
+	"sync"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/goadesign/goa/grpc/middleware"
 	ctclient "github.com/google/certificate-transparency-go/client"
 	grpcmw "github.com/grpc-ecosystem/go-grpc-middleware"
@@ -40,6 +42,7 @@ import (
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	health "google.golang.org/grpc/health/grpc_health_v1"
 )
 
 const (
@@ -50,6 +53,7 @@ type grpcServer struct {
 	*grpc.Server
 	grpcServerEndpoint string
 	caService          gw.CAServer
+	tlsCertWatcher     *fsnotify.Watcher
 }
 
 func PassFulcioConfigThruContext(cfg *config.FulcioConfig) grpc.UnaryServerInterceptor {
@@ -67,16 +71,85 @@ func PassFulcioConfigThruContext(cfg *config.FulcioConfig) grpc.UnaryServerInter
 	}
 }
 
-func createGRPCCreds(certPath, keyPath string) (grpc.ServerOption, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+type cachedTLSCert struct {
+	sync.RWMutex
+	certPath string
+	keyPath  string
+	cert     *tls.Certificate
+	Watcher  *fsnotify.Watcher
+}
+
+func newCachedTLSCert(certPath, keyPath string) (*cachedTLSCert, error) {
+	cachedTLSCert := &cachedTLSCert{
+		certPath: certPath,
+		keyPath:  keyPath,
+	}
+	if err := cachedTLSCert.UpdateCertificate(); err != nil {
+		return nil, err
+	}
+	var err error
+	cachedTLSCert.Watcher, err = fsnotify.NewWatcher()
 	if err != nil {
-		return nil, fmt.Errorf("loading GRPC tls certificate and key file: %w", err)
+		return nil, err
 	}
 
+	go func() {
+		for {
+			select {
+			case event, ok := <-cachedTLSCert.Watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Has(fsnotify.Write) {
+					log.Logger.Info("fsnotify grpc-tls-certificate write event detected")
+					if err := cachedTLSCert.UpdateCertificate(); err != nil {
+						log.Logger.Error(err)
+					}
+				}
+			case err, ok := <-cachedTLSCert.Watcher.Errors:
+				if !ok {
+					return
+				}
+				log.Logger.Error("fsnotify grpc-tls-certificate error:", err)
+			}
+		}
+	}()
+
+	// Add a path.
+	if err = cachedTLSCert.Watcher.Add(certPath); err != nil {
+		return nil, err
+	}
+	return cachedTLSCert, nil
+}
+
+func (c *cachedTLSCert) GetCertificate() *tls.Certificate {
+	// get reader lock
+	c.RLock()
+	defer c.RUnlock()
+	return c.cert
+}
+
+func (c *cachedTLSCert) UpdateCertificate() error {
+	// get writer lock
+	c.Lock()
+	defer c.Unlock()
+
+	cert, err := tls.LoadX509KeyPair(c.certPath, c.keyPath)
+	if err != nil {
+		return fmt.Errorf("loading GRPC tls certificate and key file: %w", err)
+	}
+
+	c.cert = &cert
+	return nil
+}
+
+func (c *cachedTLSCert) GRPCCreds() grpc.ServerOption {
 	return grpc.Creds(credentials.NewTLS(&tls.Config{
-		Certificates: []tls.Certificate{cert},
-		MinVersion:   tls.VersionTLS13,
-	})), nil
+		GetCertificate: func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			return c.GetCertificate(), nil
+		},
+		MinVersion: tls.VersionTLS13,
+	}))
 }
 
 func createGRPCServer(cfg *config.FulcioConfig, ctClient *ctclient.LogClient, baseca ca.CertificateAuthority, ip identity.IssuerPool) (*grpcServer, error) {
@@ -94,22 +167,27 @@ func createGRPCServer(cfg *config.FulcioConfig, ctClient *ctclient.LogClient, ba
 		grpc.MaxRecvMsgSize(int(maxMsgSize)),
 	}
 
+	var tlsCertWatcher *fsnotify.Watcher
 	if viper.IsSet("grpc-tls-certificate") && viper.IsSet("grpc-tls-key") {
-		creds, err := createGRPCCreds(viper.GetString("grpc-tls-certificate"), viper.GetString("grpc-tls-key"))
+		cachedTLSCert, err := newCachedTLSCert(viper.GetString("grpc-tls-certificate"), viper.GetString("grpc-tls-key"))
 		if err != nil {
 			return nil, err
 		}
-		serverOpts = append(serverOpts, creds)
+
+		tlsCertWatcher = cachedTLSCert.Watcher
+		serverOpts = append(serverOpts, cachedTLSCert.GRPCCreds())
 	}
 
 	myServer := grpc.NewServer(serverOpts...)
 
 	grpcCAServer := server.NewGRPCCAServer(ctClient, baseca, ip)
+
+	health.RegisterHealthServer(myServer, grpcCAServer)
 	// Register your gRPC service implementations.
 	gw.RegisterCAServer(myServer, grpcCAServer)
 
 	grpcServerEndpoint := fmt.Sprintf("%s:%s", viper.GetString("grpc-host"), viper.GetString("grpc-port"))
-	return &grpcServer{myServer, grpcServerEndpoint, grpcCAServer}, nil
+	return &grpcServer{myServer, grpcServerEndpoint, grpcCAServer, tlsCertWatcher}, nil
 }
 
 func (g *grpcServer) setupPrometheus(reg *prometheus.Registry) {
@@ -129,6 +207,9 @@ func (g *grpcServer) startTCPListener() {
 	g.grpcServerEndpoint = lis.Addr().String()
 	log.Logger.Infof("listening on grpc at %s", g.grpcServerEndpoint)
 	go func() {
+		if g.tlsCertWatcher != nil {
+			defer g.tlsCertWatcher.Close()
+		}
 		if err := g.Server.Serve(lis); err != nil {
 			log.Logger.Errorf("error shutting down grpcServer: %w", err)
 		}
@@ -161,6 +242,10 @@ func (g *grpcServer) startUnixListener() {
 	}()
 }
 
+func (g *grpcServer) ExposesGRPCTLS() bool {
+	return viper.IsSet("grpc-tls-certificate") && viper.IsSet("grpc-tls-key")
+}
+
 func createLegacyGRPCServer(cfg *config.FulcioConfig, v2Server gw.CAServer) (*grpcServer, error) {
 	logger, opts := log.SetupGRPCLogging()
 
@@ -179,7 +264,7 @@ func createLegacyGRPCServer(cfg *config.FulcioConfig, v2Server gw.CAServer) (*gr
 	// Register your gRPC service implementations.
 	gw_legacy.RegisterCAServer(myServer, legacyGRPCCAServer)
 
-	return &grpcServer{myServer, LegacyUnixDomainSocket, v2Server}, nil
+	return &grpcServer{myServer, LegacyUnixDomainSocket, v2Server, nil}, nil
 }
 
 func panicRecoveryHandler(ctx context.Context, p interface{}) error {
diff --git a/cmd/app/http.go b/cmd/app/http.go
index 9eeaced..87249fd 100644
--- a/cmd/app/http.go
+++ b/cmd/app/http.go
@@ -17,6 +17,7 @@ package app
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net/http"
@@ -32,7 +33,9 @@ import (
 	"github.com/sigstore/fulcio/pkg/log"
 	"github.com/sigstore/fulcio/pkg/server"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	health "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/proto"
 )
@@ -48,10 +51,22 @@ func extractOIDCTokenFromAuthHeader(_ context.Context, req *http.Request) metada
 }
 
 func createHTTPServer(ctx context.Context, serverEndpoint string, grpcServer, legacyGRPCServer *grpcServer) httpServer {
+	opts := []grpc.DialOption{}
+	if grpcServer.ExposesGRPCTLS() {
+		/* #nosec G402 */ // InsecureSkipVerify is only used for the HTTP server to call the TLS-enabled grpc endpoint.
+		opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
+	} else {
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	}
+	cc, err := grpc.Dial(grpcServer.grpcServerEndpoint, opts...)
+	if err != nil {
+		log.Logger.Fatal(err)
+	}
+
 	mux := runtime.NewServeMux(runtime.WithMetadata(extractOIDCTokenFromAuthHeader),
-		runtime.WithForwardResponseOption(setResponseCodeModifier))
+		runtime.WithForwardResponseOption(setResponseCodeModifier),
+		runtime.WithHealthzEndpoint(health.NewHealthClient(cc)))
 
-	opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
 	if err := gw.RegisterCAHandlerFromEndpoint(ctx, mux, grpcServer.grpcServerEndpoint, opts); err != nil {
 		log.Logger.Fatal(err)
 	}
diff --git a/config/deployment.yaml b/config/deployment.yaml
index 4aba59d..0979e18 100644
--- a/config/deployment.yaml
+++ b/config/deployment.yaml
@@ -65,6 +65,15 @@ spec:
           readOnly: true
         - name: oidc-info
           mountPath: /var/run/fulcio
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 5555
+          initialDelaySeconds: 5
+        readinessProbe:
+          grpc:
+            port: 5554
+          initialDelaySeconds: 5
         resources:
           requests:
             memory: "1G"
diff --git a/docker-compose.yml b/docker-compose.yml
index 74366da..99ca7a5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,7 +39,7 @@ services:
       - ~/.config/gcloud:/root/.config/gcloud/:z # for GCP authentication
       - ./config/config.jsn:/etc/fulcio-config/config.json:z
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5555/ping"]
+      test: ["CMD", "curl", "-f", "http://localhost:5555/healthz"]
       interval: 10s
       timeout: 3s
       retries: 3
@@ -62,7 +62,7 @@ services:
     volumes:
       - ./config/dex:/etc/config/:ro
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8888/auth/healthz"]
+      test: ["CMD", "wget", "-O", "/dev/null", "http://localhost:8888/auth/healthz"]
       interval: 10s
       timeout: 3s
       retries: 3
diff --git a/go.mod b/go.mod
index b713a65..79f03de 100644
--- a/go.mod
+++ b/go.mod
@@ -35,9 +35,9 @@ require (
 	github.com/spiffe/go-spiffe/v2 v2.1.6
 	go.step.sm/crypto v0.32.2
 	go.uber.org/zap v1.24.0
-	google.golang.org/api v0.129.0
+	google.golang.org/api v0.130.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc
-	google.golang.org/grpc v1.56.1
+	google.golang.org/grpc v1.56.2
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/square/go-jose.v2 v2.6.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -140,17 +140,17 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	goa.design/goa v2.2.5+incompatible // indirect
-	golang.org/x/crypto v0.10.0 // indirect
-	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/oauth2 v0.9.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
-	golang.org/x/term v0.9.0 // indirect
-	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/go.sum b/go.sum
index b1f1d40..360c5f8 100644
--- a/go.sum
+++ b/go.sum
@@ -511,8 +511,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -584,8 +584,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -653,13 +653,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
-golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -670,8 +670,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -753,8 +753,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
 google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.129.0 h1:2XbdjjNfFPXQyufzQVwPf1RRnHH8Den2pfNE2jw7L8w=
-google.golang.org/api v0.129.0/go.mod h1:dFjiXlanKwWE3612X97llhsoI36FAoIiRj3aTl5b/zE=
+google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
+google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -805,8 +805,8 @@ google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdL
 google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -827,8 +827,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
-google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
+google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
diff --git a/hack/tools/go.mod b/hack/tools/go.mod
index 7507eb1..0e8de68 100644
--- a/hack/tools/go.mod
+++ b/hack/tools/go.mod
@@ -3,7 +3,7 @@ module github.com/sigstore/fulcio/hack/tools
 go 1.20
 
 require (
-	github.com/googleapis/api-linter v1.52.3
+	github.com/googleapis/api-linter v1.52.4
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0
 	google.golang.org/protobuf v1.31.0
@@ -27,9 +27,9 @@ require (
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	google.golang.org/grpc v1.55.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
diff --git a/hack/tools/go.sum b/hack/tools/go.sum
index 8a36d34..69623aa 100644
--- a/hack/tools/go.sum
+++ b/hack/tools/go.sum
@@ -19,8 +19,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/googleapis/api-linter v1.52.3 h1:CJM7IhKyLrsDb3VG80OnN2rchlkDY40/wSWoPq8+068=
-github.com/googleapis/api-linter v1.52.3/go.mod h1:IS2/M4y4gdTL65CTc0i7ftAZm8mHmPXExPp3e8+e/L0=
+github.com/googleapis/api-linter v1.52.4 h1:kHd1eL4KEjlSn/zcbH+sKPbEUxQkNNKfLDzWeiZCekg=
+github.com/googleapis/api-linter v1.52.4/go.mod h1:8NLgBx7i+8QuKiDEeoVlInXZWlvacv/tTcDtHevlpMU=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c=
@@ -54,13 +54,13 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cO
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
-golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
 google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
-google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
+google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
diff --git a/pkg/server/grpc_server.go b/pkg/server/grpc_server.go
index 30343f3..df1f090 100644
--- a/pkg/server/grpc_server.go
+++ b/pkg/server/grpc_server.go
@@ -32,18 +32,21 @@ import (
 	"github.com/sigstore/fulcio/pkg/log"
 	"github.com/sigstore/sigstore/pkg/cryptoutils"
 	"google.golang.org/grpc/codes"
+	health "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
 )
 
-type grpcCAServer struct {
+type GRPCCAServer struct {
 	fulciogrpc.UnimplementedCAServer
+	health.HealthServer
 	ct *ctclient.LogClient
 	ca certauth.CertificateAuthority
 	identity.IssuerPool
 }
 
-func NewGRPCCAServer(ct *ctclient.LogClient, ca certauth.CertificateAuthority, ip identity.IssuerPool) fulciogrpc.CAServer {
-	return &grpcCAServer{
+func NewGRPCCAServer(ct *ctclient.LogClient, ca certauth.CertificateAuthority, ip identity.IssuerPool) *GRPCCAServer {
+	return &GRPCCAServer{
 		ct:         ct,
 		ca:         ca,
 		IssuerPool: ip,
@@ -54,7 +57,7 @@ const (
 	MetadataOIDCTokenKey = "oidcidentitytoken"
 )
 
-func (g *grpcCAServer) CreateSigningCertificate(ctx context.Context, request *fulciogrpc.CreateSigningCertificateRequest) (*fulciogrpc.SigningCertificate, error) {
+func (g *GRPCCAServer) CreateSigningCertificate(ctx context.Context, request *fulciogrpc.CreateSigningCertificateRequest) (*fulciogrpc.SigningCertificate, error) {
 	logger := log.ContextLogger(ctx)
 
 	// OIDC token either is passed in gRPC field or was extracted from HTTP headers
@@ -228,7 +231,7 @@ func (g *grpcCAServer) CreateSigningCertificate(ctx context.Context, request *fu
 	return result, nil
 }
 
-func (g *grpcCAServer) GetTrustBundle(ctx context.Context, _ *fulciogrpc.GetTrustBundleRequest) (*fulciogrpc.TrustBundle, error) {
+func (g *GRPCCAServer) GetTrustBundle(ctx context.Context, _ *fulciogrpc.GetTrustBundleRequest) (*fulciogrpc.TrustBundle, error) {
 	trustBundle, err := g.ca.TrustBundle(ctx)
 	if err != nil {
 		return nil, handleFulcioGRPCError(ctx, codes.Internal, err, retrieveTrustBundleCAError)
@@ -252,7 +255,7 @@ func (g *grpcCAServer) GetTrustBundle(ctx context.Context, _ *fulciogrpc.GetTrus
 	return resp, nil
 }
 
-func (g *grpcCAServer) GetConfiguration(ctx context.Context, _ *fulciogrpc.GetConfigurationRequest) (*fulciogrpc.Configuration, error) {
+func (g *GRPCCAServer) GetConfiguration(ctx context.Context, _ *fulciogrpc.GetConfigurationRequest) (*fulciogrpc.Configuration, error) {
 	cfg := config.FromContext(ctx)
 	if cfg == nil {
 		err := errors.New("configuration not loaded")
@@ -263,3 +266,11 @@ func (g *grpcCAServer) GetConfiguration(ctx context.Context, _ *fulciogrpc.GetCo
 		Issuers: cfg.ToIssuers(),
 	}, nil
 }
+
+func (g *GRPCCAServer) Check(_ context.Context, _ *health.HealthCheckRequest) (*health.HealthCheckResponse, error) {
+	return &health.HealthCheckResponse{Status: health.HealthCheckResponse_SERVING}, nil
+}
+
+func (g *GRPCCAServer) Watch(_ *health.HealthCheckRequest, _ health.Health_WatchServer) error {
+	return status.Error(codes.Unimplemented, "unimplemented")
+}


Test patch:
diff --git a/cmd/app/grpc_test.go b/cmd/app/grpc_test.go
index 26dadd6..ea41980 100644
--- a/cmd/app/grpc_test.go
+++ b/cmd/app/grpc_test.go
@@ -16,9 +16,11 @@
 package app
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 )
 
 const keyPEM = `-----BEGIN PRIVATE KEY-----
@@ -108,7 +110,41 @@ tSLmTWsb+j/Oxljalf+rAlItYk297HN0xMvlkHkB80O5Un6OMCHAjJmfOVZal2Y5
 o4ZDR+PzKEbU8eUQbooS
 -----END CERTIFICATE-----`
 
-func TestCreateGRPCCreds(t *testing.T) {
+const renewedCertPEM = `-----BEGIN CERTIFICATE-----
+MIIFqzCCA5OgAwIBAgIUYKKd201v0q4S0FVSdKvRvrC9JQIwDQYJKoZIhvcNAQEL
+BQAwZTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMREwDwYDVQQHDAhLaXJrbGFu
+ZDERMA8GA1UECgwIU2lnc3RvcmUxDzANBgNVBAsMBkZ1bGNpbzESMBAGA1UEAwwJ
+bG9jYWxob3N0MB4XDTIzMDcwMTE4NTUzM1oXDTMzMDYyODE4NTUzM1owZTELMAkG
+A1UEBhMCVVMxCzAJBgNVBAgMAldBMREwDwYDVQQHDAhLaXJrbGFuZDERMA8GA1UE
+CgwIU2lnc3RvcmUxDzANBgNVBAsMBkZ1bGNpbzESMBAGA1UEAwwJbG9jYWxob3N0
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvc3MNQYRO5ytG+8FsPn5
+0Z8koUG4sYPf6ZLTMP78+sRYfz2ggZaP46Hl3f571AB8nXSBRbIc9byDgOVpOs9t
+zRnYA6tyv9cgtOCtgFfuctHFdeWpJXQr4wWhB0oUspmu66cmFKYfdwrfnrjvRkZ2
++33fGK0hC6EXtNIX7sg+Y98jT1iW0AIiBZxMBf8p5d5fTEfodVN/NZ0FN58/TOPO
+jmkStqta+fZrPL02TVZ/IdDx7RSWVqD+KcqOJdSneuCt5qniQgcLwBMIk5ymyMLP
++yKl1GQXHfagxh3e84HlE93XgVCThk9XK8HSQerUmH0oklIx2PgcSS0FPjsz49Us
+Kd7QweZAzcaaaoapK0QkXRvUZLSwPgOjriJtd6Pi5S7xN91DnGmyDHA7BGgCtew7
+1BUgW2AzWXJq3EX0kHjMaEgHCL59SUw/pOlMiNXMC6hnUSH5lnY2isNs9+DUU+Xa
+/Z9ME+B0SiRCaRGq7ZUdXiHuaN+DiRj3hX1VO96wVjvZAh0JklI6pVB7cz6HvOwx
+iAtSiXxqIQZkyac3lP939tAFzLVvpqSqaHzUF8bqBSkWxy8iZVW9EJiIb8wAVE8R
+Sl4WssnUrneMfXjxsyQ271H6DIDLWP4BHtorqcN0vGnOE37N6DjrOJaaTyaThn2q
+Kjmt6ghqvTY0CRVpyQz3szUCAwEAAaNTMFEwHQYDVR0OBBYEFGgoph9DIwXUHUT0
+8y7CtcviGmPhMB8GA1UdIwQYMBaAFGgoph9DIwXUHUT08y7CtcviGmPhMA8GA1Ud
+EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJp6m3a6nMlpzZ9doLh9WOmj
+dCz68oStA0HMflHGkGgtZ9BDNWQI5EL8rnsdj4l5h4nte7s4AQrtoHGdT/QkIwUu
+1Zv3XBypubeof2iUJ+UhIz2Lm9vqt1hOT1HM3U7/4HCf/730y7LqwWIm580PLw1v
+73C4kizrzg4T7C/4Jy5/lF5PzmJOVPWd0LDomWzbpw0pM2h8cYUY02HVEaeVOfHP
+lCoTW7cL36q3I/0RAyjGmK8nNGNpJTB4xjdTT4TJRotkhcKEsTVfIrVJFGlVMDoT
+Fe9T6rAnYQ2TlnPRhp9tDqiRb3Y027nJouddjulgGTRudUAzNkg7lVWJSDc4PwO2
+7gm7I5/mbil6bI1r4djV0FPJZZI7EHgOM8OmKKqo5sLN0WQigZ8GSH1KHOuR1d2j
+m6GJOdUJ7+ZQ+tej0pwNERMSl0+OY+FtsFMusLXoIUUTyaOs1cpFO0ifqZSp9eUP
+50QDoeDGYS0T/0RicNDXMTltE24G3L7mHiPa5rr4tlYvVHYeoez7qFtG5LvDvBH8
+OVZzfoJGB4MbgrFxymai/9i+hdYadt2UHL9BfnUUInDhi/2l0MB1a45DYOzf5Zf0
+IYjZh058kDhlL7WOkAhPdvm2wD9KAm4FDInw49PYxpasmDKaOTf1WeWfNa5CVOYp
+wpLmyTovgQl/NcXO7caU
+-----END CERTIFICATE-----`
+
+func TestCachedTLSCert(t *testing.T) {
 	dir := t.TempDir()
 
 	// not PKI material
@@ -166,10 +202,31 @@ func TestCreateGRPCCreds(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			_, err := createGRPCCreds(tc.certPath, tc.keyPath)
+			cachedCert, err := newCachedTLSCert(tc.certPath, tc.keyPath)
 			if tc.success != (err == nil) {
 				t.Errorf("unexpected result: %v", err)
 			}
+			if tc.success {
+				cert := cachedCert.GetCertificate()
+				if cert == nil {
+					t.Fatal("unexpected error reading tls.Certificate object")
+				}
+
+				// update the cert on disk to a renewed value (representing same public/private keypair)
+				os.WriteFile(tc.certPath, []byte(renewedCertPEM), 0644)
+
+				// sleep for a second to let goroutine fire for fsnotify event
+				time.Sleep(1 * time.Second)
+
+				renewedCert := cachedCert.GetCertificate()
+				if renewedCert == nil {
+					t.Fatal("unexpected error reading renewed tls.Certificate object")
+				}
+
+				if bytes.Equal(cert.Certificate[0], renewedCert.Certificate[0]) {
+					t.Fatal("got same certificate after overwriting renewed cert to same file")
+				}
+			}
 		})
 	}
 }
diff --git a/test/prometheus/main.go b/test/prometheus/main.go
index ee0f408..22864cc 100644
--- a/test/prometheus/main.go
+++ b/test/prometheus/main.go
@@ -75,31 +75,27 @@ func checkLatency(latency *dto.MetricFamily) error {
 	if *latency.Type != *dto.MetricType_HISTOGRAM.Enum() {
 		return fmt.Errorf("Wrong type, wanted %+v, got: %+v", dto.MetricType_HISTOGRAM.Enum(), latency.Type)
 	}
-	if len(latency.Metric) != 1 {
-		return fmt.Errorf("Got multiple entries, or none for metric, wanted one, got: %+v", latency.Metric)
-	}
-	// Make sure there's a 'post' and it's a 201.
-	var code string
-	var method string
-	for _, value := range latency.Metric[0].Label {
-		if *value.Name == "code" {
-			code = *value.Value
+
+	for _, metric := range latency.Metric {
+		var code string
+		var method string
+		for _, value := range metric.Label {
+			if *value.Name == "code" {
+				code = *value.Value
+			}
+			if *value.Name == "method" {
+				method = *value.Value
+			}
 		}
-		if *value.Name == "method" {
-			method = *value.Value
+		if code == "201" && method == "post" {
+			if *metric.Histogram.SampleCount != 1 {
+				return fmt.Errorf("Unexpected samplecount, wanted 1, got %d", *metric.Histogram.SampleCount)
+			}
+			return nil
 		}
 	}
-	if code != "201" {
-		return fmt.Errorf("unexpected code, wanted 201, got %s", code)
-	}
-	if method != "post" {
-		return fmt.Errorf("unexpected method, wanted post, got %s", method)
-	}
 
-	if *latency.Metric[0].Histogram.SampleCount != 1 {
-		return fmt.Errorf("Unexpected samplecount, wanted 1, got %d", *latency.Metric[0].Histogram.SampleCount)
-	}
-	return nil
+	return fmt.Errorf("Got multiple entries, or none for metric, wanted one, got: %+v", latency.Metric)
 }
 
 func checkCertCount(certCount *dto.MetricFamily) error {