Repository: edgelesssys/contrast
Pull request #1285: coordinator: add startup, liveness and readiness probes
Author: login: miampf
Labels: name: breaking change
description: A user-affecting breaking change
Created: 2025-03-11T14:08:55Z | Merged: 2025-04-11T09:31:30Z

This PR adds the probes specified in [RFC 10](https://github.com/edgelesssys/contrast/blob/main/rfc/010-distributed-coordinator.md).

It also removes the `CONTRAST_METRICS_PORT` environment variable. Instead, contrast prometheus metrics are activated by setting the `CONTRAST_METRICS` variable and are always served on port 9102. An http server on this port will now always be started to serve the probes, `CONTRAST_METRICS` only adds the prometheus service to this server.


Diff:
diff --git a/coordinator/internal/probes/probes.go b/coordinator/internal/probes/probes.go
new file mode 100644
index 00000000000..32a0da25e0a
--- /dev/null
+++ b/coordinator/internal/probes/probes.go
@@ -0,0 +1,59 @@
+// Copyright 2025 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted bool
+	MeshapiStarted bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !h.UserapiStarted || !h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	_, err := h.Hist.HasLatest()
+	if err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+// ReadinessHandler is the http handler for `/probes/readiness`.
+type ReadinessHandler struct {
+	Authority auth
+}
+
+func (h ReadinessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	state, err := h.Authority.GetState()
+	if err != nil || state == nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check that coordinator isn't in recovery mode
+	w.WriteHeader(http.StatusOK)
+}
+
+type auth interface {
+	GetState() (*authority.State, error)
+}
diff --git a/coordinator/internal/probes/probes_test.go b/coordinator/internal/probes/probes_test.go
new file mode 100644
index 00000000000..fd0c1714642
--- /dev/null
+++ b/coordinator/internal/probes/probes_test.go
@@ -0,0 +1,251 @@
+// Copyright 2025 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStartupProbe(t *testing.T) {
+	testCases := map[string]struct {
+		userapiStartedFirst  bool
+		meshapiStartedFirst  bool
+		userapiStartedSecond bool
+		meshapiStartedSecond bool
+		want503First         bool
+		want503Second        bool
+	}{
+		"all immediately online": {
+			userapiStartedFirst:  true,
+			meshapiStartedFirst:  true,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         false,
+			want503Second:        false,
+		},
+		"userapi never starts": {
+			userapiStartedFirst:  false,
+			meshapiStartedFirst:  true,
+			userapiStartedSecond: false,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        true,
+		},
+		"meshapi never starts": {
+			userapiStartedFirst:  true,
+			meshapiStartedFirst:  false,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: false,
+			want503First:         true,
+			want503Second:        true,
+		},
+		"userapi starts later": {
+			userapiStartedFirst:  false,
+			meshapiStartedFirst:  true,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        false,
+		},
+		"meshapi starts later": {
+			userapiStartedFirst:  true,
+			meshapiStartedFirst:  false,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        false,
+		},
+		"both start later": {
+			userapiStartedFirst:  false,
+			meshapiStartedFirst:  false,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			req := httptest.NewRequest(http.MethodGet, "/probes/startup", nil)
+			resp := httptest.NewRecorder()
+
+			mux := http.NewServeMux()
+
+			handler := StartupHandler{MeshapiStarted: tc.meshapiStartedFirst, UserapiStarted: tc.userapiStartedFirst}
+
+			userapiStarted := &handler.UserapiStarted
+			meshapiStarted := &handler.MeshapiStarted
+
+			mux.Handle("/probes/startup", &handler)
+
+			mux.ServeHTTP(resp, req)
+
+			if tc.want503First {
+				assert.Equal(http.StatusServiceUnavailable, resp.Code)
+			} else {
+				assert.Equal(http.StatusOK, resp.Code)
+			}
+
+			resp = httptest.NewRecorder()
+			*userapiStarted = tc.userapiStartedSecond
+			*meshapiStarted = tc.meshapiStartedSecond
+
+			mux.ServeHTTP(resp, req)
+
+			if tc.want503Second {
+				assert.Equal(http.StatusServiceUnavailable, resp.Code)
+			} else {
+				assert.Equal(http.StatusOK, resp.Code)
+			}
+		})
+	}
+}
+
+func TestLivenessProbe(t *testing.T) {
+	testCases := map[string]struct {
+		hasLatest bool
+		err       error
+		want503   bool
+	}{
+		"store accessible but empty": {
+			want503: false,
+		},
+		"store inaccessible": {
+			err:     assert.AnError,
+			want503: true,
+		},
+		"transition exists": {
+			hasLatest: true,
+			want503:   false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			req := httptest.NewRequest(http.MethodGet, "/probes/liveness", nil)
+			resp := httptest.NewRecorder()
+
+			mux := http.NewServeMux()
+
+			store := mockStore{
+				hasLatest:      tc.hasLatest,
+				hasLatestError: tc.err,
+			}
+
+			hist := history.NewWithStore(&slog.Logger{}, store)
+
+			handler := LivenessHandler{Hist: hist}
+			mux.Handle("/probes/liveness", handler)
+
+			mux.ServeHTTP(resp, req)
+			if tc.want503 {
+				assert.Equal(http.StatusServiceUnavailable, resp.Code)
+			} else {
+				assert.Equal(http.StatusOK, resp.Code)
+			}
+		})
+	}
+}
+
+func TestReadinessProbe(t *testing.T) {
+	testCases := map[string]struct {
+		hasActiveManifest bool
+		getStateFails     bool
+		want503           bool
+	}{
+		"manifest exists": {
+			hasActiveManifest: true,
+			want503:           false,
+		},
+		"manifest doesn't exist": {
+			hasActiveManifest: false,
+			want503:           true,
+		},
+		"get state fails while manifest exists": {
+			hasActiveManifest: true,
+			getStateFails:     true,
+			want503:           true,
+		},
+		"get state fails while no manifest exists": {
+			hasActiveManifest: false,
+			getStateFails:     true,
+			want503:           true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			req := httptest.NewRequest(http.MethodGet, "/probes/readiness", nil)
+			resp := httptest.NewRecorder()
+
+			auth := mockAuth{hasState: tc.hasActiveManifest, fails: tc.getStateFails}
+
+			mux := http.NewServeMux()
+
+			handler := ReadinessHandler{Authority: auth}
+			mux.Handle("/probes/readiness", handler)
+
+			mux.ServeHTTP(resp, req)
+			if tc.want503 {
+				assert.Equal(http.StatusServiceUnavailable, resp.Code)
+			} else {
+				assert.Equal(http.StatusOK, resp.Code)
+			}
+		})
+	}
+}
+
+type mockAuth struct {
+	hasState bool
+	fails    bool
+}
+
+func (a mockAuth) GetState() (*authority.State, error) {
+	if a.fails {
+		return nil, assert.AnError
+	}
+	if !a.hasState {
+		return nil, nil
+	}
+	return &authority.State{}, nil
+}
+
+type mockStore struct {
+	hasLatest      bool
+	hasLatestError error
+}
+
+func (s mockStore) Get(_ string) ([]byte, error) {
+	return nil, nil
+}
+
+func (s mockStore) Set(_ string, _ []byte) error {
+	return nil
+}
+
+func (s mockStore) Has(_ string) (bool, error) {
+	return s.hasLatest, s.hasLatestError
+}
+
+func (s mockStore) CompareAndSwap(_ string, _, _ []byte) error {
+	return nil
+}
+
+func (s mockStore) Watch(_ string) (_ <-chan []byte, _ func(), _ error) {
+	return nil, nil, nil
+}
diff --git a/coordinator/main.go b/coordinator/main.go
index 7030c9b5636..4a1e2fc8e78 100644
--- a/coordinator/main.go
+++ b/coordinator/main.go
@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"sync"
 	"syscall"
 	"time"
@@ -18,6 +19,7 @@ import (
 	"github.com/edgelesssys/contrast/coordinator/history"
 	"github.com/edgelesssys/contrast/coordinator/internal/authority"
 	meshapiserver "github.com/edgelesssys/contrast/coordinator/internal/meshapi"
+	"github.com/edgelesssys/contrast/coordinator/internal/probes"
 	"github.com/edgelesssys/contrast/internal/atls"
 	"github.com/edgelesssys/contrast/internal/atls/issuer"
 	"github.com/edgelesssys/contrast/internal/grpc/atlscredentials"
@@ -35,7 +37,8 @@ import (
 )
 
 const (
-	metricsPortEnvVar = "CONTRAST_METRICS_PORT"
+	metricsEnvVar       = "CONTRAST_METRICS"
+	probeAndMetricsPort = 9102
 )
 
 func main() {
@@ -65,7 +68,6 @@ func run() (retErr error) {
 		return fmt.Errorf("setting up mount: %w", err)
 	}
 
-	metricsPort := os.Getenv(metricsPortEnvVar)
 	promRegistry := prometheus.NewRegistry()
 	serverMetrics := newServerMetrics(promRegistry)
 
@@ -92,30 +94,37 @@ func run() (retErr error) {
 	meshapi.RegisterMeshAPIServer(meshAPIServer, meshapiserver.New(logger))
 	serverMetrics.InitializeMetrics(meshAPIServer)
 
-	metricsServer := &http.Server{}
+	httpServer := &http.Server{}
+
+	startupHandler := probes.StartupHandler{UserapiStarted: false, MeshapiStarted: false}
+	livenessHandler := probes.LivenessHandler{Hist: hist}
+	readinessHandler := probes.ReadinessHandler{Authority: meshAuth}
+
+	userapiStarted := &startupHandler.UserapiStarted
+	meshapiStarted := &startupHandler.MeshapiStarted
 
 	eg, ctx := errgroup.WithContext(ctx)
 
 	eg.Go(func() error {
-		if metricsPort == "" {
-			return nil
-		}
-		if metricsPort == userapi.Port || metricsPort == meshapi.Port {
-			return fmt.Errorf("invalid port for metrics endpoint: %s", metricsPort)
-		}
-		logger.Info("Starting prometheus /metrics endpoint on port " + metricsPort)
+		_, enableMetrics := os.LookupEnv(metricsEnvVar)
 		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.InstrumentMetricHandler(
-			promRegistry, promhttp.HandlerFor(
-				promRegistry,
-				promhttp.HandlerOpts{Registry: promRegistry},
-			),
-		))
-		metricsServer.Addr = ":" + metricsPort
-		metricsServer.Handler = mux
-		if err := metricsServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
-			logger.Error("Serving Prometheus /metrics endpoint", "err", err)
-			return fmt.Errorf("serving Prometheus endpoint: %w", err)
+		if enableMetrics {
+			logger.Info("Starting prometheus /metrics endpoint on port " + strconv.Itoa(probeAndMetricsPort))
+			mux.Handle("/metrics", promhttp.InstrumentMetricHandler(
+				promRegistry, promhttp.HandlerFor(
+					promRegistry,
+					promhttp.HandlerOpts{Registry: promRegistry},
+				),
+			))
+		}
+		mux.Handle("/probe/startup", &startupHandler)
+		mux.Handle("/probe/liveness", &livenessHandler)
+		mux.Handle("/probe/readiness", &readinessHandler)
+		httpServer.Addr = ":" + strconv.Itoa(probeAndMetricsPort)
+		httpServer.Handler = mux
+		if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			logger.Error("Starting http server", "err", err)
+			return fmt.Errorf("starting http server: %w", err)
 		}
 		return nil
 	})
@@ -126,6 +135,7 @@ func run() (retErr error) {
 		if err != nil {
 			return fmt.Errorf("failed to listen: %w", err)
 		}
+		*userapiStarted = true
 		if err := userAPIServer.Serve(lis); err != nil {
 			logger.Error("Serving Coordinator API", "err", err)
 			return fmt.Errorf("serving Coordinator API: %w", err)
@@ -139,6 +149,7 @@ func run() (retErr error) {
 		if err != nil {
 			return fmt.Errorf("failed to listen: %w", err)
 		}
+		*meshapiStarted = true
 		if err := meshAPIServer.Serve(lis); err != nil {
 			logger.Error("Serving Coordinator API", "err", err)
 			return fmt.Errorf("serving Coordinator API: %w", err)
@@ -164,7 +175,7 @@ func run() (retErr error) {
 		gracefulStopGRPC(ctx, wg, userAPIServer) //nolint:contextcheck
 		gracefulStopGRPC(ctx, wg, meshAPIServer) //nolint:contextcheck
 		wg.Wait()
-		return metricsServer.Shutdown(ctx) //nolint:contextcheck
+		return httpServer.Shutdown(ctx) //nolint:contextcheck
 	})
 
 	return eg.Wait()
diff --git a/docs/docs/architecture/observability.md b/docs/docs/architecture/observability.md
index d6e0265dc6a..4587256790a 100644
--- a/docs/docs/architecture/observability.md
+++ b/docs/docs/architecture/observability.md
@@ -8,10 +8,9 @@ called labels.
 
 ## Exposed metrics
 
-The metrics can be accessed at the Coordinator pod at the port specified in the
-`CONTRAST_METRICS_PORT` environment variable under the `/metrics` endpoint. By
-default, this environment variable isn't specified, hence no metrics will be
-exposed.
+The metrics can be accessed at the Coordinator pod on port 9102 using the `/metrics` endpoint.
+By default, metrics are disabled. You can enable them by setting the `CONTRAST_METRICS`
+environment variable.
 
 The Coordinator exports gRPC metrics under the prefix `contrast_grpc_server_`.
 These metrics are labeled with the gRPC service name and method name.
diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go
index a9d2b4e054f..c76e86c8dd8 100644
--- a/e2e/internal/contrasttest/contrasttest.go
+++ b/e2e/internal/contrasttest/contrasttest.go
@@ -150,7 +150,7 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) {
 	resources = kuberesource.PatchImages(resources, ct.ImageReplacements)
 	resources = kuberesource.PatchNamespaces(resources, ct.Namespace)
 	resources = kuberesource.PatchServiceMeshAdminInterface(resources, 9901)
-	resources = kuberesource.PatchCoordinatorMetrics(resources, 9102)
+	resources = kuberesource.PatchCoordinatorMetrics(resources)
 	resources = kuberesource.AddLogging(resources, "debug", "*")
 	unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources)
 	require.NoError(err)
diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go
index 2b9a5250252..b5c783a0c46 100644
--- a/e2e/openssl/openssl_test.go
+++ b/e2e/openssl/openssl_test.go
@@ -13,6 +13,7 @@ import (
 	"flag"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"testing"
 	"time"
@@ -24,6 +25,7 @@ import (
 	"github.com/edgelesssys/contrast/internal/platforms"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // namespace the tests are executed in.
@@ -62,7 +64,7 @@ func TestOpenSSL(t *testing.T) {
 
 	require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")
 
-	t.Run("check coordinator metrics endpoint", func(t *testing.T) {
+	t.Run("check coordinator metrics and probe endpoints", func(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), ct.FactorPlatformTimeout(1*time.Minute))
 		defer cancel()
 
@@ -78,9 +80,43 @@ func TestOpenSSL(t *testing.T) {
 		require.NoError(err)
 		require.NotEmpty(coordinatorPods, "pod not found: %s/%s", ct.Namespace, "coordinator")
 
+		// deploy an additional port forwarder for metrics and probes
+		stableNetworkID := fmt.Sprintf("%s.coordinator.%s.svc.cluster.local", coordinatorPods[0].Name, ct.Namespace)
+		t.Logf("Constructed stable network ID %q", stableNetworkID)
+		additionalForwarder := kuberesource.
+			PortForwarder("coordinator-metrics", ct.Namespace).
+			WithListenPorts([]int32{9102}).
+			WithForwardTarget(stableNetworkID)
+
+		patchedForwarder, err := kuberesource.ResourcesToUnstructured(kuberesource.PatchImages([]any{additionalForwarder.PodApplyConfiguration}, ct.ImageReplacements))
+		require.NoError(err)
+		require.NoError(ct.Kubeclient.Apply(ctx, patchedForwarder...))
+
+		t.Cleanup(func() {
+			_ = ct.Kubeclient.Client.CoreV1().Pods(ct.Namespace).Delete(context.Background(), "port-forwarder-coordinator-metrics", metav1.DeleteOptions{})
+		})
+
 		argv := []string{"/bin/sh", "-c", "curl --fail " + net.JoinHostPort(coordinatorPods[0].Status.PodIP, "9102") + "/metrics"}
 		_, stderr, err := ct.Kubeclient.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv)
 		require.NoError(err, "stderr: %q", stderr)
+
+		for _, endpoint := range []string{"/probe/startup", "/probe/liveness", "/probe/readiness"} {
+			require.NoError(ct.Kubeclient.WithForwardedPort(ctx, ct.Namespace, "port-forwarder-coordinator-metrics", "9102", func(addr string) error {
+				req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+addr+endpoint, nil)
+				if err != nil {
+					return err
+				}
+				resp, err := http.DefaultClient.Do(req)
+				if err != nil {
+					return err
+				}
+				defer resp.Body.Close()
+				if resp.StatusCode != http.StatusOK {
+					return fmt.Errorf("unexpected status code from probe %q: %d", endpoint, resp.StatusCode)
+				}
+				return nil
+			}))
+		}
 	})
 
 	for cert, pool := range map[string]*x509.CertPool{
diff --git a/internal/kuberesource/mutators.go b/internal/kuberesource/mutators.go
index 7a23953fbe4..976447ace89 100644
--- a/internal/kuberesource/mutators.go
+++ b/internal/kuberesource/mutators.go
@@ -421,19 +421,19 @@ func PatchServiceMeshAdminInterface(resources []any, port int32) []any {
 	return out
 }
 
-// PatchCoordinatorMetrics enables Coordinator metrics on the specified port.
-func PatchCoordinatorMetrics(resources []any, port int32) []any {
+// PatchCoordinatorMetrics enables Coordinator metrics on port 9102.
+func PatchCoordinatorMetrics(resources []any) []any {
 	for _, resource := range resources {
 		switch r := resource.(type) {
 		case *applyappsv1.StatefulSetApplyConfiguration:
 			if r.Spec != nil && r.Spec.Template != nil && r.Spec.Template.Spec != nil &&
 				len(r.Spec.Template.Spec.Containers) > 0 &&
 				r.Spec.Template.Annotations[contrastRoleAnnotationKey] == "coordinator" {
-				r.Spec.Template.Spec.Containers[0].WithEnv(NewEnvVar("CONTRAST_METRICS_PORT", fmt.Sprint(port)))
+				r.Spec.Template.Spec.Containers[0].WithEnv(NewEnvVar("CONTRAST_METRICS", "1"))
 				r.Spec.Template.Spec.Containers[0].WithPorts(
 					ContainerPort().
 						WithName("prometheus").
-						WithContainerPort(port),
+						WithContainerPort(9102),
 				)
 			}
 		}


Reviews:
user: login: burgerdev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T16:06:00Z
user: login: miampf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:34:11Z
user: login: burgerdev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T10:59:43Z
user: login: miampf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T11:20:04Z
user: login: miampf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T12:14:59Z
user: login: burgerdev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T12:39:33Z
user: login: miampf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T12:43:43Z
user: login: burgerdev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T12:44:41Z
user: login: miampf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T14:44:15Z
user: login: burgerdev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-26T15:44:41Z
user: login: miampf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T10:21:29Z
user: login: burgerdev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-03T14:30:55Z
user: login: burgerdev
body: lgtm, thanks a lot!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-09T10:48:13Z
user: login: katexochen
body: commit structure lgtm
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-09T16:29:00Z

Review comments:
path: coordinator/main.go
user: login: burgerdev
body: Commit message has a gratuitous exclamation mark.
created_at: 2025-03-13T15:36:26Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	read, err := h.Hist.HasLatest()
+	if err != nil || !read {
path: coordinator/internal/probes/probes.go
user: login: burgerdev
body: The liveness handler should return 200 if the transaction store can be read, but is empty. See https://kubernetes.io/docs/concepts/configuration/liveness-readiness-startup-probes/#liveness-probe:

> Liveness probes determine when to restart a container.

A coordinator that does not have a manifest set yet should not be restarted.

You can test the error case by implementing `Stat` from https://pkg.go.dev/github.com/spf13/afero#Fs in a mock.

created_at: 2025-03-13T15:47:54Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
path: coordinator/internal/probes/probes.go
user: login: burgerdev
body: If we need to use pointers here, we need to check whether they are `nil`.
created_at: 2025-03-13T15:48:52Z
author_association: MEMBER
diff_hunk: @@ -81,6 +81,16 @@ func TestOpenSSL(t *testing.T) {
 		argv := []string{"/bin/sh", "-c", "curl --fail " + net.JoinHostPort(coordinatorPods[0].Status.PodIP, "9102") + "/metrics"}
 		_, stderr, err := ct.Kubeclient.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv)
 		require.NoError(err, "stderr: %q", stderr)
+
+		for _, endpoint := range []string{"/probe/startup", "/probe/liveness", "/probe/readiness"} {
+			argv := []string{"/bin/sh", "-c", "curl --silent --output /dev/null -w \"%{http_code}\" " + net.JoinHostPort(coordinatorPods[0].Status.PodIP, "9102") + endpoint}
path: e2e/openssl/openssl_test.go
user: login: burgerdev
body: You could use `ct.Kubeclient.WithForwardedPort` and do `http.Get` directly - that should make response handling clearer.
created_at: 2025-03-13T15:56:44Z
author_association: MEMBER
path: internal/kuberesource/mutators.go
user: login: burgerdev
body: The commit should have the prefix `kuberesource:`.
created_at: 2025-03-13T16:00:10Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
path: coordinator/internal/probes/probes.go
user: login: burgerdev
body: ```suggestion
// Copyright 2025 Edgeless Systems GmbH
```
created_at: 2025-03-13T16:01:10Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,228 @@
+// Copyright 2024 Edgeless Systems GmbH
path: coordinator/internal/probes/probes_test.go
user: login: burgerdev
body: ```suggestion
// Copyright 2025 Edgeless Systems GmbH
```
created_at: 2025-03-13T16:01:21Z
author_association: MEMBER
diff_hunk: @@ -8,10 +8,9 @@ called labels.
 
 ## Exposed metrics
 
-The metrics can be accessed at the Coordinator pod at the port specified in the
-`CONTRAST_METRICS_PORT` environment variable under the `/metrics` endpoint. By
-default, this environment variable isn't specified, hence no metrics will be
-exposed.
+The metrics can be accessed at the Coordinator pod at port 9102 `/metrics` endpoint.
path: docs/docs/architecture/observability.md
user: login: burgerdev
body: ```suggestion
The metrics can be accessed at the Coordinator pod on port 9102 using the `/metrics` endpoint.
```
created_at: 2025-03-13T16:02:14Z
author_association: MEMBER
diff_hunk: @@ -85,30 +87,36 @@ func run() (retErr error) {
 	if err != nil {
 		return fmt.Errorf("creating mesh API server: %w", err)
 	}
-	metricsServer := &http.Server{}
+	httpServer := &http.Server{}
+	userapiStarted := false
+	meshapiStarted := false
+
+	startupHandler := probes.StartupHandler{UserapiStarted: &userapiStarted, MeshapiStarted: &meshapiStarted}
path: coordinator/main.go
user: login: burgerdev
body: If `StartupHandler` did not have pointer fields, you could do
```suggestion
	startupHandler := probes.StartupHandler{}
	userapiStarted := &startupHandler.UserAPIStarted
	meshapiStarted := &startupHandler.MeshAPIStarted
```
created_at: 2025-03-13T16:05:07Z
author_association: MEMBER
path: coordinator/main.go
user: login: miampf
body: I wouldn't say it's gratuitous, after all, this _is_ a breaking change. At the end of the day I'm gonna squash and merge anyways so if you want me to I can omit the exclamation mark there, but I don't see a reason to remove it from a commit in the PR since this makes it easier for people looking for the introduction of the change to find.
created_at: 2025-03-18T10:34:11Z
author_association: CONTRIBUTOR
path: coordinator/main.go
user: login: burgerdev
body: I didn't realize that you were indicating a breaking change with this. Not sure where this convention comes from, but afaik we don't use it in Contrast.
created_at: 2025-03-18T10:59:43Z
author_association: MEMBER
path: coordinator/main.go
user: login: miampf
body: This convention comes directly from [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) (see "Summary > 3. BREAKING CHANGE"). If this goes against commit guidelines we use in contrast, I can remove it or replace it with a more obvious `BREAKING CHANGE:` in the footer.
created_at: 2025-03-18T11:20:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	read, err := h.Hist.HasLatest()
+	if err != nil || !read {
path: coordinator/internal/probes/probes.go
user: login: miampf
body: Do I understand correctly that the liveness probe should return 200 if I can successfully stat the `transitions` directory?
created_at: 2025-03-18T12:14:59Z
author_association: CONTRIBUTOR
path: coordinator/main.go
user: login: burgerdev
body: We don't have any explicit guidelines - I'm only aware of a custom of prefixing the message with the relevant subsystem it happens in. We haven't used `fix|feat|chore` etc.

Breaking changes are signaled with the PR label, which is used for generating the release changelog. 
created_at: 2025-03-18T12:39:33Z
author_association: MEMBER
path: coordinator/main.go
user: login: miampf
body: Okay, I'll rename the commit and for merging I'll use the PR title as the commit message
created_at: 2025-03-18T12:43:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	read, err := h.Hist.HasLatest()
+	if err != nil || !read {
path: coordinator/internal/probes/probes.go
user: login: burgerdev
body: At this level, we don't have a notion of directories - the `Store` is just a key-value store, and our keys are constructed with `/`-separated prefixes. The probe should report `Live` when the store is accessible, which I'd implement as a call to `HasLatest` that does not error out. For testing, we use the afero store, usually with the `MemFs` backend. For testing the probe, I'd just write a custom backend that mocks `Stat` with what you need and use that with the afero store. 
created_at: 2025-03-18T12:44:40Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	read, err := h.Hist.HasLatest()
+	if err != nil || !read {
path: coordinator/internal/probes/probes.go
user: login: miampf
body: Okay, thanks!
created_at: 2025-03-18T14:44:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	read, err := h.Hist.HasLatest()
+	if err != nil || !read {
path: coordinator/internal/probes/probes.go
user: login: burgerdev
body: You still need to remove `|| !read` here.
created_at: 2025-03-26T15:44:41Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,59 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"net/http"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+)
+
+// StartupHandler is the http handler for `/probes/startup`.
+type StartupHandler struct {
+	UserapiStarted *bool
+	MeshapiStarted *bool
+}
+
+func (h StartupHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	if !*h.UserapiStarted || !*h.MeshapiStarted {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	// TODO(miampf): Check if peer recovery was attempted once
+	w.WriteHeader(http.StatusOK)
+}
+
+// LivenessHandler is the http handler for `/probes/liveness`.
+type LivenessHandler struct {
+	Hist *history.History
+}
+
+func (h LivenessHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	read, err := h.Hist.HasLatest()
+	if err != nil || !read {
path: coordinator/internal/probes/probes.go
user: login: miampf
body: I know, in general I still need to get to that part. Am currently still working on the e2e rewrite, I can hopefully do this in the course of the next week.
created_at: 2025-03-27T10:21:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,254 @@
+// Copyright 2025 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package probes
+
+import (
+	"errors"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/edgelesssys/contrast/coordinator/history"
+	"github.com/edgelesssys/contrast/coordinator/internal/authority"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStartupProbe(t *testing.T) {
+	testCases := map[string]struct {
+		userapiStartedFirst  bool
+		meshapiStartedFirst  bool
+		userapiStartedSecond bool
+		meshapiStartedSecond bool
+		want503First         bool
+		want503Second        bool
+	}{
+		"all immediately online": {
+			userapiStartedFirst:  true,
+			meshapiStartedFirst:  true,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         false,
+			want503Second:        false,
+		},
+		"userapi never starts": {
+			userapiStartedFirst:  false,
+			meshapiStartedFirst:  true,
+			userapiStartedSecond: false,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        true,
+		},
+		"meshapi never starts": {
+			userapiStartedFirst:  true,
+			meshapiStartedFirst:  false,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: false,
+			want503First:         true,
+			want503Second:        true,
+		},
+		"userapi starts later": {
+			userapiStartedFirst:  false,
+			meshapiStartedFirst:  true,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        false,
+		},
+		"meshapi starts later": {
+			userapiStartedFirst:  true,
+			meshapiStartedFirst:  false,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        false,
+		},
+		"both start later": {
+			userapiStartedFirst:  false,
+			meshapiStartedFirst:  false,
+			userapiStartedSecond: true,
+			meshapiStartedSecond: true,
+			want503First:         true,
+			want503Second:        false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			req := httptest.NewRequest(http.MethodGet, "/probes/startup", nil)
+			resp := httptest.NewRecorder()
+
+			mux := http.NewServeMux()
+
+			handler := StartupHandler{MeshapiStarted: tc.meshapiStartedFirst, UserapiStarted: tc.userapiStartedFirst}
+
+			userapiStarted := &handler.UserapiStarted
+			meshapiStarted := &handler.MeshapiStarted
+
+			mux.Handle("/probes/startup", &handler)
+
+			mux.ServeHTTP(resp, req)
+
+			if tc.want503First {
+				assert.Equal(http.StatusServiceUnavailable, resp.Code)
+			} else {
+				assert.Equal(http.StatusOK, resp.Code)
+			}
+
+			resp = httptest.NewRecorder()
+			*userapiStarted = tc.userapiStartedSecond
+			*meshapiStarted = tc.meshapiStartedSecond
+
+			mux.ServeHTTP(resp, req)
+
+			if tc.want503Second {
+				assert.Equal(http.StatusServiceUnavailable, resp.Code)
+			} else {
+				assert.Equal(http.StatusOK, resp.Code)
+			}
+		})
+	}
+}
+
+func TestLivenessProbe(t *testing.T) {
+	someError := errors.New("someError")
path: coordinator/internal/probes/probes_test.go
user: login: burgerdev
body: ```suggestion
	someError := assert.AnError
```
created_at: 2025-04-03T14:24:42Z
author_association: MEMBER
diff_hunk: @@ -130,6 +139,7 @@ func run() (retErr error) {
 			logger.Error("Serving Coordinator API", "err", err)
 			return fmt.Errorf("serving Coordinator API: %w", err)
 		}
+		*userapiStarted = true
path: coordinator/main.go
user: login: burgerdev
body: I fear that this is too late - `Serve` returns if there's a fatal error while serving, or the server is closed. Needs to move to before the `Serve` call. Same below.
created_at: 2025-04-03T14:28:57Z
author_association: MEMBER
diff_hunk: @@ -78,9 +80,43 @@ func TestOpenSSL(t *testing.T) {
 		require.NoError(err)
 		require.NotEmpty(coordinatorPods, "pod not found: %s/%s", ct.Namespace, "coordinator")
 
+		// deploy an additional port forwarder for metrics and probes
+		stableNetworkID := fmt.Sprintf("%s.coordinator.%s.svc.cluster.local", coordinatorPods[0].Name, ct.Namespace)
+		t.Logf("Constructed stable network ID %q", stableNetworkID)
+		additionalForwarder := kuberesource.
+			PortForwarder("coordinator-metrics", ct.Namespace).
+			WithListenPorts([]int32{9102}).
+			WithForwardTarget(stableNetworkID)
+
+		patchedForwarder, err := kuberesource.ResourcesToUnstructured(kuberesource.PatchImages([]any{additionalForwarder.PodApplyConfiguration}, ct.ImageReplacements))
+		require.NoError(err)
+		require.NoError(ct.Kubeclient.Apply(ctx, patchedForwarder...))
+
+		t.Cleanup(func() {
+			_ = ct.Kubeclient.Client.CoreV1().Pods(ct.Namespace).Delete(context.Background(), "port-forwarder-coordinator-metrics", metav1.DeleteOptions{})
+		})
+
 		argv := []string{"/bin/sh", "-c", "curl --fail " + net.JoinHostPort(coordinatorPods[0].Status.PodIP, "9102") + "/metrics"}
 		_, stderr, err := ct.Kubeclient.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv)
 		require.NoError(err, "stderr: %q", stderr)
+
+		for _, endpoint := range []string{"/probe/startup", "/probe/liveness", "/probe/readiness"} {
+			require.NoError(ct.Kubeclient.WithForwardedPort(ctx, ct.Namespace, "port-forwarder-coordinator-metrics", "9102", func(addr string) error {
+				req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+addr+endpoint, nil)
+				if err != nil {
+					return err
+				}
+				resp, err := http.DefaultClient.Do(req)
+				if err != nil {
+					return err
+				}
+				defer resp.Body.Close()
+				if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusServiceUnavailable {
path: e2e/openssl/openssl_test.go
user: login: burgerdev
body: ```suggestion
				if resp.StatusCode != http.StatusOK {
```
At this point in the test, the Coordinator should be ready.
created_at: 2025-04-03T14:30:28Z
author_association: MEMBER

PR comments:
user: login: miampf
created_at: 2025-03-11T14:14:24Z
body: I labeled this as a breaking change since the environment variable changes are breaking, however, feel free to remove the label if this is considered "to overboard" :)
author_association: CONTRIBUTOR
user: login: katexochen
created_at: 2025-03-12T10:27:15Z
body: Please cleanup the git history and provide a pr description meaningful to the user. Specifically document how the breaking change affects the user.
author_association: MEMBER
user: login: github-actions[bot]
created_at: 2025-03-13T14:49:04Z
body: [PR Preview Action](https://github.com/rossjrw/pr-preview-action) v1.6.0
:---:
Preview removed because the pull request was closed.
2025-04-11 09:34 UTC
<!-- Sticky Pull Request Commentpr-preview -->
author_association: NONE
user: login: burgerdev
created_at: 2025-04-02T15:04:14Z
body: The test failure is caused by the additional `Set`, afaict - one pod is started with the old mesh cert, one with the new, or something similar.

Fix:

```diff
diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go
index 26c63d592..ae23ebf05 100644
--- a/e2e/openssl/openssl_test.go
+++ b/e2e/openssl/openssl_test.go
@@ -15,7 +15,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"path"
 	"testing"
 	"time"
 
@@ -66,7 +65,7 @@ func TestOpenSSL(t *testing.T) {
 	require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")
 
 	t.Run("check coordinator metrics and probe endpoints", func(t *testing.T) {
-		ctx, cancel := context.WithTimeout(context.Background(), ct.FactorPlatformTimeout(10*time.Minute))
+		ctx, cancel := context.WithTimeout(context.Background(), ct.FactorPlatformTimeout(1*time.Minute))
 		defer cancel()
 
 		require := require.New(t)
@@ -82,7 +81,6 @@ func TestOpenSSL(t *testing.T) {
 		require.NotEmpty(coordinatorPods, "pod not found: %s/%s", ct.Namespace, "coordinator")
 
 		// deploy an additional port forwarder for metrics and probes
-		// FIXME: "TestOpenSSL/go_dial_frontend_with_mesh_CA_cert" fails because of the deployment
 		stableNetworkID := fmt.Sprintf("%s.coordinator.%s.svc.cluster.local", coordinatorPods[0].Name, ct.Namespace)
 		t.Logf("Constructed stable network ID %q", stableNetworkID)
 		additionalForwarder := kuberesource.
@@ -90,17 +88,13 @@ func TestOpenSSL(t *testing.T) {
 			WithListenPorts([]int32{9102}).
 			WithForwardTarget(stableNetworkID)
 
-		resourceBytes, err := kuberesource.EncodeResources(append(resources, additionalForwarder.PodApplyConfiguration)...)
+		patchedForwarder, err := kuberesource.ResourcesToUnstructured(kuberesource.PatchImages([]any{additionalForwarder.PodApplyConfiguration}, ct.ImageReplacements))
 		require.NoError(err)
-		require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yml"), resourceBytes, 0o644))
-
-		require.True(t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")
-		require.True(t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests")
-		require.True(t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests")
-		require.True(t.Run("verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")
+		require.NoError(ct.Kubeclient.Apply(ctx, patchedForwarder...))
 
 		t.Cleanup(func() {
-			require.NoError(ct.Kubeclient.Client.CoreV1().Pods(ct.Namespace).Delete(context.Background(), "port-forwarder-coordinator-metrics", metav1.DeleteOptions{}))
+			_ = ct.Kubeclient.Client.CoreV1().Pods(ct.Namespace).
+				Delete(context.Background(), "port-forwarder-coordinator-metrics", metav1.DeleteOptions{})
 		})
 
 		argv := []string{"/bin/sh", "-c", "curl --fail " + net.JoinHostPort(coordinatorPods[0].Status.PodIP, "9102") + "/metrics"}
```
author_association: MEMBER
user: login: burgerdev
created_at: 2025-04-09T10:49:36Z
body: Remember to squash the changes before merging.
author_association: MEMBER
user: login: miampf
created_at: 2025-04-09T12:12:35Z
body: > Remember to squash the changes before merging.

Gonna rebase and merge because @katexochen requested me too, I'm gonna prepare the commit history for this.
author_association: CONTRIBUTOR
user: login: katexochen
created_at: 2025-04-09T16:30:16Z
body: This cannot have `breaking change` and `no changelog` label. How should the user be informed of the breaking change if not through the changelog?
author_association: MEMBER