Repository: kubefleet-dev/kubefleet
Pull request #52: feat: add new work applier metrics
Author: login: michaelawyu
Created: 2025-05-06T13:55:37Z | Merged: 2025-05-13T12:24:21Z

### Description of your changes

This PR adds new work applier metrics.

An old metric, migrated from the old work applier, has been removed, as the annotation required to emit data points has not been set since we moved away from v1alpha1 API (i.e., the annotation is only set in the v1alpha1 CRP controller).

I have:

- [x] Run `make reviewable` to ensure this PR is ready for review.

### How has this code been tested

- [x] Unit tests

### Special notes for your reviewer

Please see the comments in the PR code change for further explanation on the choice of the metrics.



Diff:
diff --git a/cmd/memberagent/main.go b/cmd/memberagent/main.go
index 680e00263..d9b7ebe91 100644
--- a/cmd/memberagent/main.go
+++ b/cmd/memberagent/main.go
@@ -103,7 +103,12 @@ func init() {
 	utilruntime.Must(placementv1beta1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 
-	metrics.Registry.MustRegister(fleetmetrics.JoinResultMetrics, fleetmetrics.LeaveResultMetrics, fleetmetrics.WorkApplyTime)
+	metrics.Registry.MustRegister(
+		fleetmetrics.JoinResultMetrics,
+		fleetmetrics.LeaveResultMetrics,
+		fleetmetrics.FleetWorkProcessingRequestsTotal,
+		fleetmetrics.FleetManifestProcessingRequestsTotal,
+	)
 }
 
 func main() {
diff --git a/pkg/controllers/workapplier/controller.go b/pkg/controllers/workapplier/controller.go
index 0590a1fc2..e87d077df 100644
--- a/pkg/controllers/workapplier/controller.go
+++ b/pkg/controllers/workapplier/controller.go
@@ -461,8 +461,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	// Track the availability information.
 	r.trackInMemberClusterObjAvailability(ctx, bundles, workRef)
 
-	trackWorkApplyLatencyMetric(work)
-
 	// Refresh the status of the Work object.
 	if err := r.refreshWorkStatus(ctx, work, bundles); err != nil {
 		return ctrl.Result{}, err
@@ -473,6 +471,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, err
 	}
 
+	trackWorkAndManifestProcessingRequestMetrics(work)
+
 	// If the Work object is not yet available, reconcile again.
 	if !isWorkObjectAvailable(work) {
 		klog.V(2).InfoS("Work object is not yet in an available state; requeue to monitor its availability", "work", workRef)
diff --git a/pkg/controllers/workapplier/controller_test.go b/pkg/controllers/workapplier/controller_test.go
index 0a705faf2..e6b6433ee 100644
--- a/pkg/controllers/workapplier/controller_test.go
+++ b/pkg/controllers/workapplier/controller_test.go
@@ -26,15 +26,16 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/utils/ptr"
-
-	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/ptr"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 )
 
 const (
@@ -188,6 +189,12 @@ func TestMain(m *testing.M) {
 	// Initialize the variables.
 	initializeVariables()
 
+	// Register the metrics.
+	ctrlmetrics.Registry.MustRegister(
+		metrics.FleetWorkProcessingRequestsTotal,
+		metrics.FleetManifestProcessingRequestsTotal,
+	)
+
 	os.Exit(m.Run())
 }
 
diff --git a/pkg/controllers/workapplier/metrics.go b/pkg/controllers/workapplier/metrics.go
index a72843ae1..6d29a4e02 100644
--- a/pkg/controllers/workapplier/metrics.go
+++ b/pkg/controllers/workapplier/metrics.go
@@ -17,29 +17,193 @@ limitations under the License.
 package workapplier
 
 import (
-	"time"
+	"fmt"
 
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 
 	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils"
+	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller"
 )
 
-func trackWorkApplyLatencyMetric(work *fleetv1beta1.Work) {
-	// Calculate the work apply latency.
-	lastUpdateTime, ok := work.GetAnnotations()[utils.LastWorkUpdateTimeAnnotationKey]
-	if ok {
-		workUpdateTime, parseErr := time.Parse(time.RFC3339, lastUpdateTime)
-		if parseErr != nil {
-			klog.ErrorS(parseErr, "Failed to parse the last update timestamp on the work", "work", klog.KObj(work))
-			return
-		}
+// Note (chenyu1):
+// the metrics for the work applier are added the assumptions that:
+// a) the applier should provide low-cardinality metrics only to keep the memory footprint low,
+//    as there might be a high number (up to 10K) of works/manifests to process in a member cluster; and
+// b) keep the overhead of metrics collection low, esp. considering that the applier runs in
+//    parallel (5 concurrent reconciliations by default) and is on the critical path of Fleet workflows.
+//
+// TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
+// metric collection for better observability.
+
+// Some of the label values for the work/manifest processing request counter metric.
+const (
+	workOrManifestStatusSkipped = "Skipped"
+	workOrManifestStatusUnknown = "Unknown"
+
+	manifestDriftOrDiffDetectionStatusFound    = "Found"
+	manifestDriftOrDiffDetectionStatusNotFound = "NotFound"
+)
+
+func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
+	// Increment the work processing request counter.
+
+	var workApplyStatus string
+	var shouldSkip bool
+	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
+	switch {
+	case workAppliedCond == nil:
+		workApplyStatus = workOrManifestStatusSkipped
+	case workAppliedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: applied condition is stale"))
+		shouldSkip = true
+	case workAppliedCond.Status == metav1.ConditionUnknown:
+		// At this point Fleet does not set the Applied condition to the Unknown status; this
+		// branch is added just for completeness reasons.
+		workApplyStatus = workOrManifestStatusUnknown
+	case len(workAppliedCond.Reason) == 0:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: applied condition has no reason (work=%v)", klog.KObj(work)))
+		shouldSkip = true
+	default:
+		workApplyStatus = workAppliedCond.Reason
+	}
 
-		latency := time.Since(workUpdateTime)
-		metrics.WorkApplyTime.WithLabelValues(work.GetName()).Observe(latency.Seconds())
-		klog.V(2).InfoS("Work has been applied", "work", klog.KObj(work), "latency", latency.Milliseconds())
+	var workAvailabilityStatus string
+	workAvailableCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeAvailable)
+	switch {
+	case workAvailableCond == nil:
+		workAvailabilityStatus = workOrManifestStatusSkipped
+	case workAvailableCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: available condition is stale"))
+		shouldSkip = true
+	case workAvailableCond.Status == metav1.ConditionUnknown:
+		// At this point Fleet does not set the Available condition to the Unknown status; this
+		// branch is added just for completeness reasons.
+		workAvailabilityStatus = workOrManifestStatusUnknown
+	case len(workAvailableCond.Reason) == 0:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: available condition has no reason (work=%v)", klog.KObj(work)))
+	default:
+		workAvailabilityStatus = workAvailableCond.Reason
 	}
 
-	klog.V(2).InfoS("No last update timestamp found on the Work object", "work", klog.KObj(work))
+	var workDiffReportedStatus string
+	workDiffReportedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeDiffReported)
+	switch {
+	case workDiffReportedCond == nil:
+		workDiffReportedStatus = workOrManifestStatusSkipped
+	case workDiffReportedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: diff reported condition is stale"))
+		shouldSkip = true
+	case workDiffReportedCond.Status == metav1.ConditionUnknown:
+		// At this point Fleet does not set the DiffReported condition to the Unknown status; this
+		// branch is added just for completeness reasons.
+		workDiffReportedStatus = workOrManifestStatusUnknown
+	case len(workDiffReportedCond.Reason) == 0:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: diff reported condition has no reason (work=%v)", klog.KObj(work)))
+	default:
+		workDiffReportedStatus = workDiffReportedCond.Reason
+	}
+
+	// Do a sanity check; if any of the work conditions is stale; do not emit any metric data point.
+	if shouldSkip {
+		// An error has been logged earlier.
+		return
+	}
+
+	metrics.FleetWorkProcessingRequestsTotal.WithLabelValues(
+		workApplyStatus,
+		workAvailabilityStatus,
+		workDiffReportedStatus,
+	).Inc()
+
+	// Increment the manifest processing request counter.
+	for idx := range work.Status.ManifestConditions {
+		manifestCond := &work.Status.ManifestConditions[idx]
+
+		// No generation checks are added here, as the observed generation reported
+		// on manifest conditions is the generation of the corresponding object in the
+		// member cluster, not the generation of the work object. Plus, as this
+		// method is called right after the status is refreshed, all conditions are
+		// expected to be up-to-date.
+
+		var manifestApplyStatus string
+		manifestAppliedCond := meta.FindStatusCondition(manifestCond.Conditions, fleetv1beta1.WorkConditionTypeApplied)
+		switch {
+		case manifestAppliedCond == nil:
+			manifestApplyStatus = workOrManifestStatusSkipped
+		case manifestAppliedCond.Status == metav1.ConditionUnknown:
+			// At this point Fleet does not set the Applied condition to the Unknown status; this
+			// branch is added just for completeness reasons.
+			manifestApplyStatus = workOrManifestStatusUnknown
+		case len(manifestAppliedCond.Reason) == 0:
+			// Normally this should never occur; this branch is added just for completeness reasons.
+			_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track manifest metrics: applied condition has no reason (manifest=%v, work=%v)", manifestCond.Identifier, klog.KObj(work)))
+			continue
+		default:
+			manifestApplyStatus = manifestAppliedCond.Reason
+		}
+
+		var manifestAvailabilityStatus string
+		manifestAvailableCond := meta.FindStatusCondition(manifestCond.Conditions, fleetv1beta1.WorkConditionTypeAvailable)
+		switch {
+		case manifestAvailableCond == nil:
+			manifestAvailabilityStatus = workOrManifestStatusSkipped
+		case manifestAvailableCond.Status == metav1.ConditionUnknown:
+			// At this point Fleet does not set the Available condition to the Unknown status; this
+			// branch is added just for completeness reasons.
+			manifestAvailabilityStatus = workOrManifestStatusUnknown
+		case len(manifestAvailableCond.Reason) == 0:
+			// Normally this should never occur; this branch is added just for completeness reasons.
+			_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track manifest metrics: available condition has no reason (manifest=%v, work=%v)", manifestCond.Identifier, klog.KObj(work)))
+			continue
+		default:
+			manifestAvailabilityStatus = manifestAvailableCond.Reason
+		}
+
+		var manifestDiffReportedStatus string
+		manifestDiffReportedCond := meta.FindStatusCondition(manifestCond.Conditions, fleetv1beta1.WorkConditionTypeDiffReported)
+		switch {
+		case manifestDiffReportedCond == nil:
+			manifestDiffReportedStatus = workOrManifestStatusSkipped
+		case manifestDiffReportedCond.Status == metav1.ConditionUnknown:
+			// At this point Fleet does not set the DiffReported condition to the Unknown status; this
+			// branch is added just for completeness reasons.
+			manifestDiffReportedStatus = workOrManifestStatusUnknown
+		case len(manifestDiffReportedCond.Reason) == 0:
+			// Normally this should never occur; this branch is added just for completeness reasons.
+			_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track manifest metrics: diff reported condition has no reason (manifest=%v, work=%v)", manifestCond.Identifier, klog.KObj(work)))
+			continue
+		default:
+			manifestDiffReportedStatus = manifestDiffReportedCond.Reason
+		}
+
+		manifestDriftDetectionStatus := manifestDriftOrDiffDetectionStatusNotFound
+		if manifestCond.DriftDetails != nil {
+			manifestDriftDetectionStatus = manifestDriftOrDiffDetectionStatusFound
+		}
+
+		manifestDiffDetectionStatus := manifestDriftOrDiffDetectionStatusNotFound
+		if manifestCond.DiffDetails != nil {
+			manifestDiffDetectionStatus = manifestDriftOrDiffDetectionStatusFound
+		}
+
+		metrics.FleetManifestProcessingRequestsTotal.WithLabelValues(
+			manifestApplyStatus,
+			manifestAvailabilityStatus,
+			manifestDiffReportedStatus,
+			manifestDriftDetectionStatus,
+			manifestDiffDetectionStatus,
+		).Inc()
+	}
 }
diff --git a/pkg/controllers/workapplier/metrics_test.go b/pkg/controllers/workapplier/metrics_test.go
new file mode 100644
index 000000000..62894fe5e
--- /dev/null
+++ b/pkg/controllers/workapplier/metrics_test.go
@@ -0,0 +1,547 @@
+/*
+Copyright 2025 The KubeFleet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package workapplier
+
+import (
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTrackWorkAndManifestProcessingRequestMetrics(t *testing.T) {
+	workMetricMetadata := `
+		# HELP fleet_work_processing_requests_total Total number of processing requests of work objects, including retries and periodic checks
+		# TYPE fleet_work_processing_requests_total counter
+	`
+
+	manifestMetricMetadata := `
+		# HELP fleet_manifest_processing_requests_total Total number of processing requests of manifest objects, including retries and periodic checks
+		# TYPE fleet_manifest_processing_requests_total counter
+	`
+
+	testCases := []struct {
+		name                    string
+		work                    *placementv1beta1.Work
+		wantWorkMetricCount     int
+		wantManifestMetricCount int
+		wantWorkCounter         string
+		wantManifestCounter     string
+	}{
+		{
+			name: "applied and available work, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Reason: WorkAllManifestsAppliedReason,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Reason: WorkAllManifestsAvailableReason,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+									Status: metav1.ConditionTrue,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 1,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 1,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "work not applied, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Reason: WorkNotAllManifestsAppliedReason,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFailedToApply),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 2,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 2,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "work applied but not available, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Reason: WorkNotAllManifestsAppliedReason,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Reason: WorkNotAllManifestsAvailableReason,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable),
+									Status: metav1.ConditionFalse,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 3,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 3,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "work diff reported, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Reason: WorkAllManifestsDiffReportedReason,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 4,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="AllManifestsDiffReported"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 4,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="NoDiffFound",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "work diff reporting failed, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Reason: WorkNotAllManifestsDiffReportedReason,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingReportDiffResultTypeFailed),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="AllManifestsDiffReported"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="SomeManifestsHaveNotReportedDiff"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 5,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Failed",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="NoDiffFound",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "applied failed, found drifts, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Reason: WorkNotAllManifestsAppliedReason,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFoundDrifts),
+								},
+							},
+							DriftDetails: &placementv1beta1.DriftDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="AllManifestsDiffReported"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="SomeManifestsHaveNotReportedDiff"} 1
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 6,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 2
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="FoundDrifts",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="Found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Failed",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="NoDiffFound",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "diff reported, found diffs, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Reason: WorkNotAllManifestsDiffReportedReason,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+							DiffDetails: &placementv1beta1.DiffDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="AllManifestsDiffReported"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="SomeManifestsHaveNotReportedDiff"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+			`,
+			wantManifestMetricCount: 7,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 2
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="FoundDrifts",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="Found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="Found",diff_reporting_status="FoundDiff",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Failed",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="NoDiffFound",drift_detection_status="NotFound"} 2
+			`,
+		},
+		// The cases below normally would never occur.
+		{
+			name: "with unknown status",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionUnknown,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionUnknown,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="AllManifestsDiffReported"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="SomeManifestsHaveNotReportedDiff"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Unknown",availability_status="Unknown",diff_reporting_status="Unknown"} 1
+			`,
+			wantManifestMetricCount: 8,
+			wantManifestCounter: `	
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 2
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="FoundDrifts",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="Found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="Found",diff_reporting_status="FoundDiff",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Failed",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="NoDiffFound",drift_detection_status="NotFound"} 2
+            	fleet_manifest_processing_requests_total{apply_status="Unknown",availability_status="Unknown",diff_detection_status="NotFound",diff_reporting_status="Unknown",drift_detection_status="NotFound"} 1
+			`,
+		},
+		{
+			name: "with out-of-date conditions",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       workName,
+					Generation: 2,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:               placementv1beta1.WorkConditionTypeApplied,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeAvailable,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeDiffReported,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingAvailabilityResultTypeFailed),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="AllManifestsApplied",availability_status="AllManifestsAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="AllManifestsDiffReported"} 1
+            	fleet_work_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_reporting_status="SomeManifestsHaveNotReportedDiff"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="Skipped",diff_reporting_status="Skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="SomeManifestsAreNotApplied",availability_status="SomeManifestsAreNotAvailable",diff_reporting_status="Skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="Unknown",availability_status="Unknown",diff_reporting_status="Unknown"} 1
+			`,
+			wantManifestMetricCount: 8,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="Available",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 2
+            	fleet_manifest_processing_requests_total{apply_status="Applied",availability_status="ManifestNotAvailableYet",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="FoundDrifts",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="Found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="ManifestApplyFailed",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Skipped",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="Found",diff_reporting_status="FoundDiff",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="Failed",drift_detection_status="NotFound"} 1
+            	fleet_manifest_processing_requests_total{apply_status="Skipped",availability_status="Skipped",diff_detection_status="NotFound",diff_reporting_status="NoDiffFound",drift_detection_status="NotFound"} 2
+            	fleet_manifest_processing_requests_total{apply_status="Unknown",availability_status="Unknown",diff_detection_status="NotFound",diff_reporting_status="Unknown",drift_detection_status="NotFound"} 1
+			`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			trackWorkAndManifestProcessingRequestMetrics(tc.work)
+
+			// Collect the metrics.
+			if c := testutil.CollectAndCount(metrics.FleetWorkProcessingRequestsTotal); c != tc.wantWorkMetricCount {
+				t.Fatalf("unexpected work metric count: got %d, want %d", c, tc.wantWorkMetricCount)
+			}
+
+			if err := testutil.CollectAndCompare(
+				metrics.FleetWorkProcessingRequestsTotal,
+				strings.NewReader(workMetricMetadata+tc.wantWorkCounter),
+			); err != nil {
+				t.Fatalf("unexpected work counter value:\n%v", err)
+			}
+
+			if c := testutil.CollectAndCount(metrics.FleetManifestProcessingRequestsTotal); c != tc.wantManifestMetricCount {
+				t.Fatalf("unexpected manifest metric count: got %d, want %d", c, tc.wantManifestMetricCount)
+			}
+
+			if err := testutil.CollectAndCompare(
+				metrics.FleetManifestProcessingRequestsTotal,
+				strings.NewReader(manifestMetricMetadata+tc.wantManifestCounter),
+			); err != nil {
+				t.Fatalf("unexpected manifest counter value:\n%v", err)
+			}
+		})
+	}
+}
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index 9c832ecd3..efe5a0226 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -86,3 +86,42 @@ var (
 		Help: "Number of currently running scheduling loop",
 	}, []string{})
 )
+
+// The work applier related metrics.
+var (
+	// FleetWorkProcessingRequestsTotal is a prometheus metric which counts the
+	// total number of work object processing requests.
+	//
+	// The following labels are available:
+	// * apply_status: the apply status of the processing request; values can
+	//   be "applied", "failed", or "skipped".
+	// * availability_status: the availability check status of the processing request;
+	//   values can be "available", "unavailable", or "skipped".
+	// * diff_reporting_status: the diff reporting status of the processing request;
+	//   values can be "reported", "failed", or "skipped".
+	FleetWorkProcessingRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "fleet_work_processing_requests_total",
+		Help: "Total number of processing requests of work objects, including retries and periodic checks",
+	}, []string{"apply_status", "availability_status", "diff_reporting_status"})
+
+	// FleetManifestProcessingRequestsTotal is a prometheus metric which counts the
+	// total number of manifest object processing requests.
+	//
+	// The following labels are available:
+	// * apply_status: the apply status of the processing request; see the list of
+	//   apply result types in the work applier source code for possible values.
+	// * availability_status: the availability check status of the processing request;
+	//   see the list of availability check result types in the work applier source code
+	//   for possible values.
+	// * diff_reporting_status: the diff reporting status of the processing request;
+	//   see the list of diff reporting result types in the work applier source code
+	//   for possible values.
+	// * drift_detection_status: the drift detection status of the processing request;
+	//   values can be "found" and "not_found".
+	// * diff_detection_status: the diff detection status of the processing request;
+	//   values can be "found" and "not_found".
+	FleetManifestProcessingRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "fleet_manifest_processing_requests_total",
+		Help: "Total number of processing requests of manifest objects, including retries and periodic checks",
+	}, []string{"apply_status", "availability_status", "diff_reporting_status", "drift_detection_status", "diff_detection_status"})
+)


Reviews:
user: login: britaniar
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-06T15:14:24Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-07T13:55:49Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-07T14:04:03Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-07T14:09:20Z
user: login: ryanzhang-oss
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T02:02:26Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T07:15:32Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T07:16:26Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T07:17:37Z
user: login: britaniar
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T16:33:49Z
user: login: britaniar
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T16:41:30Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-09T14:58:33Z
user: login: michaelawyu
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-09T15:00:24Z
user: login: britaniar
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-12T22:44:39Z

Review comments:
diff_hunk: @@ -0,0 +1,539 @@
+/*
+Copyright 2025 The KubeFleet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package workapplier
+
+import (
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTrackWorkAndManifestProcessingRequestMetrics(t *testing.T) {
+	workMetricMetadata := `
+		# HELP fleet_work_processing_requests_total Total number of processing requests of work objects, including retries and periodic checks
+		# TYPE fleet_work_processing_requests_total counter
+	`
+
+	manifestMetricMetadata := `
+		# HELP fleet_manifest_processing_requests_total Total number of processing requests of manifest objects, including retries and periodic checks
+		# TYPE fleet_manifest_processing_requests_total counter
+	`
+
+	testCases := []struct {
+		name                    string
+		work                    *placementv1beta1.Work
+		wantWorkMetricCount     int
+		wantManifestMetricCount int
+		wantWorkCounter         string
+		wantManifestCounter     string
+	}{
+		{
+			name: "applied and available work, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+									Status: metav1.ConditionTrue,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 1,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 1,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work not applied, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFailedToApply),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 2,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 2,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+				fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work applied but not available, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable),
+									Status: metav1.ConditionFalse,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 3,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 3,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+           		fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reported, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 4,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+           		fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 4,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reporting failed, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingReportDiffResultTypeFailed),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 5,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "applied failed, found drifts, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFoundDrifts),
+								},
+							},
+							DriftDetails: &placementv1beta1.DriftDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 6,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "diff reported, found diffs, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+							DiffDetails: &placementv1beta1.DiffDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+			`,
+			wantManifestMetricCount: 7,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+			`,
+		},
+		// The cases below normally would never occur.
+		{
+			name: "with unknown status",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionUnknown,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionUnknown,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 1
+			`,
+			wantManifestMetricCount: 8,
+			wantManifestCounter: `
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "with out-of-date conditions",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       workName,
+					Generation: 2,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:               placementv1beta1.WorkConditionTypeApplied,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeAvailable,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeDiffReported,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingAvailabilityResultTypeFailed),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 2
+			`,
+			wantManifestMetricCount: 9,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="appliedwithfaileddriftdetection",availability_status="failed",diff_detection_status="not_found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+	}
+
+	for _, tc := range testCases {
path: pkg/controllers/workapplier/metrics_test.go
user: login: britaniar
body: Hi Chen, would it be possible to test the metrics through integration test like updaterun does? We kind established how we wanted to test them and updaterun seems most similar to your case.
https://github.com/jwtty/kubefleet/blob/00ca14e5c8ed158488e3cdf16d3ccfef74b3f08c/pkg/controllers/updaterun/controller_integration_test.go#L256
created_at: 2025-05-06T15:13:55Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,539 @@
+/*
+Copyright 2025 The KubeFleet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package workapplier
+
+import (
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTrackWorkAndManifestProcessingRequestMetrics(t *testing.T) {
+	workMetricMetadata := `
+		# HELP fleet_work_processing_requests_total Total number of processing requests of work objects, including retries and periodic checks
+		# TYPE fleet_work_processing_requests_total counter
+	`
+
+	manifestMetricMetadata := `
+		# HELP fleet_manifest_processing_requests_total Total number of processing requests of manifest objects, including retries and periodic checks
+		# TYPE fleet_manifest_processing_requests_total counter
+	`
+
+	testCases := []struct {
+		name                    string
+		work                    *placementv1beta1.Work
+		wantWorkMetricCount     int
+		wantManifestMetricCount int
+		wantWorkCounter         string
+		wantManifestCounter     string
+	}{
+		{
+			name: "applied and available work, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+									Status: metav1.ConditionTrue,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 1,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 1,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work not applied, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFailedToApply),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 2,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 2,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+				fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work applied but not available, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable),
+									Status: metav1.ConditionFalse,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 3,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 3,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+           		fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reported, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 4,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+           		fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 4,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reporting failed, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingReportDiffResultTypeFailed),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 5,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "applied failed, found drifts, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFoundDrifts),
+								},
+							},
+							DriftDetails: &placementv1beta1.DriftDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 6,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "diff reported, found diffs, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+							DiffDetails: &placementv1beta1.DiffDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+			`,
+			wantManifestMetricCount: 7,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+			`,
+		},
+		// The cases below normally would never occur.
+		{
+			name: "with unknown status",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionUnknown,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionUnknown,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 1
+			`,
+			wantManifestMetricCount: 8,
+			wantManifestCounter: `
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "with out-of-date conditions",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       workName,
+					Generation: 2,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:               placementv1beta1.WorkConditionTypeApplied,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeAvailable,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeDiffReported,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingAvailabilityResultTypeFailed),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 2
+			`,
+			wantManifestMetricCount: 9,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="appliedwithfaileddriftdetection",availability_status="failed",diff_detection_status="not_found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+	}
+
+	for _, tc := range testCases {
path: pkg/controllers/workapplier/metrics_test.go
user: login: michaelawyu
body: Hi Britania! I fear that this pattern might not necessarily apply to the work applier metrics -> the problem is that the metrics we add at this moment for the work applier is a counter that counts individual processing requests (and their results), and in the integration tests we do not have a way to reliably tell how many requests have been processed as the controller might have requeues, failures, etc.
created_at: 2025-05-07T13:55:49Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,539 @@
+/*
+Copyright 2025 The KubeFleet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package workapplier
+
+import (
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTrackWorkAndManifestProcessingRequestMetrics(t *testing.T) {
+	workMetricMetadata := `
+		# HELP fleet_work_processing_requests_total Total number of processing requests of work objects, including retries and periodic checks
+		# TYPE fleet_work_processing_requests_total counter
+	`
+
+	manifestMetricMetadata := `
+		# HELP fleet_manifest_processing_requests_total Total number of processing requests of manifest objects, including retries and periodic checks
+		# TYPE fleet_manifest_processing_requests_total counter
+	`
+
+	testCases := []struct {
+		name                    string
+		work                    *placementv1beta1.Work
+		wantWorkMetricCount     int
+		wantManifestMetricCount int
+		wantWorkCounter         string
+		wantManifestCounter     string
+	}{
+		{
+			name: "applied and available work, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+									Status: metav1.ConditionTrue,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 1,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 1,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work not applied, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFailedToApply),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 2,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 2,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+				fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work applied but not available, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable),
+									Status: metav1.ConditionFalse,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 3,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 3,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+           		fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reported, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 4,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+           		fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 4,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reporting failed, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingReportDiffResultTypeFailed),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 5,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "applied failed, found drifts, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFoundDrifts),
+								},
+							},
+							DriftDetails: &placementv1beta1.DriftDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 6,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "diff reported, found diffs, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+							DiffDetails: &placementv1beta1.DiffDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+			`,
+			wantManifestMetricCount: 7,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+			`,
+		},
+		// The cases below normally would never occur.
+		{
+			name: "with unknown status",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionUnknown,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionUnknown,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 1
+			`,
+			wantManifestMetricCount: 8,
+			wantManifestCounter: `
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "with out-of-date conditions",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       workName,
+					Generation: 2,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:               placementv1beta1.WorkConditionTypeApplied,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeAvailable,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeDiffReported,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingAvailabilityResultTypeFailed),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 2
+			`,
+			wantManifestMetricCount: 9,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="appliedwithfaileddriftdetection",availability_status="failed",diff_detection_status="not_found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+	}
+
+	for _, tc := range testCases {
path: pkg/controllers/workapplier/metrics_test.go
user: login: michaelawyu
body:  For the staged updaterun ITs, the metric added (if I understand it correctly) is a gauge for timestamps, and the test comparing logic is essentially checking if a timestamp has been set:

```go
cmp.Comparer(func(a, b *prometheusclientmodel.Gauge) bool {
			return (a.GetValue() > 0) == (b.GetValue() > 0)
		}),
```

This makes sense for this specific metric I guess, and the labels (names, etc.) for the metric are generally guaranteed to be consistent in a IT spec. But for the work applier and its metrics situation are a bit different; the labels (applied or not, available or not, etc.) and their respective counters might be in state of a flux during a IT spec due to controllers retrying/requeueing/failing loops. With that being said, we could still add metric ITs, but for work applier the test would be merely checking if some data has been emitted (or we need to write some complex comparisons to ignore data points on an individual basis), which might not be perfect (or a bit overkilling) just in my personal opinion.
created_at: 2025-05-07T14:04:03Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,539 @@
+/*
+Copyright 2025 The KubeFleet Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package workapplier
+
+import (
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTrackWorkAndManifestProcessingRequestMetrics(t *testing.T) {
+	workMetricMetadata := `
+		# HELP fleet_work_processing_requests_total Total number of processing requests of work objects, including retries and periodic checks
+		# TYPE fleet_work_processing_requests_total counter
+	`
+
+	manifestMetricMetadata := `
+		# HELP fleet_manifest_processing_requests_total Total number of processing requests of manifest objects, including retries and periodic checks
+		# TYPE fleet_manifest_processing_requests_total counter
+	`
+
+	testCases := []struct {
+		name                    string
+		work                    *placementv1beta1.Work
+		wantWorkMetricCount     int
+		wantManifestMetricCount int
+		wantWorkCounter         string
+		wantManifestCounter     string
+	}{
+		{
+			name: "applied and available work, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+									Status: metav1.ConditionTrue,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 1,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 1,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work not applied, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFailedToApply),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 2,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 2,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+				fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work applied but not available, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionTrue,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+									Status: metav1.ConditionTrue,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Reason: string(ManifestProcessingAvailabilityResultTypeNotYetAvailable),
+									Status: metav1.ConditionFalse,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 3,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+				fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+			`,
+			wantManifestMetricCount: 3,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+           		fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reported, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 4,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+           		fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 4,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "work diff reporting failed, single manifest",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingReportDiffResultTypeFailed),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 5,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "applied failed, found drifts, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionFalse,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingApplyResultTypeFoundDrifts),
+								},
+							},
+							DriftDetails: &placementv1beta1.DriftDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeApplied),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingAvailabilityResultTypeAvailable),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 1
+			`,
+			wantManifestMetricCount: 6,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "diff reported, found diffs, multiple manifests",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionTrue,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+							DiffDetails: &placementv1beta1.DiffDetails{},
+						},
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeNoDiffFound),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 5,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+			`,
+			wantManifestMetricCount: 7,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+			`,
+		},
+		// The cases below normally would never occur.
+		{
+			name: "with unknown status",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: workName,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:   placementv1beta1.WorkConditionTypeApplied,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeAvailable,
+							Status: metav1.ConditionUnknown,
+						},
+						{
+							Type:   placementv1beta1.WorkConditionTypeDiffReported,
+							Status: metav1.ConditionUnknown,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionUnknown,
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionUnknown,
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 1
+			`,
+			wantManifestMetricCount: 8,
+			wantManifestCounter: `
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+		{
+			name: "with out-of-date conditions",
+			work: &placementv1beta1.Work{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       workName,
+					Generation: 2,
+				},
+				Status: placementv1beta1.WorkStatus{
+					Conditions: []metav1.Condition{
+						{
+							Type:               placementv1beta1.WorkConditionTypeApplied,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeAvailable,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+						{
+							Type:               placementv1beta1.WorkConditionTypeDiffReported,
+							Status:             metav1.ConditionTrue,
+							ObservedGeneration: 1,
+						},
+					},
+					ManifestConditions: []placementv1beta1.ManifestCondition{
+						{
+							Conditions: []metav1.Condition{
+								{
+									Type:   placementv1beta1.WorkConditionTypeApplied,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingApplyResultTypeAppliedWithFailedDriftDetection),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeAvailable,
+									Status: metav1.ConditionFalse,
+									Reason: string(ManifestProcessingAvailabilityResultTypeFailed),
+								},
+								{
+									Type:   placementv1beta1.WorkConditionTypeDiffReported,
+									Status: metav1.ConditionTrue,
+									Reason: string(ManifestProcessingReportDiffResultTypeFoundDiff),
+								},
+							},
+						},
+					},
+				},
+			},
+			wantWorkMetricCount: 6,
+			wantWorkCounter: `
+				fleet_work_processing_requests_total{apply_status="applied",availability_status="available",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="applied",availability_status="unavailable",diff_reporting_status="skipped"} 1
+            	fleet_work_processing_requests_total{apply_status="failed",availability_status="skipped",diff_reporting_status="skipped"} 2
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="failed"} 1
+            	fleet_work_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_reporting_status="reported"} 2
+            	fleet_work_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_reporting_status="unknown"} 2
+			`,
+			wantManifestMetricCount: 9,
+			wantManifestCounter: `
+				fleet_manifest_processing_requests_total{apply_status="applied",availability_status="available",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="applied",availability_status="manifestnotavailableyet",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="appliedwithfaileddriftdetection",availability_status="failed",diff_detection_status="not_found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="founddrifts",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="manifestapplyfailed",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="skipped",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="found",diff_reporting_status="founddiff",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="failed",drift_detection_status="not_found"} 1
+            	fleet_manifest_processing_requests_total{apply_status="skipped",availability_status="skipped",diff_detection_status="not_found",diff_reporting_status="nodifffound",drift_detection_status="not_found"} 2
+            	fleet_manifest_processing_requests_total{apply_status="unknown",availability_status="unknown",diff_detection_status="not_found",diff_reporting_status="unknown",drift_detection_status="not_found"} 1
+			`,
+		},
+	}
+
+	for _, tc := range testCases {
path: pkg/controllers/workapplier/metrics_test.go
user: login: michaelawyu
body: But I totally understand your concern; if you would like to discuss this further, please let me know 🙏
created_at: 2025-05-07T14:09:20Z
author_association: MEMBER
diff_hunk: @@ -17,29 +17,194 @@ limitations under the License.
 package workapplier
 
 import (
-	"time"
+	"strings"
 
-	"k8s.io/klog/v2"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 )
 
-func trackWorkApplyLatencyMetric(work *fleetv1beta1.Work) {
-	// Calculate the work apply latency.
-	lastUpdateTime, ok := work.GetAnnotations()[utils.LastWorkUpdateTimeAnnotationKey]
-	if ok {
-		workUpdateTime, parseErr := time.Parse(time.RFC3339, lastUpdateTime)
-		if parseErr != nil {
-			klog.ErrorS(parseErr, "Failed to parse the last update timestamp on the work", "work", klog.KObj(work))
-			return
-		}
+// Note (chenyu1):
+// the metrics for the work applier are added the assumptions that:
+// a) the applier should provide low-cardinality metrics only to keep the memory footprint low,
+//    as there might be a high number (up to 10K) of works/manifests to process in a member cluster; and
+// b) keep the overhead of metrics collection low, esp. considering that the applier runs in
+//    parallel (5 concurrent reconciliations by default) and is on the critical path of Fleet workflows.
+//
+// TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
+// metric collection for better observability.
+
+// The label values for the work processing request counter metric.
+const (
+	// The values for the apply status label.
+	workApplyStatusApplied = "applied"
+	workApplyStatusFailed  = "failed"
+	workApplyStatusUnknown = "unknown"
+	workApplyStatusSkipped = "skipped"
+
+	// The values for the availability status label.
+	workAvailabilityStatusAvailable   = "available"
+	workAvailabilityStatusUnavailable = "unavailable"
+	workAvailabilityStatusUnknown     = "unknown"
+	workAvailabilityStatusSkipped     = "skipped"
+
+	// The values for the diff reported status label.
+	workDiffReportedStatusReported    = "reported"
+	workDiffReportedStatusNotReported = "failed"
+	workDiffReportedStatusUnknown     = "unknown"
+	workDiffReportedStatusSkipped     = "skipped"
+)
+
+// Some of the label values for the manifest processing request counter metric.
+const (
+	// The values for the manifest apply status label.
+	manifestApplyStatusUnknown = "unknown"
+	manifestApplyStatusSkipped = "skipped"
+
+	// The values for the manifest availability status label.
+	manifestAvailabilityStatusSkipped = "skipped"
+	manifestAvailabilityStatusUnknown = "unknown"
+
+	// The values for the manifest diff reported status label.
+	manifestDiffReportedStatusSkipped = "skipped"
+	manifestDiffReportedStatusUnknown = "unknown"
+
+	// The values for the manifest drift detection status label.
+	manifestDriftDetectionStatusFound    = "found"
+	manifestDriftDetectionStatusNotFound = "not_found"
+
+	// The values for the manifest diff detection status label.
+	manifestDiffDetectionStatusFound    = "found"
+	manifestDiffDetectionStatusNotFound = "not_found"
+)
+
+func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
+	// Increment the work processing request counter.
+
+	var workApplyStatus string
+	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
+	switch {
+	case workAppliedCond == nil:
+		workApplyStatus = workApplyStatusSkipped
+	case workAppliedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workApplyStatus = workApplyStatusUnknown
+	case workAppliedCond.Status == metav1.ConditionTrue:
+		workApplyStatus = workApplyStatusApplied
path: pkg/controllers/workapplier/metrics.go
user: login: ryanzhang-oss
body: can we use the reason of the condition?
created_at: 2025-05-08T01:11:11Z
author_association: MEMBER
diff_hunk: @@ -17,29 +17,194 @@ limitations under the License.
 package workapplier
 
 import (
-	"time"
+	"strings"
 
-	"k8s.io/klog/v2"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 )
 
-func trackWorkApplyLatencyMetric(work *fleetv1beta1.Work) {
-	// Calculate the work apply latency.
-	lastUpdateTime, ok := work.GetAnnotations()[utils.LastWorkUpdateTimeAnnotationKey]
-	if ok {
-		workUpdateTime, parseErr := time.Parse(time.RFC3339, lastUpdateTime)
-		if parseErr != nil {
-			klog.ErrorS(parseErr, "Failed to parse the last update timestamp on the work", "work", klog.KObj(work))
-			return
-		}
+// Note (chenyu1):
+// the metrics for the work applier are added the assumptions that:
+// a) the applier should provide low-cardinality metrics only to keep the memory footprint low,
+//    as there might be a high number (up to 10K) of works/manifests to process in a member cluster; and
+// b) keep the overhead of metrics collection low, esp. considering that the applier runs in
+//    parallel (5 concurrent reconciliations by default) and is on the critical path of Fleet workflows.
+//
+// TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
+// metric collection for better observability.
+
+// The label values for the work processing request counter metric.
+const (
+	// The values for the apply status label.
+	workApplyStatusApplied = "applied"
+	workApplyStatusFailed  = "failed"
+	workApplyStatusUnknown = "unknown"
+	workApplyStatusSkipped = "skipped"
+
+	// The values for the availability status label.
+	workAvailabilityStatusAvailable   = "available"
+	workAvailabilityStatusUnavailable = "unavailable"
+	workAvailabilityStatusUnknown     = "unknown"
+	workAvailabilityStatusSkipped     = "skipped"
+
+	// The values for the diff reported status label.
+	workDiffReportedStatusReported    = "reported"
+	workDiffReportedStatusNotReported = "failed"
+	workDiffReportedStatusUnknown     = "unknown"
+	workDiffReportedStatusSkipped     = "skipped"
+)
+
+// Some of the label values for the manifest processing request counter metric.
+const (
+	// The values for the manifest apply status label.
+	manifestApplyStatusUnknown = "unknown"
+	manifestApplyStatusSkipped = "skipped"
+
+	// The values for the manifest availability status label.
+	manifestAvailabilityStatusSkipped = "skipped"
+	manifestAvailabilityStatusUnknown = "unknown"
+
+	// The values for the manifest diff reported status label.
+	manifestDiffReportedStatusSkipped = "skipped"
+	manifestDiffReportedStatusUnknown = "unknown"
+
+	// The values for the manifest drift detection status label.
+	manifestDriftDetectionStatusFound    = "found"
+	manifestDriftDetectionStatusNotFound = "not_found"
+
+	// The values for the manifest diff detection status label.
+	manifestDiffDetectionStatusFound    = "found"
+	manifestDiffDetectionStatusNotFound = "not_found"
+)
+
+func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
+	// Increment the work processing request counter.
+
+	var workApplyStatus string
+	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
+	switch {
+	case workAppliedCond == nil:
+		workApplyStatus = workApplyStatusSkipped
+	case workAppliedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workApplyStatus = workApplyStatusUnknown
+	case workAppliedCond.Status == metav1.ConditionTrue:
+		workApplyStatus = workApplyStatusApplied
+	case workAppliedCond.Status == metav1.ConditionFalse:
+		workApplyStatus = workApplyStatusFailed
+	default:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		workApplyStatus = workApplyStatusUnknown
+	}
+
+	var workAvailabilityStatus string
+	workAvailableCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeAvailable)
+	switch {
+	case workAvailableCond == nil:
+		workAvailabilityStatus = workAvailabilityStatusSkipped
+	case workAvailableCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workAvailabilityStatus = workAvailabilityStatusUnknown
+	case workAvailableCond.Status == metav1.ConditionTrue:
+		workAvailabilityStatus = workAvailabilityStatusAvailable
+	case workAvailableCond.Status == metav1.ConditionFalse:
+		workAvailabilityStatus = workAvailabilityStatusUnavailable
+	default:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		workAvailabilityStatus = workAvailabilityStatusUnknown
+	}
 
-		latency := time.Since(workUpdateTime)
-		metrics.WorkApplyTime.WithLabelValues(work.GetName()).Observe(latency.Seconds())
-		klog.V(2).InfoS("Work has been applied", "work", klog.KObj(work), "latency", latency.Milliseconds())
+	var workDiffReportedStatus string
+	workDiffReportedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeDiffReported)
+	switch {
+	case workDiffReportedCond == nil:
+		workDiffReportedStatus = workDiffReportedStatusSkipped
+	case workDiffReportedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workDiffReportedStatus = workDiffReportedStatusUnknown
+	case workDiffReportedCond.Status == metav1.ConditionTrue:
+		workDiffReportedStatus = workDiffReportedStatusReported
+	case workDiffReportedCond.Status == metav1.ConditionFalse:
+		workDiffReportedStatus = workDiffReportedStatusNotReported
+	default:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		workDiffReportedStatus = workDiffReportedStatusUnknown
 	}
 
-	klog.V(2).InfoS("No last update timestamp found on the Work object", "work", klog.KObj(work))
+	metrics.FleetWorkProcessingRequestsTotal.WithLabelValues(
+		workApplyStatus,
+		workAvailabilityStatus,
+		workDiffReportedStatus,
+	).Inc()
+
+	// Increment the manifest processing request counter.
+	for idx := range work.Status.ManifestConditions {
path: pkg/controllers/workapplier/metrics.go
user: login: ryanzhang-oss
body: does it make sense to add manifest condition if the work generation is not the latest? Is it possible for it to be not latest? If not, add an unexpected error?
created_at: 2025-05-08T01:12:45Z
author_association: MEMBER
diff_hunk: @@ -103,7 +103,12 @@ func init() {
 	utilruntime.Must(placementv1beta1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 
-	metrics.Registry.MustRegister(fleetmetrics.JoinResultMetrics, fleetmetrics.LeaveResultMetrics, fleetmetrics.WorkApplyTime)
+	metrics.Registry.MustRegister(
path: cmd/memberagent/main.go
user: login: ryanzhang-oss
body: I think the latency of apply is still something we want to track (not in the PR).
created_at: 2025-05-08T01:16:16Z
author_association: MEMBER
diff_hunk: @@ -17,29 +17,194 @@ limitations under the License.
 package workapplier
 
 import (
-	"time"
+	"strings"
 
-	"k8s.io/klog/v2"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 )
 
-func trackWorkApplyLatencyMetric(work *fleetv1beta1.Work) {
-	// Calculate the work apply latency.
-	lastUpdateTime, ok := work.GetAnnotations()[utils.LastWorkUpdateTimeAnnotationKey]
-	if ok {
-		workUpdateTime, parseErr := time.Parse(time.RFC3339, lastUpdateTime)
-		if parseErr != nil {
-			klog.ErrorS(parseErr, "Failed to parse the last update timestamp on the work", "work", klog.KObj(work))
-			return
-		}
+// Note (chenyu1):
+// the metrics for the work applier are added the assumptions that:
+// a) the applier should provide low-cardinality metrics only to keep the memory footprint low,
+//    as there might be a high number (up to 10K) of works/manifests to process in a member cluster; and
+// b) keep the overhead of metrics collection low, esp. considering that the applier runs in
+//    parallel (5 concurrent reconciliations by default) and is on the critical path of Fleet workflows.
+//
+// TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
+// metric collection for better observability.
+
+// The label values for the work processing request counter metric.
+const (
+	// The values for the apply status label.
+	workApplyStatusApplied = "applied"
+	workApplyStatusFailed  = "failed"
+	workApplyStatusUnknown = "unknown"
+	workApplyStatusSkipped = "skipped"
+
+	// The values for the availability status label.
+	workAvailabilityStatusAvailable   = "available"
+	workAvailabilityStatusUnavailable = "unavailable"
+	workAvailabilityStatusUnknown     = "unknown"
+	workAvailabilityStatusSkipped     = "skipped"
+
+	// The values for the diff reported status label.
+	workDiffReportedStatusReported    = "reported"
+	workDiffReportedStatusNotReported = "failed"
+	workDiffReportedStatusUnknown     = "unknown"
+	workDiffReportedStatusSkipped     = "skipped"
+)
+
+// Some of the label values for the manifest processing request counter metric.
+const (
+	// The values for the manifest apply status label.
+	manifestApplyStatusUnknown = "unknown"
+	manifestApplyStatusSkipped = "skipped"
+
+	// The values for the manifest availability status label.
+	manifestAvailabilityStatusSkipped = "skipped"
+	manifestAvailabilityStatusUnknown = "unknown"
+
+	// The values for the manifest diff reported status label.
+	manifestDiffReportedStatusSkipped = "skipped"
+	manifestDiffReportedStatusUnknown = "unknown"
+
+	// The values for the manifest drift detection status label.
+	manifestDriftDetectionStatusFound    = "found"
+	manifestDriftDetectionStatusNotFound = "not_found"
+
+	// The values for the manifest diff detection status label.
+	manifestDiffDetectionStatusFound    = "found"
+	manifestDiffDetectionStatusNotFound = "not_found"
+)
+
+func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
+	// Increment the work processing request counter.
+
+	var workApplyStatus string
+	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
+	switch {
+	case workAppliedCond == nil:
+		workApplyStatus = workApplyStatusSkipped
+	case workAppliedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workApplyStatus = workApplyStatusUnknown
+	case workAppliedCond.Status == metav1.ConditionTrue:
+		workApplyStatus = workApplyStatusApplied
path: pkg/controllers/workapplier/metrics.go
user: login: michaelawyu
body: Sure, I will make the edits.
created_at: 2025-05-08T07:15:32Z
author_association: MEMBER
diff_hunk: @@ -17,29 +17,194 @@ limitations under the License.
 package workapplier
 
 import (
-	"time"
+	"strings"
 
-	"k8s.io/klog/v2"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 )
 
-func trackWorkApplyLatencyMetric(work *fleetv1beta1.Work) {
-	// Calculate the work apply latency.
-	lastUpdateTime, ok := work.GetAnnotations()[utils.LastWorkUpdateTimeAnnotationKey]
-	if ok {
-		workUpdateTime, parseErr := time.Parse(time.RFC3339, lastUpdateTime)
-		if parseErr != nil {
-			klog.ErrorS(parseErr, "Failed to parse the last update timestamp on the work", "work", klog.KObj(work))
-			return
-		}
+// Note (chenyu1):
+// the metrics for the work applier are added the assumptions that:
+// a) the applier should provide low-cardinality metrics only to keep the memory footprint low,
+//    as there might be a high number (up to 10K) of works/manifests to process in a member cluster; and
+// b) keep the overhead of metrics collection low, esp. considering that the applier runs in
+//    parallel (5 concurrent reconciliations by default) and is on the critical path of Fleet workflows.
+//
+// TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
+// metric collection for better observability.
+
+// The label values for the work processing request counter metric.
+const (
+	// The values for the apply status label.
+	workApplyStatusApplied = "applied"
+	workApplyStatusFailed  = "failed"
+	workApplyStatusUnknown = "unknown"
+	workApplyStatusSkipped = "skipped"
+
+	// The values for the availability status label.
+	workAvailabilityStatusAvailable   = "available"
+	workAvailabilityStatusUnavailable = "unavailable"
+	workAvailabilityStatusUnknown     = "unknown"
+	workAvailabilityStatusSkipped     = "skipped"
+
+	// The values for the diff reported status label.
+	workDiffReportedStatusReported    = "reported"
+	workDiffReportedStatusNotReported = "failed"
+	workDiffReportedStatusUnknown     = "unknown"
+	workDiffReportedStatusSkipped     = "skipped"
+)
+
+// Some of the label values for the manifest processing request counter metric.
+const (
+	// The values for the manifest apply status label.
+	manifestApplyStatusUnknown = "unknown"
+	manifestApplyStatusSkipped = "skipped"
+
+	// The values for the manifest availability status label.
+	manifestAvailabilityStatusSkipped = "skipped"
+	manifestAvailabilityStatusUnknown = "unknown"
+
+	// The values for the manifest diff reported status label.
+	manifestDiffReportedStatusSkipped = "skipped"
+	manifestDiffReportedStatusUnknown = "unknown"
+
+	// The values for the manifest drift detection status label.
+	manifestDriftDetectionStatusFound    = "found"
+	manifestDriftDetectionStatusNotFound = "not_found"
+
+	// The values for the manifest diff detection status label.
+	manifestDiffDetectionStatusFound    = "found"
+	manifestDiffDetectionStatusNotFound = "not_found"
+)
+
+func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
+	// Increment the work processing request counter.
+
+	var workApplyStatus string
+	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
+	switch {
+	case workAppliedCond == nil:
+		workApplyStatus = workApplyStatusSkipped
+	case workAppliedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workApplyStatus = workApplyStatusUnknown
+	case workAppliedCond.Status == metav1.ConditionTrue:
+		workApplyStatus = workApplyStatusApplied
+	case workAppliedCond.Status == metav1.ConditionFalse:
+		workApplyStatus = workApplyStatusFailed
+	default:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		workApplyStatus = workApplyStatusUnknown
+	}
+
+	var workAvailabilityStatus string
+	workAvailableCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeAvailable)
+	switch {
+	case workAvailableCond == nil:
+		workAvailabilityStatus = workAvailabilityStatusSkipped
+	case workAvailableCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workAvailabilityStatus = workAvailabilityStatusUnknown
+	case workAvailableCond.Status == metav1.ConditionTrue:
+		workAvailabilityStatus = workAvailabilityStatusAvailable
+	case workAvailableCond.Status == metav1.ConditionFalse:
+		workAvailabilityStatus = workAvailabilityStatusUnavailable
+	default:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		workAvailabilityStatus = workAvailabilityStatusUnknown
+	}
 
-		latency := time.Since(workUpdateTime)
-		metrics.WorkApplyTime.WithLabelValues(work.GetName()).Observe(latency.Seconds())
-		klog.V(2).InfoS("Work has been applied", "work", klog.KObj(work), "latency", latency.Milliseconds())
+	var workDiffReportedStatus string
+	workDiffReportedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeDiffReported)
+	switch {
+	case workDiffReportedCond == nil:
+		workDiffReportedStatus = workDiffReportedStatusSkipped
+	case workDiffReportedCond.ObservedGeneration != work.Generation:
+		// Normally this should never occur; as the method is called right after the status
+		// is refreshed.
+		workDiffReportedStatus = workDiffReportedStatusUnknown
+	case workDiffReportedCond.Status == metav1.ConditionTrue:
+		workDiffReportedStatus = workDiffReportedStatusReported
+	case workDiffReportedCond.Status == metav1.ConditionFalse:
+		workDiffReportedStatus = workDiffReportedStatusNotReported
+	default:
+		// Normally this should never occur; this branch is added just for completeness reasons.
+		workDiffReportedStatus = workDiffReportedStatusUnknown
 	}
 
-	klog.V(2).InfoS("No last update timestamp found on the Work object", "work", klog.KObj(work))
+	metrics.FleetWorkProcessingRequestsTotal.WithLabelValues(
+		workApplyStatus,
+		workAvailabilityStatus,
+		workDiffReportedStatus,
+	).Inc()
+
+	// Increment the manifest processing request counter.
+	for idx := range work.Status.ManifestConditions {
path: pkg/controllers/workapplier/metrics.go
user: login: michaelawyu
body: Hi Ryan! This method is called only after the status refreshing methods so it's kind of guaranteed to have fresh status conditions; but I could add a sanity check just in case.
created_at: 2025-05-08T07:16:25Z
author_association: MEMBER
diff_hunk: @@ -103,7 +103,12 @@ func init() {
 	utilruntime.Must(placementv1beta1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 
-	metrics.Registry.MustRegister(fleetmetrics.JoinResultMetrics, fleetmetrics.LeaveResultMetrics, fleetmetrics.WorkApplyTime)
+	metrics.Registry.MustRegister(
path: cmd/memberagent/main.go
user: login: michaelawyu
body: Yeah, that is true. It's a bit tricky to implement the whole-flow latency though, but doing a partial bookkeeping (e.g., from work generator to the work applier) would still be relatively easy I think
created_at: 2025-05-08T07:17:37Z
author_association: MEMBER
diff_hunk: @@ -36,105 +39,87 @@ import (
 // TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
 // metric collection for better observability.
 
-// The label values for the work processing request counter metric.
+// Some of the label values for the work/manifest processing request counter metric.
 const (
-	// The values for the apply status label.
-	workApplyStatusApplied = "applied"
-	workApplyStatusFailed  = "failed"
-	workApplyStatusUnknown = "unknown"
-	workApplyStatusSkipped = "skipped"
-
-	// The values for the availability status label.
-	workAvailabilityStatusAvailable   = "available"
-	workAvailabilityStatusUnavailable = "unavailable"
-	workAvailabilityStatusUnknown     = "unknown"
-	workAvailabilityStatusSkipped     = "skipped"
-
-	// The values for the diff reported status label.
-	workDiffReportedStatusReported    = "reported"
-	workDiffReportedStatusNotReported = "failed"
-	workDiffReportedStatusUnknown     = "unknown"
-	workDiffReportedStatusSkipped     = "skipped"
-)
-
-// Some of the label values for the manifest processing request counter metric.
-const (
-	// The values for the manifest apply status label.
-	manifestApplyStatusUnknown = "unknown"
-	manifestApplyStatusSkipped = "skipped"
-
-	// The values for the manifest availability status label.
-	manifestAvailabilityStatusSkipped = "skipped"
-	manifestAvailabilityStatusUnknown = "unknown"
+	workOrManifestStatusSkipped = "skipped"
+	workOrManifestStatusUnknown = "unknown"
 
-	// The values for the manifest diff reported status label.
-	manifestDiffReportedStatusSkipped = "skipped"
-	manifestDiffReportedStatusUnknown = "unknown"
-
-	// The values for the manifest drift detection status label.
-	manifestDriftDetectionStatusFound    = "found"
-	manifestDriftDetectionStatusNotFound = "not_found"
-
-	// The values for the manifest diff detection status label.
-	manifestDiffDetectionStatusFound    = "found"
-	manifestDiffDetectionStatusNotFound = "not_found"
+	manifestDriftOrDiffDetectionStatusFound    = "found"
+	manifestDriftOrDiffDetectionStatusNotFound = "not_found"
 )
 
 func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
 	// Increment the work processing request counter.
 
 	var workApplyStatus string
+	var shouldSkip bool
 	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
 	switch {
 	case workAppliedCond == nil:
-		workApplyStatus = workApplyStatusSkipped
+		workApplyStatus = workOrManifestStatusSkipped
 	case workAppliedCond.ObservedGeneration != work.Generation:
 		// Normally this should never occur; as the method is called right after the status
 		// is refreshed.
-		workApplyStatus = workApplyStatusUnknown
-	case workAppliedCond.Status == metav1.ConditionTrue:
-		workApplyStatus = workApplyStatusApplied
-	case workAppliedCond.Status == metav1.ConditionFalse:
-		workApplyStatus = workApplyStatusFailed
-	default:
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: applied condition is stale"))
+		shouldSkip = true
+	case workAppliedCond.Status == metav1.ConditionUnknown:
+		// At this point Fleet does not set the Applied condition to the Unknown status; this
+		// branch is added just for completeness reasons.
+		workApplyStatus = workOrManifestStatusUnknown
+	case len(workAppliedCond.Reason) == 0:
 		// Normally this should never occur; this branch is added just for completeness reasons.
-		workApplyStatus = workApplyStatusUnknown
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: applied condition has no reason (work=%v)", klog.KObj(work)))
+		shouldSkip = true
+	default:
+		workApplyStatus = strings.ToLower(workAppliedCond.Reason)
path: pkg/controllers/workapplier/metrics.go
user: login: britaniar
body: why do we make it all lowercase? is it not harder to read this way?
created_at: 2025-05-08T16:33:49Z
author_association: MEMBER
diff_hunk: @@ -36,105 +39,87 @@ import (
 // TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
 // metric collection for better observability.
 
-// The label values for the work processing request counter metric.
+// Some of the label values for the work/manifest processing request counter metric.
 const (
-	// The values for the apply status label.
-	workApplyStatusApplied = "applied"
-	workApplyStatusFailed  = "failed"
-	workApplyStatusUnknown = "unknown"
-	workApplyStatusSkipped = "skipped"
-
-	// The values for the availability status label.
-	workAvailabilityStatusAvailable   = "available"
-	workAvailabilityStatusUnavailable = "unavailable"
-	workAvailabilityStatusUnknown     = "unknown"
-	workAvailabilityStatusSkipped     = "skipped"
-
-	// The values for the diff reported status label.
-	workDiffReportedStatusReported    = "reported"
-	workDiffReportedStatusNotReported = "failed"
-	workDiffReportedStatusUnknown     = "unknown"
-	workDiffReportedStatusSkipped     = "skipped"
-)
-
-// Some of the label values for the manifest processing request counter metric.
-const (
-	// The values for the manifest apply status label.
-	manifestApplyStatusUnknown = "unknown"
-	manifestApplyStatusSkipped = "skipped"
-
-	// The values for the manifest availability status label.
-	manifestAvailabilityStatusSkipped = "skipped"
-	manifestAvailabilityStatusUnknown = "unknown"
+	workOrManifestStatusSkipped = "skipped"
+	workOrManifestStatusUnknown = "unknown"
 
-	// The values for the manifest diff reported status label.
-	manifestDiffReportedStatusSkipped = "skipped"
-	manifestDiffReportedStatusUnknown = "unknown"
-
-	// The values for the manifest drift detection status label.
-	manifestDriftDetectionStatusFound    = "found"
-	manifestDriftDetectionStatusNotFound = "not_found"
-
-	// The values for the manifest diff detection status label.
-	manifestDiffDetectionStatusFound    = "found"
-	manifestDiffDetectionStatusNotFound = "not_found"
+	manifestDriftOrDiffDetectionStatusFound    = "found"
+	manifestDriftOrDiffDetectionStatusNotFound = "not_found"
path: pkg/controllers/workapplier/metrics.go
user: login: britaniar
body: I think we should try to be consistent with the reason. Our reasons don't use "_"
```suggestion
	manifestDriftOrDiffDetectionStatusNotFound = "notfound"
```
created_at: 2025-05-08T16:41:30Z
author_association: MEMBER
diff_hunk: @@ -36,105 +39,87 @@ import (
 // TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
 // metric collection for better observability.
 
-// The label values for the work processing request counter metric.
+// Some of the label values for the work/manifest processing request counter metric.
 const (
-	// The values for the apply status label.
-	workApplyStatusApplied = "applied"
-	workApplyStatusFailed  = "failed"
-	workApplyStatusUnknown = "unknown"
-	workApplyStatusSkipped = "skipped"
-
-	// The values for the availability status label.
-	workAvailabilityStatusAvailable   = "available"
-	workAvailabilityStatusUnavailable = "unavailable"
-	workAvailabilityStatusUnknown     = "unknown"
-	workAvailabilityStatusSkipped     = "skipped"
-
-	// The values for the diff reported status label.
-	workDiffReportedStatusReported    = "reported"
-	workDiffReportedStatusNotReported = "failed"
-	workDiffReportedStatusUnknown     = "unknown"
-	workDiffReportedStatusSkipped     = "skipped"
-)
-
-// Some of the label values for the manifest processing request counter metric.
-const (
-	// The values for the manifest apply status label.
-	manifestApplyStatusUnknown = "unknown"
-	manifestApplyStatusSkipped = "skipped"
-
-	// The values for the manifest availability status label.
-	manifestAvailabilityStatusSkipped = "skipped"
-	manifestAvailabilityStatusUnknown = "unknown"
+	workOrManifestStatusSkipped = "skipped"
+	workOrManifestStatusUnknown = "unknown"
 
-	// The values for the manifest diff reported status label.
-	manifestDiffReportedStatusSkipped = "skipped"
-	manifestDiffReportedStatusUnknown = "unknown"
-
-	// The values for the manifest drift detection status label.
-	manifestDriftDetectionStatusFound    = "found"
-	manifestDriftDetectionStatusNotFound = "not_found"
-
-	// The values for the manifest diff detection status label.
-	manifestDiffDetectionStatusFound    = "found"
-	manifestDiffDetectionStatusNotFound = "not_found"
+	manifestDriftOrDiffDetectionStatusFound    = "found"
+	manifestDriftOrDiffDetectionStatusNotFound = "not_found"
 )
 
 func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) {
 	// Increment the work processing request counter.
 
 	var workApplyStatus string
+	var shouldSkip bool
 	workAppliedCond := meta.FindStatusCondition(work.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied)
 	switch {
 	case workAppliedCond == nil:
-		workApplyStatus = workApplyStatusSkipped
+		workApplyStatus = workOrManifestStatusSkipped
 	case workAppliedCond.ObservedGeneration != work.Generation:
 		// Normally this should never occur; as the method is called right after the status
 		// is refreshed.
-		workApplyStatus = workApplyStatusUnknown
-	case workAppliedCond.Status == metav1.ConditionTrue:
-		workApplyStatus = workApplyStatusApplied
-	case workAppliedCond.Status == metav1.ConditionFalse:
-		workApplyStatus = workApplyStatusFailed
-	default:
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: applied condition is stale"))
+		shouldSkip = true
+	case workAppliedCond.Status == metav1.ConditionUnknown:
+		// At this point Fleet does not set the Applied condition to the Unknown status; this
+		// branch is added just for completeness reasons.
+		workApplyStatus = workOrManifestStatusUnknown
+	case len(workAppliedCond.Reason) == 0:
 		// Normally this should never occur; this branch is added just for completeness reasons.
-		workApplyStatus = workApplyStatusUnknown
+		_ = controller.NewUnexpectedBehaviorError(fmt.Errorf("failed to track work metrics: applied condition has no reason (work=%v)", klog.KObj(work)))
+		shouldSkip = true
+	default:
+		workApplyStatus = strings.ToLower(workAppliedCond.Reason)
path: pkg/controllers/workapplier/metrics.go
user: login: michaelawyu
body: Oh, that's because Prometheus actually recommends using snake case (`_`, all lower letters) as mentioned in the docs; I do not particularly have a strong preference on this though. Let me make the edits.
created_at: 2025-05-09T14:58:33Z
author_association: MEMBER
diff_hunk: @@ -36,105 +39,87 @@ import (
 // TO-DO (chenyu1): evaluate if there is any need for high-cardinality options and/or stateful
 // metric collection for better observability.
 
-// The label values for the work processing request counter metric.
+// Some of the label values for the work/manifest processing request counter metric.
 const (
-	// The values for the apply status label.
-	workApplyStatusApplied = "applied"
-	workApplyStatusFailed  = "failed"
-	workApplyStatusUnknown = "unknown"
-	workApplyStatusSkipped = "skipped"
-
-	// The values for the availability status label.
-	workAvailabilityStatusAvailable   = "available"
-	workAvailabilityStatusUnavailable = "unavailable"
-	workAvailabilityStatusUnknown     = "unknown"
-	workAvailabilityStatusSkipped     = "skipped"
-
-	// The values for the diff reported status label.
-	workDiffReportedStatusReported    = "reported"
-	workDiffReportedStatusNotReported = "failed"
-	workDiffReportedStatusUnknown     = "unknown"
-	workDiffReportedStatusSkipped     = "skipped"
-)
-
-// Some of the label values for the manifest processing request counter metric.
-const (
-	// The values for the manifest apply status label.
-	manifestApplyStatusUnknown = "unknown"
-	manifestApplyStatusSkipped = "skipped"
-
-	// The values for the manifest availability status label.
-	manifestAvailabilityStatusSkipped = "skipped"
-	manifestAvailabilityStatusUnknown = "unknown"
+	workOrManifestStatusSkipped = "skipped"
+	workOrManifestStatusUnknown = "unknown"
 
-	// The values for the manifest diff reported status label.
-	manifestDiffReportedStatusSkipped = "skipped"
-	manifestDiffReportedStatusUnknown = "unknown"
-
-	// The values for the manifest drift detection status label.
-	manifestDriftDetectionStatusFound    = "found"
-	manifestDriftDetectionStatusNotFound = "not_found"
-
-	// The values for the manifest diff detection status label.
-	manifestDiffDetectionStatusFound    = "found"
-	manifestDiffDetectionStatusNotFound = "not_found"
+	manifestDriftOrDiffDetectionStatusFound    = "found"
+	manifestDriftOrDiffDetectionStatusNotFound = "not_found"
path: pkg/controllers/workapplier/metrics.go
user: login: michaelawyu
body: Same as the issue before; it's added here because Prometheus prefers snake case. But also as mentioned before I do not have a strong preference on this part. Let me make it camelCase then.
created_at: 2025-05-09T15:00:24Z
author_association: MEMBER

PR comments:
user: login: michaelawyu
created_at: 2025-05-06T13:59:01Z
body: Note: as mentioned in the PR description, the original work applier metric has been removed as it hasn't been set for a long while.

At this moment it might no longer be possible for us to capture the whole workflow latency (i.e., from the moment CRP controller detects a resource change to the moment where the manifests are eventually applied), due to a variety of factors (e.g., rolling out). It's still possible to add a metric that tracks the latency between the moment where work generator prepares a work object to the moment where manifests are finally applied, though how useful such a metric can be remains a topic to further discuss.

Note that the controller runtime default metrics for the work applier are always available, so we do know the latency on work applier reconciliations.
author_association: MEMBER
user: login: codecov[bot]
created_at: 2025-05-07T15:46:33Z
body: ## [Codecov](https://app.codecov.io/gh/kubefleet-dev/kubefleet/pull/52?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kubefleet-dev) Report
Attention: Patch coverage is `85.23490%` with `22 lines` in your changes missing coverage. Please review.
| [Files with missing lines](https://app.codecov.io/gh/kubefleet-dev/kubefleet/pull/52?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kubefleet-dev) | Patch % | Lines |
|---|---|---|
| [pkg/controllers/workapplier/metrics.go](https://app.codecov.io/gh/kubefleet-dev/kubefleet/pull/52?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kubefleet-dev#diff-cGtnL2NvbnRyb2xsZXJzL3dvcmthcHBsaWVyL21ldHJpY3MuZ28=) | 84.50% | [22 Missing :warning: ](https://app.codecov.io/gh/kubefleet-dev/kubefleet/pull/52?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kubefleet-dev) |

:loudspeaker: Thoughts on this report? [Let us know!](https://github.com/codecov/feedback/issues/255)
author_association: NONE