Repository: koordinator-sh/koordinator

Issue:
[proposal] Expose Reservation CR Metrics for Resource Utilization Monitoring
<!-- Please only use this template for submitting proposal -->

**What is your proposal**:

  Expose Reservation CR Metrics for Resource Utilization Monitoring

**Why is this needed**:

  On GPU-enabled nodes, CPU and memory resources are frequently ​​underutilized​​. To optimize resource efficiency, CPU-intensive tasks are scheduled on these nodes. However, this leads to ​​GPU task scheduling failures​​ when CPU tasks monopolize excessive CPU/memory resources, starving GPU workloads of necessary compute capacity. Implement a ​​persistent reservation mechanism​​ that:
1. Permanently reserves a dedicated portion of CPU/memory resources ​​exclusively for GPU tasks​​ on each GPU node.
2. Exposes granular metrics for each reservation to monitor: Resource allocation status (e.g., allocated CPU cores, memory)

**Is there a suggested solution, if so, please add it**:

  Expose related resource usage through metrics, similar to the elasticquota plugin.

Hints from issue comments:
@googs1025: Reservation cr is an object provided to resources that need Reservation. I am not sure why we need to care about its usage? 🤔 @saintube: @qinfustu CMIIW, I suppose you want to record the metrics about the reservations' requests to calculate the resource allocation rate of the cluster. It is a good point! And we can add a [GCGaugeVec](https://github.com/koordinator-sh/koordinator/blob/main/pkg/util/metrics/expire_metric.go#L43) named `scheduler_reservation_requests` for it. Please feel free to contribute!@qinfustu: @saintube Correct. The first task is to calculate the cluster resource allocation rate. Additionally, as described in my earlier scenario, reservations are persistently reserving node resources for GPU pods. We need to calculate the allocated value of these reservations to evaluate whether the request configuration is properly set. Add scheduler_reservation_spec and scheduler_reservation_status metrics to expose request, allocatable, and allocated resource data.@zk2k2: Hello I would like to work on this issue, please assign it to me!@qinfustu: @zk2k2 Excellent! I've already developed a version that's in internal use. I'll reference your implementation for future iterations.@saintube: > Hello I would like to work on this issue, please assign it to me!

Welcome!
/assign @zk2k2 @googs1025: @zk2k2 Are you still working on this issue? 😄 
If not I want to take this. I understand the context of this requirement.@zk2k2: @googs1025 No sadly I got too busy, it's all yours, good luck!@googs1025: After some thought, it seems that we should record the indicators in this way, but I am not sure if it is too complicated? 🤔 

```yaml
koordinator_reservation_resource_utilization{namespace="default", reservation_name="reservation-1", resource="cpu"} 0.75
koordinator_reservation_resource_utilization{namespace="default", reservation_name="reservation-1", resource="memory"} 0.5
koordinator_reservation_resource_utilization{namespace="default", reservation_name="reservation-1", resource="nvidia.com/gpu"} 0.5

```@googs1025: /assign

Solution patch:
diff --git a/pkg/scheduler/metrics/metrics.go b/pkg/scheduler/metrics/metrics.go
index 57058a3d..b940412e 100644
--- a/pkg/scheduler/metrics/metrics.go
+++ b/pkg/scheduler/metrics/metrics.go
@@ -48,6 +48,18 @@ var (
 			Name:      "reservation_status_phase",
 			Help:      `The current number of reservations in each status phase (e.g. Pending, Available, Succeeded, Failed)`,
 		}, []string{"name", "phase"}))
+	ReservationResource = utilmetrics.NewGCGaugeVec(
+		"reservation_resource",
+		prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Subsystem: schedulermetrics.SchedulerSubsystem,
+				Name:      "reservation_resource",
+				Help:      "Resource metrics for a reservation, including allocatable, allocated, and utilization with unit.",
+			},
+			[]string{"type", "name", "resource", "unit"},
+		),
+	)
+
 	ElasticQuotaProcessLatency = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
 			Subsystem: schedulermetrics.SchedulerSubsystem,
@@ -101,12 +113,29 @@ var (
 
 	gcMetricsList = []prometheus.Collector{
 		ReservationStatusPhase.GetGaugeVec(),
+		ReservationResource.GetGaugeVec(),
 	}
 )
 
 const (
-	reservationNameKey  = "name"
-	reservationPhaseKey = "phase"
+	reservationNameKey         = "name"
+	reservationPhaseKey        = "phase"
+	reservationResourceKey     = "resource"
+	reservationResourceTypeKey = "type"
+	reservationResourceUnitKey = "unit"
+)
+
+const (
+	UnitCore  = "core"
+	UnitGiB   = "Gi"
+	UnitRaw   = "raw"
+	UnitRatio = "ratio"
+)
+
+const (
+	TypeAllocatable = "allocatable"
+	TypeAllocated   = "allocated"
+	TypeUtilization = "utilization"
 )
 
 // RecordReservationPhase records the phase of a reservation as a metric.
@@ -119,6 +148,17 @@ func RecordReservationPhase(name string, phase string, value float64) {
 	ReservationStatusPhase.WithSet(labels, value)
 }
 
+// RecordReservationResourceByTypeWithUnit records the resource record of a reservation as a metric.
+func RecordReservationResourceByTypeWithUnit(name, resource, typ, unit string, value float64) {
+	labels := prometheus.Labels{
+		reservationResourceTypeKey: typ,
+		reservationNameKey:         name,
+		reservationResourceKey:     resource,
+		reservationResourceUnitKey: unit,
+	}
+	ReservationResource.WithSet(labels, value)
+}
+
 var registerMetrics sync.Once
 
 // Register all metrics.
diff --git a/pkg/scheduler/plugins/elasticquota/controller.go b/pkg/scheduler/plugins/elasticquota/controller.go
index 855c6626..31b8e8cf 100644
--- a/pkg/scheduler/plugins/elasticquota/controller.go
+++ b/pkg/scheduler/plugins/elasticquota/controller.go
@@ -270,6 +270,24 @@ func (ctrl *Controller) syncElasticQuotaStatusMetricsWorker() {
 }
 
 func syncElasticQuotaMetrics(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoSummary) {
+	resourceField, quotaLabels := getResourceFieldAndQuotaLabels(eq, summary)
+	for k, v := range resourceField {
+		decorateResource(eq, v)
+		k = strings.TrimPrefix(k, extension.QuotaKoordinatorPrefix+"/")
+		RecordElasticQuotaMetric(ElasticQuotaStatusMetric, v, k, quotaLabels)
+	}
+}
+
+func deleteElasticQuotaMetrics(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoSummary) {
+	resourceField, quotaLabels := getResourceFieldAndQuotaLabels(eq, summary)
+	for k, v := range resourceField {
+		decorateResource(eq, v)
+		k = strings.TrimPrefix(k, extension.QuotaKoordinatorPrefix+"/")
+		DeleteElasticQuotaMetric(ElasticQuotaStatusMetric, v, k, quotaLabels)
+	}
+}
+
+func getResourceFieldAndQuotaLabels(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoSummary) (map[string]v1.ResourceList, map[string]string) {
 	quotaLabels := map[string]string{
 		"name":      summary.Name,
 		"tree":      summary.Tree,
@@ -294,20 +312,9 @@ func syncElasticQuotaMetrics(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoS
 			m[extension.AnnotationUnschedulableResource] = unschedulable
 		}
 	}
-
-	for k, v := range m {
-		decorateResource(eq, v)
-
-		k = strings.TrimPrefix(k, extension.QuotaKoordinatorPrefix+"/")
-		RecordElasticQuotaMetric(ElasticQuotaStatusMetric, v, k, quotaLabels)
-	}
-	decorateResource(eq, summary.Used)
-	RecordElasticQuotaMetric(ElasticQuotaStatusMetric, summary.Used, "used", quotaLabels)
-
-	decorateResource(eq, summary.Min)
-	decorateResource(eq, summary.Max)
-	decorateResource(eq, summary.SharedWeight)
-	RecordElasticQuotaMetric(ElasticQuotaSpecMetric, summary.Min, "min", quotaLabels)
-	RecordElasticQuotaMetric(ElasticQuotaSpecMetric, summary.Max, "max", quotaLabels)
-	RecordElasticQuotaMetric(ElasticQuotaSpecMetric, summary.SharedWeight, "sharedWeight", quotaLabels)
+	m["used"] = summary.Used
+	m["min"] = summary.Min
+	m["max"] = summary.Max
+	m["sharedWeight"] = summary.SharedWeight
+	return m, quotaLabels
 }
diff --git a/pkg/scheduler/plugins/elasticquota/metrics.go b/pkg/scheduler/plugins/elasticquota/metrics.go
index 70ad6ddc..eeb28e6e 100644
--- a/pkg/scheduler/plugins/elasticquota/metrics.go
+++ b/pkg/scheduler/plugins/elasticquota/metrics.go
@@ -75,6 +75,17 @@ func RecordElasticQuotaMetric(gaugeVec *metrics.GaugeVec, resources corev1.Resou
 func recordResourceMetric(gaugeVec *metrics.GaugeVec, value int64, resourceName, field string, labels map[string]string) {
 	labels["resource"] = resourceName
 	labels["field"] = field
-
 	gaugeVec.With(labels).Set(float64(value))
 }
+
+func DeleteElasticQuotaMetric(gaugeVec *metrics.GaugeVec, resources corev1.ResourceList, field string, labels map[string]string) {
+	for resourceName := range resources {
+		deleteResourceMetric(gaugeVec, string(resourceName), field, labels)
+	}
+}
+
+func deleteResourceMetric(gaugeVec *metrics.GaugeVec, resourceName, field string, labels map[string]string) {
+	labels["resource"] = resourceName
+	labels["field"] = field
+	gaugeVec.Delete(labels)
+}
diff --git a/pkg/scheduler/plugins/elasticquota/quota_handler.go b/pkg/scheduler/plugins/elasticquota/quota_handler.go
index 3fc14d00..18d52031 100644
--- a/pkg/scheduler/plugins/elasticquota/quota_handler.go
+++ b/pkg/scheduler/plugins/elasticquota/quota_handler.go
@@ -104,7 +104,10 @@ func (g *Plugin) OnQuotaDelete(obj interface{}) {
 		klog.Errorf("quota is nil")
 		return
 	}
-
+	summary, _ := g.GetQuotaSummary(quota.Name, false)
+	if summary != nil {
+		deleteElasticQuotaMetrics(quota, summary)
+	}
 	klog.V(5).Infof("OnQuotaDeleteFunc delete quota: %v", quota.Name)
 	g.deleteQuotaToTreeMap(quota.Name)
 	mgr := g.GetGroupQuotaManagerForTree(quota.Labels[extension.LabelQuotaTreeID])
diff --git a/pkg/scheduler/plugins/reservation/controller/controller.go b/pkg/scheduler/plugins/reservation/controller/controller.go
index fd92d91a..c0f2c3a6 100644
--- a/pkg/scheduler/plugins/reservation/controller/controller.go
+++ b/pkg/scheduler/plugins/reservation/controller/controller.go
@@ -255,6 +255,7 @@ func (c *Controller) syncStatus(reservation *schedulingv1alpha1.Reservation) err
 	if apiext.IsReservationAllocateOnce(reservation) {
 		reservationutil.SetReservationSucceeded(reservation)
 	}
+	RecordReservationResource(reservation) // must be called after actualAllocated
 
 	return c.updateReservationStatus(reservation)
 }
@@ -337,3 +338,54 @@ func RecordReservationPhases(reservation *schedulingv1alpha1.Reservation) {
 		metrics.RecordReservationPhase(reservation.Name, phase.name, boolFloat64(isCurrentPhase))
 	}
 }
+
+func RecordReservationResource(reservation *schedulingv1alpha1.Reservation) {
+	if reservation.Status.Allocatable == nil {
+		return
+	}
+	const (
+		MilliCorePerCore = 1000
+		BytesPerGiB      = 1024 * 1024 * 1024
+	)
+
+	resources := quotav1.ResourceNames(reservation.Status.Allocatable)
+	for _, resourceName := range resources {
+		allocatable := reservation.Status.Allocatable[resourceName]
+		allocated := reservation.Status.Allocated[resourceName]
+
+		var allocatableVal, allocatedVal float64
+		var unit string
+
+		switch resourceName {
+		case corev1.ResourceCPU:
+			// mCPU -> Core
+			allocatableVal = float64(allocatable.MilliValue()) / MilliCorePerCore
+			allocatedVal = float64(allocated.MilliValue()) / MilliCorePerCore
+			unit = metrics.UnitCore
+		case corev1.ResourceMemory:
+			// bytes -> GiB
+			allocatableVal = float64(allocatable.Value()) / BytesPerGiB
+			allocatedVal = float64(allocated.Value()) / BytesPerGiB
+			unit = metrics.UnitGiB
+		default:
+			allocatableVal = float64(allocatable.Value())
+			allocatedVal = float64(allocated.Value())
+			unit = metrics.UnitRaw
+		}
+
+		// allocatable
+		metrics.RecordReservationResourceByTypeWithUnit(
+			reservation.Name, string(resourceName), metrics.TypeAllocatable, unit, allocatableVal)
+
+		// allocated
+		metrics.RecordReservationResourceByTypeWithUnit(
+			reservation.Name, string(resourceName), metrics.TypeAllocated, unit, allocatedVal)
+
+		// utilization
+		if allocatableVal > 0 {
+			utilization := allocatedVal / allocatableVal
+			metrics.RecordReservationResourceByTypeWithUnit(
+				reservation.Name, string(resourceName), metrics.TypeUtilization, metrics.UnitRatio, utilization)
+		}
+	}
+}



Test patch:
diff --git a/pkg/scheduler/plugins/elasticquota/controller_test.go b/pkg/scheduler/plugins/elasticquota/controller_test.go
index d90540e3..abe76df1 100644
--- a/pkg/scheduler/plugins/elasticquota/controller_test.go
+++ b/pkg/scheduler/plugins/elasticquota/controller_test.go
@@ -365,7 +365,53 @@ func Test_updateElasticQuotaStatusIfChanged(t *testing.T) {
 	}
 }
 
+func Test_deleteElasticQuotaMetrics(t *testing.T) {
+	eq, summary := getEqAndSummary()
+	syncElasticQuotaMetrics(eq, summary)
+	dtoMetrics := getMetrics(t)
+	assert.Equal(t, 22, len(dtoMetrics))
+	deleteElasticQuotaMetrics(eq, summary)
+	dtoMetricsD := getMetrics(t)
+	assert.Equal(t, 0, len(dtoMetricsD))
+}
+
 func Test_syncElasticQuotaMetrics(t *testing.T) {
+	eq, summary := getEqAndSummary()
+	syncElasticQuotaMetrics(eq, summary)
+	dtoMetrics := getMetrics(t)
+	expect := []string{
+		`{"label":[{"name":"field","value":"allocated"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":7000}}`,
+		`{"label":[{"name":"field","value":"allocated"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"child-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":6000}}`,
+		`{"label":[{"name":"field","value":"child-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"guaranteed"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":8000}}`,
+		`{"label":[{"name":"field","value":"guaranteed"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"max"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":4000}}`,
+		`{"label":[{"name":"field","value":"max"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":200}}`,
+		`{"label":[{"name":"field","value":"min"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":2000}}`,
+		`{"label":[{"name":"field","value":"min"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":100}}`,
+		`{"label":[{"name":"field","value":"non-preemptible-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":3000}}`,
+		`{"label":[{"name":"field","value":"non-preemptible-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"non-preemptible-used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":2000}}`,
+		`{"label":[{"name":"field","value":"non-preemptible-used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":4000}}`,
+		`{"label":[{"name":"field","value":"request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"runtime"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":5000}}`,
+		`{"label":[{"name":"field","value":"runtime"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+		`{"label":[{"name":"field","value":"unschedulable-resource"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":4000}}`,
+		`{"label":[{"name":"field","value":"unschedulable-resource"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":8}}`,
+		`{"label":[{"name":"field","value":"used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":1000}}`,
+		`{"label":[{"name":"field","value":"used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
+	}
+	assert.Equal(t, len(expect), len(dtoMetrics))
+	for i := range dtoMetrics {
+		v := dtoMetrics[i]
+		jsonStrBytes, _ := json.Marshal(v)
+		assert.Equal(t, expect[i], string(jsonStrBytes))
+	}
+}
+
+func getEqAndSummary() (*v1alpha1.ElasticQuota, *core.QuotaInfoSummary) {
 	eq := &v1alpha1.ElasticQuota{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
@@ -396,7 +442,10 @@ func Test_syncElasticQuotaMetrics(t *testing.T) {
 		Allocated:             MakeResourceList().CPU(7).Mem(50).Obj(),
 		Guaranteed:            MakeResourceList().CPU(8).Mem(50).Obj(),
 	}
-	syncElasticQuotaMetrics(eq, summary)
+	return eq, summary
+}
+
+func getMetrics(t *testing.T) []*dto.Metric {
 	metricsCh := make(chan prometheus.Metric, 10)
 	go func() {
 		ElasticQuotaSpecMetric.Collect(metricsCh)
@@ -424,36 +473,7 @@ loopChan:
 	sort.Slice(dtoMetrics, func(i, j int) bool {
 		return dtoMetrics[i].String() < dtoMetrics[j].String()
 	})
-	expect := []string{
-		`{"label":[{"name":"field","value":"allocated"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":7000}}`,
-		`{"label":[{"name":"field","value":"allocated"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"child-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":6000}}`,
-		`{"label":[{"name":"field","value":"child-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"guaranteed"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":8000}}`,
-		`{"label":[{"name":"field","value":"guaranteed"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"max"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":4000}}`,
-		`{"label":[{"name":"field","value":"max"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":200}}`,
-		`{"label":[{"name":"field","value":"min"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":2000}}`,
-		`{"label":[{"name":"field","value":"min"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":100}}`,
-		`{"label":[{"name":"field","value":"non-preemptible-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":3000}}`,
-		`{"label":[{"name":"field","value":"non-preemptible-request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"non-preemptible-used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":2000}}`,
-		`{"label":[{"name":"field","value":"non-preemptible-used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":4000}}`,
-		`{"label":[{"name":"field","value":"request"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"runtime"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":5000}}`,
-		`{"label":[{"name":"field","value":"runtime"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-		`{"label":[{"name":"field","value":"unschedulable-resource"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":4000}}`,
-		`{"label":[{"name":"field","value":"unschedulable-resource"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":8}}`,
-		`{"label":[{"name":"field","value":"used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"cpu"},{"name":"tree","value":"tree-1"}],"gauge":{"value":1000}}`,
-		`{"label":[{"name":"field","value":"used"},{"name":"is_parent","value":"true"},{"name":"name","value":"test-eq"},{"name":"parent","value":"root"},{"name":"resource","value":"memory"},{"name":"tree","value":"tree-1"}],"gauge":{"value":50}}`,
-	}
-	assert.Equal(t, len(expect), len(dtoMetrics))
-	for i := range dtoMetrics {
-		v := dtoMetrics[i]
-		jsonStrBytes, _ := json.Marshal(v)
-		assert.Equal(t, expect[i], string(jsonStrBytes))
-	}
+	return dtoMetrics
 }
 
 type eqWrapper struct{ *v1alpha1.ElasticQuota }
diff --git a/pkg/scheduler/plugins/reservation/controller/controller_test.go b/pkg/scheduler/plugins/reservation/controller/controller_test.go
index 0c01acee..8d833c13 100644
--- a/pkg/scheduler/plugins/reservation/controller/controller_test.go
+++ b/pkg/scheduler/plugins/reservation/controller/controller_test.go
@@ -224,7 +224,10 @@ func TestSyncStatus(t *testing.T) {
 	// register metrics
 	metricsRegistry := basemetrics.NewKubeRegistry()
 	metrics.ReservationStatusPhase.GetGaugeVec().Reset()
-	metricsRegistry.Registerer().MustRegister(metrics.ReservationStatusPhase.GetGaugeVec())
+	metrics.ReservationResource.GetGaugeVec().Reset()
+	metricsRegistry.Registerer().MustRegister(
+		metrics.ReservationStatusPhase.GetGaugeVec(),
+		metrics.ReservationResource.GetGaugeVec())
 
 	reservation := &schedulingv1alpha1.Reservation{
 		ObjectMeta: metav1.ObjectMeta{
@@ -348,4 +351,24 @@ func TestSyncStatus(t *testing.T) {
 	if err := testutil.GatherAndCompare(metricsRegistry, strings.NewReader(expectedSuccessedMetrics), "scheduler_reservation_status_phase"); err != nil {
 		t.Error(err)
 	}
+
+	expectedUtilizationMetrics := fmt.Sprintf(`
+# HELP scheduler_reservation_resource Resource metrics for a reservation, including allocatable, allocated, and utilization with unit.
+# TYPE scheduler_reservation_resource gauge
+scheduler_reservation_resource{name="%s",resource="cpu",type="allocatable",unit="core"} 10
+scheduler_reservation_resource{name="%s",resource="cpu",type="allocated",unit="core"} 8
+scheduler_reservation_resource{name="%s",resource="cpu",type="utilization",unit="ratio"} 0.8
+scheduler_reservation_resource{name="%s",resource="memory",type="allocatable",unit="Gi"} 10
+scheduler_reservation_resource{name="%s",resource="memory",type="allocated",unit="Gi"} 8
+scheduler_reservation_resource{name="%s",resource="memory",type="utilization",unit="ratio"} 0.8
+`,
+		"normalReservation", "normalReservation",
+		"normalReservation", "normalReservation",
+		"normalReservation", "normalReservation",
+	)
+
+	if err := testutil.GatherAndCompare(metricsRegistry, strings.NewReader(expectedUtilizationMetrics),
+		"scheduler_reservation_resource"); err != nil {
+		t.Error(err)
+	}
 }