Repository: spidernet-io/spiderpool
Pull request #1678 (2023-04-19 08:54:50)

add debug level metrics
Signed-off-by: Icarus9913 [icaruswu66@qq.com](mailto:icaruswu66@qq.com)


sonarcloud[bot]: SonarCloud Quality Gate failed.&nbsp; &nbsp; [![Quality Gate failed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/failed-16px.png 'Quality Gate failed')](https://sonarcloud.io/dashboard?id=spidernet-io_spiderpool&pullRequest=1678)

[![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=BUG) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=BUG)  
[![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=VULNERABILITY)  
[![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=SECURITY_HOTSPOT) [![E](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/E-16px.png 'E')](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=SECURITY_HOTSPOT) [5 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=SECURITY_HOTSPOT)  
[![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=CODE_SMELL)

[![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1678) No Coverage information  
[![4.8%](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/5-16px.png '4.8%')](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1678&metric=new_duplicated_lines_density&view=list) [4.8% Duplication](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1678&metric=new_duplicated_lines_density&view=list)


sonarcloud[bot]: SonarCloud Quality Gate failed.&nbsp; &nbsp; [![Quality Gate failed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/failed-16px.png 'Quality Gate failed')](https://sonarcloud.io/dashboard?id=spidernet-io_spiderpool&pullRequest=1678)

[![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=BUG) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=BUG)  
[![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=VULNERABILITY)  
[![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=SECURITY_HOTSPOT) [![E](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/E-16px.png 'E')](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=SECURITY_HOTSPOT) [5 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=SECURITY_HOTSPOT)  
[![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1678&resolved=false&types=CODE_SMELL)

[![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1678) No Coverage information  
[![4.8%](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/5-16px.png '4.8%')](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1678&metric=new_duplicated_lines_density&view=list) [4.8% Duplication](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1678&metric=new_duplicated_lines_density&view=list)



Patch:
diff --git a/cmd/spiderpool-agent/cmd/config.go b/cmd/spiderpool-agent/cmd/config.go
index ccede13f3..326d5f7a2 100644
--- a/cmd/spiderpool-agent/cmd/config.go
+++ b/cmd/spiderpool-agent/cmd/config.go
@@ -54,7 +54,8 @@ var envInfo = []envConf{
 	{"GOLANG_ENV_MAXPROCS", "8", false, nil, nil, &agentContext.Cfg.GoMaxProcs},
 
 	{"SPIDERPOOL_LOG_LEVEL", logutils.LogInfoLevelStr, true, &agentContext.Cfg.LogLevel, nil, nil},
-	{"SPIDERPOOL_ENABLED_METRIC", "false", false, nil, &agentContext.Cfg.EnabledMetric, nil},
+	{"SPIDERPOOL_ENABLED_METRIC", "false", false, nil, &agentContext.Cfg.EnableMetric, nil},
+	{"SPIDERPOOL_ENABLED_DEBUG_METRIC", "false", false, nil, &agentContext.Cfg.EnableDebugLevelMetric, nil},
 	{"SPIDERPOOL_HEALTH_PORT", "5710", true, &agentContext.Cfg.HttpPort, nil, nil},
 	{"SPIDERPOOL_METRIC_HTTP_PORT", "5711", true, &agentContext.Cfg.MetricHttpPort, nil, nil},
 	{"SPIDERPOOL_GOPS_LISTEN_PORT", "5712", false, &agentContext.Cfg.GopsListenPort, nil, nil},
@@ -75,8 +76,9 @@ type Config struct {
 	ConfigPath string
 
 	// env
-	LogLevel      string
-	EnabledMetric bool
+	LogLevel               string
+	EnableMetric           bool
+	EnableDebugLevelMetric bool
 
 	HttpPort         string
 	MetricHttpPort   string
diff --git a/cmd/spiderpool-agent/cmd/metrics_server.go b/cmd/spiderpool-agent/cmd/metrics_server.go
index 884c14c7a..183563d7d 100644
--- a/cmd/spiderpool-agent/cmd/metrics_server.go
+++ b/cmd/spiderpool-agent/cmd/metrics_server.go
@@ -14,7 +14,8 @@ import (
 
 // initAgentMetricsServer will start an opentelemetry http server for spiderpool agent.
 func initAgentMetricsServer(ctx context.Context) {
-	metricController, err := metric.InitMetricController(ctx, constant.SpiderpoolAgent, agentContext.Cfg.EnabledMetric)
+	metricController, err := metric.InitMetric(ctx, constant.SpiderpoolAgent,
+		agentContext.Cfg.EnableMetric, agentContext.Cfg.EnableDebugLevelMetric)
 	if nil != err {
 		logger.Fatal(err.Error())
 	}
@@ -24,7 +25,7 @@ func initAgentMetricsServer(ctx context.Context) {
 		logger.Fatal(err.Error())
 	}
 
-	if agentContext.Cfg.EnabledMetric {
+	if agentContext.Cfg.EnableMetric {
 		metricsSrv := &http.Server{
 			Addr:    fmt.Sprintf(":%s", agentContext.Cfg.MetricHttpPort),
 			Handler: metricController,
diff --git a/cmd/spiderpool-controller/cmd/config.go b/cmd/spiderpool-controller/cmd/config.go
index 44e727127..b46e29c62 100644
--- a/cmd/spiderpool-controller/cmd/config.go
+++ b/cmd/spiderpool-controller/cmd/config.go
@@ -53,7 +53,9 @@ var envInfo = []envConf{
 	{"GOLANG_ENV_MAXPROCS", "8", false, nil, nil, &controllerContext.Cfg.GoMaxProcs},
 
 	{"SPIDERPOOL_LOG_LEVEL", logutils.LogInfoLevelStr, true, &controllerContext.Cfg.LogLevel, nil, nil},
-	{"SPIDERPOOL_ENABLED_METRIC", "false", false, nil, &controllerContext.Cfg.EnabledMetric, nil},
+	{"SPIDERPOOL_ENABLED_METRIC", "false", false, nil, &controllerContext.Cfg.EnableMetric, nil},
+	{"SPIDERPOOL_ENABLED_DEBUG_METRIC", "false", false, nil, &controllerContext.Cfg.EnableDebugLevelMetric, nil},
+	{"SPIDERPOOL_METRIC_RENEW_PERIOD", "120", true, nil, nil, &controllerContext.Cfg.MetricRenewPeriod},
 	{"SPIDERPOOL_HEALTH_PORT", "5720", true, &controllerContext.Cfg.HttpPort, nil, nil},
 	{"SPIDERPOOL_METRIC_HTTP_PORT", "5721", true, &controllerContext.Cfg.MetricHttpPort, nil, nil},
 	{"SPIDERPOOL_WEBHOOK_PORT", "5722", true, &controllerContext.Cfg.WebhookPort, nil, nil},
@@ -104,8 +106,10 @@ type Config struct {
 	TlsServerKeyPath  string
 
 	// env
-	LogLevel      string
-	EnabledMetric bool
+	LogLevel               string
+	EnableMetric           bool
+	EnableDebugLevelMetric bool
+	MetricRenewPeriod      int
 
 	HttpPort         string
 	MetricHttpPort   string
diff --git a/cmd/spiderpool-controller/cmd/daemon.go b/cmd/spiderpool-controller/cmd/daemon.go
index 54d8e4f9a..2b7c5d1df 100644
--- a/cmd/spiderpool-controller/cmd/daemon.go
+++ b/cmd/spiderpool-controller/cmd/daemon.go
@@ -184,6 +184,8 @@ func DaemonMain() {
 
 	setupInformers()
 
+	monitorMetrics(controllerContext.InnerCtx)
+
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
 	WatchSignal(sigCh)
diff --git a/cmd/spiderpool-controller/cmd/metrics_server.go b/cmd/spiderpool-controller/cmd/metrics_server.go
index 5526f4270..bd05cb5c5 100644
--- a/cmd/spiderpool-controller/cmd/metrics_server.go
+++ b/cmd/spiderpool-controller/cmd/metrics_server.go
@@ -7,14 +7,19 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/spidernet-io/spiderpool/pkg/constant"
+	spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
 	"github.com/spidernet-io/spiderpool/pkg/metric"
 )
 
 // initControllerMetricsServer will start an opentelemetry http server for spiderpool controller.
 func initControllerMetricsServer(ctx context.Context) {
-	metricController, err := metric.InitMetricController(ctx, constant.SpiderpoolController, controllerContext.Cfg.EnabledMetric)
+	metricController, err := metric.InitMetric(ctx, constant.SpiderpoolController,
+		controllerContext.Cfg.EnableMetric, controllerContext.Cfg.EnableDebugLevelMetric)
 	if nil != err {
 		logger.Fatal(err.Error())
 	}
@@ -24,7 +29,7 @@ func initControllerMetricsServer(ctx context.Context) {
 		logger.Fatal(err.Error())
 	}
 
-	if controllerContext.Cfg.EnabledMetric {
+	if controllerContext.Cfg.EnableMetric {
 		metricsSrv := &http.Server{
 			Addr:    fmt.Sprintf(":%s", controllerContext.Cfg.MetricHttpPort),
 			Handler: metricController,
@@ -43,3 +48,36 @@ func initControllerMetricsServer(ctx context.Context) {
 		controllerContext.MetricsHttpServer = metricsSrv
 	}
 }
+
+func monitorMetrics(ctx context.Context) {
+	if !controllerContext.Cfg.EnableMetric {
+		return
+	}
+
+	renewPeriod := time.Duration(controllerContext.Cfg.MetricRenewPeriod) * time.Second
+	client := controllerContext.CRDManager.GetClient()
+
+	// record IPPool counts metric
+	go wait.UntilWithContext(ctx, func(ctx context.Context) {
+		var poolList spiderpoolv2beta1.SpiderIPPoolList
+		err := client.List(ctx, &poolList)
+		if nil != err {
+			logger.Sugar().Errorf("failed to monitor metric TotalIPPoolCounts, error: %v", err)
+			return
+		}
+		metric.TotalIPPoolCounts.Record(int64(len(poolList.Items)))
+	}, renewPeriod)
+
+	if controllerContext.Cfg.EnableSpiderSubnet {
+		// record Subnet counts metric
+		go wait.UntilWithContext(ctx, func(ctx context.Context) {
+			var subnetList spiderpoolv2beta1.SpiderSubnetList
+			err := client.List(ctx, &subnetList)
+			if nil != err {
+				logger.Sugar().Errorf("failed to monitor metric TotalSubnetCounts, error: %v", err)
+				return
+			}
+			metric.TotalSubnetCounts.Record(int64(len(subnetList.Items)))
+		}, renewPeriod)
+	}
+}
diff --git a/docs/concepts/metrics.md b/docs/concepts/metrics.md
index d86de147c..a7edfb380 100644
--- a/docs/concepts/metrics.md
+++ b/docs/concepts/metrics.md
@@ -7,19 +7,21 @@ And spiderpool metrics provide the insight of Spiderpool Agent and Spiderpool Co
 
 The metrics of spiderpool controller is set by the following pod environment:
 
-| environment                 | description    | default |
-| --------------------------- | -------------- | ------- |
-| SPIDERPOOL_ENABLED_METRIC   | enable metrics | false   |
-| SPIDERPOOL_METRIC_HTTP_PORT | metrics port   | 5721    |
+| environment                     | description                | default |
+|---------------------------------|----------------------------| ------- |
+| SPIDERPOOL_ENABLED_METRIC       | enable metrics             | false   |
+| SPIDERPOOL_ENABLED_DEBUG_METRIC | enable debug level metrics | false   |
+| SPIDERPOOL_METRIC_HTTP_PORT     | metrics port               | 5721    |
 
 ## spiderpool agent
 
 The metrics of spiderpool agent is set by the following pod environment:
 
-| environment                 | description    | default |
-| --------------------------- | -------------- | ------- |
-| SPIDERPOOL_ENABLED_METRIC   | enable metrics | false   |
-| SPIDERPOOL_METRIC_HTTP_PORT | metrics port   | 5721    |
+| environment                     | description                | default |
+|---------------------------------|----------------------------|---------|
+| SPIDERPOOL_ENABLED_METRIC       | enable metrics             | false   |
+| SPIDERPOOL_ENABLED_DEBUG_METRIC | enable debug level metrics | false   |
+| SPIDERPOOL_METRIC_HTTP_PORT     | metrics port               | 5711    |
 
 ## Get Started
 
@@ -47,48 +49,55 @@ For example, let's enable spiderpool agent metrics by running `helm upgrade --se
 Spiderpool agent exports some metrics related with IPAM allocation and release. Currently, those include:
 
 
-| Name                                                      | description                                                                                                 |
-| --------------------------------------------------------- |-------------------------------------------------------------------------------------------------------------|
-| spiderpool_ipam_allocation_counts                         | Number of IPAM allocation requests that Spiderpool Agent received , prometheus type: counter                |
-| spiderpool_ipam_allocation_failure_counts                 | Number of Spiderpool Agent IPAM allocation failures, prometheus type: counter                               |
-| spiderpool_ipam_allocation_update_ippool_conflict_counts  | Number of Spiderpool Agent IPAM allocation update IPPool conflicts, prometheus type: counter                |
-| spiderpool_ipam_allocation_err_internal_counts            | Number of Spiderpool Agent IPAM allocation internal errors, prometheus type: counter                        |
-| spiderpool_ipam_allocation_err_no_available_pool_counts   | Number of Spiderpool Agent IPAM allocation no available IPPool errors, prometheus type: counter             |
-| spiderpool_ipam_allocation_err_retries_exhausted_counts   | Number of Spiderpool Agent IPAM allocation retries exhausted errors, prometheus type: counter               |
-| spiderpool_ipam_allocation_err_ip_used_out_counts         | Number of Spiderpool Agent IPAM allocation IP addresses used out errors, prometheus type: counter           |
-| spiderpool_ipam_allocation_average_duration_seconds       | The average duration of all Spiderpool Agent allocation processes, prometheus type: gauge                   |
-| spiderpool_ipam_allocation_max_duration_seconds           | The maximum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge           |
-| spiderpool_ipam_allocation_min_duration_seconds           | The minimum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge           |
-| spiderpool_ipam_allocation_latest_duration_seconds        | The latest duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge            |
-| spiderpool_ipam_allocation_duration_seconds               | Histogram of IPAM allocation duration in seconds, prometheus type: histogram                                |
-| spiderpool_ipam_allocation_average_limit_duration_seconds | The average duration of all Spiderpool Agent allocation queuing, prometheus type: gauge                     |
-| spiderpool_ipam_allocation_max_limit_duration_seconds     | The maximum duration of Spiderpool Agent allocation queuing, prometheus type: gauge                         |
-| spiderpool_ipam_allocation_min_limit_duration_seconds     | The minimum duration of Spiderpool Agent allocation queuing, prometheus type: gauge                         |
-| spiderpool_ipam_allocation_latest_limit_duration_seconds  | The latest duration of Spiderpool Agent allocation queuing, prometheus type: gauge                          |
-| spiderpool_ipam_allocation_limit_duration_seconds         | Histogram of IPAM allocation queuing duration in seconds, prometheus type: histogram                        |
-| spiderpool_ipam_release_counts                            | Count of the number of Spiderpool Agent received the IPAM release requests, prometheus type: counter        |
-| spiderpool_ipam_release_failure_counts                    | Number of Spiderpool Agent IPAM release failure, prometheus type: counter                                   |
-| spiderpool_ipam_release_update_ippool_conflict_counts     | Number of Spiderpool Agent IPAM release update IPPool conflicts, prometheus type: counter                   |
-| spiderpool_ipam_release_err_internal_counts               | Number of Spiderpool Agent IPAM releasing internal error, prometheus type: counter                          |
-| spiderpool_ipam_release_err_retries_exhausted_counts      | Number of Spiderpool Agent IPAM releasing retries exhausted error, prometheus type: counter                 |
-| spiderpool_ipam_release_average_duration_seconds          | The average duration of all Spiderpool Agent release processes, prometheus type: gauge                      |
-| spiderpool_ipam_release_max_duration_seconds              | The maximum duration of Spiderpool Agent release process (per-process), prometheus type: gauge              |
-| spiderpool_ipam_release_min_duration_seconds              | The minimum duration of Spiderpool Agent release process (per-process), prometheus type: gauge              |
-| spiderpool_ipam_release_latest_duration_seconds           | The latest duration of Spiderpool Agent release process (per-process), prometheus type: gauge               |
-| spiderpool_ipam_release_duration_seconds                  | Histogram of IPAM release duration in seconds, prometheus type: histogram                                   |
-| spiderpool_ipam_release_average_limit_duration_seconds    | The average duration of all Spiderpool Agent release queuing, prometheus type: gauge                        |
-| spiderpool_ipam_release_max_limit_duration_seconds        | The maximum duration of Spiderpool Agent release queuing, prometheus type: gauge                            |
-| spiderpool_ipam_release_min_limit_duration_seconds        | The minimum duration of Spiderpool Agent release queuing, prometheus type: gauge                            |
-| spiderpool_ipam_release_latest_limit_duration_seconds     | The latest duration of Spiderpool Agent release queuing, prometheus type: gauge                             |
-| spiderpool_ipam_release_limit_duration_seconds            | Histogram of IPAM release queuing duration in seconds, prometheus type: histogram                           |
-| spiderpool_auto_pool_waited_for_available_counts          | Number of Spiderpool Agent IPAM allocation wait for auto-created IPPool available, prometheus type: counter |
+| Name                                                      | description                                                                                                                       |
+|-----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|
+| spiderpool_ipam_allocation_counts                         | Number of IPAM allocation requests that Spiderpool Agent received , prometheus type: counter                                      |
+| spiderpool_ipam_allocation_failure_counts                 | Number of Spiderpool Agent IPAM allocation failures, prometheus type: counter                                                     |
+| spiderpool_ipam_allocation_update_ippool_conflict_counts  | Number of Spiderpool Agent IPAM allocation update IPPool conflicts, prometheus type: counter                                      |
+| spiderpool_ipam_allocation_err_internal_counts            | Number of Spiderpool Agent IPAM allocation internal errors, prometheus type: counter                                              |
+| spiderpool_ipam_allocation_err_no_available_pool_counts   | Number of Spiderpool Agent IPAM allocation no available IPPool errors, prometheus type: counter                                   |
+| spiderpool_ipam_allocation_err_retries_exhausted_counts   | Number of Spiderpool Agent IPAM allocation retries exhausted errors, prometheus type: counter                                     |
+| spiderpool_ipam_allocation_err_ip_used_out_counts         | Number of Spiderpool Agent IPAM allocation IP addresses used out errors, prometheus type: counter                                 |
+| spiderpool_ipam_allocation_average_duration_seconds       | The average duration of all Spiderpool Agent allocation processes, prometheus type: gauge                                         |
+| spiderpool_ipam_allocation_max_duration_seconds           | The maximum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge                                 |
+| spiderpool_ipam_allocation_min_duration_seconds           | The minimum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge                                 |
+| spiderpool_ipam_allocation_latest_duration_seconds        | The latest duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge                                  |
+| spiderpool_ipam_allocation_duration_seconds               | Histogram of IPAM allocation duration in seconds, prometheus type: histogram                                                      |
+| spiderpool_ipam_allocation_average_limit_duration_seconds | The average duration of all Spiderpool Agent allocation queuing, prometheus type: gauge                                           |
+| spiderpool_ipam_allocation_max_limit_duration_seconds     | The maximum duration of Spiderpool Agent allocation queuing, prometheus type: gauge                                               |
+| spiderpool_ipam_allocation_min_limit_duration_seconds     | The minimum duration of Spiderpool Agent allocation queuing, prometheus type: gauge                                               |
+| spiderpool_ipam_allocation_latest_limit_duration_seconds  | The latest duration of Spiderpool Agent allocation queuing, prometheus type: gauge                                                |
+| spiderpool_ipam_allocation_limit_duration_seconds         | Histogram of IPAM allocation queuing duration in seconds, prometheus type: histogram                                              |
+| spiderpool_ipam_release_counts                            | Count of the number of Spiderpool Agent received the IPAM release requests, prometheus type: counter                              |
+| spiderpool_ipam_release_failure_counts                    | Number of Spiderpool Agent IPAM release failure, prometheus type: counter                                                         |
+| spiderpool_ipam_release_update_ippool_conflict_counts     | Number of Spiderpool Agent IPAM release update IPPool conflicts, prometheus type: counter                                         |
+| spiderpool_ipam_release_err_internal_counts               | Number of Spiderpool Agent IPAM releasing internal error, prometheus type: counter                                                |
+| spiderpool_ipam_release_err_retries_exhausted_counts      | Number of Spiderpool Agent IPAM releasing retries exhausted error, prometheus type: counter                                       |
+| spiderpool_ipam_release_average_duration_seconds          | The average duration of all Spiderpool Agent release processes, prometheus type: gauge                                            |
+| spiderpool_ipam_release_max_duration_seconds              | The maximum duration of Spiderpool Agent release process (per-process), prometheus type: gauge                                    |
+| spiderpool_ipam_release_min_duration_seconds              | The minimum duration of Spiderpool Agent release process (per-process), prometheus type: gauge                                    |
+| spiderpool_ipam_release_latest_duration_seconds           | The latest duration of Spiderpool Agent release process (per-process), prometheus type: gauge                                     |
+| spiderpool_ipam_release_duration_seconds                  | Histogram of IPAM release duration in seconds, prometheus type: histogram                                                         |
+| spiderpool_ipam_release_average_limit_duration_seconds    | The average duration of all Spiderpool Agent release queuing, prometheus type: gauge                                              |
+| spiderpool_ipam_release_max_limit_duration_seconds        | The maximum duration of Spiderpool Agent release queuing, prometheus type: gauge                                                  |
+| spiderpool_ipam_release_min_limit_duration_seconds        | The minimum duration of Spiderpool Agent release queuing, prometheus type: gauge                                                  |
+| spiderpool_ipam_release_latest_limit_duration_seconds     | The latest duration of Spiderpool Agent release queuing, prometheus type: gauge                                                   |
+| spiderpool_ipam_release_limit_duration_seconds            | Histogram of IPAM release queuing duration in seconds, prometheus type: histogram                                                 |
+| spiderpool_debug_auto_pool_waited_for_available_counts    | Number of Spiderpool Agent IPAM allocation wait for auto-created IPPool available, prometheus type: counter. (debug level metric) |
 
 ### Spiderpool Controller
 
 Spiderpool controller exports some metrics related with SpiderIPPool IP garbage collection. Currently, those include:
 
-| Name                                             | description                                                                                             |
-|--------------------------------------------------|---------------------------------------------------------------------------------------------------------|
-| spiderpool_ip_gc_counts                          | Number of Spiderpool Controller IP garbage collection, prometheus type: counter                         |
-| spiderpool_ip_gc_failure_counts                  | Number of Spiderpool Controller IP garbage collection failures, prometheus type: counter                |
-| spiderpool_subnet_ippool_counts                  | Number of SpiderSubnet corresponding IPPools number, prometheus type: gauge                             |
+| Name                                                   | description                                                                                                        |
+|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|
+| spiderpool_ip_gc_counts                                | Number of Spiderpool Controller IP garbage collection, prometheus type: counter.                                   |
+| spiderpool_ip_gc_failure_counts                        | Number of Spiderpool Controller IP garbage collection failures, prometheus type: counter.                          |
+| spiderpool_total_ippool_counts                         | Number of Spiderpool IPPools, prometheus type: gauge.                                                              |
+| spiderpool_debug_ippool_total_ip_counts                | Number of Spiderpool IPPool corresponding total IPs (per-IPPool), prometheus type: gauge. (debug level metric)     |
+| spiderpool_debug_ippool_available_ip_counts            | Number of Spiderpool IPPool corresponding availbale IPs (per-IPPool), prometheus type: gauge. (debug level metric) |
+| spiderpool_total_subnet_counts                         | Number of Spiderpool Subnets, prometheus type: gauge.                                                              |
+| spiderpool_debug_subnet_ippool_counts                  | Number of Spiderpool Subnet corresponding IPPools (per-Subnet), prometheus type: gauge. (debug level metric)       |
+| spiderpool_debug_subnet_total_ip_counts                | Number of Spiderpool Subnet corresponding total IPs (per-Subnet), prometheus type: gauge. (debug level metric)     |
+| spiderpool_debug_subnet_available_ip_counts            | Number of Spiderpool Subnet corresponding availbale IPs (per-Subnet), prometheus type: gauge. (debug level metric) |
+| spiderpool_debug_auto_pool_waited_for_available_counts | Number of waiting for auto-created IPPool available, prometheus type: couter. (debug level metric)                 |
diff --git a/pkg/ippoolmanager/ippool_informer.go b/pkg/ippoolmanager/ippool_informer.go
index 484ea4b5b..31975f13a 100644
--- a/pkg/ippoolmanager/ippool_informer.go
+++ b/pkg/ippoolmanager/ippool_informer.go
@@ -5,11 +5,11 @@ package ippoolmanager
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -31,6 +31,7 @@ import (
 	informers "github.com/spidernet-io/spiderpool/pkg/k8s/client/informers/externalversions/spiderpool.spidernet.io/v2beta1"
 	listers "github.com/spidernet-io/spiderpool/pkg/k8s/client/listers/spiderpool.spidernet.io/v2beta1"
 	"github.com/spidernet-io/spiderpool/pkg/logutils"
+	"github.com/spidernet-io/spiderpool/pkg/metric"
 )
 
 var informerLogger *zap.Logger
@@ -262,7 +263,15 @@ func (ic *IPPoolController) processNextWorkItem() bool {
 	return true
 }
 
-func (ic *IPPoolController) handleIPPool(ctx context.Context, pool *spiderpoolv2beta1.SpiderIPPool) error {
+func (ic *IPPoolController) handleIPPool(ctx context.Context, pool *spiderpoolv2beta1.SpiderIPPool) (err error) {
+	defer func() {
+		if err == nil {
+			attr := attribute.String(constant.KindSpiderIPPool, pool.Name)
+			metric.IPPoolTotalIPCounts.Add(ctx, *pool.Status.TotalIPCount, attr)
+			metric.IPPoolAvailableIPCounts.Add(ctx, (*pool.Status.TotalIPCount)-(*pool.Status.AllocatedIPCount), attr)
+		}
+	}()
+
 	// checkout the Auto-created IPPools whether need to scale or clean up legacies
 	if ic.EnableSpiderSubnet && IsAutoCreatedIPPool(pool) {
 		err := ic.cleanAutoIPPoolLegacy(ctx, pool)
@@ -272,7 +281,7 @@ func (ic *IPPoolController) handleIPPool(ctx context.Context, pool *spiderpoolv2
 	}
 
 	// update the IPPool status properties
-	err := ic.syncHandler(ctx, pool)
+	err = ic.syncHandler(ctx, pool)
 	if nil != err {
 		return err
 	}
@@ -280,64 +289,44 @@ func (ic *IPPoolController) handleIPPool(ctx context.Context, pool *spiderpoolv2
 	return nil
 }
 
-// syncHandleAllIPPool will calculate and update the provided SpiderIPPool status AllocatedIPCount or TotalIPCount.
+// syncHandler will calculate and update the provided SpiderIPPool status AllocatedIPCount or TotalIPCount.
 // And it will also remove finalizer once the IPPool is dying and no longer being used.
 func (ic *IPPoolController) syncHandler(ctx context.Context, pool *spiderpoolv2beta1.SpiderIPPool) error {
-	if pool.DeletionTimestamp != nil {
-		//remove finalizer to delete the dying IPPool when the IPPool is no longer being used
-		var shouldDelete bool
-		if pool.Status.AllocatedIPs == nil {
-			shouldDelete = true
-		} else {
-			var poolAllocatedIPs spiderpoolv2beta1.PoolIPAllocations
-			err := json.Unmarshal([]byte(*pool.Status.AllocatedIPs), &poolAllocatedIPs)
-			if nil != err {
-				return fmt.Errorf("%w: failed to parse IPPool %s Status AllocatedIPs: %v",
-					constant.ErrWrongInput, pool.Name, err)
-			}
-			if len(poolAllocatedIPs) == 0 {
-				shouldDelete = true
-			}
+	//remove finalizer to delete the dying IPPool when the IPPool is no longer being used
+	if pool.DeletionTimestamp != nil && pool.Status.AllocatedIPs == nil {
+		err := ic.removeFinalizer(ctx, pool)
+		if client.IgnoreNotFound(err) != nil {
+			return fmt.Errorf("failed to remove SpiderIPPool '%s' finalizer: %w", pool.Name, err)
 		}
 
-		if shouldDelete {
-			err := ic.removeFinalizer(ctx, pool)
-			if nil != err {
-				if apierrors.IsNotFound(err) {
-					informerLogger.Sugar().Debugf("SpiderIPPool '%s' is already deleted", pool.Name)
-					return nil
-				}
-				return fmt.Errorf("failed to remove SpiderIPPool '%s' finalizer: %w", pool.Name, err)
-			}
+		informerLogger.Sugar().Infof("remove SpiderIPPool '%s' finalizer successfully", pool.Name)
+		return nil
+	}
 
-			informerLogger.Sugar().Infof("remove SpiderIPPool '%s' finalizer successfully", pool.Name)
-		}
-	} else {
-		needUpdate := false
-		// initial the original data
-		if pool.Status.AllocatedIPCount == nil {
-			needUpdate = true
-			pool.Status.AllocatedIPCount = pointer.Int64(0)
-			informerLogger.Sugar().Infof("initial SpiderIPPool '%s' status AllocatedIPCount to 0", pool.Name)
-		}
+	needUpdate := false
+	// initial the original data
+	if pool.Status.AllocatedIPCount == nil {
+		needUpdate = true
+		pool.Status.AllocatedIPCount = pointer.Int64(0)
+		informerLogger.Sugar().Infof("initial SpiderIPPool '%s' status AllocatedIPCount to 0", pool.Name)
+	}
 
-		totalIPs, err := spiderpoolip.AssembleTotalIPs(*pool.Spec.IPVersion, pool.Spec.IPs, pool.Spec.ExcludeIPs)
-		if nil != err {
-			return fmt.Errorf("%w: failed to calculate SpiderIPPool '%s' total IP count, error: %v", constant.ErrWrongInput, pool.Name, err)
-		}
+	totalIPs, err := spiderpoolip.AssembleTotalIPs(*pool.Spec.IPVersion, pool.Spec.IPs, pool.Spec.ExcludeIPs)
+	if nil != err {
+		return fmt.Errorf("%w: failed to calculate SpiderIPPool '%s' total IP count, error: %v", constant.ErrWrongInput, pool.Name, err)
+	}
 
-		if pool.Status.TotalIPCount == nil || *pool.Status.TotalIPCount != int64(len(totalIPs)) {
-			needUpdate = true
-			pool.Status.TotalIPCount = pointer.Int64(int64(len(totalIPs)))
-		}
+	if pool.Status.TotalIPCount == nil || *pool.Status.TotalIPCount != int64(len(totalIPs)) {
+		needUpdate = true
+		pool.Status.TotalIPCount = pointer.Int64(int64(len(totalIPs)))
+	}
 
-		if needUpdate {
-			err = ic.client.Status().Update(ctx, pool)
-			if nil != err {
-				return fmt.Errorf("failed to update pool: %w", err)
-			}
-			informerLogger.Sugar().Debugf("update SpiderIPPool '%s' status TotalIPCount to '%d' successfully", pool.Name, *pool.Status.TotalIPCount)
+	if needUpdate {
+		err = ic.client.Status().Update(ctx, pool)
+		if nil != err {
+			return fmt.Errorf("failed to update pool: %w", err)
 		}
+		informerLogger.Sugar().Debugf("update SpiderIPPool '%s' status TotalIPCount to '%d' successfully", pool.Name, *pool.Status.TotalIPCount)
 	}
 
 	return nil
diff --git a/pkg/metric/metrics.go b/pkg/metric/metrics.go
index 1957e3999..0610aeef9 100644
--- a/pkg/metric/metrics.go
+++ b/pkg/metric/metrics.go
@@ -22,20 +22,22 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 )
 
+const debugMetrics = "debug-metrics"
+
 var (
 	// meter is a global creator of metric instruments.
-	meter api.Meter
+	meter, debugLevelMeter api.Meter
 	// globalEnableMetric determines whether to use metric or not
 	globalEnableMetric bool
 )
 
-// InitMetricController will set up meter with the input param(required) and create a prometheus exporter.
-// returns http handler and error
-func InitMetricController(ctx context.Context, meterName string, enableMetric bool) (http.Handler, error) {
+// InitMetric will set up meter with the input param(required) and create a prometheus exporter.
+func InitMetric(ctx context.Context, meterName string, enableMetric, enableDebugLevelMetrics bool) (http.Handler, error) {
 	if len(meterName) == 0 {
 		return nil, fmt.Errorf("failed to init metric controller, meter name is asked to be set")
 	}
 
+	globalEnableMetric = enableMetric
 	otelResource, err := resource.New(ctx,
 		resource.WithAttributes(
 			semconv.ServiceNameKey.String(constant.SpiderpoolAPIGroup),
@@ -53,7 +55,7 @@ func InitMetricController(ctx context.Context, meterName string, enableMetric bo
 		sdkmetric.WithResource(otelResource),
 		sdkmetric.WithView(sdkmetric.NewView(
 			sdkmetric.Instrument{
-				Name: fmt.Sprintf(MetricPrefix + "*"),
+				Name: fmt.Sprintf(metricPrefix + "*"),
 				Kind: sdkmetric.InstrumentKindHistogram,
 			},
 			sdkmetric.Stream{Aggregation: aggregation.ExplicitBucketHistogram{
@@ -63,54 +65,75 @@ func InitMetricController(ctx context.Context, meterName string, enableMetric bo
 	)
 	global.SetMeterProvider(provider)
 
-	globalEnableMetric = enableMetric
+	debugMeterName := fmt.Sprintf("%s-%s", meterName, debugMetrics)
+	meter = api.NewNoopMeterProvider().Meter(meterName)
+	debugLevelMeter = api.NewNoopMeterProvider().Meter(debugMeterName)
 	if globalEnableMetric {
 		meter = global.Meter(meterName)
-	} else {
-		meter = api.NewNoopMeterProvider().Meter(meterName)
+		if enableDebugLevelMetrics {
+			debugLevelMeter = global.Meter(debugMeterName)
+		}
 	}
 
 	return promhttp.Handler(), nil
 }
 
-// NewMetricInt64Counter will create otel Int64Counter metric.
+// newMetricInt64Counter will create otel Int64Counter metric.
 // The first param metricName is required and the second param is optional.
-func NewMetricInt64Counter(metricName string, description string) (instrument.Int64Counter, error) {
+func newMetricInt64Counter(metricName string, description string, isDebugLevel bool) (instrument.Int64Counter, error) {
 	if len(metricName) == 0 {
 		return nil, fmt.Errorf("failed to create metric Int64Counter, metric name is asked to be set")
 	}
-	return meter.Int64Counter(metricName, instrument.WithDescription(description))
+
+	m := meter
+	if isDebugLevel {
+		m = debugLevelMeter
+	}
+	return m.Int64Counter(metricName, instrument.WithDescription(description))
 }
 
-// NewMetricFloat64Histogram will create otel Float64Histogram metric.
+// newMetricFloat64Histogram will create otel Float64Histogram metric.
 // The first param metricName is required and the second param is optional.
 // Notice: if you want to match the quantile {0.1, 0.3, 0.5, 1, 3, 5, 7, 10, 15}, please let the metric name match regex "*_histogram",
 // otherwise it will match the  otel default quantile.
-func NewMetricFloat64Histogram(metricName string, description string) (instrument.Float64Histogram, error) {
+func newMetricFloat64Histogram(metricName string, description string, isDebugLevel bool) (instrument.Float64Histogram, error) {
 	if len(metricName) == 0 {
 		return nil, fmt.Errorf("failed to create metric Float64Histogram, metric name is asked to be set")
 	}
-	return meter.Float64Histogram(metricName, instrument.WithDescription(description))
+
+	m := meter
+	if isDebugLevel {
+		m = debugLevelMeter
+	}
+	return m.Float64Histogram(metricName, instrument.WithDescription(description))
 }
 
-// NewMetricFloat64Gauge will create otel Float64Gauge metric.
+// newMetricFloat64Gauge will create otel Float64Gauge metric.
 // The first param metricName is required and the second param is optional.
-func NewMetricFloat64Gauge(metricName string, description string) (instrument.Float64ObservableGauge, error) {
+func newMetricFloat64Gauge(metricName string, description string, isDebugLevel bool) (instrument.Float64ObservableGauge, error) {
 	if len(metricName) == 0 {
 		return nil, fmt.Errorf("failed to create metric Float64Guage, metric name is asked to be set")
 	}
 
-	return meter.Float64ObservableGauge(metricName, instrument.WithDescription(description))
+	m := meter
+	if isDebugLevel {
+		m = debugLevelMeter
+	}
+	return m.Float64ObservableGauge(metricName, instrument.WithDescription(description))
 }
 
-// NewMetricInt64Gauge will create otel Int64Gauge metric.
+// newMetricInt64Gauge will create otel Int64Gauge metric.
 // The first param metricName is required and the second param is optional.
-func NewMetricInt64Gauge(metricName string, description string) (instrument.Int64ObservableGauge, error) {
+func newMetricInt64Gauge(metricName string, description string, isDebugLevel bool) (instrument.Int64ObservableGauge, error) {
 	if len(metricName) == 0 {
 		return nil, fmt.Errorf("failed to create metric Float64Guage, metric name is asked to be set")
 	}
 
-	return meter.Int64ObservableGauge(metricName, instrument.WithDescription(description))
+	m := meter
+	if isDebugLevel {
+		m = debugLevelMeter
+	}
+	return m.Int64ObservableGauge(metricName, instrument.WithDescription(description))
 }
 
 var _ TimeRecorder = &timeRecorder{}
diff --git a/pkg/metric/metrics_instance.go b/pkg/metric/metrics_instance.go
index 03404132a..25ca1f844 100644
--- a/pkg/metric/metrics_instance.go
+++ b/pkg/metric/metrics_instance.go
@@ -14,60 +14,67 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/lock"
 )
 
-const MetricPrefix = "spiderpool_"
+const metricPrefix = "spiderpool_"
+const debugPrefix = "debug_"
 
 const (
 	// spiderpool agent ipam allocation metrics name
-	ipam_allocation_counts                        = MetricPrefix + "ipam_allocation_counts"
-	ipam_allocation_failure_counts                = MetricPrefix + "ipam_allocation_failure_counts"
-	ipam_allocation_update_ippool_conflict_counts = MetricPrefix + "ipam_allocation_update_ippool_conflict_counts"
-	ipam_allocation_err_internal_counts           = MetricPrefix + "ipam_allocation_err_internal_counts"
-	ipam_allocation_err_no_available_pool_counts  = MetricPrefix + "ipam_allocation_err_no_available_pool_counts"
-	ipam_allocation_err_retries_exhausted_counts  = MetricPrefix + "ipam_allocation_err_retries_exhausted_counts"
-	ipam_allocation_err_ip_used_out_counts        = MetricPrefix + "ipam_allocation_err_ip_used_out_counts"
-
-	ipam_allocation_average_duration_seconds = MetricPrefix + "ipam_allocation_average_duration_seconds"
-	ipam_allocation_max_duration_seconds     = MetricPrefix + "ipam_allocation_max_duration_seconds"
-	ipam_allocation_min_duration_seconds     = MetricPrefix + "ipam_allocation_min_duration_seconds"
-	ipam_allocation_latest_duration_seconds  = MetricPrefix + "ipam_allocation_latest_duration_seconds"
-	ipam_allocation_duration_seconds         = MetricPrefix + "ipam_allocation_duration_seconds"
-
-	ipam_allocation_average_limit_duration_seconds = MetricPrefix + "ipam_allocation_average_limit_duration_seconds"
-	ipam_allocation_max_limit_duration_seconds     = MetricPrefix + "ipam_allocation_max_limit_duration_seconds"
-	ipam_allocation_min_limit_duration_seconds     = MetricPrefix + "ipam_allocation_min_limit_duration_seconds"
-	ipam_allocation_latest_limit_duration_seconds  = MetricPrefix + "ipam_allocation_latest_limit_duration_seconds"
-	ipam_allocation_limit_duration_seconds         = MetricPrefix + "ipam_allocation_limit_duration_seconds"
+	ipam_allocation_counts                        = metricPrefix + "ipam_allocation_counts"
+	ipam_allocation_failure_counts                = metricPrefix + "ipam_allocation_failure_counts"
+	ipam_allocation_update_ippool_conflict_counts = metricPrefix + "ipam_allocation_update_ippool_conflict_counts"
+	ipam_allocation_err_internal_counts           = metricPrefix + "ipam_allocation_err_internal_counts"
+	ipam_allocation_err_no_available_pool_counts  = metricPrefix + "ipam_allocation_err_no_available_pool_counts"
+	ipam_allocation_err_retries_exhausted_counts  = metricPrefix + "ipam_allocation_err_retries_exhausted_counts"
+	ipam_allocation_err_ip_used_out_counts        = metricPrefix + "ipam_allocation_err_ip_used_out_counts"
+
+	ipam_allocation_average_duration_seconds = metricPrefix + "ipam_allocation_average_duration_seconds"
+	ipam_allocation_max_duration_seconds     = metricPrefix + "ipam_allocation_max_duration_seconds"
+	ipam_allocation_min_duration_seconds     = metricPrefix + "ipam_allocation_min_duration_seconds"
+	ipam_allocation_latest_duration_seconds  = metricPrefix + "ipam_allocation_latest_duration_seconds"
+	ipam_allocation_duration_seconds         = metricPrefix + "ipam_allocation_duration_seconds"
+
+	ipam_allocation_average_limit_duration_seconds = metricPrefix + "ipam_allocation_average_limit_duration_seconds"
+	ipam_allocation_max_limit_duration_seconds     = metricPrefix + "ipam_allocation_max_limit_duration_seconds"
+	ipam_allocation_min_limit_duration_seconds     = metricPrefix + "ipam_allocation_min_limit_duration_seconds"
+	ipam_allocation_latest_limit_duration_seconds  = metricPrefix + "ipam_allocation_latest_limit_duration_seconds"
+	ipam_allocation_limit_duration_seconds         = metricPrefix + "ipam_allocation_limit_duration_seconds"
 
 	// spiderpool agent ipam release metrics name
-	ipam_release_counts                        = MetricPrefix + "ipam_release_counts"
-	ipam_release_failure_counts                = MetricPrefix + "ipam_release_failure_counts"
-	ipam_release_update_ippool_conflict_counts = MetricPrefix + "ipam_release_update_ippool_conflict_counts"
-	ipam_release_err_internal_counts           = MetricPrefix + "ipam_release_err_internal_counts"
-	ipam_release_err_retries_exhausted_counts  = MetricPrefix + "ipam_release_err_retries_exhausted_counts"
-
-	ipam_release_average_duration_seconds = MetricPrefix + "ipam_release_average_duration_seconds"
-	ipam_release_max_duration_seconds     = MetricPrefix + "ipam_release_max_duration_seconds"
-	ipam_release_min_duration_seconds     = MetricPrefix + "ipam_release_min_duration_seconds"
-	ipam_release_latest_duration_seconds  = MetricPrefix + "ipam_release_latest_duration_seconds"
-	ipam_release_duration_seconds         = MetricPrefix + "ipam_release_duration_seconds"
-
-	ipam_release_average_limit_duration_seconds = MetricPrefix + "ipam_release_average_limit_duration_seconds"
-	ipam_release_max_limit_duration_seconds     = MetricPrefix + "ipam_release_max_limit_duration_seconds"
-	ipam_release_min_limit_duration_seconds     = MetricPrefix + "ipam_release_min_limit_duration_seconds"
-	ipam_release_latest_limit_duration_seconds  = MetricPrefix + "ipam_release_latest_limit_duration_seconds"
-	ipam_release_limit_duration_seconds         = MetricPrefix + "ipam_release_limit_duration_seconds"
+	ipam_release_counts                        = metricPrefix + "ipam_release_counts"
+	ipam_release_failure_counts                = metricPrefix + "ipam_release_failure_counts"
+	ipam_release_update_ippool_conflict_counts = metricPrefix + "ipam_release_update_ippool_conflict_counts"
+	ipam_release_err_internal_counts           = metricPrefix + "ipam_release_err_internal_counts"
+	ipam_release_err_retries_exhausted_counts  = metricPrefix + "ipam_release_err_retries_exhausted_counts"
+
+	ipam_release_average_duration_seconds = metricPrefix + "ipam_release_average_duration_seconds"
+	ipam_release_max_duration_seconds     = metricPrefix + "ipam_release_max_duration_seconds"
+	ipam_release_min_duration_seconds     = metricPrefix + "ipam_release_min_duration_seconds"
+	ipam_release_latest_duration_seconds  = metricPrefix + "ipam_release_latest_duration_seconds"
+	ipam_release_duration_seconds         = metricPrefix + "ipam_release_duration_seconds"
+
+	ipam_release_average_limit_duration_seconds = metricPrefix + "ipam_release_average_limit_duration_seconds"
+	ipam_release_max_limit_duration_seconds     = metricPrefix + "ipam_release_max_limit_duration_seconds"
+	ipam_release_min_limit_duration_seconds     = metricPrefix + "ipam_release_min_limit_duration_seconds"
+	ipam_release_latest_limit_duration_seconds  = metricPrefix + "ipam_release_latest_limit_duration_seconds"
+	ipam_release_limit_duration_seconds         = metricPrefix + "ipam_release_limit_duration_seconds"
 
 	// spiderpool controller IP GC metrics name
-	ip_gc_counts         = MetricPrefix + "ip_gc_counts"
-	ip_gc_failure_counts = MetricPrefix + "ip_gc_failure_counts"
-
-	// spiderpool controller SpiderSubnet feature
-	subnet_ippool_counts                  = MetricPrefix + "subnet_ippool_counts"
-	auto_pool_waited_for_available_counts = MetricPrefix + "auto_pool_waited_for_available_counts"
+	ip_gc_counts         = metricPrefix + "ip_gc_counts"
+	ip_gc_failure_counts = metricPrefix + "ip_gc_failure_counts"
+
+	// spiderpool IPPool and Subnet metrics and these include some debug level metrics
+	total_ippool_counts                   = metricPrefix + "total_ippool_counts"
+	ippool_total_ip_counts                = metricPrefix + debugPrefix + "ippool_total_ip_counts"
+	ippool_available_ip_counts            = metricPrefix + debugPrefix + "ippool_available_ip_counts"
+	total_subnet_counts                   = metricPrefix + "total_subnet_counts"
+	subnet_ippool_counts                  = metricPrefix + debugPrefix + "subnet_ippool_counts"
+	subnet_total_ip_counts                = metricPrefix + debugPrefix + "subnet_total_ip_counts"
+	subnet_available_ip_counts            = metricPrefix + debugPrefix + "subnet_available_ip_counts"
+	auto_pool_waited_for_available_counts = metricPrefix + debugPrefix + "auto_pool_waited_for_available_counts"
 )
 
 var (
-	// spiderpool agent ipam allocation metrics
+	// ipam allocation metrics in spiderpool-agent
 	IpamAllocationTotalCounts                   instrument.Int64Counter
 	IpamAllocationFailureCounts                 instrument.Int64Counter
 	IpamAllocationUpdateIPPoolConflictCounts    instrument.Int64Counter
@@ -86,7 +93,7 @@ var (
 	ipamAllocationLatestLimitDurationSeconds    = new(asyncFloat64Gauge)
 	ipamAllocationLimitDurationSecondsHistogram instrument.Float64Histogram
 
-	// spiderpool agent ipam release metrics
+	// ipam release metrics in spiderpool-agent
 	IpamReleaseTotalCounts                   instrument.Int64Counter
 	IpamReleaseFailureCounts                 instrument.Int64Counter
 	IpamReleaseUpdateIPPoolConflictCounts    instrument.Int64Counter
@@ -103,13 +110,20 @@ var (
 	ipamReleaseLatestLimitDurationSeconds    = new(asyncFloat64Gauge)
 	ipamReleaseLimitDurationSecondsHistogram instrument.Float64Histogram
 
-	// spiderpool controller IP GC metrics
+	// IP GC metrics in spiderpool-controller
 	IPGCTotalCounts   instrument.Int64Counter
 	IPGCFailureCounts instrument.Int64Counter
 
-	SubnetPoolCounts = new(asyncInt64Gauge)
+	// IPPool&Subnet metrics in spiderpool-controller
+	TotalIPPoolCounts       = new(asyncInt64Gauge)
+	IPPoolTotalIPCounts     instrument.Int64Counter
+	IPPoolAvailableIPCounts instrument.Int64Counter
+	TotalSubnetCounts       = new(asyncInt64Gauge)
+	SubnetPoolCounts        = new(asyncInt64Gauge)
+	SubnetTotalIPCounts     instrument.Int64Counter
+	SubnetAvailableIPCounts instrument.Int64Counter
 
-	// SpiderSubnet feature
+	// SpiderSubnet feature performance monitoring metric in spiderpool-agent
 	AutoPoolWaitedForAvailableCounts instrument.Int64Counter
 )
 
@@ -122,14 +136,19 @@ type asyncFloat64Gauge struct {
 }
 
 // initGauge will new an otel float64 gauge metric and register a call back function
-func (a *asyncFloat64Gauge) initGauge(metricName string, description string) error {
-	tmpGauge, err := NewMetricFloat64Gauge(metricName, description)
+func (a *asyncFloat64Gauge) initGauge(metricName string, description string, isDebugLevel bool) error {
+	m := meter
+	if isDebugLevel {
+		m = debugLevelMeter
+	}
+
+	tmpGauge, err := newMetricFloat64Gauge(metricName, description, isDebugLevel)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool metric '%s', error: %v", metricName, err)
 	}
 
 	a.gaugeMetric = tmpGauge
-	_, err = meter.RegisterCallback(func(_ context.Context, observer api.Observer) error {
+	_, err = m.RegisterCallback(func(_ context.Context, observer api.Observer) error {
 		observer.ObserveFloat64(a.gaugeMetric,
 			a.observerValueToReport,
 			a.observerAttrsToReport...,
@@ -163,14 +182,19 @@ type asyncInt64Gauge struct {
 }
 
 // initGauge will new an otel int64 gauge metric and register a call back function
-func (a *asyncInt64Gauge) initGauge(metricName string, description string) error {
-	tmpGauge, err := NewMetricInt64Gauge(metricName, description)
+func (a *asyncInt64Gauge) initGauge(metricName string, description string, isDebugLevel bool) error {
+	m := meter
+	if isDebugLevel {
+		m = debugLevelMeter
+	}
+
+	tmpGauge, err := newMetricInt64Gauge(metricName, description, isDebugLevel)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool metric '%s', error: %v", metricName, err)
 	}
 
 	a.gaugeMetric = tmpGauge
-	_, err = meter.RegisterCallback(func(_ context.Context, observer api.Observer) error {
+	_, err = m.RegisterCallback(func(_ context.Context, observer api.Observer) error {
 		observer.ObserveInt64(a.gaugeMetric,
 			a.observerValueToReport,
 			a.observerAttrsToReport...,
@@ -207,7 +231,7 @@ func InitSpiderpoolAgentMetrics(ctx context.Context) error {
 		return err
 	}
 
-	autoPoolWaitedForAvailableCounts, err := NewMetricInt64Counter(auto_pool_waited_for_available_counts, "ipam waited for auto-created IPPool available counts")
+	autoPoolWaitedForAvailableCounts, err := newMetricInt64Counter(auto_pool_waited_for_available_counts, "ipam waited for auto-created IPPool available counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", auto_pool_waited_for_available_counts, err)
 	}
@@ -223,7 +247,7 @@ func InitSpiderpoolControllerMetrics(ctx context.Context) error {
 		return err
 	}
 
-	err = SubnetPoolCounts.initGauge(subnet_ippool_counts, "spider subnet corresponding ippools counts")
+	err = initSpiderpoolControllerCRMetrics(ctx)
 	if nil != err {
 		return err
 	}
@@ -234,111 +258,111 @@ func InitSpiderpoolControllerMetrics(ctx context.Context) error {
 // initSpiderpoolAgentAllocationMetrics will init spiderpool-agent IPAM allocation metrics
 func initSpiderpoolAgentAllocationMetrics(ctx context.Context) error {
 	// spiderpool agent ipam allocation total counts, metric type "int64 counter"
-	allocationTotalCounts, err := NewMetricInt64Counter(ipam_allocation_counts, "spiderpool agent ipam allocation total counts")
+	allocationTotalCounts, err := newMetricInt64Counter(ipam_allocation_counts, "spiderpool agent ipam allocation total counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_counts, err)
 	}
 	IpamAllocationTotalCounts = allocationTotalCounts
 
 	// spiderpool agent ipam allocation failure counts, metric type "int64 counter"
-	allocationFailureCounts, err := NewMetricInt64Counter(ipam_allocation_failure_counts, "spiderpool agent ipam allocation failure counts")
+	allocationFailureCounts, err := newMetricInt64Counter(ipam_allocation_failure_counts, "spiderpool agent ipam allocation failure counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_failure_counts, err)
 	}
 	IpamAllocationFailureCounts = allocationFailureCounts
 
 	// spiderpool agent ipam allocation update IPPool conflict counts, metric type "int64 counter"
-	allocationUpdateIPPoolConflictCounts, err := NewMetricInt64Counter(ipam_allocation_update_ippool_conflict_counts, "spiderpool agent ipam allocation update IPPool conflict counts")
+	allocationUpdateIPPoolConflictCounts, err := newMetricInt64Counter(ipam_allocation_update_ippool_conflict_counts, "spiderpool agent ipam allocation update IPPool conflict counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_update_ippool_conflict_counts, err)
 	}
 	IpamAllocationUpdateIPPoolConflictCounts = allocationUpdateIPPoolConflictCounts
 
 	// spiderpool agent ipam allocation internal error counts, metric type "int64 counter"
-	allocationErrInternalCounts, err := NewMetricInt64Counter(ipam_allocation_err_internal_counts, "spiderpool agent ipam allocation internal error counts")
+	allocationErrInternalCounts, err := newMetricInt64Counter(ipam_allocation_err_internal_counts, "spiderpool agent ipam allocation internal error counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_err_internal_counts, err)
 	}
 	IpamAllocationErrInternalCounts = allocationErrInternalCounts
 
 	// spiderpool agent ipam allocation no available IPPool error counts, metric type "int64 counter"
-	allocationErrNoAvailablePoolCounts, err := NewMetricInt64Counter(ipam_allocation_err_no_available_pool_counts, "spiderpool agent ipam allocation no available IPPool error counts")
+	allocationErrNoAvailablePoolCounts, err := newMetricInt64Counter(ipam_allocation_err_no_available_pool_counts, "spiderpool agent ipam allocation no available IPPool error counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_err_no_available_pool_counts, err)
 	}
 	IpamAllocationErrNoAvailablePoolCounts = allocationErrNoAvailablePoolCounts
 
 	// spiderpool agent ipam allocation retries exhausted error counts, metric type "int64 counter"
-	allocationErrRetriesExhaustedCounts, err := NewMetricInt64Counter(ipam_allocation_err_retries_exhausted_counts, "spiderpool agent ipam allocation retries exhausted error counts")
+	allocationErrRetriesExhaustedCounts, err := newMetricInt64Counter(ipam_allocation_err_retries_exhausted_counts, "spiderpool agent ipam allocation retries exhausted error counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_err_retries_exhausted_counts, err)
 	}
 	IpamAllocationErrRetriesExhaustedCounts = allocationErrRetriesExhaustedCounts
 
 	// spiderpool agent ipam allocation IP addresses used out error counts, metric type "int64 counter"
-	allocationErrIPUsedOutCounts, err := NewMetricInt64Counter(ipam_allocation_err_ip_used_out_counts, "spiderpool agent ipam allocation IP addresses used out error counts")
+	allocationErrIPUsedOutCounts, err := newMetricInt64Counter(ipam_allocation_err_ip_used_out_counts, "spiderpool agent ipam allocation IP addresses used out error counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_err_ip_used_out_counts, err)
 	}
 	IpamAllocationErrIPUsedOutCounts = allocationErrIPUsedOutCounts
 
 	// spiderpool agent ipam average allocation duration, metric type "float64 gauge"
-	err = ipamAllocationAverageDurationSeconds.initGauge(ipam_allocation_average_duration_seconds, "spiderpool agent ipam average allocation duration")
+	err = ipamAllocationAverageDurationSeconds.initGauge(ipam_allocation_average_duration_seconds, "spiderpool agent ipam average allocation duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam maximum allocation duration, metric type "float64 gauge"
-	err = ipamAllocationMaxDurationSeconds.initGauge(ipam_allocation_max_duration_seconds, "spiderpool agent ipam maximum allocation duration")
+	err = ipamAllocationMaxDurationSeconds.initGauge(ipam_allocation_max_duration_seconds, "spiderpool agent ipam maximum allocation duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam minimum allocation duration, metric type "float64 gauge"
-	err = ipamAllocationMinDurationSeconds.initGauge(ipam_allocation_min_duration_seconds, "spiderpool agent ipam minimum allocation duration")
+	err = ipamAllocationMinDurationSeconds.initGauge(ipam_allocation_min_duration_seconds, "spiderpool agent ipam minimum allocation duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam latest allocation duration, metric type "float64 gauge"
-	err = ipamAllocationLatestDurationSeconds.initGauge(ipam_allocation_latest_duration_seconds, "spiderpool agent ipam latest allocation duration")
+	err = ipamAllocationLatestDurationSeconds.initGauge(ipam_allocation_latest_duration_seconds, "spiderpool agent ipam latest allocation duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam allocation duration bucket, metric type "float64 histogram"
-	allocationHistogram, err := NewMetricFloat64Histogram(ipam_allocation_duration_seconds, "histogram of spiderpool agent ipam allocation duration")
+	allocationHistogram, err := newMetricFloat64Histogram(ipam_allocation_duration_seconds, "histogram of spiderpool agent ipam allocation duration", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_duration_seconds, err)
 	}
 	ipamAllocationDurationSecondsHistogram = allocationHistogram
 
 	// spiderpool agent ipam average allocation limit duration, metric type "float64 gauge"
-	err = ipamAllocationAverageLimitDurationSeconds.initGauge(ipam_allocation_average_limit_duration_seconds, "spiderpool agent ipam average allocation limit duration")
+	err = ipamAllocationAverageLimitDurationSeconds.initGauge(ipam_allocation_average_limit_duration_seconds, "spiderpool agent ipam average allocation limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam maximum allocation limit duration, metric type "float64 gauge"
-	err = ipamAllocationMaxLimitDurationSeconds.initGauge(ipam_allocation_max_limit_duration_seconds, "spiderpool agent ipam maximum allocation limit duration")
+	err = ipamAllocationMaxLimitDurationSeconds.initGauge(ipam_allocation_max_limit_duration_seconds, "spiderpool agent ipam maximum allocation limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam minimum allocation limit duration, metric type "float64 gauge"
-	err = ipamAllocationMinLimitDurationSeconds.initGauge(ipam_allocation_min_limit_duration_seconds, "spiderpool agent ipam minimum allocation limit duration")
+	err = ipamAllocationMinLimitDurationSeconds.initGauge(ipam_allocation_min_limit_duration_seconds, "spiderpool agent ipam minimum allocation limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam latest allocation limit duration, metric type "float64 gauge"
-	err = ipamAllocationLatestLimitDurationSeconds.initGauge(ipam_allocation_latest_limit_duration_seconds, "spiderpool agent ipam latest allocation limit duration")
+	err = ipamAllocationLatestLimitDurationSeconds.initGauge(ipam_allocation_latest_limit_duration_seconds, "spiderpool agent ipam latest allocation limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam allocation limit duration bucket, metric type "float64 histogram"
-	allocationLimitHistogram, err := NewMetricFloat64Histogram(ipam_allocation_limit_duration_seconds, "histogram of spiderpool agent ipam allocation limit duration")
+	allocationLimitHistogram, err := newMetricFloat64Histogram(ipam_allocation_limit_duration_seconds, "histogram of spiderpool agent ipam allocation limit duration", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_limit_duration_seconds, err)
 	}
@@ -350,97 +374,97 @@ func initSpiderpoolAgentAllocationMetrics(ctx context.Context) error {
 // initSpiderpoolAgentReleaseMetrics will init spiderpool-agent IPAM release metrics
 func initSpiderpoolAgentReleaseMetrics(ctx context.Context) error {
 	// spiderpool agent ipam release total counts, metric type "int64 counter"
-	releaseTotalCounts, err := NewMetricInt64Counter(ipam_release_counts, "spiderpool agent ipam release total counts")
+	releaseTotalCounts, err := newMetricInt64Counter(ipam_release_counts, "spiderpool agent ipam release total counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_counts, err)
 	}
 	IpamReleaseTotalCounts = releaseTotalCounts
 
 	// spiderpool agent ipam release failure counts, metric type "int64 counter"
-	releaseFailureCounts, err := NewMetricInt64Counter(ipam_release_failure_counts, "spiderpool agent ipam release failure counts")
+	releaseFailureCounts, err := newMetricInt64Counter(ipam_release_failure_counts, "spiderpool agent ipam release failure counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_failure_counts, err)
 	}
 	IpamReleaseFailureCounts = releaseFailureCounts
 
 	// spiderpool agent ipam release update IPPool conflict counts, metric type "int64 counter"
-	releaseUpdateIPPoolConflictCounts, err := NewMetricInt64Counter(ipam_release_update_ippool_conflict_counts, "spiderpool agent ipam release update IPPool conflict counts")
+	releaseUpdateIPPoolConflictCounts, err := newMetricInt64Counter(ipam_release_update_ippool_conflict_counts, "spiderpool agent ipam release update IPPool conflict counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_update_ippool_conflict_counts, err)
 	}
 	IpamReleaseUpdateIPPoolConflictCounts = releaseUpdateIPPoolConflictCounts
 
 	// spiderpool agent ipam releasing internal error counts, metric type "int64 counter"
-	releasingErrInternalCounts, err := NewMetricInt64Counter(ipam_release_err_internal_counts, "spiderpool agent ipam release internal error counts")
+	releasingErrInternalCounts, err := newMetricInt64Counter(ipam_release_err_internal_counts, "spiderpool agent ipam release internal error counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_err_internal_counts, err)
 	}
 	IpamReleaseErrInternalCounts = releasingErrInternalCounts
 
 	// spiderpool agent ipam releasing retries exhausted error counts, metric type "int64 counter"
-	releasingErrRetriesExhaustedCounts, err := NewMetricInt64Counter(ipam_release_err_retries_exhausted_counts, "spiderpool agent ipam release retries exhausted error counts")
+	releasingErrRetriesExhaustedCounts, err := newMetricInt64Counter(ipam_release_err_retries_exhausted_counts, "spiderpool agent ipam release retries exhausted error counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_err_retries_exhausted_counts, err)
 	}
 	IpamReleaseErrRetriesExhaustedCounts = releasingErrRetriesExhaustedCounts
 
 	// spiderpool agent ipam average release duration, metric type "float64 gauge"
-	err = ipamReleaseAverageDurationSeconds.initGauge(ipam_release_average_duration_seconds, "spiderpool agent ipam average release duration")
+	err = ipamReleaseAverageDurationSeconds.initGauge(ipam_release_average_duration_seconds, "spiderpool agent ipam average release duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam maximum release duration, metric type "float64 gauge"
-	err = ipamReleaseMaxDurationSeconds.initGauge(ipam_release_max_duration_seconds, "spiderpool agent ipam maximum release duration")
+	err = ipamReleaseMaxDurationSeconds.initGauge(ipam_release_max_duration_seconds, "spiderpool agent ipam maximum release duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam minimum allocation duration, metric type "float64 gauge"
-	err = ipamReleaseMinDurationSeconds.initGauge(ipam_release_min_duration_seconds, "spiderpool agent ipam minimum release duration")
+	err = ipamReleaseMinDurationSeconds.initGauge(ipam_release_min_duration_seconds, "spiderpool agent ipam minimum release duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam latest release duration, metric type "float64 gauge"
-	err = ipamReleaseLatestDurationSeconds.initGauge(ipam_release_latest_duration_seconds, "spiderpool agent ipam latest release duration")
+	err = ipamReleaseLatestDurationSeconds.initGauge(ipam_release_latest_duration_seconds, "spiderpool agent ipam latest release duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam allocation duration bucket, metric type "float64 histogram"
-	releaseHistogram, err := NewMetricFloat64Histogram(ipam_release_duration_seconds, "histogram of spiderpool agent ipam release duration")
+	releaseHistogram, err := newMetricFloat64Histogram(ipam_release_duration_seconds, "histogram of spiderpool agent ipam release duration", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_duration_seconds, err)
 	}
 	ipamReleaseDurationSecondsHistogram = releaseHistogram
 
 	// spiderpool agent ipam average release limit duration, metric type "float64 gauge"
-	err = ipamReleaseAverageLimitDurationSeconds.initGauge(ipam_release_average_limit_duration_seconds, "spiderpool agent ipam average release limit duration")
+	err = ipamReleaseAverageLimitDurationSeconds.initGauge(ipam_release_average_limit_duration_seconds, "spiderpool agent ipam average release limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam maximum release limit duration, metric type "float64 gauge"
-	err = ipamReleaseMaxLimitDurationSeconds.initGauge(ipam_release_max_limit_duration_seconds, "spiderpool agent ipam maximum release limit duration")
+	err = ipamReleaseMaxLimitDurationSeconds.initGauge(ipam_release_max_limit_duration_seconds, "spiderpool agent ipam maximum release limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam minimum allocation limit duration, metric type "float64 gauge"
-	err = ipamReleaseMinLimitDurationSeconds.initGauge(ipam_release_min_limit_duration_seconds, "spiderpool agent ipam minimum release limit duration")
+	err = ipamReleaseMinLimitDurationSeconds.initGauge(ipam_release_min_limit_duration_seconds, "spiderpool agent ipam minimum release limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam latest release limit duration, metric type "float64 gauge"
-	err = ipamReleaseLatestLimitDurationSeconds.initGauge(ipam_release_latest_limit_duration_seconds, "spiderpool agent ipam latest release limit duration")
+	err = ipamReleaseLatestLimitDurationSeconds.initGauge(ipam_release_latest_limit_duration_seconds, "spiderpool agent ipam latest release limit duration", false)
 	if nil != err {
 		return err
 	}
 
 	// spiderpool agent ipam allocation limit duration bucket, metric type "float64 histogram"
-	releaseLimitHistogram, err := NewMetricFloat64Histogram(ipam_release_limit_duration_seconds, "histogram of spiderpool agent ipam release limit duration")
+	releaseLimitHistogram, err := newMetricFloat64Histogram(ipam_release_limit_duration_seconds, "histogram of spiderpool agent ipam release limit duration", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_limit_duration_seconds, err)
 	}
@@ -451,19 +475,21 @@ func initSpiderpoolAgentReleaseMetrics(ctx context.Context) error {
 
 // initSpiderpoolControllerGCMetrics will init spiderpool-controller IP gc metrics
 func initSpiderpoolControllerGCMetrics(ctx context.Context) error {
-	ipGCTotalCounts, err := NewMetricInt64Counter(ip_gc_counts, "spiderpool controller ip gc total counts")
+	ipGCTotalCounts, err := newMetricInt64Counter(ip_gc_counts, "spiderpool controller ip gc total counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", ip_gc_counts, err)
 	}
 	IPGCTotalCounts = ipGCTotalCounts
+	IPGCTotalCounts.Add(ctx, 0)
 
-	ipGCFailureCounts, err := NewMetricInt64Counter(ip_gc_failure_counts, "spiderpool controller ip gc total counts")
+	ipGCFailureCounts, err := newMetricInt64Counter(ip_gc_failure_counts, "spiderpool controller ip gc total counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", ip_gc_failure_counts, err)
 	}
 	IPGCFailureCounts = ipGCFailureCounts
+	ipGCFailureCounts.Add(ctx, 0)
 
-	releaseUpdateIPPoolConflictCounts, err := NewMetricInt64Counter(ipam_release_update_ippool_conflict_counts, "spiderpool controller gc release update IPPool conflict counts")
+	releaseUpdateIPPoolConflictCounts, err := newMetricInt64Counter(ipam_release_update_ippool_conflict_counts, "spiderpool controller gc release update IPPool conflict counts", false)
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_release_update_ippool_conflict_counts, err)
 	}
@@ -471,3 +497,46 @@ func initSpiderpoolControllerGCMetrics(ctx context.Context) error {
 
 	return nil
 }
+
+func initSpiderpoolControllerCRMetrics(ctx context.Context) error {
+	err := TotalSubnetCounts.initGauge(total_subnet_counts, "spiderpool total SpiderSubnet counts", false)
+	if nil != err {
+		return err
+	}
+
+	subnetTotalIPCounts, err := newMetricInt64Counter(subnet_total_ip_counts, "spiderpool single SpiderSubnet corresponding total IP counts", true)
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", subnet_total_ip_counts, err)
+	}
+	SubnetTotalIPCounts = subnetTotalIPCounts
+
+	subnetAvailableIPCounts, err := newMetricInt64Counter(subnet_available_ip_counts, "spiderpool single SpiderSubnet corresponding available IP counts", true)
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", subnet_available_ip_counts, err)
+	}
+	SubnetAvailableIPCounts = subnetAvailableIPCounts
+
+	err = TotalIPPoolCounts.initGauge(total_ippool_counts, "spiderpool total SpiderIPPool counts", false)
+	if nil != err {
+		return err
+	}
+
+	poolTotalIPCounts, err := newMetricInt64Counter(ippool_total_ip_counts, "spiderpool single SpiderIPPool corresponding total IP counts", true)
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", ippool_total_ip_counts, err)
+	}
+	IPPoolTotalIPCounts = poolTotalIPCounts
+
+	poolAvailableIPCounts, err := newMetricInt64Counter(ippool_available_ip_counts, "spiderpool single SpiderIPPool corresponding available IP counts", true)
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", ippool_available_ip_counts, err)
+	}
+	IPPoolAvailableIPCounts = poolAvailableIPCounts
+
+	err = SubnetPoolCounts.initGauge(subnet_ippool_counts, "spider subnet corresponding ippools counts", true)
+	if nil != err {
+		return err
+	}
+
+	return nil
+}
diff --git a/pkg/metric/metrics_ipam.go b/pkg/metric/metrics_ipam.go
index 6d28c5b9f..e238d1d87 100644
--- a/pkg/metric/metrics_ipam.go
+++ b/pkg/metric/metrics_ipam.go
@@ -9,7 +9,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/lock"
 )
 
-// AllocDurationConstruct is Singleton
+// IPAMDurationConstruct is Singleton
 var IPAMDurationConstruct = new(ipamDurationConstruct)
 
 type ipamDurationConstruct struct {
diff --git a/pkg/subnetmanager/subnet_informer.go b/pkg/subnetmanager/subnet_informer.go
index b18d1bd60..5624e889d 100644
--- a/pkg/subnetmanager/subnet_informer.go
+++ b/pkg/subnetmanager/subnet_informer.go
@@ -347,13 +347,21 @@ func (sc *SubnetController) processDynamicNextWorkItem(ctx context.Context) bool
 	return true
 }
 
-func (sc *SubnetController) syncHandler(ctx context.Context, subnetName string) error {
+func (sc *SubnetController) syncHandler(ctx context.Context, subnetName string) (err error) {
 	subnet, err := sc.SubnetsLister.Get(subnetName)
 	if err != nil {
 		return client.IgnoreNotFound(err)
 	}
 
 	subnetCopy := subnet.DeepCopy()
+	defer func() {
+		if err == nil {
+			attr := attribute.String(constant.KindSpiderSubnet, subnetName)
+			metric.SubnetTotalIPCounts.Add(ctx, *subnetCopy.Status.TotalIPCount, attr)
+			metric.SubnetAvailableIPCounts.Add(ctx, (*subnetCopy.Status.TotalIPCount)-(*subnetCopy.Status.AllocatedIPCount), attr)
+		}
+	}()
+
 	if err := sc.syncMetadata(ctx, subnetCopy); err != nil {
 		return fmt.Errorf("failed to sync metadata of Subnet: %v", err)
 	}
diff --git a/pkg/subnetmanager/subnet_validate.go b/pkg/subnetmanager/subnet_validate.go
index 6cf01c969..770500bdd 100644
--- a/pkg/subnetmanager/subnet_validate.go
+++ b/pkg/subnetmanager/subnet_validate.go
@@ -185,7 +185,6 @@ func (sw *SubnetWebhook) validateSubnetCIDR(ctx context.Context, subnet *spiderp
 		)
 	}
 
-	// TODO(iiiceoo): Use label selector.
 	subnetList := spiderpoolv2beta1.SpiderSubnetList{}
 	if err := sw.APIReader.List(ctx, &subnetList); err != nil {
 		return field.InternalError(subnetField, fmt.Errorf("failed to list Subnets: %v", err))


Test patch:
diff --git a/pkg/ippoolmanager/ippool_manager_suite_test.go b/pkg/ippoolmanager/ippool_manager_suite_test.go
index e3a87ac78..5f3967868 100644
--- a/pkg/ippoolmanager/ippool_manager_suite_test.go
+++ b/pkg/ippoolmanager/ippool_manager_suite_test.go
@@ -46,7 +46,7 @@ func TestIPPoolManager(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	_, err := metric.InitMetricController(context.TODO(), constant.SpiderpoolAgent, false)
+	_, err := metric.InitMetric(context.TODO(), constant.SpiderpoolAgent, false, false)
 	Expect(err).NotTo(HaveOccurred())
 	err = metric.InitSpiderpoolAgentMetrics(context.TODO())
 	Expect(err).NotTo(HaveOccurred())
diff --git a/pkg/subnetmanager/subnet_manager_suite_test.go b/pkg/subnetmanager/subnet_manager_suite_test.go
index e56060ebc..c3f394e59 100644
--- a/pkg/subnetmanager/subnet_manager_suite_test.go
+++ b/pkg/subnetmanager/subnet_manager_suite_test.go
@@ -4,6 +4,7 @@
 package subnetmanager_test
 
 import (
+	"context"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -16,8 +17,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
+	"github.com/spidernet-io/spiderpool/pkg/constant"
 	electionmock "github.com/spidernet-io/spiderpool/pkg/election/mock"
 	spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
+	"github.com/spidernet-io/spiderpool/pkg/metric"
 	reservedipmanagermock "github.com/spidernet-io/spiderpool/pkg/reservedipmanager/mock"
 	"github.com/spidernet-io/spiderpool/pkg/subnetmanager"
 )
@@ -42,8 +45,13 @@ func TestSubnetManager(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
+	_, err := metric.InitMetric(context.TODO(), constant.SpiderpoolController, false, false)
+	Expect(err).NotTo(HaveOccurred())
+	err = metric.InitSpiderpoolControllerMetrics(context.TODO())
+	Expect(err).NotTo(HaveOccurred())
+
 	scheme = runtime.NewScheme()
-	err := spiderpoolv2beta1.AddToScheme(scheme)
+	err = spiderpoolv2beta1.AddToScheme(scheme)
 	Expect(err).NotTo(HaveOccurred())
 
 	fakeClient = fake.NewClientBuilder().