Repository: spidernet-io/spiderpool
Pull request #1506 (2023-03-21 04:19:22)

change statefulset pod validation
Signed-off-by: Icarus9913 [icaruswu66@qq.com](mailto:icaruswu66@qq.com)

sonarcloud[bot]: SonarCloud Quality Gate failed.&nbsp; &nbsp; [![Quality Gate failed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/failed-16px.png 'Quality Gate failed')](https://sonarcloud.io/dashboard?id=spidernet-io_spiderpool&pullRequest=1506)

[![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=BUG) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=BUG)  
[![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=VULNERABILITY)  
[![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=SECURITY_HOTSPOT) [![E](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/E-16px.png 'E')](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=SECURITY_HOTSPOT) [5 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=SECURITY_HOTSPOT)  
[![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=CODE_SMELL) [0 Code Smells](https://sonarcloud.io/project/issues?id=spidernet-io_spiderpool&pullRequest=1506&resolved=false&types=CODE_SMELL)

[![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1506) No Coverage information  
[![3.6%](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/5-16px.png '3.6%')](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1506&metric=new_duplicated_lines_density&view=list) [3.6% Duplication](https://sonarcloud.io/component_measures?id=spidernet-io_spiderpool&pullRequest=1506&metric=new_duplicated_lines_density&view=list)



Patch:
diff --git a/docs/concepts/metrics.md b/docs/concepts/metrics.md
index d86de147c..4566bff5f 100644
--- a/docs/concepts/metrics.md
+++ b/docs/concepts/metrics.md
@@ -47,48 +47,60 @@ For example, let's enable spiderpool agent metrics by running `helm upgrade --se
 Spiderpool agent exports some metrics related with IPAM allocation and release. Currently, those include:
 
 
-| Name                                                      | description                                                                                                 |
-| --------------------------------------------------------- |-------------------------------------------------------------------------------------------------------------|
-| spiderpool_ipam_allocation_counts                         | Number of IPAM allocation requests that Spiderpool Agent received , prometheus type: counter                |
-| spiderpool_ipam_allocation_failure_counts                 | Number of Spiderpool Agent IPAM allocation failures, prometheus type: counter                               |
-| spiderpool_ipam_allocation_update_ippool_conflict_counts  | Number of Spiderpool Agent IPAM allocation update IPPool conflicts, prometheus type: counter                |
-| spiderpool_ipam_allocation_err_internal_counts            | Number of Spiderpool Agent IPAM allocation internal errors, prometheus type: counter                        |
-| spiderpool_ipam_allocation_err_no_available_pool_counts   | Number of Spiderpool Agent IPAM allocation no available IPPool errors, prometheus type: counter             |
-| spiderpool_ipam_allocation_err_retries_exhausted_counts   | Number of Spiderpool Agent IPAM allocation retries exhausted errors, prometheus type: counter               |
-| spiderpool_ipam_allocation_err_ip_used_out_counts         | Number of Spiderpool Agent IPAM allocation IP addresses used out errors, prometheus type: counter           |
-| spiderpool_ipam_allocation_average_duration_seconds       | The average duration of all Spiderpool Agent allocation processes, prometheus type: gauge                   |
-| spiderpool_ipam_allocation_max_duration_seconds           | The maximum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge           |
-| spiderpool_ipam_allocation_min_duration_seconds           | The minimum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge           |
-| spiderpool_ipam_allocation_latest_duration_seconds        | The latest duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge            |
-| spiderpool_ipam_allocation_duration_seconds               | Histogram of IPAM allocation duration in seconds, prometheus type: histogram                                |
-| spiderpool_ipam_allocation_average_limit_duration_seconds | The average duration of all Spiderpool Agent allocation queuing, prometheus type: gauge                     |
-| spiderpool_ipam_allocation_max_limit_duration_seconds     | The maximum duration of Spiderpool Agent allocation queuing, prometheus type: gauge                         |
-| spiderpool_ipam_allocation_min_limit_duration_seconds     | The minimum duration of Spiderpool Agent allocation queuing, prometheus type: gauge                         |
-| spiderpool_ipam_allocation_latest_limit_duration_seconds  | The latest duration of Spiderpool Agent allocation queuing, prometheus type: gauge                          |
-| spiderpool_ipam_allocation_limit_duration_seconds         | Histogram of IPAM allocation queuing duration in seconds, prometheus type: histogram                        |
-| spiderpool_ipam_release_counts                            | Count of the number of Spiderpool Agent received the IPAM release requests, prometheus type: counter        |
-| spiderpool_ipam_release_failure_counts                    | Number of Spiderpool Agent IPAM release failure, prometheus type: counter                                   |
-| spiderpool_ipam_release_update_ippool_conflict_counts     | Number of Spiderpool Agent IPAM release update IPPool conflicts, prometheus type: counter                   |
-| spiderpool_ipam_release_err_internal_counts               | Number of Spiderpool Agent IPAM releasing internal error, prometheus type: counter                          |
-| spiderpool_ipam_release_err_retries_exhausted_counts      | Number of Spiderpool Agent IPAM releasing retries exhausted error, prometheus type: counter                 |
-| spiderpool_ipam_release_average_duration_seconds          | The average duration of all Spiderpool Agent release processes, prometheus type: gauge                      |
-| spiderpool_ipam_release_max_duration_seconds              | The maximum duration of Spiderpool Agent release process (per-process), prometheus type: gauge              |
-| spiderpool_ipam_release_min_duration_seconds              | The minimum duration of Spiderpool Agent release process (per-process), prometheus type: gauge              |
-| spiderpool_ipam_release_latest_duration_seconds           | The latest duration of Spiderpool Agent release process (per-process), prometheus type: gauge               |
-| spiderpool_ipam_release_duration_seconds                  | Histogram of IPAM release duration in seconds, prometheus type: histogram                                   |
-| spiderpool_ipam_release_average_limit_duration_seconds    | The average duration of all Spiderpool Agent release queuing, prometheus type: gauge                        |
-| spiderpool_ipam_release_max_limit_duration_seconds        | The maximum duration of Spiderpool Agent release queuing, prometheus type: gauge                            |
-| spiderpool_ipam_release_min_limit_duration_seconds        | The minimum duration of Spiderpool Agent release queuing, prometheus type: gauge                            |
-| spiderpool_ipam_release_latest_limit_duration_seconds     | The latest duration of Spiderpool Agent release queuing, prometheus type: gauge                             |
-| spiderpool_ipam_release_limit_duration_seconds            | Histogram of IPAM release queuing duration in seconds, prometheus type: histogram                           |
+| Name                                                      | description                                                  |
+| --------------------------------------------------------- | ------------------------------------------------------------ |
+| spiderpool_ipam_allocation_counts                         | Number of IPAM allocation requests that Spiderpool Agent received , prometheus type: counter |
+| spiderpool_ipam_allocation_failure_counts                 | Number of Spiderpool Agent IPAM allocation failures, prometheus type: counter |
+| spiderpool_ipam_allocation_rollback_failure_counts        | Number of Spiderpool Agent IPAM allocation rollback failures, prometheus type: counter |
+| spiderpool_ipam_allocation_update_ippool_conflict_counts  | Number of Spiderpool Agent IPAM allocation update IPPool conflicts, prometheus type: counter |
+| spiderpool_ipam_allocation_err_internal_counts            | Number of Spiderpool Agent IPAM allocation internal errors, prometheus type: counter |
+| spiderpool_ipam_allocation_err_no_available_pool_counts   | Number of Spiderpool Agent IPAM allocation no available IPPool errors, prometheus type: counter |
+| spiderpool_ipam_allocation_err_retries_exhausted_counts   | Number of Spiderpool Agent IPAM allocation retries exhausted errors, prometheus type: counter |
+| spiderpool_ipam_allocation_err_ip_used_out_counts         | Number of Spiderpool Agent IPAM allocation IP addresses used out errors, prometheus type: counter |
+| spiderpool_ipam_allocation_average_duration_seconds       | The average duration of all Spiderpool Agent allocation processes, prometheus type: gauge |
+| spiderpool_ipam_allocation_max_duration_seconds           | The maximum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge |
+| spiderpool_ipam_allocation_min_duration_seconds           | The minimum duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge |
+| spiderpool_ipam_allocation_latest_duration_seconds        | The latest duration of Spiderpool Agent allocation process (per-process), prometheus type: gauge |
+| spiderpool_ipam_allocation_duration_seconds               | Histogram of IPAM allocation duration in seconds, prometheus type: histogram |
+| spiderpool_ipam_allocation_average_limit_duration_seconds | The average duration of all Spiderpool Agent allocation queuing, prometheus type: gauge |
+| spiderpool_ipam_allocation_max_limit_duration_seconds     | The maximum duration of Spiderpool Agent allocation queuing, prometheus type: gauge |
+| spiderpool_ipam_allocation_min_limit_duration_seconds     | The minimum duration of Spiderpool Agent allocation queuing, prometheus type: gauge |
+| spiderpool_ipam_allocation_latest_limit_duration_seconds  | The latest duration of Spiderpool Agent allocation queuing, prometheus type: gauge |
+| spiderpool_ipam_allocation_limit_duration_seconds         | Histogram of IPAM allocation queuing duration in seconds, prometheus type: histogram |
+| spiderpool_ipam_release_counts                            | Count of the number of Spiderpool Agent received the IPAM release requests, prometheus type: counter |
+| spiderpool_ipam_release_failure_counts                    | Number of Spiderpool Agent IPAM release failure, prometheus type: counter |
+| spiderpool_ipam_release_update_ippool_conflict_counts     | Number of Spiderpool Agent IPAM release update IPPool conflicts, prometheus type: counter |
+| spiderpool_ipam_release_err_internal_counts               | Number of Spiderpool Agent IPAM releasing internal error, prometheus type: counter |
+| spiderpool_ipam_release_err_retries_exhausted_counts      | Number of Spiderpool Agent IPAM releasing retries exhausted error, prometheus type: counter |
+| spiderpool_ipam_release_average_duration_seconds          | The average duration of all Spiderpool Agent release processes, prometheus type: gauge |
+| spiderpool_ipam_release_max_duration_seconds              | The maximum duration of Spiderpool Agent release process (per-process), prometheus type: gauge |
+| spiderpool_ipam_release_min_duration_seconds              | The minimum duration of Spiderpool Agent release process (per-process), prometheus type: gauge |
+| spiderpool_ipam_release_latest_duration_seconds           | The latest duration of Spiderpool Agent release process (per-process), prometheus type: gauge |
+| spiderpool_ipam_release_duration_seconds                  | Histogram of IPAM release duration in seconds, prometheus type: histogram |
+| spiderpool_ipam_release_average_limit_duration_seconds    | The average duration of all Spiderpool Agent release queuing, prometheus type: gauge |
+| spiderpool_ipam_release_max_limit_duration_seconds        | The maximum duration of Spiderpool Agent release queuing, prometheus type: gauge |
+| spiderpool_ipam_release_min_limit_duration_seconds        | The minimum duration of Spiderpool Agent release queuing, prometheus type: gauge |
+| spiderpool_ipam_release_latest_limit_duration_seconds     | The latest duration of Spiderpool Agent release queuing, prometheus type: gauge |
+| spiderpool_ipam_release_limit_duration_seconds            | Histogram of IPAM release queuing duration in seconds, prometheus type: histogram |
 | spiderpool_auto_pool_waited_for_available_counts          | Number of Spiderpool Agent IPAM allocation wait for auto-created IPPool available, prometheus type: counter |
 
 ### Spiderpool Controller
 
 Spiderpool controller exports some metrics related with SpiderIPPool IP garbage collection. Currently, those include:
 
-| Name                                             | description                                                                                             |
-|--------------------------------------------------|---------------------------------------------------------------------------------------------------------|
-| spiderpool_ip_gc_counts                          | Number of Spiderpool Controller IP garbage collection, prometheus type: counter                         |
-| spiderpool_ip_gc_failure_counts                  | Number of Spiderpool Controller IP garbage collection failures, prometheus type: counter                |
-| spiderpool_subnet_ippool_counts                  | Number of SpiderSubnet corresponding IPPools number, prometheus type: gauge                             |
+| Name                                                   | description                                                  |
+| ------------------------------------------------------ | ------------------------------------------------------------ |
+| spiderpool_ip_gc_counts                                | Number of Spiderpool Controller IP garbage collection, prometheus type: counter |
+| spiderpool_ip_gc_failure_counts                        | Number of Spiderpool Controller IP garbage collection failures, prometheus type: counter |
+| spiderpool_subnet_ippool_counts                        | Number of SpiderSubnet corresponding IPPools number, prometheus type: gauge |
+| spiderpool_ippool_informer_conflict_counts             | Number of Spiderpool Controller IPPool object status update operation conflict number, prometheus type: counter |
+| spiderpool_auto_pool_creation_average_duration_seconds | The average duration of All new auto-created IPPools creation and mark duration, prometheus type: gauge |
+| spiderpool_auto_pool_creation_max_duration_seconds     | The maximum duration of auto-created IPPool creation and mark duration (per-process), prometheus type: gauge |
+| spiderpool_auto_pool_creation_min_duration_seconds     | The minimum duration of auto-created IPPool creation and mark duration (per-process), prometheus type: gauge |
+| spiderpool_auto_pool_creation_latest_duration_seconds  | The latest duration of auto-created IPPool creation and mark duration (per-process), prometheus type: gauge |
+| spiderpool_auto_pool_scale_average_duration_seconds    | The average duration of All auto-created IPPools scale duration, prometheus type: gauge |
+| spiderpool_auto_pool_scale_max_duration_seconds        | The maximum duration of auto-created IPPool scale duration (per-process), prometheus type: gauge |
+| spiderpool_auto_pool_scale_min_duration_seconds        | The minimum duration of auto-created IPPool scale duration (per-process), prometheus type: gauge |
+| spiderpool_auto_pool_scale_latest_duration_seconds     | The latest duration of auto-created IPPool scale duration (per-process), prometheus type: gauge |
+| spiderpool_auto_pool_scale_duration_seconds            | Histogram of new auto-created IPPool scale duration in seconds, prometheus type: histogram |
+| spiderpool_auto_pool_scale_conflict_counts             | Number of Spiderpool Controller auto-created IPPool scale operation conflicts, prometheus type: counter |
diff --git a/pkg/ipam/pool_selections.go b/pkg/ipam/pool_selections.go
index a843ee02a..305b53cf9 100644
--- a/pkg/ipam/pool_selections.go
+++ b/pkg/ipam/pool_selections.go
@@ -23,7 +23,6 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
 	"github.com/spidernet-io/spiderpool/pkg/logutils"
-	"github.com/spidernet-io/spiderpool/pkg/metric"
 	"github.com/spidernet-io/spiderpool/pkg/namespacemanager"
 	"github.com/spidernet-io/spiderpool/pkg/types"
 )
@@ -130,19 +129,10 @@ func (i *ipam) getPoolFromSubnetAnno(ctx context.Context, pod *corev1.Pod, nic s
 	// If the application is an orphan pod and do not find any IPPool, it will return immediately to inform you to create IPPool.
 	findSubnetIPPool := func(subnetName string, ipVersion types.IPVersion) (*spiderpoolv2beta1.SpiderIPPool, error) {
 		poolName := subnetmanagercontrollers.SubnetPoolName(podController.Name, ipVersion, nic, podController.UID)
-		var isRetried bool
-		defer func() {
-			if isRetried {
-				metric.AutoPoolWaitedForAvailableCounts.Add(ctx, 1)
-			}
-		}()
 
 		var pool *spiderpoolv2beta1.SpiderIPPool
 		var err error
 		for j := 1; j <= i.config.OperationRetries; j++ {
-			if j > 1 {
-				isRetried = true
-			}
 			// third-party controller applications
 			if !slices.Contains(constant.K8sAPIVersions, podController.APIVersion) || !slices.Contains(constant.K8sKinds, podController.Kind) {
 				pool, err = i.applyThirdControllerAutoPool(ctx, subnetName, poolName, podController, types.AutoPoolProperty{
diff --git a/pkg/metric/metrics_instance.go b/pkg/metric/metrics_instance.go
index 03404132a..a3b847fb7 100644
--- a/pkg/metric/metrics_instance.go
+++ b/pkg/metric/metrics_instance.go
@@ -20,6 +20,7 @@ const (
 	// spiderpool agent ipam allocation metrics name
 	ipam_allocation_counts                        = MetricPrefix + "ipam_allocation_counts"
 	ipam_allocation_failure_counts                = MetricPrefix + "ipam_allocation_failure_counts"
+	ipam_allocation_rollback_failure_counts       = MetricPrefix + "ipam_allocation_rollback_failure_counts"
 	ipam_allocation_update_ippool_conflict_counts = MetricPrefix + "ipam_allocation_update_ippool_conflict_counts"
 	ipam_allocation_err_internal_counts           = MetricPrefix + "ipam_allocation_err_internal_counts"
 	ipam_allocation_err_no_available_pool_counts  = MetricPrefix + "ipam_allocation_err_no_available_pool_counts"
@@ -61,15 +62,29 @@ const (
 	ip_gc_counts         = MetricPrefix + "ip_gc_counts"
 	ip_gc_failure_counts = MetricPrefix + "ip_gc_failure_counts"
 
+	subnet_ippool_counts = MetricPrefix + "subnet_ippool_counts"
+
 	// spiderpool controller SpiderSubnet feature
-	subnet_ippool_counts                  = MetricPrefix + "subnet_ippool_counts"
-	auto_pool_waited_for_available_counts = MetricPrefix + "auto_pool_waited_for_available_counts"
+	ippool_informer_conflict_counts             = MetricPrefix + "ippool_informer_conflict_counts"
+	auto_pool_creation_average_duration_seconds = MetricPrefix + "auto_pool_creation_average_duration_seconds"
+	auto_pool_creation_max_duration_seconds     = MetricPrefix + "auto_pool_creation_max_duration_seconds"
+	auto_pool_creation_min_duration_seconds     = MetricPrefix + "auto_pool_creation_min_duration_seconds"
+	auto_pool_creation_latest_duration_seconds  = MetricPrefix + "auto_pool_creation_latest_duration_seconds"
+	auto_pool_creation_duration_seconds         = MetricPrefix + "auto_pool_creation_duration_seconds"
+	auto_pool_scale_average_duration_seconds    = MetricPrefix + "auto_pool_scale_average_duration_seconds"
+	auto_pool_scale_max_duration_seconds        = MetricPrefix + "auto_pool_scale_max_duration_seconds"
+	auto_pool_scale_min_duration_seconds        = MetricPrefix + "auto_pool_scale_min_duration_seconds"
+	auto_pool_scale_latest_duration_seconds     = MetricPrefix + "auto_pool_scale_latest_duration_seconds"
+	auto_pool_scale_duration_seconds            = MetricPrefix + "auto_pool_scale_duration_seconds"
+	auto_pool_scale_conflict_counts             = MetricPrefix + "auto_pool_scale_conflict_counts"
+	auto_pool_waited_for_available_counts       = MetricPrefix + "auto_pool_waited_for_available_counts"
 )
 
 var (
 	// spiderpool agent ipam allocation metrics
 	IpamAllocationTotalCounts                   instrument.Int64Counter
 	IpamAllocationFailureCounts                 instrument.Int64Counter
+	IpamAllocationRollbackFailureCounts         instrument.Int64Counter
 	IpamAllocationUpdateIPPoolConflictCounts    instrument.Int64Counter
 	IpamAllocationErrInternalCounts             instrument.Int64Counter
 	IpamAllocationErrNoAvailablePoolCounts      instrument.Int64Counter
@@ -110,7 +125,19 @@ var (
 	SubnetPoolCounts = new(asyncInt64Gauge)
 
 	// SpiderSubnet feature
-	AutoPoolWaitedForAvailableCounts instrument.Int64Counter
+	IPPoolInformerConflictCounts             instrument.Int64Counter
+	autoPoolCreationAverageDurationSeconds   = new(asyncFloat64Gauge)
+	autoPoolCreationMaxDurationSeconds       = new(asyncFloat64Gauge)
+	autoPoolCreationMinDurationSeconds       = new(asyncFloat64Gauge)
+	autoPoolCreationLatestDurationSeconds    = new(asyncFloat64Gauge)
+	autoPoolCreationDurationSecondsHistogram instrument.Float64Histogram
+	autoPoolScaleAverageDurationSeconds      = new(asyncFloat64Gauge)
+	autoPoolScaleMaxDurationSeconds          = new(asyncFloat64Gauge)
+	autoPoolScaleMinDurationSeconds          = new(asyncFloat64Gauge)
+	autoPoolScaleLatestDurationSeconds       = new(asyncFloat64Gauge)
+	autoPoolScaleDurationSecondsHistogram    instrument.Float64Histogram
+	AutoPoolScaleConflictCounts              instrument.Int64Counter
+	AutoPoolWaitedForAvailableCounts         instrument.Int64Counter
 )
 
 // asyncFloat64Gauge is custom otel float64 gauge
@@ -207,6 +234,11 @@ func InitSpiderpoolAgentMetrics(ctx context.Context) error {
 		return err
 	}
 
+	err = initAutoPoolCreationMetrics(ctx)
+	if nil != err {
+		return err
+	}
+
 	autoPoolWaitedForAvailableCounts, err := NewMetricInt64Counter(auto_pool_waited_for_available_counts, "ipam waited for auto-created IPPool available counts")
 	if nil != err {
 		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", auto_pool_waited_for_available_counts, err)
@@ -223,11 +255,27 @@ func InitSpiderpoolControllerMetrics(ctx context.Context) error {
 		return err
 	}
 
+	err = initAutoPoolCreationMetrics(ctx)
+	if nil != err {
+		return err
+	}
+
+	err = initAutoPoolScaleMetrics(ctx)
+	if nil != err {
+		return err
+	}
+
 	err = SubnetPoolCounts.initGauge(subnet_ippool_counts, "spider subnet corresponding ippools counts")
 	if nil != err {
 		return err
 	}
 
+	poolInformerConflictCounts, err := NewMetricInt64Counter(ippool_informer_conflict_counts, "ippool informer operation conflict counts")
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", ippool_informer_conflict_counts, err)
+	}
+	IPPoolInformerConflictCounts = poolInformerConflictCounts
+
 	return nil
 }
 
@@ -247,6 +295,13 @@ func initSpiderpoolAgentAllocationMetrics(ctx context.Context) error {
 	}
 	IpamAllocationFailureCounts = allocationFailureCounts
 
+	// spiderpool agent ipam allocation rollback failure counts, metric type "int64 counter"
+	allocationRollbackFailureCounts, err := NewMetricInt64Counter(ipam_allocation_rollback_failure_counts, "spiderpool agent ipam allocation rollback failure counts")
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool agent metric '%s', error: %v", ipam_allocation_rollback_failure_counts, err)
+	}
+	IpamAllocationRollbackFailureCounts = allocationRollbackFailureCounts
+
 	// spiderpool agent ipam allocation update IPPool conflict counts, metric type "int64 counter"
 	allocationUpdateIPPoolConflictCounts, err := NewMetricInt64Counter(ipam_allocation_update_ippool_conflict_counts, "spiderpool agent ipam allocation update IPPool conflict counts")
 	if nil != err {
@@ -471,3 +526,72 @@ func initSpiderpoolControllerGCMetrics(ctx context.Context) error {
 
 	return nil
 }
+
+// initAutoPoolCreationMetrics will init auto-created IPPool creation metrics
+// Notice: this metrics serve for both Spiderpool-agent and Spiderpool-controller components
+func initAutoPoolCreationMetrics(ctx context.Context) error {
+	err := autoPoolCreationAverageDurationSeconds.initGauge(auto_pool_creation_average_duration_seconds, "auto-created IPPool creation average duration")
+	if nil != err {
+		return err
+	}
+
+	err = autoPoolCreationMaxDurationSeconds.initGauge(auto_pool_creation_max_duration_seconds, "auto-created IPPool creation max duration")
+	if nil != err {
+		return err
+	}
+
+	err = autoPoolCreationMinDurationSeconds.initGauge(auto_pool_creation_min_duration_seconds, "auto-created IPPool creation min duration")
+	if nil != err {
+		return err
+	}
+
+	err = autoPoolCreationLatestDurationSeconds.initGauge(auto_pool_creation_latest_duration_seconds, "auto-created IPPool creation latest duration")
+	if nil != err {
+		return err
+	}
+
+	autoPoolCreationHistogram, err := NewMetricFloat64Histogram(auto_pool_creation_duration_seconds, "histogram of auto-created IPPool creation duration")
+	if nil != err {
+		return err
+	}
+	autoPoolCreationDurationSecondsHistogram = autoPoolCreationHistogram
+
+	return nil
+}
+
+// initAutoPoolScaleMetrics will init spiderpool-controller IPPool informer auto-created IPPool scale metrics
+func initAutoPoolScaleMetrics(ctx context.Context) error {
+	err := autoPoolScaleAverageDurationSeconds.initGauge(auto_pool_scale_average_duration_seconds, "auto-created IPPool scale average duration")
+	if nil != err {
+		return err
+	}
+
+	err = autoPoolScaleMaxDurationSeconds.initGauge(auto_pool_scale_max_duration_seconds, "auto-created IPPool scale max duration")
+	if nil != err {
+		return err
+	}
+
+	err = autoPoolScaleMinDurationSeconds.initGauge(auto_pool_scale_min_duration_seconds, "auto-created IPPool scale min duration")
+	if nil != err {
+		return err
+	}
+
+	err = autoPoolScaleLatestDurationSeconds.initGauge(auto_pool_scale_latest_duration_seconds, "auto-created IPPool scale latest duration")
+	if nil != err {
+		return err
+	}
+
+	autoPoolScaleHistogram, err := NewMetricFloat64Histogram(auto_pool_scale_duration_seconds, "histogram of auto-created IPPool scale duration")
+	if nil != err {
+		return fmt.Errorf("")
+	}
+	autoPoolScaleDurationSecondsHistogram = autoPoolScaleHistogram
+
+	autoPoolScaleConflictCounts, err := NewMetricInt64Counter(auto_pool_scale_conflict_counts, "scale auto-created IPPool conflict counts")
+	if nil != err {
+		return fmt.Errorf("failed to new spiderpool controller metric '%s', error: %v", auto_pool_scale_conflict_counts, err)
+	}
+	AutoPoolScaleConflictCounts = autoPoolScaleConflictCounts
+
+	return nil
+}
diff --git a/pkg/metric/metrics_ipam.go b/pkg/metric/metrics_ipam.go
index 6d28c5b9f..3499bab0d 100644
--- a/pkg/metric/metrics_ipam.go
+++ b/pkg/metric/metrics_ipam.go
@@ -35,32 +35,34 @@ func (idc *ipamDurationConstruct) RecordIPAMAllocationDuration(ctx context.Conte
 		return
 	}
 
-	// latest allocation duration
-	ipamAllocationLatestDurationSeconds.Record(allocationDuration)
-
-	// allocation duration histogram
-	ipamAllocationDurationSecondsHistogram.Record(ctx, allocationDuration)
-
-	idc.allocate.cacheLock.Lock()
-
-	// IPAM average allocation duration
-	idc.allocate.avgDuration = (idc.allocate.avgDuration*float64(idc.allocate.counts) + allocationDuration) / float64(idc.allocate.counts+1)
-	idc.allocate.counts++
-	ipamAllocationAverageDurationSeconds.Record(idc.allocate.avgDuration)
-
-	// IPAM maximum allocation duration
-	if allocationDuration > idc.allocate.maxDuration {
-		idc.allocate.maxDuration = allocationDuration
-		ipamAllocationMaxDurationSeconds.Record(idc.allocate.maxDuration)
-	}
-
-	// IPAM minimum allocation duration
-	if idc.allocate.counts == 1 || allocationDuration < idc.allocate.minDuration {
-		idc.allocate.minDuration = allocationDuration
-		ipamAllocationMinDurationSeconds.Record(idc.allocate.minDuration)
-	}
-
-	idc.allocate.cacheLock.Unlock()
+	go func() {
+		// latest allocation duration
+		ipamAllocationLatestDurationSeconds.Record(allocationDuration)
+
+		// allocation duration histogram
+		ipamAllocationDurationSecondsHistogram.Record(ctx, allocationDuration)
+
+		idc.allocate.cacheLock.Lock()
+
+		// IPAM average allocation duration
+		idc.allocate.avgDuration = (idc.allocate.avgDuration*float64(idc.allocate.counts) + allocationDuration) / float64(idc.allocate.counts+1)
+		idc.allocate.counts++
+		ipamAllocationAverageDurationSeconds.Record(idc.allocate.avgDuration)
+
+		// IPAM maximum allocation duration
+		if allocationDuration > idc.allocate.maxDuration {
+			idc.allocate.maxDuration = allocationDuration
+			ipamAllocationMaxDurationSeconds.Record(idc.allocate.maxDuration)
+		}
+
+		// IPAM minimum allocation duration
+		if idc.allocate.counts == 1 || allocationDuration < idc.allocate.minDuration {
+			idc.allocate.minDuration = allocationDuration
+			ipamAllocationMinDurationSeconds.Record(idc.allocate.minDuration)
+		}
+
+		idc.allocate.cacheLock.Unlock()
+	}()
 }
 
 // RecordIPAMReleaseDuration serves for spiderpool agent IPAM allocation.
@@ -69,32 +71,34 @@ func (idc *ipamDurationConstruct) RecordIPAMReleaseDuration(ctx context.Context,
 		return
 	}
 
-	// latest release duration
-	ipamReleaseLatestDurationSeconds.Record(releaseDuration)
+	go func() {
+		// latest release duration
+		ipamReleaseLatestDurationSeconds.Record(releaseDuration)
 
-	// release duration histogram
-	ipamReleaseDurationSecondsHistogram.Record(ctx, releaseDuration)
+		// release duration histogram
+		ipamReleaseDurationSecondsHistogram.Record(ctx, releaseDuration)
 
-	idc.release.cacheLock.Lock()
+		idc.release.cacheLock.Lock()
 
-	// IPAM average release duration
-	idc.release.avgDuration = (idc.release.avgDuration*float64(idc.release.counts) + releaseDuration) / float64(idc.release.counts+1)
-	idc.release.counts++
-	ipamReleaseAverageDurationSeconds.Record(idc.release.avgDuration)
+		// IPAM average release duration
+		idc.release.avgDuration = (idc.release.avgDuration*float64(idc.release.counts) + releaseDuration) / float64(idc.release.counts+1)
+		idc.release.counts++
+		ipamReleaseAverageDurationSeconds.Record(idc.release.avgDuration)
 
-	// IPAM maximum release duration
-	if releaseDuration > idc.release.maxDuration {
-		idc.release.maxDuration = releaseDuration
-		ipamReleaseMaxDurationSeconds.Record(idc.release.maxDuration)
-	}
+		// IPAM maximum release duration
+		if releaseDuration > idc.release.maxDuration {
+			idc.release.maxDuration = releaseDuration
+			ipamReleaseMaxDurationSeconds.Record(idc.release.maxDuration)
+		}
 
-	// IPAM minimum release duration
-	if idc.release.counts == 1 || releaseDuration < idc.release.minDuration {
-		idc.release.minDuration = releaseDuration
-		ipamReleaseMinDurationSeconds.Record(idc.release.minDuration)
-	}
+		// IPAM minimum release duration
+		if idc.release.counts == 1 || releaseDuration < idc.release.minDuration {
+			idc.release.minDuration = releaseDuration
+			ipamReleaseMinDurationSeconds.Record(idc.release.minDuration)
+		}
 
-	idc.release.cacheLock.Unlock()
+		idc.release.cacheLock.Unlock()
+	}()
 }
 
 func (idc *ipamDurationConstruct) RecordIPAMAllocationLimitDuration(ctx context.Context, limitDuration float64) {
@@ -102,25 +106,27 @@ func (idc *ipamDurationConstruct) RecordIPAMAllocationLimitDuration(ctx context.
 		return
 	}
 
-	ipamAllocationLatestLimitDurationSeconds.Record(limitDuration)
-	ipamAllocationLimitDurationSecondsHistogram.Record(ctx, limitDuration)
+	go func() {
+		ipamAllocationLatestLimitDurationSeconds.Record(limitDuration)
+		ipamAllocationLimitDurationSecondsHistogram.Record(ctx, limitDuration)
 
-	idc.allocateLimit.cacheLock.Lock()
-	defer idc.allocateLimit.cacheLock.Unlock()
+		idc.allocateLimit.cacheLock.Lock()
+		defer idc.allocateLimit.cacheLock.Unlock()
 
-	idc.allocateLimit.avgDuration = (idc.allocateLimit.avgDuration*float64(idc.allocateLimit.counts) + limitDuration) / float64(idc.allocateLimit.counts+1)
-	idc.allocateLimit.counts++
-	ipamAllocationAverageLimitDurationSeconds.Record(idc.allocateLimit.avgDuration)
+		idc.allocateLimit.avgDuration = (idc.allocateLimit.avgDuration*float64(idc.allocateLimit.counts) + limitDuration) / float64(idc.allocateLimit.counts+1)
+		idc.allocateLimit.counts++
+		ipamAllocationAverageLimitDurationSeconds.Record(idc.allocateLimit.avgDuration)
 
-	if limitDuration > idc.allocateLimit.maxDuration {
-		idc.allocateLimit.maxDuration = limitDuration
-		ipamAllocationMaxLimitDurationSeconds.Record(idc.allocateLimit.maxDuration)
-	}
+		if limitDuration > idc.allocateLimit.maxDuration {
+			idc.allocateLimit.maxDuration = limitDuration
+			ipamAllocationMaxLimitDurationSeconds.Record(idc.allocateLimit.maxDuration)
+		}
 
-	if idc.allocateLimit.counts == 1 || limitDuration < idc.allocateLimit.minDuration {
-		idc.allocateLimit.minDuration = limitDuration
-		ipamAllocationMinLimitDurationSeconds.Record(idc.allocateLimit.minDuration)
-	}
+		if idc.allocateLimit.counts == 1 || limitDuration < idc.allocateLimit.minDuration {
+			idc.allocateLimit.minDuration = limitDuration
+			ipamAllocationMinLimitDurationSeconds.Record(idc.allocateLimit.minDuration)
+		}
+	}()
 }
 
 func (idc *ipamDurationConstruct) RecordIPAMReleaseLimitDuration(ctx context.Context, limitDuration float64) {
@@ -128,23 +134,25 @@ func (idc *ipamDurationConstruct) RecordIPAMReleaseLimitDuration(ctx context.Con
 		return
 	}
 
-	ipamReleaseLatestLimitDurationSeconds.Record(limitDuration)
-	ipamReleaseLimitDurationSecondsHistogram.Record(ctx, limitDuration)
+	go func() {
+		ipamReleaseLatestLimitDurationSeconds.Record(limitDuration)
+		ipamReleaseLimitDurationSecondsHistogram.Record(ctx, limitDuration)
 
-	idc.releaseLimit.cacheLock.Lock()
-	defer idc.releaseLimit.cacheLock.Unlock()
+		idc.releaseLimit.cacheLock.Lock()
+		defer idc.releaseLimit.cacheLock.Unlock()
 
-	idc.releaseLimit.avgDuration = (idc.releaseLimit.avgDuration*float64(idc.releaseLimit.counts) + limitDuration) / float64(idc.releaseLimit.counts+1)
-	idc.releaseLimit.counts++
-	ipamReleaseAverageLimitDurationSeconds.Record(idc.releaseLimit.avgDuration)
+		idc.releaseLimit.avgDuration = (idc.releaseLimit.avgDuration*float64(idc.releaseLimit.counts) + limitDuration) / float64(idc.releaseLimit.counts+1)
+		idc.releaseLimit.counts++
+		ipamReleaseAverageLimitDurationSeconds.Record(idc.releaseLimit.avgDuration)
 
-	if limitDuration > idc.releaseLimit.maxDuration {
-		idc.releaseLimit.maxDuration = limitDuration
-		ipamReleaseMaxLimitDurationSeconds.Record(idc.releaseLimit.maxDuration)
-	}
+		if limitDuration > idc.releaseLimit.maxDuration {
+			idc.releaseLimit.maxDuration = limitDuration
+			ipamReleaseMaxLimitDurationSeconds.Record(idc.releaseLimit.maxDuration)
+		}
 
-	if idc.releaseLimit.counts == 1 || limitDuration < idc.releaseLimit.minDuration {
-		idc.releaseLimit.minDuration = limitDuration
-		ipamReleaseMinLimitDurationSeconds.Record(idc.releaseLimit.minDuration)
-	}
+		if idc.releaseLimit.counts == 1 || limitDuration < idc.releaseLimit.minDuration {
+			idc.releaseLimit.minDuration = limitDuration
+			ipamReleaseMinLimitDurationSeconds.Record(idc.releaseLimit.minDuration)
+		}
+	}()
 }
diff --git a/pkg/metric/metrics_subnet.go b/pkg/metric/metrics_subnet.go
new file mode 100644
index 000000000..da87db239
--- /dev/null
+++ b/pkg/metric/metrics_subnet.go
@@ -0,0 +1,106 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package metric
+
+import (
+	"context"
+
+	"github.com/spidernet-io/spiderpool/pkg/lock"
+)
+
+// AutoPoolCreationDurationConstruct is Singleton
+var AutoPoolCreationDurationConstruct = new(autoPoolCreationDurationConstruct)
+
+// AutoPoolScaleDurationConstruct is Singleton
+var AutoPoolScaleDurationConstruct = new(autoPoolScaleDurationConstruct)
+
+type autoPoolCreationDurationConstruct struct {
+	cacheLock lock.RWMutex
+
+	creationAvgDuration float64
+	maxCreationDuration float64
+	minCreationDuration float64
+
+	creationCounts int
+}
+
+func (a *autoPoolCreationDurationConstruct) RecordAutoPoolCreationDuration(ctx context.Context, duration float64) {
+	if !globalEnableMetric {
+		return
+	}
+
+	go func() {
+		// latest auto-created IPPool creation duration
+		autoPoolCreationLatestDurationSeconds.Record(duration)
+
+		// auto-created IPPool creation duration histogram
+		autoPoolCreationDurationSecondsHistogram.Record(ctx, duration)
+
+		a.cacheLock.Lock()
+
+		// auto-created IPPool creation average duration
+		a.creationAvgDuration = (a.creationAvgDuration*float64(a.creationCounts) + duration) / float64(a.creationCounts+1)
+		a.creationCounts++
+		autoPoolCreationAverageDurationSeconds.Record(a.creationAvgDuration)
+
+		// auto-created IPPool creation maximum duration
+		if duration > a.maxCreationDuration {
+			a.maxCreationDuration = duration
+			autoPoolCreationMaxDurationSeconds.Record(a.maxCreationDuration)
+		}
+
+		// auto-created IPPool creation minimum duration
+		if a.creationCounts == 1 || duration < a.minCreationDuration {
+			a.minCreationDuration = duration
+			autoPoolCreationMinDurationSeconds.Record(a.minCreationDuration)
+		}
+
+		a.cacheLock.Unlock()
+	}()
+}
+
+type autoPoolScaleDurationConstruct struct {
+	cacheLock lock.RWMutex
+
+	scaleAvgDuration float64
+	maxScaleDuration float64
+	minScaleDuration float64
+
+	scaleCounts int
+}
+
+func (a *autoPoolScaleDurationConstruct) RecordAutoPoolScaleDuration(ctx context.Context, duration float64) {
+	if !globalEnableMetric {
+		return
+	}
+
+	go func() {
+		// latest auto-created IPPool creation duration
+		autoPoolScaleLatestDurationSeconds.Record(duration)
+
+		// auto-created IPPool creation duration histogram
+		autoPoolScaleDurationSecondsHistogram.Record(ctx, duration)
+
+		a.cacheLock.Lock()
+
+		// auto-created IPPool creation average duration
+		a.scaleAvgDuration = (a.scaleAvgDuration*float64(a.scaleCounts) + duration) / float64(a.scaleCounts+1)
+		a.scaleCounts++
+		autoPoolScaleAverageDurationSeconds.Record(a.scaleAvgDuration)
+
+		// auto-created IPPool creation maximum duration
+		if duration > a.maxScaleDuration {
+			a.maxScaleDuration = duration
+			autoPoolScaleMaxDurationSeconds.Record(a.maxScaleDuration)
+		}
+
+		// auto-created IPPool creation minimum duration
+		if a.scaleCounts == 1 || duration < a.minScaleDuration {
+			a.minScaleDuration = duration
+			autoPoolScaleMinDurationSeconds.Record(a.minScaleDuration)
+		}
+
+		a.cacheLock.Unlock()
+	}()
+}
diff --git a/pkg/statefulsetmanager/sts_manager.go b/pkg/statefulsetmanager/sts_manager.go
index 735723d71..b1b872d4c 100644
--- a/pkg/statefulsetmanager/sts_manager.go
+++ b/pkg/statefulsetmanager/sts_manager.go
@@ -77,7 +77,7 @@ func (sm *statefulSetManager) IsValidStatefulSetPod(ctx context.Context, namespa
 
 	stsName, replicas, found := getStatefulSetNameAndOrdinal(podName)
 	if !found {
-		return false, fmt.Errorf("failed to parse the name and replica of its StatefulSet controller from the name of Pod '%s/%s'", namespace, podName)
+		return false, nil
 	}
 
 	sts, err := sm.GetStatefulSetByName(ctx, namespace, stsName, constant.IgnoreCache)
diff --git a/pkg/subnetmanager/subnet_informer.go b/pkg/subnetmanager/subnet_informer.go
index a93797ba4..37d64d1ad 100644
--- a/pkg/subnetmanager/subnet_informer.go
+++ b/pkg/subnetmanager/subnet_informer.go
@@ -9,7 +9,6 @@ import (
 	"reflect"
 	"time"
 
-	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -41,7 +40,6 @@ import (
 	informers "github.com/spidernet-io/spiderpool/pkg/k8s/client/informers/externalversions/spiderpool.spidernet.io/v2beta1"
 	listers "github.com/spidernet-io/spiderpool/pkg/k8s/client/listers/spiderpool.spidernet.io/v2beta1"
 	"github.com/spidernet-io/spiderpool/pkg/logutils"
-	"github.com/spidernet-io/spiderpool/pkg/metric"
 	spiderpooltypes "github.com/spidernet-io/spiderpool/pkg/types"
 	"github.com/spidernet-io/spiderpool/pkg/utils/convert"
 )
@@ -411,9 +409,6 @@ func (sc *SubnetController) syncControlledIPPoolIPs(ctx context.Context, subnet
 		return err
 	}
 
-	// record the metric of how many IPPools the Subnet has.
-	metric.SubnetPoolCounts.Record(int64(len(ipPools)), attribute.String(constant.KindSpiderSubnet, subnet.Name))
-
 	subnetTotalIPs, err := spiderpoolip.AssembleTotalIPs(*subnet.Spec.IPVersion, subnet.Spec.IPs, subnet.Spec.ExcludeIPs)
 	if err != nil {
 		return err


Test patch:
diff --git a/pkg/statefulsetmanager/sts_manager_test.go b/pkg/statefulsetmanager/sts_manager_test.go
index 42c26e675..5b12fb1f0 100644
--- a/pkg/statefulsetmanager/sts_manager_test.go
+++ b/pkg/statefulsetmanager/sts_manager_test.go
@@ -231,12 +231,12 @@ var _ = Describe("StatefulSetManager", Label("sts_manager_test"), func() {
 				Expect(valid).To(BeFalse())
 			})
 
-			It("failed to parse replica string to int due to some unknown errors", func() {
+			It("invalid StatefulSet pod with bad pod name replica parsing", func() {
 				patches := gomonkey.ApplyFuncReturn(strconv.ParseInt, int64(0), constant.ErrUnknown)
 				defer patches.Reset()
 
 				valid, err := stsManager.IsValidStatefulSetPod(ctx, stsT.Namespace, fmt.Sprintf("%s-%d", stsName, 0), constant.KindStatefulSet)
-				Expect(err).To(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred())
 				Expect(valid).To(BeFalse())
 			})