Repository: tikv/pd
Pull request #9331: feat(resourcemanager): implement keyspace manager for resource groups
Author: login: JmPotato
Labels: name: component/keyspace
description: Key space.
name: size/XXL
description: Denotes a PR that changes 1000+ lines, ignoring generated files.
name: release-note-none
description: Denotes a PR that doesn't merit a release note.
name: component/resource-control
name: lgtm
name: approved
name: dco-signoff: yes
description: Indicates the PR's author has signed the dco.
Created: 2025-05-19T08:24:19Z | Merged: 2025-05-21T02:41:01Z

<!--

Thank you for working on PD! Please read PD's [CONTRIBUTING](https://github.com/tikv/pd/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.

PR Title Format:
1. pkg [, pkg2, pkg3]: what's changed
2. *: what's changed

-->

### What problem does this PR solve?

<!--

Please create an issue first to describe the problem.
There MUST be one line starting with "Issue Number:  " and 
linking the relevant issues via the "close" or "ref".
For more info, check https://github.com/tikv/pd/blob/master/CONTRIBUTING.md#linking-issues.

-->
Issue Number: ref #9296.

### What is changed and how does it work?

<!--

You could use the "commit message" code block to add more description to the final commit message.
For more info, check https://github.com/tikv/pd/blob/master/CONTRIBUTING.md#format-of-the-commit-message.

-->

```commit-message
Add keyspace-level resource group management.
```

### Check List

<!-- Remove the items that are not applicable. -->

Tests

<!-- At least one of these tests must be included. -->

- Unit test
- Integration test

### Release note

<!--

A bugfix or a new feature needs a release note. If there is no need to give a release note, just leave it with the `None`.

Please refer to [Release Notes Language Style Guide](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/release-notes-style-guide.html) to write a quality release note.

-->

```release-note
None.
```



Diff:
diff --git a/errors.toml b/errors.toml
index 2508836f7fe..78fb79384e6 100644
--- a/errors.toml
+++ b/errors.toml
@@ -846,6 +846,11 @@ error = '''
 invalid group settings, please check the group name, priority and the number of resources
 '''
 
+["PD:resourcemanager:ErrKeyspaceNotExists"]
+error = '''
+the keyspace does not exist with id %d
+'''
+
 ["PD:scatter:ErrEmptyRegion"]
 error = '''
 empty region
diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go
index 8f350fe16cf..f186195cdd0 100644
--- a/pkg/errs/errno.go
+++ b/pkg/errs/errno.go
@@ -525,6 +525,7 @@ var (
 
 // Resource Manager errors
 var (
+	ErrKeyspaceNotExists      = errors.Normalize("the keyspace does not exist with id %d", errors.RFCCodeText("PD:resourcemanager:ErrKeyspaceNotExists"))
 	ErrResourceGroupNotExists = errors.Normalize("the %s resource group does not exist", errors.RFCCodeText("PD:resourcemanager:ErrGroupNotExists"))
 	ErrDeleteReservedGroup    = errors.Normalize("cannot delete reserved group", errors.RFCCodeText("PD:resourcemanager:ErrDeleteReservedGroup"))
 	ErrInvalidGroup           = errors.Normalize("invalid group settings, please check the group name, priority and the number of resources", errors.RFCCodeText("PD:resourcemanager:ErrInvalidGroup"))
diff --git a/pkg/mcs/resourcemanager/server/apis/v1/api.go b/pkg/mcs/resourcemanager/server/apis/v1/api.go
index e142dbbc69a..2198f338efd 100644
--- a/pkg/mcs/resourcemanager/server/apis/v1/api.go
+++ b/pkg/mcs/resourcemanager/server/apis/v1/api.go
@@ -30,6 +30,7 @@ import (
 
 	rmserver "github.com/tikv/pd/pkg/mcs/resourcemanager/server"
 	"github.com/tikv/pd/pkg/mcs/utils"
+	"github.com/tikv/pd/pkg/mcs/utils/constant"
 	"github.com/tikv/pd/pkg/utils/apiutil"
 	"github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi"
 	"github.com/tikv/pd/pkg/utils/reflectutil"
@@ -161,7 +162,7 @@ func (s *Service) putResourceGroup(c *gin.Context) {
 //	@Router		/config/group/{name} [get]
 func (s *Service) getResourceGroup(c *gin.Context) {
 	withStats := strings.EqualFold(c.Query("with_stats"), "true")
-	group := s.manager.GetResourceGroup(c.Param("name"), withStats)
+	group := s.manager.GetResourceGroup(constant.NullKeyspaceID, c.Param("name"), withStats)
 	if group == nil {
 		c.String(http.StatusNotFound, errors.New("resource group not found").Error())
 	}
@@ -178,7 +179,7 @@ func (s *Service) getResourceGroup(c *gin.Context) {
 //	@Router		/config/groups [get]
 func (s *Service) getResourceGroupList(c *gin.Context) {
 	withStats := strings.EqualFold(c.Query("with_stats"), "true")
-	groups := s.manager.GetResourceGroupList(withStats)
+	groups := s.manager.GetResourceGroupList(constant.NullKeyspaceID, withStats)
 	c.IndentedJSON(http.StatusOK, groups)
 }
 
@@ -191,7 +192,7 @@ func (s *Service) getResourceGroupList(c *gin.Context) {
 //	@Failure	404		{string}	error
 //	@Router		/config/group/{name} [delete]
 func (s *Service) deleteResourceGroup(c *gin.Context) {
-	if err := s.manager.DeleteResourceGroup(c.Param("name")); err != nil {
+	if err := s.manager.DeleteResourceGroup(constant.NullKeyspaceID, c.Param("name")); err != nil {
 		c.String(http.StatusNotFound, err.Error())
 	}
 	c.String(http.StatusOK, "Success!")
diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go
index 4cc162f7145..0eecf86fd89 100644
--- a/pkg/mcs/resourcemanager/server/grpc_service.go
+++ b/pkg/mcs/resourcemanager/server/grpc_service.go
@@ -31,6 +31,7 @@ import (
 	bs "github.com/tikv/pd/pkg/basicserver"
 	"github.com/tikv/pd/pkg/errs"
 	"github.com/tikv/pd/pkg/mcs/registry"
+	"github.com/tikv/pd/pkg/mcs/utils/constant"
 	"github.com/tikv/pd/pkg/utils/apiutil"
 )
 
@@ -93,7 +94,7 @@ func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroup
 	if err := s.checkServing(); err != nil {
 		return nil, err
 	}
-	rg := s.manager.GetResourceGroup(req.ResourceGroupName, req.WithRuStats)
+	rg := s.manager.GetResourceGroup(constant.NullKeyspaceID, req.ResourceGroupName, req.WithRuStats)
 	if rg == nil {
 		return nil, errors.New("resource group not found")
 	}
@@ -107,7 +108,7 @@ func (s *Service) ListResourceGroups(_ context.Context, req *rmpb.ListResourceGr
 	if err := s.checkServing(); err != nil {
 		return nil, err
 	}
-	groups := s.manager.GetResourceGroupList(req.WithRuStats)
+	groups := s.manager.GetResourceGroupList(constant.NullKeyspaceID, req.WithRuStats)
 	resp := &rmpb.ListResourceGroupsResponse{
 		Groups: make([]*rmpb.ResourceGroup, 0, len(groups)),
 	}
@@ -134,7 +135,7 @@ func (s *Service) DeleteResourceGroup(_ context.Context, req *rmpb.DeleteResourc
 	if err := s.checkServing(); err != nil {
 		return nil, err
 	}
-	err := s.manager.DeleteResourceGroup(req.ResourceGroupName)
+	err := s.manager.DeleteResourceGroup(constant.NullKeyspaceID, req.ResourceGroupName)
 	if err != nil {
 		return nil, err
 	}
@@ -180,24 +181,17 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu
 		for _, req := range request.Requests {
 			resourceGroupName := req.GetResourceGroupName()
 			// Get the resource group from manager to acquire token buckets.
-			rg := s.manager.GetMutableResourceGroup(resourceGroupName)
+			rg := s.manager.GetMutableResourceGroup(constant.NullKeyspaceID, resourceGroupName)
 			if rg == nil {
 				log.Warn("resource group not found", zap.String("resource-group", resourceGroupName))
 				continue
 			}
 			// Send the consumption to update the metrics.
-			isBackground := req.GetIsBackground()
-			isTiFlash := req.GetIsTiflash()
-			if isBackground && isTiFlash {
-				return errors.New("background and tiflash cannot be true at the same time")
+			err = s.manager.dispatchConsumption(req)
+			if err != nil {
+				return err
 			}
-			s.manager.consumptionDispatcher <- struct {
-				resourceGroupName string
-				*rmpb.Consumption
-				isBackground bool
-				isTiFlash    bool
-			}{resourceGroupName, req.GetConsumptionSinceLastRequest(), isBackground, isTiFlash}
-			if isBackground {
+			if req.GetIsBackground() {
 				continue
 			}
 			now := time.Now()
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
new file mode 100644
index 00000000000..0fedd2e01af
--- /dev/null
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -0,0 +1,239 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"encoding/json"
+	"math"
+	"sort"
+
+	"github.com/gogo/protobuf/proto"
+	"go.uber.org/zap"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+	"github.com/pingcap/log"
+
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/storage/endpoint"
+	"github.com/tikv/pd/pkg/utils/syncutil"
+)
+
+const (
+	defaultConsumptionChanSize = 1024
+
+	reservedDefaultGroupName = "default"
+	maxGroupNameLength       = 32
+	middlePriority           = 8
+	maxPriority              = 16
+	unlimitedRate            = math.MaxInt32
+	unlimitedBurstLimit      = -1
+)
+
+// consumptionItem is used to send the consumption info to the background metrics flusher.
+type consumptionItem struct {
+	keyspaceID        uint32
+	resourceGroupName string
+	*rmpb.Consumption
+	isBackground bool
+	isTiFlash    bool
+}
+
+type keyspaceResourceGroupManager struct {
+	syncutil.RWMutex
+	groups map[string]*ResourceGroup
+
+	keyspaceID uint32
+	storage    endpoint.ResourceGroupStorage
+}
+
+func newKeyspaceResourceGroupManager(keyspaceID uint32, storage endpoint.ResourceGroupStorage) *keyspaceResourceGroupManager {
+	return &keyspaceResourceGroupManager{
+		groups:     make(map[string]*ResourceGroup),
+		keyspaceID: keyspaceID,
+		storage:    storage,
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroupFromRaw(name string, rawValue string) error {
+	group := &rmpb.ResourceGroup{}
+	if err := proto.Unmarshal([]byte(rawValue), group); err != nil {
+		log.Error("failed to parse the keyspace resource group meta info",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	krgm.groups[group.Name] = FromProtoResourceGroup(group)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name string, rawValue string) error {
+	tokens := &GroupStates{}
+	if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
+		log.Error("failed to parse the keyspace resource group state",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	if group, ok := krgm.groups[name]; ok {
+		group.SetStatesIntoResourceGroup(tokens)
+	}
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
+	krgm.RLock()
+	if _, ok := krgm.groups[reservedDefaultGroupName]; ok {
+		krgm.RUnlock()
+		return
+	}
+	krgm.RUnlock()
+	defaultGroup := &ResourceGroup{
+		Name: reservedDefaultGroupName,
+		Mode: rmpb.GroupMode_RUMode,
+		RUSettings: &RequestUnitSettings{
+			RU: &GroupTokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   unlimitedRate,
+					BurstLimit: unlimitedBurstLimit,
+				},
+			},
+		},
+		Priority: middlePriority,
+	}
+	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil {
+		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+	if len(grouppb.Name) == 0 || len(grouppb.Name) > maxGroupNameLength {
+		return errs.ErrInvalidGroup
+	}
+	// Check the Priority.
+	if grouppb.GetPriority() > maxPriority {
+		return errs.ErrInvalidGroup
+	}
+	group := FromProtoResourceGroup(grouppb)
+	krgm.Lock()
+	defer krgm.Unlock()
+	if err := group.persistSettings(krgm.keyspaceID, krgm.storage); err != nil {
+		return err
+	}
+	if err := group.persistStates(krgm.keyspaceID, krgm.storage); err != nil {
+		return err
+	}
+	krgm.groups[group.Name] = group
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.ResourceGroup) error {
+	if group == nil || group.Name == "" {
+		return errs.ErrInvalidGroup
+	}
+	krgm.RLock()
+	curGroup, ok := krgm.groups[group.Name]
+	krgm.RUnlock()
+	if !ok {
+		return errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name)
+	}
+
+	err := curGroup.PatchSettings(group)
+	if err != nil {
+		return err
+	}
+	return curGroup.persistSettings(krgm.keyspaceID, krgm.storage)
+}
+
+func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
+	if name == reservedDefaultGroupName {
+		return errs.ErrDeleteReservedGroup
+	}
+	if err := krgm.storage.DeleteResourceGroupSetting(krgm.keyspaceID, name); err != nil {
+		return err
+	}
+	krgm.Lock()
+	delete(krgm.groups, name)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroup(name string, withStats bool) *ResourceGroup {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	if group, ok := krgm.groups[name]; ok {
+		return group.Clone(withStats)
+	}
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) getMutableResourceGroup(name string) *ResourceGroup {
+	krgm.Lock()
+	defer krgm.Unlock()
+	return krgm.groups[name]
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroupList(withStats, includeDefault bool) []*ResourceGroup {
+	krgm.RLock()
+	res := make([]*ResourceGroup, 0, len(krgm.groups))
+	for _, group := range krgm.groups {
+		if !includeDefault && group.Name == reservedDefaultGroupName {
+			continue
+		}
+		res = append(res, group.Clone(withStats))
+	}
+	krgm.RUnlock()
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Name < res[j].Name
+	})
+	return res
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroupNames(includeDefault bool) []string {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	res := make([]string, 0, len(krgm.groups))
+	for name := range krgm.groups {
+		if !includeDefault && name == reservedDefaultGroupName {
+			continue
+		}
+		res = append(res, name)
+	}
+	return res
+}
+
+func (krgm *keyspaceResourceGroupManager) persistResourceGroupRunningState() {
+	krgm.RLock()
+	keys := make([]string, 0, len(krgm.groups))
+	for k := range krgm.groups {
+		keys = append(keys, k)
+	}
+	krgm.RUnlock()
+	for idx := range keys {
+		krgm.RLock()
+		group, ok := krgm.groups[keys[idx]]
+		if ok {
+			if err := group.persistStates(krgm.keyspaceID, krgm.storage); err != nil {
+				log.Error("persist keyspace resource group state failed",
+					zap.Uint32("keyspace-id", krgm.keyspaceID),
+					zap.String("group-name", group.Name),
+					zap.Int("index", idx),
+					zap.Error(err))
+			}
+		}
+		krgm.RUnlock()
+	}
+}
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
new file mode 100644
index 00000000000..f3ddd755af2
--- /dev/null
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
@@ -0,0 +1,407 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/stretchr/testify/require"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+
+	"github.com/tikv/pd/pkg/storage"
+)
+
+func TestInitDefaultResourceGroup(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+	re.NotNil(krgm)
+	re.Equal(uint32(1), krgm.keyspaceID)
+	re.Empty(krgm.groups)
+
+	// No default resource group initially.
+	_, exists := krgm.groups[reservedDefaultGroupName]
+	re.False(exists)
+
+	// Initialize the default resource group.
+	krgm.initDefaultResourceGroup()
+
+	// Verify the default resource group is created.
+	defaultGroup, exists := krgm.groups[reservedDefaultGroupName]
+	re.True(exists)
+	re.Equal(reservedDefaultGroupName, defaultGroup.Name)
+	re.Equal(rmpb.GroupMode_RUMode, defaultGroup.Mode)
+	re.Equal(uint32(middlePriority), defaultGroup.Priority)
+
+	// Verify the default resource group has unlimited rate and burst limit.
+	re.Equal(uint64(unlimitedRate), defaultGroup.RUSettings.RU.Settings.FillRate)
+	re.Equal(int64(unlimitedBurstLimit), defaultGroup.RUSettings.RU.Settings.BurstLimit)
+}
+
+func TestAddResourceGroup(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Test adding invalid resource group (empty name).
+	group := &rmpb.ResourceGroup{
+		Name: "",
+		Mode: rmpb.GroupMode_RUMode,
+	}
+	err := krgm.addResourceGroup(group)
+	re.Error(err)
+	// Test adding invalid resource group (too long name).
+	group = &rmpb.ResourceGroup{
+		Name: "test_the_resource_group_name_is_too_long",
+		Mode: rmpb.GroupMode_RUMode,
+	}
+	err = krgm.addResourceGroup(group)
+	re.Error(err)
+
+	// Test adding a valid resource group.
+	group = &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+	err = krgm.addResourceGroup(group)
+	re.NoError(err)
+
+	// Verify the group was added.
+	addedGroup, exists := krgm.groups["test_group"]
+	re.True(exists)
+	re.Equal(group.GetName(), addedGroup.Name)
+	re.Equal(group.GetMode(), addedGroup.Mode)
+	re.Equal(group.GetPriority(), addedGroup.Priority)
+	re.Equal(group.GetRUSettings().GetRU().GetSettings().GetFillRate(), addedGroup.RUSettings.RU.Settings.FillRate)
+	re.Equal(group.GetRUSettings().GetRU().GetSettings().GetBurstLimit(), addedGroup.RUSettings.RU.Settings.BurstLimit)
+}
+
+func TestModifyResourceGroup(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Add a resource group first.
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+	err := krgm.addResourceGroup(group)
+	re.NoError(err)
+
+	// Modify the resource group.
+	modifiedGroup := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 10,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   200,
+					BurstLimit: 300,
+				},
+			},
+		},
+	}
+	err = krgm.modifyResourceGroup(modifiedGroup)
+	re.NoError(err)
+
+	// Verify the group was modified.
+	updatedGroup, exists := krgm.groups["test_group"]
+	re.True(exists)
+	re.Equal(modifiedGroup.GetName(), updatedGroup.Name)
+	re.Equal(modifiedGroup.GetPriority(), updatedGroup.Priority)
+	re.Equal(modifiedGroup.GetRUSettings().GetRU().GetSettings().GetFillRate(), updatedGroup.RUSettings.RU.Settings.FillRate)
+	re.Equal(modifiedGroup.GetRUSettings().GetRU().GetSettings().GetBurstLimit(), updatedGroup.RUSettings.RU.Settings.BurstLimit)
+
+	// Try to modify a non-existent group.
+	nonExistentGroup := &rmpb.ResourceGroup{
+		Name: "non_existent",
+		Mode: rmpb.GroupMode_RUMode,
+	}
+	err = krgm.modifyResourceGroup(nonExistentGroup)
+	re.Error(err)
+}
+
+func TestDeleteResourceGroup(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Add a resource group first.
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+	}
+	err := krgm.addResourceGroup(group)
+	re.NoError(err)
+
+	// Verify the group exists.
+	re.NotNil(krgm.getResourceGroup(group.GetName(), false))
+
+	// Delete the group.
+	err = krgm.deleteResourceGroup(group.GetName())
+	re.NoError(err)
+
+	// Verify the group was deleted.
+	re.Nil(krgm.getResourceGroup(group.GetName(), false))
+
+	// Try to delete the default group.
+	krgm.initDefaultResourceGroup()
+	err = krgm.deleteResourceGroup(reservedDefaultGroupName)
+	re.Error(err) // Should not be able to delete default group.
+
+	// Verify default group still exists.
+	re.NotNil(krgm.getResourceGroup(reservedDefaultGroupName, false))
+}
+
+func TestGetResourceGroup(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Add a resource group.
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+	err := krgm.addResourceGroup(group)
+	re.NoError(err)
+
+	// Get the resource group without stats.
+	retrievedGroup := krgm.getResourceGroup(group.GetName(), false)
+	re.NotNil(retrievedGroup)
+	re.Equal(group.GetName(), retrievedGroup.Name)
+	re.Equal(group.GetMode(), retrievedGroup.Mode)
+	re.Equal(group.GetPriority(), retrievedGroup.Priority)
+
+	// Get a non-existent group.
+	nonExistentGroup := krgm.getResourceGroup("non_existent", false)
+	re.Nil(nonExistentGroup)
+}
+
+func TestGetResourceGroupList(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Add some resource groups.
+	for i := 1; i <= 3; i++ {
+		name := "group" + string(rune('0'+i))
+		group := &rmpb.ResourceGroup{
+			Name:     name,
+			Mode:     rmpb.GroupMode_RUMode,
+			Priority: uint32(i),
+		}
+		err := krgm.addResourceGroup(group)
+		re.NoError(err)
+	}
+
+	// Get all resource groups.
+	groups := krgm.getResourceGroupList(false, true)
+	re.Len(groups, 3)
+
+	// Verify groups are sorted by name.
+	re.Equal("group1", groups[0].Name)
+	re.Equal("group2", groups[1].Name)
+	re.Equal("group3", groups[2].Name)
+
+	krgm.initDefaultResourceGroup()
+	groups = krgm.getResourceGroupList(false, true)
+	re.Len(groups, 4)
+	groups = krgm.getResourceGroupList(false, false)
+	re.Len(groups, 3)
+
+	names := krgm.getResourceGroupNames(true)
+	re.Len(names, 4)
+	names = krgm.getResourceGroupNames(false)
+	re.Len(names, 3)
+}
+
+func TestAddResourceGroupFromRaw(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Create a resource group.
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+
+	// Marshal to bytes.
+	data, err := proto.Marshal(group)
+	re.NoError(err)
+
+	// Add from raw.
+	err = krgm.addResourceGroupFromRaw(group.GetName(), string(data))
+	re.NoError(err)
+
+	// Verify the group was added correctly.
+	addedGroup, exists := krgm.groups[group.GetName()]
+	re.True(exists)
+	re.Equal(group.GetName(), addedGroup.Name)
+	re.Equal(group.GetMode(), addedGroup.Mode)
+	re.Equal(group.GetPriority(), addedGroup.Priority)
+
+	// Test with invalid raw value.
+	err = krgm.addResourceGroupFromRaw(group.GetName(), "invalid_data")
+	re.Error(err)
+}
+
+func TestSetRawStatesIntoResourceGroup(t *testing.T) {
+	re := require.New(t)
+
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+
+	// Add a resource group first.
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+	err := krgm.addResourceGroup(group)
+	re.NoError(err)
+
+	// Create group states.
+	tokens := 150.0
+	lastUpdate := time.Now()
+	states := &GroupStates{
+		RU: &GroupTokenBucketState{
+			Tokens:     tokens,
+			LastUpdate: &lastUpdate,
+		},
+		RUConsumption: &rmpb.Consumption{
+			RRU: 50,
+			WRU: 30,
+		},
+	}
+
+	// Marshal to JSON.
+	data, err := json.Marshal(states)
+	re.NoError(err)
+
+	// Set raw states.
+	err = krgm.setRawStatesIntoResourceGroup(group.GetName(), string(data))
+	re.NoError(err)
+
+	// Verify states were updated.
+	updatedGroup := krgm.groups[group.GetName()]
+	re.InDelta(tokens, updatedGroup.RUSettings.RU.Tokens, 0.001)
+	re.Equal(states.RUConsumption.RRU, updatedGroup.RUConsumption.RRU)
+	re.Equal(states.RUConsumption.WRU, updatedGroup.RUConsumption.WRU)
+
+	// Test with invalid raw value.
+	err = krgm.setRawStatesIntoResourceGroup(group.GetName(), "invalid_data")
+	re.Error(err)
+}
+
+func TestPersistResourceGroupRunningState(t *testing.T) {
+	re := require.New(t)
+
+	storage := storage.NewStorageWithMemoryBackend()
+	krgm := newKeyspaceResourceGroupManager(1, storage)
+
+	// Add a resource group
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+	err := krgm.addResourceGroup(group)
+	re.NoError(err)
+
+	// Check the states before persist.
+	storage.LoadResourceGroupStates(func(keyspaceID uint32, name, rawValue string) {
+		re.Equal(uint32(1), keyspaceID)
+		re.Equal(group.GetName(), name)
+		states := &GroupStates{}
+		err := json.Unmarshal([]byte(rawValue), states)
+		re.NoError(err)
+		re.Equal(0.0, states.RU.Tokens)
+	})
+
+	mutableGroup := krgm.getMutableResourceGroup(group.GetName())
+	mutableGroup.RUSettings.RU.Tokens = 100.0
+	// Persist the running state.
+	krgm.persistResourceGroupRunningState()
+
+	// Verify state was persisted.
+	storage.LoadResourceGroupStates(func(keyspaceID uint32, name, rawValue string) {
+		re.Equal(uint32(1), keyspaceID)
+		re.Equal(group.GetName(), name)
+		states := &GroupStates{}
+		err := json.Unmarshal([]byte(rawValue), states)
+		re.NoError(err)
+		re.Equal(mutableGroup.RUSettings.RU.Tokens, states.RU.Tokens)
+	})
+}
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index e6d836f921f..d51d0643713 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -18,11 +18,9 @@ import (
 	"context"
 	"encoding/json"
 	"math"
-	"sort"
 	"strings"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 
@@ -42,17 +40,12 @@ import (
 )
 
 const (
-	defaultConsumptionChanSize = 1024
+	persistLoopInterval        = 1 * time.Minute
 	metricsCleanupInterval     = time.Minute
 	metricsCleanupTimeout      = 20 * time.Minute
 	metricsAvailableRUInterval = 1 * time.Second
 	defaultCollectIntervalSec  = 20
 	tickPerSecond              = time.Second
-
-	reservedDefaultGroupName = "default"
-	middlePriority           = 8
-	unlimitedRate            = math.MaxInt32
-	unlimitedBurstLimit      = -1
 )
 
 // Manager is the manager of resource group.
@@ -60,23 +53,19 @@ type Manager struct {
 	syncutil.RWMutex
 	srv              bs.Server
 	controllerConfig *ControllerConfig
-	groups           map[string]*ResourceGroup
+	krgms            map[uint32]*keyspaceResourceGroupManager
 	storage          endpoint.ResourceGroupStorage
 	// consumptionChan is used to send the consumption
 	// info to the background metrics flusher.
-	consumptionDispatcher chan struct {
-		resourceGroupName string
-		*rmpb.Consumption
-		isBackground bool
-		isTiFlash    bool
-	}
+	consumptionDispatcher chan *consumptionItem
 	// record update time of each resource group
 	consumptionRecord map[consumptionRecordKey]time.Time
 }
 
 type consumptionRecordKey struct {
-	name   string
-	ruType string
+	keyspaceID uint32
+	name       string
+	ruType     string
 }
 
 // ConfigProvider is used to get resource manager config from the given
@@ -89,15 +78,10 @@ type ConfigProvider interface {
 // which should implement the `ConfigProvider` interface.
 func NewManager[T ConfigProvider](srv bs.Server) *Manager {
 	m := &Manager{
-		controllerConfig: srv.(T).GetControllerConfig(),
-		groups:           make(map[string]*ResourceGroup),
-		consumptionDispatcher: make(chan struct {
-			resourceGroupName string
-			*rmpb.Consumption
-			isBackground bool
-			isTiFlash    bool
-		}, defaultConsumptionChanSize),
-		consumptionRecord: make(map[consumptionRecordKey]time.Time),
+		controllerConfig:      srv.(T).GetControllerConfig(),
+		krgms:                 make(map[uint32]*keyspaceResourceGroupManager),
+		consumptionDispatcher: make(chan *consumptionItem, defaultConsumptionChanSize),
+		consumptionRecord:     make(map[consumptionRecordKey]time.Time),
 	}
 	// The first initialization after the server is started.
 	srv.AddStartCallback(func() {
@@ -118,6 +102,28 @@ func (m *Manager) GetBasicServer() bs.Server {
 	return m.srv
 }
 
+// GetStorage returns the storage.
+func (m *Manager) GetStorage() endpoint.ResourceGroupStorage {
+	return m.storage
+}
+
+func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32) *keyspaceResourceGroupManager {
+	m.Lock()
+	defer m.Unlock()
+	krgm, ok := m.krgms[keyspaceID]
+	if !ok {
+		krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage)
+		m.krgms[keyspaceID] = krgm
+	}
+	return krgm
+}
+
+func (m *Manager) getKeyspaceResourceGroupManager(keyspaceID uint32) *keyspaceResourceGroupManager {
+	m.RLock()
+	defer m.RUnlock()
+	return m.krgms[keyspaceID]
+}
+
 // Init initializes the resource group manager.
 func (m *Manager) Init(ctx context.Context) error {
 	v, err := m.storage.LoadControllerConfig()
@@ -133,56 +139,10 @@ func (m *Manager) Init(ctx context.Context) error {
 	if err := m.storage.SaveControllerConfig(m.controllerConfig); err != nil {
 		return err
 	}
-	// Load resource group meta info from storage.
-	m.Lock()
-	m.groups = make(map[string]*ResourceGroup)
-	m.Unlock()
-	handler := func(keyspaceID uint32, name string, rawValue string) {
-		group := &rmpb.ResourceGroup{}
-		if err := proto.Unmarshal([]byte(rawValue), group); err != nil {
-			log.Error("failed to parse the resource group",
-				zap.Uint32("keyspace-id", keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
-			panic(err)
-		}
-		m.groups[group.Name] = FromProtoResourceGroup(group)
-	}
-	if err := m.storage.LoadResourceGroupSettings(handler); err != nil {
-		return err
-	}
-	// Load resource group states from storage.
-	tokenHandler := func(keyspaceID uint32, name string, rawValue string) {
-		tokens := &GroupStates{}
-		if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
-			log.Error("failed to parse the resource group state",
-				zap.Uint32("keyspace-id", keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
-			panic(err)
-		}
-		if group, ok := m.groups[name]; ok {
-			group.SetStatesIntoResourceGroup(tokens)
-		}
-	}
-	if err := m.storage.LoadResourceGroupStates(tokenHandler); err != nil {
-		return err
-	}
 
-	// Add default group if it's not inited.
-	if _, ok := m.groups[reservedDefaultGroupName]; !ok {
-		defaultGroup := &ResourceGroup{
-			Name: reservedDefaultGroupName,
-			Mode: rmpb.GroupMode_RUMode,
-			RUSettings: &RequestUnitSettings{
-				RU: &GroupTokenBucket{
-					Settings: &rmpb.TokenLimitSettings{
-						FillRate:   unlimitedRate,
-						BurstLimit: unlimitedBurstLimit,
-					},
-				},
-			},
-			Priority: middlePriority,
-		}
-		if err := m.AddResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil {
-			log.Warn("init default group failed", zap.Error(err))
-		}
+	// Load keyspace resource groups from the storage.
+	if err := m.loadKeyspaceResourceGroups(); err != nil {
+		return err
 	}
 
 	// Start the background metrics flusher.
@@ -195,6 +155,53 @@ func (m *Manager) Init(ctx context.Context) error {
 	return nil
 }
 
+func (m *Manager) loadKeyspaceResourceGroups() error {
+	// Empty the keyspace resource group manager map before the loading.
+	m.Lock()
+	m.krgms = make(map[uint32]*keyspaceResourceGroupManager)
+	m.Unlock()
+	// Load keyspace resource group meta info from the storage.
+	if err := m.storage.LoadResourceGroupSettings(func(keyspaceID uint32, name string, rawValue string) {
+		err := m.getOrCreateKeyspaceResourceGroupManager(keyspaceID).addResourceGroupFromRaw(name, rawValue)
+		if err != nil {
+			log.Error("failed to add resource group to the keyspace resource group manager",
+				zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name), zap.Error(err))
+		}
+	}); err != nil {
+		return err
+	}
+	// Load keyspace resource group states from the storage.
+	if err := m.storage.LoadResourceGroupStates(func(keyspaceID uint32, name string, rawValue string) {
+		krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+		if krgm == nil {
+			log.Warn("failed to get the corresponding keyspace resource group manager",
+				zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name))
+			return
+		}
+		err := krgm.setRawStatesIntoResourceGroup(name, rawValue)
+		if err != nil {
+			log.Error("failed to set resource group state",
+				zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name), zap.Error(err))
+		}
+	}); err != nil {
+		return err
+	}
+	// Initialize the reserved keyspace resource group manager and default resource groups.
+	m.initReserved()
+	return nil
+}
+
+func (m *Manager) initReserved() {
+	// Initialize the null keyspace resource group manager if it doesn't exist.
+	m.getOrCreateKeyspaceResourceGroupManager(constant.NullKeyspaceID)
+	// Initialize the default resource group respectively for each keyspace.
+	m.RLock()
+	defer m.RUnlock()
+	for _, krgm := range m.krgms {
+		krgm.initDefaultResourceGroup()
+	}
+}
+
 // UpdateControllerConfigItem updates the controller config item.
 func (m *Manager) UpdateControllerConfigItem(key string, value any) error {
 	kp := strings.Split(key, ".")
@@ -240,96 +247,63 @@ func (m *Manager) GetControllerConfig() *ControllerConfig {
 // NOTE: AddResourceGroup should also be idempotent because tidb depends
 // on this retry mechanism.
 func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
-	// Check the name.
-	if len(grouppb.Name) == 0 || len(grouppb.Name) > 32 {
-		return errs.ErrInvalidGroup
-	}
-	// Check the Priority.
-	if grouppb.GetPriority() > 16 {
-		return errs.ErrInvalidGroup
-	}
-	group := FromProtoResourceGroup(grouppb)
-	m.Lock()
-	defer m.Unlock()
-	if err := group.persistSettings(m.storage); err != nil {
-		return err
+	keyspaceID := constant.NullKeyspaceID
+	// TODO: validate the keyspace with the given ID first.
+	krgm := m.getOrCreateKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	if err := group.persistStates(m.storage); err != nil {
-		return err
-	}
-	m.groups[group.Name] = group
-	return nil
+	return krgm.addResourceGroup(grouppb)
 }
 
 // ModifyResourceGroup modifies an existing resource group.
 func (m *Manager) ModifyResourceGroup(group *rmpb.ResourceGroup) error {
-	if group == nil || group.Name == "" {
-		return errs.ErrInvalidGroup
-	}
-	m.Lock()
-	curGroup, ok := m.groups[group.Name]
-	m.Unlock()
-	if !ok {
-		return errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name)
-	}
-
-	err := curGroup.PatchSettings(group)
-	if err != nil {
-		return err
+	keyspaceID := constant.NullKeyspaceID
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	return curGroup.persistSettings(m.storage)
+	return krgm.modifyResourceGroup(group)
 }
 
 // DeleteResourceGroup deletes a resource group.
-func (m *Manager) DeleteResourceGroup(name string) error {
-	if name == reservedDefaultGroupName {
-		return errs.ErrDeleteReservedGroup
-	}
-	if err := m.storage.DeleteResourceGroupSetting(constant.NullKeyspaceID, name); err != nil {
-		return err
+func (m *Manager) DeleteResourceGroup(keyspaceID uint32, name string) error {
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	m.Lock()
-	delete(m.groups, name)
-	m.Unlock()
-	return nil
+	return krgm.deleteResourceGroup(name)
 }
 
 // GetResourceGroup returns a copy of a resource group.
-func (m *Manager) GetResourceGroup(name string, withStats bool) *ResourceGroup {
-	m.RLock()
-	defer m.RUnlock()
-	if group, ok := m.groups[name]; ok {
-		return group.Clone(withStats)
+func (m *Manager) GetResourceGroup(keyspaceID uint32, name string, withStats bool) *ResourceGroup {
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		return nil
 	}
-	return nil
+	return krgm.getResourceGroup(name, withStats)
 }
 
 // GetMutableResourceGroup returns a mutable resource group.
-func (m *Manager) GetMutableResourceGroup(name string) *ResourceGroup {
-	m.RLock()
-	defer m.RUnlock()
-	if group, ok := m.groups[name]; ok {
-		return group
+func (m *Manager) GetMutableResourceGroup(keyspaceID uint32, name string) *ResourceGroup {
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		return nil
 	}
-	return nil
+	return krgm.getMutableResourceGroup(name)
 }
 
 // GetResourceGroupList returns copies of resource group list.
-func (m *Manager) GetResourceGroupList(withStats bool) []*ResourceGroup {
-	m.RLock()
-	res := make([]*ResourceGroup, 0, len(m.groups))
-	for _, group := range m.groups {
-		res = append(res, group.Clone(withStats))
+func (m *Manager) GetResourceGroupList(keyspaceID uint32, withStats bool) []*ResourceGroup {
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		return nil
 	}
-	m.RUnlock()
-	sort.Slice(res, func(i, j int) bool {
-		return res[i].Name < res[j].Name
-	})
-	return res
+	return krgm.getResourceGroupList(withStats, true)
 }
 
 func (m *Manager) persistLoop(ctx context.Context) {
-	ticker := time.NewTicker(time.Minute)
+	ticker := time.NewTicker(persistLoopInterval)
 	failpoint.Inject("fastPersist", func() {
 		ticker.Reset(100 * time.Millisecond)
 	})
@@ -339,31 +313,39 @@ func (m *Manager) persistLoop(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			m.persistResourceGroupRunningState()
+			for _, krgm := range m.getKeyspaceResourceGroupManagers() {
+				krgm.persistResourceGroupRunningState()
+			}
 		}
 	}
 }
 
-func (m *Manager) persistResourceGroupRunningState() {
+func (m *Manager) getKeyspaceResourceGroupManagers() []*keyspaceResourceGroupManager {
 	m.RLock()
-	keys := make([]string, 0, len(m.groups))
-	for k := range m.groups {
-		keys = append(keys, k)
-	}
-	m.RUnlock()
-	for idx := range keys {
-		m.RLock()
-		group, ok := m.groups[keys[idx]]
-		if ok {
-			if err := group.persistStates(m.storage); err != nil {
-				log.Error("persist resource group state failed", zap.Error(err))
-			}
-		}
-		m.RUnlock()
+	defer m.RUnlock()
+	krgms := make([]*keyspaceResourceGroupManager, 0, len(m.krgms))
+	for _, krgm := range m.krgms {
+		krgms = append(krgms, krgm)
+	}
+	return krgms
+}
+
+func (m *Manager) dispatchConsumption(req *rmpb.TokenBucketRequest) error {
+	isBackground := req.GetIsBackground()
+	isTiFlash := req.GetIsTiflash()
+	if isBackground && isTiFlash {
+		return errors.New("background and tiflash cannot be true at the same time")
+	}
+	m.consumptionDispatcher <- &consumptionItem{
+		keyspaceID:        constant.NullKeyspaceID,
+		resourceGroupName: req.GetResourceGroupName(),
+		Consumption:       req.GetConsumptionSinceLastRequest(),
+		isBackground:      isBackground,
+		isTiFlash:         isTiFlash,
 	}
+	return nil
 }
 
-// Receive the consumption and flush it to the metrics.
 func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 	defer logutil.LogPanic()
 	cleanUpTicker := time.NewTicker(metricsCleanupInterval)
@@ -372,7 +354,32 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 	defer availableRUTicker.Stop()
 	recordMaxTicker := time.NewTicker(tickPerSecond)
 	defer recordMaxTicker.Stop()
-	maxPerSecTrackers := make(map[string]*maxPerSecCostTracker)
+
+	maxPerSecTrackers := make(map[uint32]map[string]*maxPerSecCostTracker)
+	// getMaxPerSecTracker returns the max per sec tracker for the given keyspace ID and name.
+	// If the tracker doesn't exist, it will be created.
+	getMaxPerSecTracker := func(keyspaceID uint32, name string) *maxPerSecCostTracker {
+		trackers := maxPerSecTrackers[keyspaceID]
+		if trackers == nil {
+			trackers = make(map[string]*maxPerSecCostTracker)
+			maxPerSecTrackers[keyspaceID] = trackers
+		}
+		tracker, ok := trackers[name]
+		if !ok {
+			tracker = newMaxPerSecCostTracker(name, defaultCollectIntervalSec)
+			trackers[name] = tracker
+		}
+		return tracker
+	}
+	// deleteMaxPerSecTracker deletes the max per sec tracker for the given keyspace ID and name.
+	// If the tracker doesn't exist, it will do nothing.
+	deleteMaxPerSecTracker := func(keyspaceID uint32, name string) {
+		trackers := maxPerSecTrackers[keyspaceID]
+		if trackers == nil {
+			return
+		}
+		delete(trackers, name)
+	}
 	for {
 		select {
 		case <-ctx.Done():
@@ -391,6 +398,8 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 			}
 
 			var (
+				// TODO: add keyspace name lable to the metrics.
+				keyspaceID               = consumptionInfo.keyspaceID
 				name                     = consumptionInfo.resourceGroupName
 				rruMetrics               = readRequestUnitCost.WithLabelValues(name, name, ruLabelType)
 				wruMetrics               = writeRequestUnitCost.WithLabelValues(name, name, ruLabelType)
@@ -402,12 +411,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 				readRequestCountMetrics  = requestCount.WithLabelValues(name, name, readTypeLabel)
 				writeRequestCountMetrics = requestCount.WithLabelValues(name, name, writeTypeLabel)
 			)
-			t, ok := maxPerSecTrackers[name]
-			if !ok {
-				t = newMaxPerSecCostTracker(name, defaultCollectIntervalSec)
-				maxPerSecTrackers[name] = t
-			}
-			t.CollectConsumption(consumption)
+			getMaxPerSecTracker(keyspaceID, name).CollectConsumption(consumption)
 
 			// RU info.
 			if consumption.RRU > 0 {
@@ -439,64 +443,51 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 				writeRequestCountMetrics.Add(consumption.KvWriteRpcCount)
 			}
 
-			m.consumptionRecord[consumptionRecordKey{name: name, ruType: ruLabelType}] = time.Now()
+			m.consumptionRecord[consumptionRecordKey{keyspaceID: keyspaceID, name: name, ruType: ruLabelType}] = time.Now()
 
 			// TODO: maybe we need to distinguish background ru.
-			if rg := m.GetMutableResourceGroup(name); rg != nil {
+			if rg := m.GetMutableResourceGroup(keyspaceID, name); rg != nil {
 				rg.UpdateRUConsumption(consumptionInfo.Consumption)
 			}
 		case <-cleanUpTicker.C:
 			// Clean up the metrics that have not been updated for a long time.
 			for r, lastTime := range m.consumptionRecord {
-				if time.Since(lastTime) > metricsCleanupTimeout {
-					readRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					writeRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					sqlLayerRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					readByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					writeByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					kvCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					sqlCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					requestCount.DeleteLabelValues(r.name, r.name, readTypeLabel)
-					requestCount.DeleteLabelValues(r.name, r.name, writeTypeLabel)
-					availableRUCounter.DeleteLabelValues(r.name, r.name)
-					delete(m.consumptionRecord, r)
-					delete(maxPerSecTrackers, r.name)
-					readRequestUnitMaxPerSecCost.DeleteLabelValues(r.name)
-					writeRequestUnitMaxPerSecCost.DeleteLabelValues(r.name)
-					resourceGroupConfigGauge.DeletePartialMatch(prometheus.Labels{newResourceGroupNameLabel: r.name})
+				if time.Since(lastTime) <= metricsCleanupTimeout {
+					continue
 				}
+				readRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				writeRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				sqlLayerRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				readByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				writeByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				kvCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				sqlCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
+				requestCount.DeleteLabelValues(r.name, r.name, readTypeLabel)
+				requestCount.DeleteLabelValues(r.name, r.name, writeTypeLabel)
+				availableRUCounter.DeleteLabelValues(r.name, r.name)
+				delete(m.consumptionRecord, r)
+				deleteMaxPerSecTracker(r.keyspaceID, r.name)
+				readRequestUnitMaxPerSecCost.DeleteLabelValues(r.name)
+				writeRequestUnitMaxPerSecCost.DeleteLabelValues(r.name)
+				resourceGroupConfigGauge.DeletePartialMatch(prometheus.Labels{newResourceGroupNameLabel: r.name})
 			}
 		case <-availableRUTicker.C:
-			m.RLock()
-			groups := make([]*ResourceGroup, 0, len(m.groups))
-			for name, group := range m.groups {
-				if name == reservedDefaultGroupName {
-					continue
+			// Prevent from holding the lock too long when there're many keyspaces and resource groups.
+			for _, krgm := range m.getKeyspaceResourceGroupManagers() {
+				for _, group := range krgm.getResourceGroupList(true, false) {
+					ru := math.Max(group.getRUToken(), 0)
+					availableRUCounter.WithLabelValues(group.Name, group.Name).Set(ru)
+					resourceGroupConfigGauge.WithLabelValues(group.Name, priorityLabel).Set(group.getPriority())
+					resourceGroupConfigGauge.WithLabelValues(group.Name, ruPerSecLabel).Set(group.getFillRate())
+					resourceGroupConfigGauge.WithLabelValues(group.Name, ruCapacityLabel).Set(group.getBurstLimit())
 				}
-				groups = append(groups, group)
-			}
-			m.RUnlock()
-			// prevent many groups and hold the lock long time.
-			for _, group := range groups {
-				ru := math.Max(group.getRUToken(), 0)
-				availableRUCounter.WithLabelValues(group.Name, group.Name).Set(ru)
-				resourceGroupConfigGauge.WithLabelValues(group.Name, priorityLabel).Set(group.getPriority())
-				resourceGroupConfigGauge.WithLabelValues(group.Name, ruPerSecLabel).Set(group.getFillRate())
-				resourceGroupConfigGauge.WithLabelValues(group.Name, ruCapacityLabel).Set(group.getBurstLimit())
 			}
 		case <-recordMaxTicker.C:
 			// Record the sum of RRU and WRU every second.
-			m.RLock()
-			names := make([]string, 0, len(m.groups))
-			for name := range m.groups {
-				names = append(names, name)
-			}
-			m.RUnlock()
-			for _, name := range names {
-				if t, ok := maxPerSecTrackers[name]; !ok {
-					maxPerSecTrackers[name] = newMaxPerSecCostTracker(name, defaultCollectIntervalSec)
-				} else {
-					t.FlushMetrics()
+			for _, krgm := range m.getKeyspaceResourceGroupManagers() {
+				names := krgm.getResourceGroupNames(true)
+				for _, name := range names {
+					getMaxPerSecTracker(krgm.keyspaceID, name).FlushMetrics()
 				}
 			}
 		}
diff --git a/pkg/mcs/resourcemanager/server/manager_test.go b/pkg/mcs/resourcemanager/server/manager_test.go
new file mode 100644
index 00000000000..bc1d3120408
--- /dev/null
+++ b/pkg/mcs/resourcemanager/server/manager_test.go
@@ -0,0 +1,85 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+
+	bs "github.com/tikv/pd/pkg/basicserver"
+	"github.com/tikv/pd/pkg/mcs/utils/constant"
+	"github.com/tikv/pd/pkg/storage"
+	"github.com/tikv/pd/pkg/utils/testutil"
+)
+
+type mockConfigProvider struct{ bs.Server }
+
+func (*mockConfigProvider) GetControllerConfig() *ControllerConfig { return &ControllerConfig{} }
+
+func (*mockConfigProvider) AddStartCallback(...func()) {}
+
+func (*mockConfigProvider) AddServiceReadyCallback(...func(context.Context) error) {}
+
+func TestBackgroundMetricsFlush(t *testing.T) {
+	re := require.New(t)
+
+	storage := storage.NewStorageWithMemoryBackend()
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = storage
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	err := m.Init(ctx)
+	re.NoError(err)
+
+	// Add a resource group.
+	group := &rmpb.ResourceGroup{
+		Name:     "test_group",
+		Mode:     rmpb.GroupMode_RUMode,
+		Priority: 5,
+		RUSettings: &rmpb.GroupRequestUnitSettings{
+			RU: &rmpb.TokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   100,
+					BurstLimit: 200,
+				},
+			},
+		},
+	}
+	err = m.AddResourceGroup(group)
+	re.NoError(err)
+
+	// Send consumption to the dispatcher.
+	req := &rmpb.TokenBucketRequest{
+		ResourceGroupName: group.GetName(),
+		ConsumptionSinceLastRequest: &rmpb.Consumption{
+			RRU: 10.0,
+			WRU: 20.0,
+		},
+	}
+	m.dispatchConsumption(req)
+
+	// Verify consumption was added to the resource group.
+	testutil.Eventually(re, func() bool {
+		updatedGroup := m.GetResourceGroup(constant.NullKeyspaceID, req.GetResourceGroupName(), true)
+		re.NotNil(updatedGroup)
+		return updatedGroup.RUConsumption.RRU == req.ConsumptionSinceLastRequest.RRU &&
+			updatedGroup.RUConsumption.WRU == req.ConsumptionSinceLastRequest.WRU
+	})
+}
diff --git a/pkg/mcs/resourcemanager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go
index 5049ff14452..701061e6598 100644
--- a/pkg/mcs/resourcemanager/server/resource_group.go
+++ b/pkg/mcs/resourcemanager/server/resource_group.go
@@ -26,7 +26,6 @@ import (
 	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
 	"github.com/pingcap/log"
 
-	"github.com/tikv/pd/pkg/mcs/utils/constant"
 	"github.com/tikv/pd/pkg/storage/endpoint"
 	"github.com/tikv/pd/pkg/utils/syncutil"
 )
@@ -232,9 +231,9 @@ func (rg *ResourceGroup) IntoProtoResourceGroup() *rmpb.ResourceGroup {
 
 // persistSettings persists the resource group settings.
 // TODO: persist the state of the group separately.
-func (rg *ResourceGroup) persistSettings(storage endpoint.ResourceGroupStorage) error {
+func (rg *ResourceGroup) persistSettings(keyspaceID uint32, storage endpoint.ResourceGroupStorage) error {
 	metaGroup := rg.IntoProtoResourceGroup()
-	return storage.SaveResourceGroupSetting(constant.NullKeyspaceID, rg.Name, metaGroup)
+	return storage.SaveResourceGroupSetting(keyspaceID, rg.Name, metaGroup)
 }
 
 // GroupStates is the tokens set of a resource group.
@@ -300,7 +299,7 @@ func (rg *ResourceGroup) UpdateRUConsumption(c *rmpb.Consumption) {
 }
 
 // persistStates persists the resource group tokens.
-func (rg *ResourceGroup) persistStates(storage endpoint.ResourceGroupStorage) error {
+func (rg *ResourceGroup) persistStates(keyspaceID uint32, storage endpoint.ResourceGroupStorage) error {
 	states := rg.GetGroupStates()
-	return storage.SaveResourceGroupStates(constant.NullKeyspaceID, rg.Name, states)
+	return storage.SaveResourceGroupStates(keyspaceID, rg.Name, states)
 }


Reviews:
user: login: JmPotato
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-19T08:28:38Z
user: login: lhy1024
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T02:28:22Z
user: login: lhy1024
body: the rest LGTM
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T03:31:14Z
user: login: lhy1024
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T06:05:30Z
user: login: glorv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T06:50:59Z
user: login: rleungx
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-20T07:37:32Z
user: login: rleungx
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-21T02:28:17Z

Review comments:
diff_hunk: @@ -0,0 +1,461 @@
+// Copyright 2022 TiKV Project Authors.
path: pkg/mcs/resourcemanager/server/keyspace_manager.go
user: login: JmPotato
body: ```suggestion
// Copyright 2025 TiKV Project Authors.
```
created_at: 2025-05-19T08:28:37Z
author_association: MEMBER
diff_hunk: @@ -118,6 +83,28 @@ func (m *Manager) GetBasicServer() bs.Server {
 	return m.srv
 }
 
+// GetStorage returns the storage.
+func (m *Manager) GetStorage() endpoint.ResourceGroupStorage {
+	return m.storage
+}
+
+func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32) *keyspaceResourceGroupManager {
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
+		krgm = newKeyspaceResourceGroupManager(keyspaceID, m)
+		m.Lock()
+		m.krgms[keyspaceID] = krgm
+		m.Unlock()
+	}
+	return krgm
+}
+
path: pkg/mcs/resourcemanager/server/manager.go
user: login: lhy1024
body: To avoid a race where two goroutines might both see `krgm == nil` and create it, should we move the m.Lock() call to before the initial if krgm == nil check?
created_at: 2025-05-20T02:14:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -346,215 +310,20 @@ func (m *Manager) persistLoop(ctx context.Context) {
 
 func (m *Manager) persistResourceGroupRunningState() {
 	m.RLock()
-	keys := make([]string, 0, len(m.groups))
-	for k := range m.groups {
-		keys = append(keys, k)
+	krgms := make([]*keyspaceResourceGroupManager, 0, len(m.krgms))
+	for _, krgm := range m.krgms {
+		krgms = append(krgms, krgm)
 	}
 	m.RUnlock()
-	for idx := range keys {
-		m.RLock()
-		group, ok := m.groups[keys[idx]]
-		if ok {
-			if err := group.persistStates(m.storage); err != nil {
-				log.Error("persist resource group state failed", zap.Error(err))
-			}
-		}
-		m.RUnlock()
+	for _, krgm := range krgms {
+		krgm.persistResourceGroupRunningState()
 	}
 }
 
-// Receive the consumption and flush it to the metrics.
-func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
-	defer logutil.LogPanic()
-	cleanUpTicker := time.NewTicker(metricsCleanupInterval)
-	defer cleanUpTicker.Stop()
-	availableRUTicker := time.NewTicker(metricsAvailableRUInterval)
-	defer availableRUTicker.Stop()
-	recordMaxTicker := time.NewTicker(tickPerSecond)
-	defer recordMaxTicker.Stop()
-	maxPerSecTrackers := make(map[string]*maxPerSecCostTracker)
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case consumptionInfo := <-m.consumptionDispatcher:
-			consumption := consumptionInfo.Consumption
-			if consumption == nil {
-				continue
-			}
-			ruLabelType := defaultTypeLabel
-			if consumptionInfo.isBackground {
-				ruLabelType = backgroundTypeLabel
-			}
-			if consumptionInfo.isTiFlash {
-				ruLabelType = tiflashTypeLabel
-			}
-
-			var (
-				name                     = consumptionInfo.resourceGroupName
-				rruMetrics               = readRequestUnitCost.WithLabelValues(name, name, ruLabelType)
-				wruMetrics               = writeRequestUnitCost.WithLabelValues(name, name, ruLabelType)
-				sqlLayerRuMetrics        = sqlLayerRequestUnitCost.WithLabelValues(name, name)
-				readByteMetrics          = readByteCost.WithLabelValues(name, name, ruLabelType)
-				writeByteMetrics         = writeByteCost.WithLabelValues(name, name, ruLabelType)
-				kvCPUMetrics             = kvCPUCost.WithLabelValues(name, name, ruLabelType)
-				sqlCPUMetrics            = sqlCPUCost.WithLabelValues(name, name, ruLabelType)
-				readRequestCountMetrics  = requestCount.WithLabelValues(name, name, readTypeLabel)
-				writeRequestCountMetrics = requestCount.WithLabelValues(name, name, writeTypeLabel)
-			)
-			t, ok := maxPerSecTrackers[name]
-			if !ok {
-				t = newMaxPerSecCostTracker(name, defaultCollectIntervalSec)
-				maxPerSecTrackers[name] = t
-			}
-			t.CollectConsumption(consumption)
-
-			// RU info.
-			if consumption.RRU > 0 {
-				rruMetrics.Add(consumption.RRU)
-			}
-			if consumption.WRU > 0 {
-				wruMetrics.Add(consumption.WRU)
-			}
-			// Byte info.
-			if consumption.ReadBytes > 0 {
-				readByteMetrics.Add(consumption.ReadBytes)
-			}
-			if consumption.WriteBytes > 0 {
-				writeByteMetrics.Add(consumption.WriteBytes)
-			}
-			// CPU time info.
-			if consumption.TotalCpuTimeMs > 0 {
-				if consumption.SqlLayerCpuTimeMs > 0 {
-					sqlLayerRuMetrics.Add(consumption.SqlLayerCpuTimeMs * m.controllerConfig.RequestUnit.CPUMsCost)
-					sqlCPUMetrics.Add(consumption.SqlLayerCpuTimeMs)
-				}
-				kvCPUMetrics.Add(consumption.TotalCpuTimeMs - consumption.SqlLayerCpuTimeMs)
-			}
-			// RPC count info.
-			if consumption.KvReadRpcCount > 0 {
-				readRequestCountMetrics.Add(consumption.KvReadRpcCount)
-			}
-			if consumption.KvWriteRpcCount > 0 {
-				writeRequestCountMetrics.Add(consumption.KvWriteRpcCount)
-			}
-
-			m.consumptionRecord[consumptionRecordKey{name: name, ruType: ruLabelType}] = time.Now()
-
-			// TODO: maybe we need to distinguish background ru.
-			if rg := m.GetMutableResourceGroup(name); rg != nil {
-				rg.UpdateRUConsumption(consumptionInfo.Consumption)
-			}
-		case <-cleanUpTicker.C:
-			// Clean up the metrics that have not been updated for a long time.
-			for r, lastTime := range m.consumptionRecord {
-				if time.Since(lastTime) > metricsCleanupTimeout {
-					readRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					writeRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					sqlLayerRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					readByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					writeByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					kvCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					sqlCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
-					requestCount.DeleteLabelValues(r.name, r.name, readTypeLabel)
-					requestCount.DeleteLabelValues(r.name, r.name, writeTypeLabel)
-					availableRUCounter.DeleteLabelValues(r.name, r.name)
-					delete(m.consumptionRecord, r)
-					delete(maxPerSecTrackers, r.name)
-					readRequestUnitMaxPerSecCost.DeleteLabelValues(r.name)
-					writeRequestUnitMaxPerSecCost.DeleteLabelValues(r.name)
-					resourceGroupConfigGauge.DeletePartialMatch(prometheus.Labels{newResourceGroupNameLabel: r.name})
-				}
-			}
-		case <-availableRUTicker.C:
-			m.RLock()
-			groups := make([]*ResourceGroup, 0, len(m.groups))
-			for name, group := range m.groups {
-				if name == reservedDefaultGroupName {
-					continue
-				}
-				groups = append(groups, group)
-			}
-			m.RUnlock()
-			// prevent many groups and hold the lock long time.
-			for _, group := range groups {
-				ru := math.Max(group.getRUToken(), 0)
-				availableRUCounter.WithLabelValues(group.Name, group.Name).Set(ru)
-				resourceGroupConfigGauge.WithLabelValues(group.Name, priorityLabel).Set(group.getPriority())
-				resourceGroupConfigGauge.WithLabelValues(group.Name, ruPerSecLabel).Set(group.getFillRate())
-				resourceGroupConfigGauge.WithLabelValues(group.Name, ruCapacityLabel).Set(group.getBurstLimit())
-			}
-		case <-recordMaxTicker.C:
-			// Record the sum of RRU and WRU every second.
-			m.RLock()
-			names := make([]string, 0, len(m.groups))
-			for name := range m.groups {
-				names = append(names, name)
-			}
-			m.RUnlock()
-			for _, name := range names {
-				if t, ok := maxPerSecTrackers[name]; !ok {
-					maxPerSecTrackers[name] = newMaxPerSecCostTracker(name, defaultCollectIntervalSec)
-				} else {
-					t.FlushMetrics()
-				}
-			}
-		}
-	}
-}
-
-type maxPerSecCostTracker struct {
-	name          string
-	maxPerSecRRU  float64
-	maxPerSecWRU  float64
-	rruSum        float64
-	wruSum        float64
-	lastRRUSum    float64
-	lastWRUSum    float64
-	flushPeriod   int
-	cnt           int
-	rruMaxMetrics prometheus.Gauge
-	wruMaxMetrics prometheus.Gauge
-}
-
-func newMaxPerSecCostTracker(name string, flushPeriod int) *maxPerSecCostTracker {
-	return &maxPerSecCostTracker{
-		name:          name,
-		flushPeriod:   flushPeriod,
-		rruMaxMetrics: readRequestUnitMaxPerSecCost.WithLabelValues(name),
-		wruMaxMetrics: writeRequestUnitMaxPerSecCost.WithLabelValues(name),
-	}
-}
-
-// CollectConsumption collects the consumption info.
-func (t *maxPerSecCostTracker) CollectConsumption(consume *rmpb.Consumption) {
-	t.rruSum += consume.RRU
-	t.wruSum += consume.WRU
-}
-
-// FlushMetrics and set the maxPerSecRRU and maxPerSecWRU to the metrics.
-func (t *maxPerSecCostTracker) FlushMetrics() {
-	if t.lastRRUSum == 0 && t.lastWRUSum == 0 {
-		t.lastRRUSum = t.rruSum
-		t.lastWRUSum = t.wruSum
+func (m *Manager) dispatchConsumption(keyspaceID uint32, item *consumptionItem) {
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm == nil {
path: pkg/mcs/resourcemanager/server/manager.go
user: login: lhy1024
body: When will it be nill and do we need to add a log?
created_at: 2025-05-20T02:19:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,461 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/zap"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+	"github.com/pingcap/log"
+
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/storage/endpoint"
+	"github.com/tikv/pd/pkg/utils/logutil"
+	"github.com/tikv/pd/pkg/utils/syncutil"
+)
+
+const (
+	defaultConsumptionChanSize = 1024
+	metricsCleanupInterval     = time.Minute
+	metricsCleanupTimeout      = 20 * time.Minute
+	metricsAvailableRUInterval = 1 * time.Second
+	defaultCollectIntervalSec  = 20
+	tickPerSecond              = time.Second
+
+	reservedDefaultGroupName = "default"
+	middlePriority           = 8
+	maxPriority              = 16
+	unlimitedRate            = math.MaxInt32
+	unlimitedBurstLimit      = -1
+)
+
+// centralManager is used to provide some basic functions as a resource group manager.
+type centralManager interface {
+	GetControllerConfig() *ControllerConfig
+	GetStorage() endpoint.ResourceGroupStorage
+}
+
+// consumptionItem is used to send the consumption info to the background metrics flusher.
+type consumptionItem struct {
+	resourceGroupName string
+	*rmpb.Consumption
+	isBackground bool
+	isTiFlash    bool
+}
+
+type keyspaceResourceGroupManager struct {
+	syncutil.RWMutex
+	groups map[string]*ResourceGroup
+
+	keyspaceID uint32
+	centralMgr centralManager
+
+	// consumptionChan is used to send the consumption
+	// info to the background metrics flusher.
+	consumptionDispatcher chan *consumptionItem
+	// record update time of each resource group
+	consumptionRecord map[consumptionRecordKey]time.Time
+}
+
+func newKeyspaceResourceGroupManager(keyspaceID uint32, centralMgr centralManager) *keyspaceResourceGroupManager {
+	return &keyspaceResourceGroupManager{
+		groups:                make(map[string]*ResourceGroup),
+		keyspaceID:            keyspaceID,
+		centralMgr:            centralMgr,
+		consumptionDispatcher: make(chan *consumptionItem, defaultConsumptionChanSize),
+		consumptionRecord:     make(map[consumptionRecordKey]time.Time),
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) getControllerConfig() *ControllerConfig {
+	return krgm.centralMgr.GetControllerConfig()
+}
+
+func (krgm *keyspaceResourceGroupManager) getStorage() endpoint.ResourceGroupStorage {
+	return krgm.centralMgr.GetStorage()
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroupFromRaw(name string, rawValue string) error {
+	group := &rmpb.ResourceGroup{}
+	if err := proto.Unmarshal([]byte(rawValue), group); err != nil {
+		log.Error("failed to parse the keyspace resource group meta info",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	krgm.groups[group.Name] = FromProtoResourceGroup(group)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name string, rawValue string) error {
+	tokens := &GroupStates{}
+	if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
+		log.Error("failed to parse the keyspace resource group state",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	if group, ok := krgm.groups[name]; ok {
+		group.SetStatesIntoResourceGroup(tokens)
+	}
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
+	krgm.RLock()
+	if _, ok := krgm.groups[reservedDefaultGroupName]; ok {
+		krgm.RUnlock()
+		return
+	}
+	krgm.RUnlock()
+	defaultGroup := &ResourceGroup{
+		Name: reservedDefaultGroupName,
+		Mode: rmpb.GroupMode_RUMode,
+		RUSettings: &RequestUnitSettings{
+			RU: &GroupTokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   unlimitedRate,
+					BurstLimit: unlimitedBurstLimit,
+				},
+			},
+		},
+		Priority: middlePriority,
+	}
+	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil {
+		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+	// Check the name.
+	if len(grouppb.Name) == 0 || len(grouppb.Name) > 32 {
+		return errs.ErrInvalidGroup
+	}
+	// Check the Priority.
+	if grouppb.GetPriority() > maxPriority {
+		return errs.ErrInvalidGroup
+	}
+	group := FromProtoResourceGroup(grouppb)
+	krgm.Lock()
+	defer krgm.Unlock()
+	if err := group.persistSettings(krgm.keyspaceID, krgm.getStorage()); err != nil {
+		return err
+	}
+	if err := group.persistStates(krgm.keyspaceID, krgm.getStorage()); err != nil {
+		return err
+	}
+	krgm.groups[group.Name] = group
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.ResourceGroup) error {
+	if group == nil || group.Name == "" {
+		return errs.ErrInvalidGroup
+	}
+	krgm.Lock()
+	curGroup, ok := krgm.groups[group.Name]
+	krgm.Unlock()
+	if !ok {
+		return errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name)
+	}
+
+	err := curGroup.PatchSettings(group)
+	if err != nil {
+		return err
+	}
+	return curGroup.persistSettings(krgm.keyspaceID, krgm.getStorage())
+}
+
+func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
+	if name == reservedDefaultGroupName {
+		return errs.ErrDeleteReservedGroup
+	}
+	if err := krgm.getStorage().DeleteResourceGroupSetting(krgm.keyspaceID, name); err != nil {
+		return err
+	}
+	krgm.Lock()
+	delete(krgm.groups, name)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroup(name string, withStats bool) *ResourceGroup {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	if group, ok := krgm.groups[name]; ok {
+		return group.Clone(withStats)
+	}
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) getMutableResourceGroup(name string) *ResourceGroup {
+	krgm.Lock()
+	defer krgm.Unlock()
+	return krgm.groups[name]
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroupList(withStats bool) []*ResourceGroup {
+	krgm.RLock()
+	res := make([]*ResourceGroup, 0, len(krgm.groups))
+	for _, group := range krgm.groups {
+		res = append(res, group.Clone(withStats))
+	}
+	krgm.RUnlock()
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Name < res[j].Name
+	})
+	return res
+}
+
+func (krgm *keyspaceResourceGroupManager) getMutableResourceGroups(inlcudeDefault bool) []*ResourceGroup {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	res := make([]*ResourceGroup, 0, len(krgm.groups))
+	for _, group := range krgm.groups {
+		if !inlcudeDefault && group.Name == reservedDefaultGroupName {
+			continue
+		}
+		res = append(res, group.Clone(true))
+	}
+	return res
+}
path: pkg/mcs/resourcemanager/server/keyspace_manager.go
user: login: lhy1024
body: This function is named with `Mutable`, it returns a clone copy?
created_at: 2025-05-20T03:17:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,451 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/zap"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+	"github.com/pingcap/log"
+
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/storage/endpoint"
+	"github.com/tikv/pd/pkg/utils/logutil"
+	"github.com/tikv/pd/pkg/utils/syncutil"
+)
+
+const (
+	defaultConsumptionChanSize = 1024
+	metricsCleanupInterval     = time.Minute
+	metricsCleanupTimeout      = 20 * time.Minute
+	metricsAvailableRUInterval = 1 * time.Second
+	defaultCollectIntervalSec  = 20
+	tickPerSecond              = time.Second
+
+	reservedDefaultGroupName = "default"
+	middlePriority           = 8
+	maxPriority              = 16
+	unlimitedRate            = math.MaxInt32
+	unlimitedBurstLimit      = -1
+)
+
+// centralManager is used to provide some basic functions as a resource group manager.
+type centralManager interface {
+	GetControllerConfig() *ControllerConfig
+	GetStorage() endpoint.ResourceGroupStorage
+}
+
+// consumptionItem is used to send the consumption info to the background metrics flusher.
+type consumptionItem struct {
+	resourceGroupName string
+	*rmpb.Consumption
+	isBackground bool
+	isTiFlash    bool
+}
+
+type keyspaceResourceGroupManager struct {
+	syncutil.RWMutex
+	groups map[string]*ResourceGroup
+
+	keyspaceID uint32
+	centralMgr centralManager
+
+	// consumptionChan is used to send the consumption
+	// info to the background metrics flusher.
+	consumptionDispatcher chan *consumptionItem
+	// record update time of each resource group
+	consumptionRecord map[consumptionRecordKey]time.Time
+}
+
+func newKeyspaceResourceGroupManager(keyspaceID uint32, centralMgr centralManager) *keyspaceResourceGroupManager {
+	return &keyspaceResourceGroupManager{
+		groups:                make(map[string]*ResourceGroup),
+		keyspaceID:            keyspaceID,
+		centralMgr:            centralMgr,
+		consumptionDispatcher: make(chan *consumptionItem, defaultConsumptionChanSize),
+		consumptionRecord:     make(map[consumptionRecordKey]time.Time),
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) getControllerConfig() *ControllerConfig {
+	return krgm.centralMgr.GetControllerConfig()
+}
+
+func (krgm *keyspaceResourceGroupManager) getStorage() endpoint.ResourceGroupStorage {
+	return krgm.centralMgr.GetStorage()
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroupFromRaw(name string, rawValue string) error {
+	group := &rmpb.ResourceGroup{}
+	if err := proto.Unmarshal([]byte(rawValue), group); err != nil {
+		log.Error("failed to parse the keyspace resource group meta info",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	krgm.groups[group.Name] = FromProtoResourceGroup(group)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name string, rawValue string) error {
+	tokens := &GroupStates{}
+	if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
+		log.Error("failed to parse the keyspace resource group state",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	if group, ok := krgm.groups[name]; ok {
+		group.SetStatesIntoResourceGroup(tokens)
+	}
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
+	krgm.RLock()
+	if _, ok := krgm.groups[reservedDefaultGroupName]; ok {
+		krgm.RUnlock()
+		return
+	}
+	krgm.RUnlock()
+	defaultGroup := &ResourceGroup{
+		Name: reservedDefaultGroupName,
+		Mode: rmpb.GroupMode_RUMode,
+		RUSettings: &RequestUnitSettings{
+			RU: &GroupTokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   unlimitedRate,
+					BurstLimit: unlimitedBurstLimit,
+				},
+			},
+		},
+		Priority: middlePriority,
+	}
+	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil {
+		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+	// Check the name.
+	if len(grouppb.Name) == 0 || len(grouppb.Name) > 32 {
+		return errs.ErrInvalidGroup
+	}
+	// Check the Priority.
+	if grouppb.GetPriority() > maxPriority {
+		return errs.ErrInvalidGroup
+	}
+	group := FromProtoResourceGroup(grouppb)
+	krgm.Lock()
+	defer krgm.Unlock()
+	if err := group.persistSettings(krgm.keyspaceID, krgm.getStorage()); err != nil {
+		return err
+	}
+	if err := group.persistStates(krgm.keyspaceID, krgm.getStorage()); err != nil {
+		return err
+	}
+	krgm.groups[group.Name] = group
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.ResourceGroup) error {
+	if group == nil || group.Name == "" {
+		return errs.ErrInvalidGroup
+	}
+	krgm.Lock()
+	curGroup, ok := krgm.groups[group.Name]
+	krgm.Unlock()
+	if !ok {
+		return errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name)
+	}
+
+	err := curGroup.PatchSettings(group)
+	if err != nil {
+		return err
+	}
+	return curGroup.persistSettings(krgm.keyspaceID, krgm.getStorage())
+}
+
+func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
+	if name == reservedDefaultGroupName {
+		return errs.ErrDeleteReservedGroup
+	}
+	if err := krgm.getStorage().DeleteResourceGroupSetting(krgm.keyspaceID, name); err != nil {
+		return err
+	}
+	krgm.Lock()
+	delete(krgm.groups, name)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroup(name string, withStats bool) *ResourceGroup {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	if group, ok := krgm.groups[name]; ok {
+		return group.Clone(withStats)
+	}
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) getMutableResourceGroup(name string) *ResourceGroup {
+	krgm.Lock()
+	defer krgm.Unlock()
+	return krgm.groups[name]
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroupList(withStats, includeDefault bool) []*ResourceGroup {
+	krgm.RLock()
+	res := make([]*ResourceGroup, 0, len(krgm.groups))
+	for _, group := range krgm.groups {
+		if !includeDefault && group.Name == reservedDefaultGroupName {
+			continue
+		}
+		res = append(res, group.Clone(withStats))
+	}
+	krgm.RUnlock()
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Name < res[j].Name
+	})
+	return res
+}
+
+func (krgm *keyspaceResourceGroupManager) getResourceGroupNames(includeDefault bool) []string {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	res := make([]string, 0, len(krgm.groups))
+	for name := range krgm.groups {
+		if !includeDefault && name == reservedDefaultGroupName {
+			continue
+		}
+		res = append(res, name)
+	}
+	return res
+}
+
+func (krgm *keyspaceResourceGroupManager) persistResourceGroupRunningState() {
+	krgm.RLock()
+	keys := make([]string, 0, len(krgm.groups))
+	for k := range krgm.groups {
+		keys = append(keys, k)
+	}
+	krgm.RUnlock()
+	for idx := range keys {
+		krgm.RLock()
+		group, ok := krgm.groups[keys[idx]]
+		if ok {
+			if err := group.persistStates(krgm.keyspaceID, krgm.getStorage()); err != nil {
+				log.Error("persist keyspace resource group state failed",
+					zap.Uint32("keyspace-id", krgm.keyspaceID),
+					zap.String("group-name", group.Name),
+					zap.Int("index", idx),
+					zap.Error(err))
+			}
+		}
+		krgm.RUnlock()
+	}
+}
+
+// TODO: add keyspace name lable to the metrics.
+func (krgm *keyspaceResourceGroupManager) backgroundMetricsFlush(ctx context.Context) {
path: pkg/mcs/resourcemanager/server/keyspace_manager.go
user: login: glorv
body: Is it better to share a single background routine for the backgroundMetricsFlush for all keyspaces? if there are thousands of keyspaces while each only contains few resource groups, to many background routines may have a bad effect on performance.
created_at: 2025-05-20T06:44:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,451 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/zap"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+	"github.com/pingcap/log"
+
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/storage/endpoint"
+	"github.com/tikv/pd/pkg/utils/logutil"
+	"github.com/tikv/pd/pkg/utils/syncutil"
+)
+
+const (
+	defaultConsumptionChanSize = 1024
+	metricsCleanupInterval     = time.Minute
+	metricsCleanupTimeout      = 20 * time.Minute
+	metricsAvailableRUInterval = 1 * time.Second
+	defaultCollectIntervalSec  = 20
+	tickPerSecond              = time.Second
+
+	reservedDefaultGroupName = "default"
+	middlePriority           = 8
+	maxPriority              = 16
+	unlimitedRate            = math.MaxInt32
+	unlimitedBurstLimit      = -1
+)
+
+// centralManager is used to provide some basic functions as a resource group manager.
+type centralManager interface {
+	GetControllerConfig() *ControllerConfig
+	GetStorage() endpoint.ResourceGroupStorage
+}
+
+// consumptionItem is used to send the consumption info to the background metrics flusher.
+type consumptionItem struct {
+	resourceGroupName string
+	*rmpb.Consumption
+	isBackground bool
+	isTiFlash    bool
+}
+
+type keyspaceResourceGroupManager struct {
+	syncutil.RWMutex
+	groups map[string]*ResourceGroup
+
+	keyspaceID uint32
+	centralMgr centralManager
+
+	// consumptionChan is used to send the consumption
+	// info to the background metrics flusher.
+	consumptionDispatcher chan *consumptionItem
+	// record update time of each resource group
+	consumptionRecord map[consumptionRecordKey]time.Time
+}
+
+func newKeyspaceResourceGroupManager(keyspaceID uint32, centralMgr centralManager) *keyspaceResourceGroupManager {
+	return &keyspaceResourceGroupManager{
+		groups:                make(map[string]*ResourceGroup),
+		keyspaceID:            keyspaceID,
+		centralMgr:            centralMgr,
+		consumptionDispatcher: make(chan *consumptionItem, defaultConsumptionChanSize),
+		consumptionRecord:     make(map[consumptionRecordKey]time.Time),
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) getControllerConfig() *ControllerConfig {
+	return krgm.centralMgr.GetControllerConfig()
+}
+
+func (krgm *keyspaceResourceGroupManager) getStorage() endpoint.ResourceGroupStorage {
+	return krgm.centralMgr.GetStorage()
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroupFromRaw(name string, rawValue string) error {
+	group := &rmpb.ResourceGroup{}
+	if err := proto.Unmarshal([]byte(rawValue), group); err != nil {
+		log.Error("failed to parse the keyspace resource group meta info",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	krgm.groups[group.Name] = FromProtoResourceGroup(group)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name string, rawValue string) error {
+	tokens := &GroupStates{}
+	if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
+		log.Error("failed to parse the keyspace resource group state",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	if group, ok := krgm.groups[name]; ok {
+		group.SetStatesIntoResourceGroup(tokens)
+	}
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
+	krgm.RLock()
+	if _, ok := krgm.groups[reservedDefaultGroupName]; ok {
+		krgm.RUnlock()
+		return
+	}
+	krgm.RUnlock()
+	defaultGroup := &ResourceGroup{
+		Name: reservedDefaultGroupName,
+		Mode: rmpb.GroupMode_RUMode,
+		RUSettings: &RequestUnitSettings{
+			RU: &GroupTokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   unlimitedRate,
+					BurstLimit: unlimitedBurstLimit,
+				},
+			},
+		},
+		Priority: middlePriority,
+	}
+	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil {
+		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+	// Check the name.
+	if len(grouppb.Name) == 0 || len(grouppb.Name) > 32 {
path: pkg/mcs/resourcemanager/server/keyspace_manager.go
user: login: rleungx
body: How about using a constant?
created_at: 2025-05-20T07:31:21Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,451 @@
+// Copyright 2025 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/zap"
+
+	rmpb "github.com/pingcap/kvproto/pkg/resource_manager"
+	"github.com/pingcap/log"
+
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/storage/endpoint"
+	"github.com/tikv/pd/pkg/utils/logutil"
+	"github.com/tikv/pd/pkg/utils/syncutil"
+)
+
+const (
+	defaultConsumptionChanSize = 1024
+	metricsCleanupInterval     = time.Minute
+	metricsCleanupTimeout      = 20 * time.Minute
+	metricsAvailableRUInterval = 1 * time.Second
+	defaultCollectIntervalSec  = 20
+	tickPerSecond              = time.Second
+
+	reservedDefaultGroupName = "default"
+	middlePriority           = 8
+	maxPriority              = 16
+	unlimitedRate            = math.MaxInt32
+	unlimitedBurstLimit      = -1
+)
+
+// centralManager is used to provide some basic functions as a resource group manager.
+type centralManager interface {
+	GetControllerConfig() *ControllerConfig
+	GetStorage() endpoint.ResourceGroupStorage
+}
+
+// consumptionItem is used to send the consumption info to the background metrics flusher.
+type consumptionItem struct {
+	resourceGroupName string
+	*rmpb.Consumption
+	isBackground bool
+	isTiFlash    bool
+}
+
+type keyspaceResourceGroupManager struct {
+	syncutil.RWMutex
+	groups map[string]*ResourceGroup
+
+	keyspaceID uint32
+	centralMgr centralManager
+
+	// consumptionChan is used to send the consumption
+	// info to the background metrics flusher.
+	consumptionDispatcher chan *consumptionItem
+	// record update time of each resource group
+	consumptionRecord map[consumptionRecordKey]time.Time
+}
+
+func newKeyspaceResourceGroupManager(keyspaceID uint32, centralMgr centralManager) *keyspaceResourceGroupManager {
+	return &keyspaceResourceGroupManager{
+		groups:                make(map[string]*ResourceGroup),
+		keyspaceID:            keyspaceID,
+		centralMgr:            centralMgr,
+		consumptionDispatcher: make(chan *consumptionItem, defaultConsumptionChanSize),
+		consumptionRecord:     make(map[consumptionRecordKey]time.Time),
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) getControllerConfig() *ControllerConfig {
+	return krgm.centralMgr.GetControllerConfig()
+}
+
+func (krgm *keyspaceResourceGroupManager) getStorage() endpoint.ResourceGroupStorage {
+	return krgm.centralMgr.GetStorage()
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroupFromRaw(name string, rawValue string) error {
+	group := &rmpb.ResourceGroup{}
+	if err := proto.Unmarshal([]byte(rawValue), group); err != nil {
+		log.Error("failed to parse the keyspace resource group meta info",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	krgm.groups[group.Name] = FromProtoResourceGroup(group)
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name string, rawValue string) error {
+	tokens := &GroupStates{}
+	if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
+		log.Error("failed to parse the keyspace resource group state",
+			zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
+		return err
+	}
+	krgm.Lock()
+	if group, ok := krgm.groups[name]; ok {
+		group.SetStatesIntoResourceGroup(tokens)
+	}
+	krgm.Unlock()
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
+	krgm.RLock()
+	if _, ok := krgm.groups[reservedDefaultGroupName]; ok {
+		krgm.RUnlock()
+		return
+	}
+	krgm.RUnlock()
+	defaultGroup := &ResourceGroup{
+		Name: reservedDefaultGroupName,
+		Mode: rmpb.GroupMode_RUMode,
+		RUSettings: &RequestUnitSettings{
+			RU: &GroupTokenBucket{
+				Settings: &rmpb.TokenLimitSettings{
+					FillRate:   unlimitedRate,
+					BurstLimit: unlimitedBurstLimit,
+				},
+			},
+		},
+		Priority: middlePriority,
+	}
+	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup()); err != nil {
+		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+	}
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+	// Check the name.
+	if len(grouppb.Name) == 0 || len(grouppb.Name) > 32 {
+		return errs.ErrInvalidGroup
+	}
+	// Check the Priority.
+	if grouppb.GetPriority() > maxPriority {
+		return errs.ErrInvalidGroup
+	}
+	group := FromProtoResourceGroup(grouppb)
+	krgm.Lock()
+	defer krgm.Unlock()
+	if err := group.persistSettings(krgm.keyspaceID, krgm.getStorage()); err != nil {
+		return err
+	}
+	if err := group.persistStates(krgm.keyspaceID, krgm.getStorage()); err != nil {
+		return err
+	}
+	krgm.groups[group.Name] = group
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.ResourceGroup) error {
+	if group == nil || group.Name == "" {
+		return errs.ErrInvalidGroup
+	}
+	krgm.Lock()
path: pkg/mcs/resourcemanager/server/keyspace_manager.go
user: login: rleungx
body: ```suggestion
	krgm.RLock()
```
created_at: 2025-05-20T07:32:06Z
author_association: MEMBER

PR comments:
user: login: codecov[bot]
created_at: 2025-05-19T13:06:52Z
body: ## [Codecov](https://app.codecov.io/gh/tikv/pd/pull/9331?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tikv) Report
Attention: Patch coverage is `72.61905%` with `92 lines` in your changes missing coverage. Please review.
> Project coverage is 75.93%. Comparing base [(`04f1ae0`)](https://app.codecov.io/gh/tikv/pd/commit/04f1ae061187afe4edce5060153e3dcb51c3cd02?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tikv) to head [(`08123e6`)](https://app.codecov.io/gh/tikv/pd/commit/08123e639f07fec09a5c89ed32c342add7f66c60?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tikv).
> Report is 5 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #9331      +/-   ##
==========================================
- Coverage   75.98%   75.93%   -0.05%     
==========================================
  Files         473      474       +1     
  Lines       73625    73819     +194     
==========================================
+ Hits        55945    56057     +112     
- Misses      14181    14239      +58     
- Partials     3499     3523      +24     
```

| [Flag](https://app.codecov.io/gh/tikv/pd/pull/9331/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tikv) | Coverage Δ | |
|---|---|---|
| [unittests](https://app.codecov.io/gh/tikv/pd/pull/9331/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tikv) | `75.93% <72.61%> (-0.05%)` | :arrow_down: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=tikv#carryforward-flags-in-the-pull-request-comment) to find out more.

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: ti-chi-bot[bot]
created_at: 2025-05-21T02:28:21Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/tikv/pd/pull/9331#pullrequestreview-2852728271" title="Approved">lhy1024</a>*, *<a href="https://github.com/tikv/pd/pull/9331#pullrequestreview-2856060057" title="Approved">rleungx</a>*

The full list of commands accepted by this bot can be found [here](https://prow.tidb.net/command-help?repo=tikv%2Fpd).

The pull request process is described [here](https://book.prow.tidb.net/#/workflows/pr)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/tikv/pd/blob/master/OWNERS)~~ [lhy1024,rleungx]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: ti-chi-bot[bot]
created_at: 2025-05-21T02:28:22Z
body: [LGTM Timeline notifier]
---
Timeline:

- `2025-05-20 06:05:33.784960106 +0000 UTC m=+68970.886127356`: :ballot_box_with_check: agreed by [lhy1024](https://github.com/lhy1024).
- `2025-05-21 02:28:21.872008855 +0000 UTC m=+142338.973176076`: :ballot_box_with_check: agreed by [rleungx](https://github.com/rleungx).
author_association: CONTRIBUTOR