Repository: alibaba/sentinel-golang
Pull request #373 (2021-01-06 01:13:24)

Support resource level rule replacement in hotspot module
<!--  Thanks for submitting a pull request! Here are some tips for you:
1. Please make sure you have read and understood the contributing guidelines: https://github.com/alibaba/blob/master/CONTRIBUTING.md
2. Please make sure the PR has a corresponding issue.
-->

### Describe what this PR does / why we need it
Support resource level rule replacement in hotspot module.

### Does this pull request fix one issue?

<!--If that, add "Fixes #xxxx" below in the next line. For example, Fixes #15. Otherwise, add "NONE" -->

### Describe how you did it


### Describe how to verify it


### Special notes for reviews

codecov-io: # [Codecov](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=h1) Report
> Merging [#373](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=desc) (5431f87) into [master](https://codecov.io/gh/alibaba/sentinel-golang/commit/1eaa5c5cc8ea53d25036bf0eb36ba42719cc5b1b?el=desc) (1eaa5c5) will **increase** coverage by `0.30%`.
> The diff coverage is `73.46%`.

[![Impacted file tree graph](https://codecov.io/gh/alibaba/sentinel-golang/pull/373/graphs/tree.svg?width=650&height=150&src=pr&token=SrpzbhKA3E)](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #373      +/-   ##
==========================================
+ Coverage   50.42%   50.72%   +0.30%     
==========================================
  Files          70       70              
  Lines        4141     4189      +48     
==========================================
+ Hits         2088     2125      +37     
- Misses       1771     1780       +9     
- Partials      282      284       +2     
```


| [Impacted Files](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [core/hotspot/rule\_manager.go](https://codecov.io/gh/alibaba/sentinel-golang/pull/373/diff?src=pr&el=tree#diff-Y29yZS9ob3RzcG90L3J1bGVfbWFuYWdlci5nbw==) | `58.44% <73.46%> (+5.43%)` | :arrow_up: |
| [core/hotspot/rule.go](https://codecov.io/gh/alibaba/sentinel-golang/pull/373/diff?src=pr&el=tree#diff-Y29yZS9ob3RzcG90L3J1bGUuZ28=) | `66.66% <0.00%> (-3.34%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=footer). Last update [1eaa5c5...5431f87](https://codecov.io/gh/alibaba/sentinel-golang/pull/373?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/core/flow/rule_manager.go b/core/flow/rule_manager.go
index 041a2c2..4e5814a 100644
--- a/core/flow/rule_manager.go
+++ b/core/flow/rule_manager.go
@@ -303,7 +303,7 @@ func onResourceRuleUpdate(res string, rawResRules []*Rule) (err error) {
 	}
 	tcMux.Unlock()
 	currentRules[res] = rawResRules
-	logging.Debug("[Flow onRuleUpdate] Time statistic(ns) for updating flow rule", "timeCost", util.CurrentTimeNano()-start)
+	logging.Debug("[Flow onResourceRuleUpdate] Time statistic(ns) for updating flow rule", "timeCost", util.CurrentTimeNano()-start)
 	logging.Info("[Flow] load resource level rules", "resource", res, "validResRules", validResRules)
 	return nil
 }
diff --git a/core/hotspot/rule_manager.go b/core/hotspot/rule_manager.go
index ee383c1..3c89b3d 100644
--- a/core/hotspot/rule_manager.go
+++ b/core/hotspot/rule_manager.go
@@ -35,7 +35,7 @@ var (
 	tcGenFuncMap  = make(map[ControlBehavior]TrafficControllerGenFunc, 4)
 	tcMap         = make(trafficControllerMap)
 	tcMux         = new(sync.RWMutex)
-	currentRules  = make([]*Rule, 0)
+	currentRules  = make(map[string][]*Rule, 0)
 	updateRuleMux = new(sync.Mutex)
 )
 
@@ -82,27 +82,36 @@ func getTrafficControllersFor(res string) []TrafficShapingController {
 	return tcMap[res]
 }
 
-// LoadRules replaces old rules with the given hotspot parameter flow control rules. Return value:
-//
-// bool: indicates whether the internal map has been changed;
-// error: indicates whether occurs the error.
+// LoadRules replaces all old hotspot param flow rules with the given rules.
+// Return value:
+//   bool: indicates whether the internal map has been changed;
+//   error: indicates whether occurs the error.
 func LoadRules(rules []*Rule) (bool, error) {
+	resRulesMap := make(map[string][]*Rule, 16)
+	for _, rule := range rules {
+		resRules, exists := resRulesMap[rule.Resource]
+		if !exists {
+			resRules = make([]*Rule, 0, 1)
+		}
+		resRulesMap[rule.Resource] = append(resRules, rule)
+	}
+
 	updateRuleMux.Lock()
 	defer updateRuleMux.Unlock()
-	isEqual := reflect.DeepEqual(currentRules, rules)
+	isEqual := reflect.DeepEqual(currentRules, resRulesMap)
 	if isEqual {
 		logging.Info("[HotSpot] Load rules is the same with current rules, so ignore load operation.")
 		return false, nil
 	}
 
-	err := onRuleUpdate(rules)
+	err := onRuleUpdate(resRulesMap)
 	return true, err
 }
 
-// GetRules returns all the rules based on copy.
-// It doesn't take effect for hotspot module if user changes the rule.
+// GetRules returns all the hotspot param flow rules based on copy.
+// It doesn't take effect for hotspot module if user changes the returned rules.
 // GetRules need to compete hotspot module's global lock and the high performance losses of copy,
-// 		reduce or do not call GetRules if possible
+// 		reduce or do not call GetRules if possible.
 func GetRules() []Rule {
 	tcMux.RLock()
 	rules := rulesFrom(tcMap)
@@ -115,10 +124,10 @@ func GetRules() []Rule {
 	return ret
 }
 
-// GetRulesOfResource returns specific resource's rules based on copy.
-// It doesn't take effect for hotspot module if user changes the rule.
+// GetRulesOfResource returns specific resource's hotspot parameter flow control rules based on copy.
+// It doesn't take effect for hotspot module if user changes the returned rules.
 // GetRulesOfResource need to compete hotspot module's global lock and the high performance losses of copy,
-// 		reduce or do not call GetRulesOfResource frequently if possible
+// 		reduce or do not call GetRulesOfResource frequently if possible.
 func GetRulesOfResource(res string) []Rule {
 	tcMux.RLock()
 	resTcs := tcMap[res]
@@ -131,13 +140,19 @@ func GetRulesOfResource(res string) []Rule {
 	return ret
 }
 
-// ClearRules clears all parameter flow rules.
+// ClearRules clears all hotspot param flow rules.
 func ClearRules() error {
 	_, err := LoadRules(nil)
 	return err
 }
 
-func onRuleUpdate(rules []*Rule) (err error) {
+// ClearRulesOfResource clears resource level hotspot param flow rules.
+func ClearRulesOfResource(res string) error {
+	_, err := LoadRulesOfResource(res, nil)
+	return err
+}
+
+func onRuleUpdate(rawResRulesMap map[string][]*Rule) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			var ok bool
@@ -148,24 +163,20 @@ func onRuleUpdate(rules []*Rule) (err error) {
 		}
 	}()
 
-	newRuleMap := make(map[string][]*Rule, len(rules))
-	for _, r := range rules {
-		if err := IsValidRule(r); err != nil {
-			logging.Warn("[HotSpot onRuleUpdate] Ignoring invalid hotspot rule when loading new rules", "rule", r, "err", err.Error())
-			continue
+	// ignore invalid rules
+	validResRulesMap := make(map[string][]*Rule, len(rawResRulesMap))
+	for res, rules := range rawResRulesMap {
+		validResRules := make([]*Rule, 0, len(rules))
+		for _, rule := range rules {
+			if err := IsValidRule(rule); err != nil {
+				logging.Warn("[HotSpot onRuleUpdate] Ignoring invalid hotspot param flow rule when loading new rules", "rule", rule, "err", err.Error())
+				continue
+			}
+			validResRules = append(validResRules, rule)
 		}
-		res := r.ResourceName()
-		ruleSet, ok := newRuleMap[res]
-		if !ok {
-			ruleSet = make([]*Rule, 0, 1)
+		if len(validResRules) > 0 {
+			validResRulesMap[res] = validResRules
 		}
-		ruleSet = append(ruleSet, r)
-		newRuleMap[res] = ruleSet
-	}
-
-	m := make(trafficControllerMap)
-	for res, rules := range newRuleMap {
-		m[res] = make([]TrafficShapingController, 0, len(rules))
 	}
 
 	start := util.CurrentTimeNano()
@@ -179,49 +190,11 @@ func onRuleUpdate(rules []*Rule) (err error) {
 	}
 	tcMux.RUnlock()
 
-	for res, resRules := range newRuleMap {
-		emptyTcList := make([]TrafficShapingController, 0, 0)
-		for _, r := range resRules {
-			oldResTcs := tcMapClone[res]
-			if oldResTcs == nil {
-				oldResTcs = emptyTcList
-			}
-
-			equalIdx, reuseStatIdx := calculateReuseIndexFor(r, oldResTcs)
-			// there is equivalent rule in old traffic shaping controller slice
-			if equalIdx >= 0 {
-				equalOldTC := oldResTcs[equalIdx]
-				insertTcToTcMap(equalOldTC, res, m)
-				// remove old tc from old resTcs
-				tcMapClone[res] = append(oldResTcs[:equalIdx], oldResTcs[equalIdx+1:]...)
-				continue
-			}
-
-			// generate new traffic shaping controller
-			generator, supported := tcGenFuncMap[r.ControlBehavior]
-			if !supported {
-				logging.Warn("[HotSpot onRuleUpdate] Ignoring the frequent param flow rule due to unsupported control behavior", "rule", r)
-				continue
-			}
-			var tc TrafficShapingController
-			if reuseStatIdx >= 0 {
-				// generate new traffic shaping controller with reusable statistic metric.
-				tc = generator(r, oldResTcs[reuseStatIdx].BoundMetric())
-			} else {
-				tc = generator(r, nil)
-			}
-			if tc == nil {
-				logging.Debug("[HotSpot onRuleUpdate] Ignoring the frequent param flow rule due to bad generated traffic controller", "rule", r)
-				continue
-			}
-
-			//  remove the reused traffic shaping controller old res tcs
-			if reuseStatIdx >= 0 {
-				tcMapClone[res] = append(oldResTcs[:reuseStatIdx], oldResTcs[reuseStatIdx+1:]...)
-			}
-			insertTcToTcMap(tc, res, m)
-		}
+	m := make(trafficControllerMap, len(validResRulesMap))
+	for res, rules := range validResRulesMap {
+		m[res] = buildResourceTrafficShapingController(res, rules, tcMapClone[res])
 	}
+
 	for res, tcs := range m {
 		if len(tcs) > 0 {
 			// update resource slot chain
@@ -237,14 +210,102 @@ func onRuleUpdate(rules []*Rule) (err error) {
 
 	tcMux.Lock()
 	tcMap = m
-	currentRules = rules
 	tcMux.Unlock()
 
-	logging.Debug("[HotSpot onRuleUpdate] Time statistic(ns) for updating hotSpot rule", "timeCost", util.CurrentTimeNano()-start)
-	logRuleUpdate(newRuleMap)
+	currentRules = rawResRulesMap
+
+	logging.Debug("[HotSpot onRuleUpdate] Time statistic(ns) for updating hotspot param flow rules", "timeCost", util.CurrentTimeNano()-start)
+	logRuleUpdate(validResRulesMap)
+	return nil
+}
+
+func onResourceRuleUpdate(res string, rawResRules []*Rule) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			var ok bool
+			err, ok = r.(error)
+			if !ok {
+				err = fmt.Errorf("%v", r)
+			}
+		}
+	}()
+
+	validResRules := make([]*Rule, 0, len(rawResRules))
+	for _, rule := range rawResRules {
+		if err := IsValidRule(rule); err != nil {
+			logging.Warn("[HotSpot onResourceRuleUpdate] Ignoring invalid hotspot param flow rule", "rule", rule, "reason", err.Error())
+			continue
+		}
+		validResRules = append(validResRules, rule)
+	}
+
+	start := util.CurrentTimeNano()
+	oldResTcs := make([]TrafficShapingController, 0, 8)
+	tcMux.RLock()
+	oldResTcs = append(oldResTcs, tcMap[res]...)
+	tcMux.RUnlock()
+
+	newResTcs := buildResourceTrafficShapingController(res, validResRules, oldResTcs)
+	if len(newResTcs) > 0 {
+		// update resource slot chain
+		misc.RegisterRuleCheckSlotForResource(res, DefaultSlot)
+		for _, tc := range newResTcs {
+			if tc.BoundRule().MetricType == Concurrency {
+				misc.RegisterStatSlotForResource(res, DefaultConcurrencyStatSlot)
+				break
+			}
+		}
+	}
+
+	tcMux.Lock()
+	if len(newResTcs) == 0 {
+		delete(tcMap, res)
+	} else {
+		tcMap[res] = newResTcs
+	}
+	tcMux.Unlock()
+
+	currentRules[res] = rawResRules
+
+	logging.Debug("[HotSpot onResourceRuleUpdate] Time statistic(ns) for updating hotspot param flow rules", "timeCost", util.CurrentTimeNano()-start)
+	logging.Info("[HotSpot] load resource level hotspot param flow rules", "resource", res, "validResRules", validResRules)
 	return nil
 }
 
+// LoadRulesOfResource loads the given resource's hotspot param flow rules to the rule manager,
+// while all previous resource's rules will be replaced. The first returned value indicates whether
+// do real load operation, if the rules is the same with previous resource's rules, return false.
+func LoadRulesOfResource(res string, rules []*Rule) (bool, error) {
+	if len(res) == 0 {
+		return false, errors.New("empty resource")
+	}
+
+	updateRuleMux.Lock()
+	defer updateRuleMux.Unlock()
+
+	// clear resource rules
+	if len(rules) == 0 {
+		// clear resource's currentRules
+		delete(currentRules, res)
+		// clear tcMap
+		tcMux.Lock()
+		delete(tcMap, res)
+		tcMux.Unlock()
+		logging.Info("[HotSpot] clear resource level hotspot param flow rules", "resource", res)
+		return true, nil
+	}
+
+	// load resource level rules
+	isEqual := reflect.DeepEqual(currentRules[res], rules)
+	if isEqual {
+		logging.Info("[HotSpot] Load resource level hotspot param flow rules is the same with current resource level rules, so ignore load operation.")
+		return false, nil
+	}
+
+	err := onResourceRuleUpdate(res, rules)
+	return true, err
+}
+
 func logRuleUpdate(m map[string][]*Rule) {
 	rules := make([]*Rule, 0, 8)
 	for _, rs := range m {
@@ -254,9 +315,9 @@ func logRuleUpdate(m map[string][]*Rule) {
 		rules = append(rules, rs...)
 	}
 	if len(rules) == 0 {
-		logging.Info("[HotspotRuleManager] Hotspot rules were cleared")
+		logging.Info("[HotspotRuleManager] Hotspot param flow rules were cleared")
 	} else {
-		logging.Info("[HotspotRuleManager] Hotspot rules were loaded", "rules", rules)
+		logging.Info("[HotspotRuleManager] Hotspot param flow rules were loaded", "rules", rules)
 	}
 }
 
@@ -304,14 +365,48 @@ func calculateReuseIndexFor(r *Rule, oldResTcs []TrafficShapingController) (equa
 	return equalIdx, reuseStatIdx
 }
 
-func insertTcToTcMap(tc TrafficShapingController, res string, m trafficControllerMap) {
-	tcsOfRes, exists := m[res]
-	if !exists {
-		tcsOfRes = make([]TrafficShapingController, 0, 1)
-		m[res] = append(tcsOfRes, tc)
-	} else {
-		m[res] = append(tcsOfRes, tc)
+// buildResourceTrafficShapingController builds TrafficShapingController slice from rules. the resource of rules must be equals to res.
+func buildResourceTrafficShapingController(res string, resRules []*Rule, oldResTcs []TrafficShapingController) []TrafficShapingController {
+	newTcsOfRes := make([]TrafficShapingController, 0, len(resRules))
+	for _, rule := range resRules {
+		if res != rule.Resource {
+			logging.Error(errors.Errorf("unmatched resource name, expect: %s, actual: %s", res, rule.Resource), "Unmatched resource name in hotspot.buildResourceTrafficShapingController()", "rule", rule)
+			continue
+		}
+
+		equalIdx, reuseStatIdx := calculateReuseIndexFor(rule, oldResTcs)
+		// there is equivalent rule in old traffic shaping controller slice
+		if equalIdx >= 0 {
+			equalOldTC := oldResTcs[equalIdx]
+			newTcsOfRes = append(newTcsOfRes, equalOldTC)
+			// remove old tc from old resTcs
+			oldResTcs = append(oldResTcs[:equalIdx], oldResTcs[equalIdx+1:]...)
+			continue
+		}
+
+		// generate new traffic shaping controller
+		generator, supported := tcGenFuncMap[rule.ControlBehavior]
+		if !supported {
+			logging.Warn("[HotSpot buildResourceTrafficShapingController] Ignoring the hotspot param flow rule due to unsupported control behavior", "rule", rule)
+			continue
+		}
+		var tc TrafficShapingController
+		if reuseStatIdx >= 0 {
+			// generate new traffic shaping controller with reusable statistic metric.
+			tc = generator(rule, oldResTcs[reuseStatIdx].BoundMetric())
+			// remove the reused traffic shaping controller old res tcs
+			oldResTcs = append(oldResTcs[:reuseStatIdx], oldResTcs[reuseStatIdx+1:]...)
+		} else {
+			tc = generator(rule, nil)
+		}
+		if tc == nil {
+			logging.Debug("[HotSpot buildResourceTrafficShapingController] Ignoring the hotspot param flow rule due to bad generated traffic controller", "rule", rule)
+			continue
+		}
+
+		newTcsOfRes = append(newTcsOfRes, tc)
 	}
+	return newTcsOfRes
 }
 
 func IsValidRule(rule *Rule) error {
diff --git a/core/isolation/rule_manager.go b/core/isolation/rule_manager.go
index f1654f9..399a2fe 100644
--- a/core/isolation/rule_manager.go
+++ b/core/isolation/rule_manager.go
@@ -27,63 +27,139 @@ import (
 var (
 	ruleMap       = make(map[string][]*Rule)
 	rwMux         = &sync.RWMutex{}
-	currentRules  = make([]*Rule, 0)
+	currentRules  = make(map[string][]*Rule, 0)
 	updateRuleMux = new(sync.Mutex)
 )
 
 // LoadRules loads the given isolation rules to the rule manager, while all previous rules will be replaced.
 // the first returned value indicates whether do real load operation, if the rules is the same with previous rules, return false
 func LoadRules(rules []*Rule) (bool, error) {
+	resRulesMap := make(map[string][]*Rule, 16)
+	for _, rule := range rules {
+		resRules, exist := resRulesMap[rule.Resource]
+		if !exist {
+			resRules = make([]*Rule, 0, 1)
+		}
+		resRulesMap[rule.Resource] = append(resRules, rule)
+	}
+
 	updateRuleMux.Lock()
 	defer updateRuleMux.Unlock()
-	isEqual := reflect.DeepEqual(currentRules, rules)
+	isEqual := reflect.DeepEqual(currentRules, resRulesMap)
 	if isEqual {
 		logging.Info("[Isolation] Load rules is the same with current rules, so ignore load operation.")
 		return false, nil
 	}
 
-	err := onRuleUpdate(rules)
+	err := onRuleUpdate(resRulesMap)
 	return true, err
 }
 
-func onRuleUpdate(rules []*Rule) (err error) {
-	m := make(map[string][]*Rule, len(rules))
-	for _, r := range rules {
-		if e := IsValid(r); e != nil {
-			logging.Error(e, "Invalid isolation rule in isolation.LoadRules()", "rule", r)
-			continue
+func onRuleUpdate(rawResRulesMap map[string][]*Rule) (err error) {
+	validResRulesMap := make(map[string][]*Rule, len(rawResRulesMap))
+	for res, rules := range rawResRulesMap {
+		validResRules := make([]*Rule, 0, len(rules))
+		for _, rule := range rules {
+			if err := IsValidRule(rule); err != nil {
+				logging.Warn("[Isolation onRuleUpdate] Ignoring invalid isolation rule", "rule", rule, "reason", err.Error())
+				continue
+			}
+			validResRules = append(validResRules, rule)
 		}
-		resRules, ok := m[r.Resource]
-		if !ok {
-			resRules = make([]*Rule, 0, 1)
+		if len(validResRules) > 0 {
+			validResRulesMap[res] = validResRules
 		}
-		m[r.Resource] = append(resRules, r)
 	}
 
 	start := util.CurrentTimeNano()
 
-	for res, rs := range m {
+	for res, rs := range validResRulesMap {
 		if len(rs) > 0 {
 			// update resource slot chain
 			misc.RegisterRuleCheckSlotForResource(res, DefaultSlot)
 		}
 	}
 	rwMux.Lock()
-	ruleMap = m
-	currentRules = rules
+	ruleMap = validResRulesMap
 	rwMux.Unlock()
+	currentRules = rawResRulesMap
 
-	logging.Debug("[Isolation LoadRules] Time statistic(ns) for updating isolation rule", "timeCost", util.CurrentTimeNano()-start)
-	logRuleUpdate(m)
+	logging.Debug("[Isolation onRuleUpdate] Time statistic(ns) for updating isolation rule", "timeCost", util.CurrentTimeNano()-start)
+	logRuleUpdate(validResRulesMap)
 	return
 }
 
+// LoadRulesOfResource loads the given resource's isolation rules to the rule manager, while all previous resource's rules will be replaced.
+// the first returned value indicates whether do real load operation, if the rules is the same with previous resource's rules, return false
+func LoadRulesOfResource(res string, rules []*Rule) (bool, error) {
+	if len(res) == 0 {
+		return false, errors.New("empty resource")
+	}
+	updateRuleMux.Lock()
+	defer updateRuleMux.Unlock()
+	// clear resource rules
+	if len(rules) == 0 {
+		// clear resource's currentRules
+		delete(currentRules, res)
+		// clear ruleMap
+		rwMux.Lock()
+		delete(ruleMap, res)
+		rwMux.Unlock()
+		logging.Info("[Isolation] clear resource level rules", "resource", res)
+		return true, nil
+	}
+	// load resource level rules
+	isEqual := reflect.DeepEqual(currentRules[res], rules)
+	if isEqual {
+		logging.Info("[Isolation] Load resource level rules is the same with current resource level rules, so ignore load operation.")
+		return false, nil
+	}
+
+	err := onResourceRuleUpdate(res, rules)
+	return true, err
+}
+
+func onResourceRuleUpdate(res string, rawResRules []*Rule) (err error) {
+	validResRules := make([]*Rule, 0, len(rawResRules))
+	for _, rule := range rawResRules {
+		if err := IsValidRule(rule); err != nil {
+			logging.Warn("[Isolation onResourceRuleUpdate] Ignoring invalid flow rule", "rule", rule, "reason", err.Error())
+			continue
+		}
+		validResRules = append(validResRules, rule)
+	}
+
+	start := util.CurrentTimeNano()
+	if len(validResRules) > 0 {
+		// update resource slot chain
+		misc.RegisterRuleCheckSlotForResource(res, DefaultSlot)
+	}
+
+	rwMux.Lock()
+	if len(validResRules) == 0 {
+		delete(ruleMap, res)
+	} else {
+		ruleMap[res] = validResRules
+	}
+	rwMux.Unlock()
+	currentRules[res] = rawResRules
+	logging.Debug("[Isolation onResourceRuleUpdate] Time statistic(ns) for updating flow rule", "timeCost", util.CurrentTimeNano()-start)
+	logging.Info("[Isolation] load resource level rules", "resource", res, "validResRules", validResRules)
+	return nil
+}
+
 // ClearRules clears all the rules in isolation module.
 func ClearRules() error {
 	_, err := LoadRules(nil)
 	return err
 }
 
+// ClearRulesOfResource clears resource level rules in isolation module.
+func ClearRulesOfResource(res string) error {
+	_, err := LoadRulesOfResource(res, nil)
+	return err
+}
+
 // GetRules returns all the rules based on copy.
 // It doesn't take effect for isolation module if user changes the rule.
 func GetRules() []Rule {
@@ -157,7 +233,7 @@ func logRuleUpdate(m map[string][]*Rule) {
 }
 
 // IsValidRule checks whether the given Rule is valid.
-func IsValid(r *Rule) error {
+func IsValidRule(r *Rule) error {
 	if r == nil {
 		return errors.New("nil isolation rule")
 	}


Test patch:
diff --git a/core/hotspot/rule_manager_test.go b/core/hotspot/rule_manager_test.go
index 9571dde..291d2cb 100644
--- a/core/hotspot/rule_manager_test.go
+++ b/core/hotspot/rule_manager_test.go
@@ -23,6 +23,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func clearData() {
+	tcMap = make(trafficControllerMap)
+	currentRules = make(map[string][]*Rule, 0)
+}
+
 func Test_tcGenFuncMap(t *testing.T) {
 	t.Run("Test_tcGenFuncMap_withoutMetric", func(t *testing.T) {
 		specific := make(map[interface{}]int64)
@@ -150,7 +155,8 @@ func Test_IsValidRule(t *testing.T) {
 }
 
 func Test_onRuleUpdate(t *testing.T) {
-	tcMap = make(trafficControllerMap)
+	clearData()
+	defer clearData()
 
 	specific := make(map[interface{}]int64)
 	specific["sss"] = 1
@@ -215,7 +221,7 @@ func Test_onRuleUpdate(t *testing.T) {
 
 	updated, err := LoadRules([]*Rule{r1, r2, r3, r4})
 	if !updated || err != nil {
-		t.Errorf("Fail to prepare data, err: %+v", err)
+		t.Fatalf("Fail to prepare data, err: %+v", err)
 	}
 	assert.True(t, len(tcMap["abc"]) == 4)
 
@@ -271,7 +277,10 @@ func Test_onRuleUpdate(t *testing.T) {
 	oldTc2MetricPtrAddr := fmt.Sprintf("%p", tcMap["abc"][1].BoundMetric())
 	fmt.Println("oldTc2MetricPtr:", oldTc2MetricPtrAddr)
 
-	err = onRuleUpdate([]*Rule{r21, r22, r23})
+	rulesMap := map[string][]*Rule{
+		"abc": {r21, r22, r23},
+	}
+	err = onRuleUpdate(rulesMap)
 	assert.True(t, err == nil)
 	assert.True(t, len(tcMap) == 1)
 	abcTcs := tcMap["abc"]
@@ -290,11 +299,12 @@ func Test_onRuleUpdate(t *testing.T) {
 	assert.True(t, newTc1PtrAddr == oldTc1PtrAddr && newTc2MetricPtrAddr == oldTc2MetricPtrAddr)
 	assert.True(t, abcTcs[0].BoundRule() == r1 && abcTcs[0] == oldTc1Ptr)
 	assert.True(t, abcTcs[1].BoundMetric() == oldTc2Ptr.BoundMetric())
-
-	tcMap = make(trafficControllerMap)
 }
 
 func TestLoadRules(t *testing.T) {
+	clearData()
+	defer clearData()
+
 	t.Run("loadSameRules", func(t *testing.T) {
 		specific := make(map[interface{}]int64)
 		specific["sss"] = 1
@@ -333,3 +343,227 @@ func TestLoadRules(t *testing.T) {
 		assert.False(t, ok)
 	})
 }
+
+func TestLoadRulesOfResource(t *testing.T) {
+	clearData()
+	defer clearData()
+
+	specific := make(map[interface{}]int64)
+	specific["sss"] = 1
+	specific["123"] = 3
+	r11 := Rule{
+		ID:                "1",
+		Resource:          "abc1",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         100.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r12 := Rule{
+		ID:                "2",
+		Resource:          "abc1",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         200.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r21 := Rule{
+		ID:                "3",
+		Resource:          "abc2",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         100.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r22 := Rule{
+		ID:                "4",
+		Resource:          "abc2",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         200.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+
+	succ, err := LoadRules([]*Rule{&r11, &r12, &r21, &r22})
+	assert.True(t, succ && err == nil)
+
+	t.Run("LoadRulesOfResource_empty_resource", func(t *testing.T) {
+		succ, err = LoadRulesOfResource("", []*Rule{&r11, &r12})
+		assert.True(t, !succ && err != nil)
+	})
+
+	t.Run("LoadRulesOfResource_cache_hit", func(t *testing.T) {
+		r11Copy := r11
+		r12Copy := r12
+		succ, err = LoadRulesOfResource("abc1", []*Rule{&r11Copy, &r12Copy})
+		assert.True(t, !succ && err == nil)
+	})
+
+	t.Run("LoadRulesOfResource_clear", func(t *testing.T) {
+		succ, err = LoadRulesOfResource("abc1", []*Rule{})
+		assert.True(t, succ && err == nil)
+		assert.True(t, len(tcMap["abc1"]) == 0 && len(currentRules["abc1"]) == 0)
+		assert.True(t, len(tcMap["abc2"]) == 2 && len(currentRules["abc2"]) == 2)
+	})
+}
+
+func Test_onResourceRuleUpdate(t *testing.T) {
+	clearData()
+	defer clearData()
+
+	specific := make(map[interface{}]int64)
+	specific["sss"] = 1
+	specific["123"] = 3
+	r11 := Rule{
+		ID:                "1",
+		Resource:          "abc1",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         100.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r12 := Rule{
+		ID:                "2",
+		Resource:          "abc1",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         200.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r21 := Rule{
+		ID:                "3",
+		Resource:          "abc2",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         100.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r22 := Rule{
+		ID:                "4",
+		Resource:          "abc2",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         200.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+
+	succ, err := LoadRules([]*Rule{&r11, &r12, &r21, &r22})
+	assert.True(t, succ && err == nil)
+
+	t.Run("Test_onResourceRuleUpdate_normal", func(t *testing.T) {
+		r11Copy := r11
+		r11Copy.Threshold = 500
+		err = onResourceRuleUpdate("abc1", []*Rule{&r11Copy})
+
+		assert.True(t, len(tcMap["abc1"]) == 1)
+		assert.True(t, len(currentRules["abc1"]) == 1)
+		assert.True(t, tcMap["abc1"][0].BoundRule() == &r11Copy)
+
+		assert.True(t, len(tcMap["abc2"]) == 2)
+		assert.True(t, len(currentRules["abc2"]) == 2)
+
+		clearData()
+	})
+}
+
+func TestClearRulesOfResource(t *testing.T) {
+	clearData()
+	defer clearData()
+
+	specific := make(map[interface{}]int64)
+	specific["sss"] = 1
+	specific["123"] = 3
+	r11 := Rule{
+		ID:                "1",
+		Resource:          "abc1",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         100.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r12 := Rule{
+		ID:                "2",
+		Resource:          "abc1",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         200.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r21 := Rule{
+		ID:                "3",
+		Resource:          "abc2",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         100.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+	r22 := Rule{
+		ID:                "4",
+		Resource:          "abc2",
+		MetricType:        Concurrency,
+		ControlBehavior:   Reject,
+		ParamIndex:        0,
+		Threshold:         200.0,
+		MaxQueueingTimeMs: 0,
+		BurstCount:        10,
+		DurationInSec:     1,
+		SpecificItems:     specific,
+	}
+
+	succ, err := LoadRules([]*Rule{&r11, &r12, &r21, &r22})
+	assert.True(t, succ && err == nil)
+
+	t.Run("TestClearRulesOfResource_normal", func(t *testing.T) {
+		assert.True(t, ClearRulesOfResource("abc1") == nil)
+
+		assert.True(t, len(tcMap["abc1"]) == 0)
+		assert.True(t, len(currentRules["abc1"]) == 0)
+		assert.True(t, len(tcMap["abc2"]) == 2)
+		assert.True(t, len(currentRules["abc2"]) == 2)
+		clearData()
+	})
+}
diff --git a/core/isolation/rule_manager_test.go b/core/isolation/rule_manager_test.go
index d177bc1..471d41d 100644
--- a/core/isolation/rule_manager_test.go
+++ b/core/isolation/rule_manager_test.go
@@ -21,6 +21,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func clearData() {
+	ruleMap = make(map[string][]*Rule)
+	currentRules = make(map[string][]*Rule, 0)
+}
+
 func TestLoadRules(t *testing.T) {
 	t.Run("TestLoadRules_1", func(t *testing.T) {
 		logging.ResetGlobalLoggerLevel(logging.DebugLevel)
@@ -47,8 +52,7 @@ func TestLoadRules(t *testing.T) {
 		assert.True(t, len(ruleMap["abc2"]) == 1)
 		assert.True(t, ruleMap["abc2"][0] == r2)
 
-		err = ClearRules()
-		assert.True(t, err == nil)
+		clearData()
 	})
 
 	t.Run("loadSameRules", func(t *testing.T) {
@@ -69,5 +73,153 @@ func TestLoadRules(t *testing.T) {
 		})
 		assert.Nil(t, err)
 		assert.False(t, ok)
+		clearData()
+	})
+
+	t.Run("TestClearRules_normal", func(t *testing.T) {
+		r1 := &Rule{
+			Resource:   "abc1",
+			MetricType: Concurrency,
+			Threshold:  100,
+		}
+		r2 := &Rule{
+			Resource:   "abc2",
+			MetricType: Concurrency,
+			Threshold:  200,
+		}
+		r3 := &Rule{
+			Resource:   "abc3",
+			MetricType: MetricType(1),
+			Threshold:  200,
+		}
+
+		succ, err := LoadRules([]*Rule{r1, r2, r3})
+		assert.True(t, succ && err == nil)
+
+		assert.True(t, ClearRules() == nil)
+
+		assert.True(t, len(ruleMap["abc1"]) == 0)
+		assert.True(t, len(currentRules["abc1"]) == 0)
+		assert.True(t, len(ruleMap["abc2"]) == 0)
+		assert.True(t, len(currentRules["abc2"]) == 0)
+		assert.True(t, len(ruleMap["abc3"]) == 0)
+		assert.True(t, len(currentRules["abc3"]) == 0)
+		clearData()
+	})
+}
+
+func TestLoadRulesOfResource(t *testing.T) {
+	r1 := &Rule{
+		Resource:   "abc1",
+		MetricType: Concurrency,
+		Threshold:  100,
+	}
+	r2 := &Rule{
+		Resource:   "abc2",
+		MetricType: Concurrency,
+		Threshold:  200,
+	}
+	r3 := &Rule{
+		Resource:   "abc3",
+		MetricType: MetricType(1),
+		Threshold:  200,
+	}
+
+	r4 := &Rule{
+		Resource:   "abc2",
+		MetricType: MetricType(1),
+		Threshold:  300,
+	}
+
+	succ, err := LoadRules([]*Rule{r1, r2, r3, r4})
+	assert.True(t, succ && err == nil)
+
+	t.Run("LoadRulesOfResource_empty_resource", func(t *testing.T) {
+		succ, err = LoadRulesOfResource("", []*Rule{r1, r2})
+		assert.True(t, !succ && err != nil)
+	})
+
+	t.Run("LoadRulesOfResource_cache_hit", func(t *testing.T) {
+		r111 := *r2
+		r122 := *r4
+		succ, err = LoadRulesOfResource("abc2", []*Rule{&r111, &r122})
+		assert.True(t, !succ && err == nil)
+	})
+
+	t.Run("LoadRulesOfResource_clear", func(t *testing.T) {
+		succ, err = LoadRulesOfResource("abc1", []*Rule{})
+		assert.True(t, succ && err == nil)
+		assert.True(t, len(ruleMap["abc1"]) == 0 && len(currentRules["abc1"]) == 0)
+		assert.True(t, len(ruleMap["abc2"]) == 1 && len(currentRules["abc2"]) == 2)
+	})
+	clearData()
+}
+
+func Test_ResourceRuleUpdate(t *testing.T) {
+	logging.ResetGlobalLoggerLevel(logging.DebugLevel)
+	t.Run("Test_onResourceRuleUpdate_normal", func(t *testing.T) {
+		r1 := &Rule{
+			Resource:   "abc1",
+			MetricType: Concurrency,
+			Threshold:  100,
+		}
+		r2 := &Rule{
+			Resource:   "abc2",
+			MetricType: Concurrency,
+			Threshold:  200,
+		}
+		r3 := &Rule{
+			Resource:   "abc3",
+			MetricType: MetricType(1),
+			Threshold:  200,
+		}
+
+		succ, err := LoadRules([]*Rule{r1, r2, r3})
+		assert.True(t, succ && err == nil)
+
+		r111 := r1
+		r111.Threshold = 100
+		err = onResourceRuleUpdate("abc1", []*Rule{r111})
+
+		assert.True(t, err == nil)
+		assert.True(t, len(ruleMap["abc1"]) == 1)
+		assert.True(t, len(currentRules["abc1"]) == 1)
+		assert.True(t, ruleMap["abc1"][0] == r111)
+
+		assert.True(t, len(ruleMap["abc2"]) == 1)
+		assert.True(t, len(currentRules["abc2"]) == 1)
+
+		clearData()
+	})
+
+	t.Run("TestClearRulesOfResource_normal", func(t *testing.T) {
+		r1 := &Rule{
+			Resource:   "abc1",
+			MetricType: Concurrency,
+			Threshold:  100,
+		}
+		r2 := &Rule{
+			Resource:   "abc2",
+			MetricType: Concurrency,
+			Threshold:  200,
+		}
+		r3 := &Rule{
+			Resource:   "abc3",
+			MetricType: MetricType(1),
+			Threshold:  200,
+		}
+
+		succ, err := LoadRules([]*Rule{r1, r2, r3})
+		assert.True(t, succ && err == nil)
+
+		assert.True(t, ClearRulesOfResource("abc1") == nil)
+
+		assert.True(t, len(ruleMap["abc1"]) == 0)
+		assert.True(t, len(currentRules["abc1"]) == 0)
+		assert.True(t, len(ruleMap["abc2"]) == 1)
+		assert.True(t, len(currentRules["abc2"]) == 1)
+		assert.True(t, len(ruleMap["abc3"]) == 0)
+		assert.True(t, len(currentRules["abc3"]) == 1)
+		clearData()
 	})
 }