Repository: pingcap/pd (go)

Issue:
pd-ctl: tell PD to remove a bad peer explicitly
In the TiKV disaster recovery, the data of a peer may be corrupted, and we must remove this peer from the region. So we should tell PD to remove the bad peer at first.

After we send the command, we don't need to wait for the response because this is an asynchronous flow, we can fetch the region status later to see whether the ConfChange is executed or not.

Pull request description:
api, pdctl: support add/remove region peer.
Fix #786.

Patch:
diff --git a/pdctl/command/operator.go b/pdctl/command/operator.go
index 054ac10de..a93d00e87 100644
--- a/pdctl/command/operator.go
+++ b/pdctl/command/operator.go
@@ -76,6 +76,8 @@ func NewAddOperatorCommand() *cobra.Command {
 	c.AddCommand(NewTransferLeaderCommand())
 	c.AddCommand(NewTransferRegionCommand())
 	c.AddCommand(NewTransferPeerCommand())
+	c.AddCommand(NewAddPeerCommand())
+	c.AddCommand(NewRemovePeerCommand())
 	return c
 }
 
@@ -167,6 +169,64 @@ func transferPeerCommandFunc(cmd *cobra.Command, args []string) {
 	postJSON(cmd, operatorsPrefix, input)
 }
 
+// NewAddPeerCommand returns a command to add region peer.
+func NewAddPeerCommand() *cobra.Command {
+	c := &cobra.Command{
+		Use:   "add-peer <region_id> <to_store_id>",
+		Short: "add a region peer on specified store",
+		Run:   addPeerCommandFunc,
+	}
+	return c
+}
+
+func addPeerCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) != 2 {
+		fmt.Println(cmd.UsageString())
+		return
+	}
+
+	ids, err := parseUint64s(args)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	input := make(map[string]interface{})
+	input["name"] = cmd.Name()
+	input["region_id"] = ids[0]
+	input["store_id"] = ids[1]
+	postJSON(cmd, operatorsPrefix, input)
+}
+
+// NewRemovePeerCommand returns a command to add region peer.
+func NewRemovePeerCommand() *cobra.Command {
+	c := &cobra.Command{
+		Use:   "remove-peer <region_id> <from_store_id>",
+		Short: "remove a region peer on specified store",
+		Run:   removePeerCommandFunc,
+	}
+	return c
+}
+
+func removePeerCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) != 2 {
+		fmt.Println(cmd.UsageString())
+		return
+	}
+
+	ids, err := parseUint64s(args)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	input := make(map[string]interface{})
+	input["name"] = cmd.Name()
+	input["region_id"] = ids[0]
+	input["store_id"] = ids[1]
+	postJSON(cmd, operatorsPrefix, input)
+}
+
 // NewRemoveOperatorCommand returns a command to remove operators.
 func NewRemoveOperatorCommand() *cobra.Command {
 	c := &cobra.Command{
diff --git a/server/api/operator.go b/server/api/operator.go
index 42a3dc26a..0ee1e2b9b 100644
--- a/server/api/operator.go
+++ b/server/api/operator.go
@@ -156,6 +156,36 @@ func (h *operatorHandler) Post(w http.ResponseWriter, r *http.Request) {
 			h.r.JSON(w, http.StatusInternalServerError, err.Error())
 			return
 		}
+	case "add-peer":
+		regionID, ok := input["region_id"].(float64)
+		if !ok {
+			h.r.JSON(w, http.StatusBadRequest, "missing region id")
+			return
+		}
+		storeID, ok := input["store_id"].(float64)
+		if !ok {
+			h.r.JSON(w, http.StatusBadRequest, "invalid store id to transfer peer to")
+			return
+		}
+		if err := h.AddAddPeerOperator(uint64(regionID), uint64(storeID)); err != nil {
+			h.r.JSON(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+	case "remove-peer":
+		regionID, ok := input["region_id"].(float64)
+		if !ok {
+			h.r.JSON(w, http.StatusBadRequest, "missing region id")
+			return
+		}
+		storeID, ok := input["store_id"].(float64)
+		if !ok {
+			h.r.JSON(w, http.StatusBadRequest, "invalid store id to transfer peer to")
+			return
+		}
+		if err := h.AddRemovePeerOperator(uint64(regionID), uint64(storeID)); err != nil {
+			h.r.JSON(w, http.StatusInternalServerError, err.Error())
+			return
+		}
 	default:
 		h.r.JSON(w, http.StatusBadRequest, "unknown operator")
 		return
diff --git a/server/cluster_worker.go b/server/cluster_worker.go
index c585ba8f2..8f27fae7f 100644
--- a/server/cluster_worker.go
+++ b/server/cluster_worker.go
@@ -24,7 +24,12 @@ import (
 	"github.com/pingcap/pd/server/core"
 )
 
-func (c *RaftCluster) handleRegionHeartbeat(region *core.RegionInfo) error {
+// HandleRegionHeartbeat processes RegionInfo reports from client.
+func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error {
+	if err := c.cachedCluster.handleRegionHeartbeat(region); err != nil {
+		return errors.Trace(err)
+	}
+
 	// If the region peer count is 0, then we should not handle this.
 	if len(region.GetPeers()) == 0 {
 		log.Warnf("invalid region, zero region peer count - %v", region)
diff --git a/server/grpc_service.go b/server/grpc_service.go
index ebcb1c186..d25999b11 100644
--- a/server/grpc_service.go
+++ b/server/grpc_service.go
@@ -332,14 +332,7 @@ func (s *Server) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error {
 			continue
 		}
 
-		err = cluster.cachedCluster.handleRegionHeartbeat(region)
-		if err != nil {
-			msg := errors.Trace(err).Error()
-			hbStreams.sendErr(region, pdpb.ErrorType_UNKNOWN, msg, storeLabel)
-			continue
-		}
-
-		err = cluster.handleRegionHeartbeat(region)
+		err = cluster.HandleRegionHeartbeat(region)
 		if err != nil {
 			msg := errors.Trace(err).Error()
 			hbStreams.sendErr(region, pdpb.ErrorType_UNKNOWN, msg, storeLabel)
diff --git a/server/handler.go b/server/handler.go
index 0150b31a8..1b74c0620 100644
--- a/server/handler.go
+++ b/server/handler.go
@@ -341,3 +341,54 @@ func (h *Handler) AddTransferPeerOperator(regionID uint64, fromStoreID, toStoreI
 	c.addOperator(op)
 	return nil
 }
+
+// AddAddPeerOperator adds an operator to add peer.
+func (h *Handler) AddAddPeerOperator(regionID uint64, toStoreID uint64) error {
+	c, err := h.getCoordinator()
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	region := c.cluster.GetRegion(regionID)
+	if region == nil {
+		return errRegionNotFound(regionID)
+	}
+
+	if region.GetStorePeer(toStoreID) != nil {
+		return errors.Errorf("region already has peer in store %v", toStoreID)
+	}
+
+	if c.cluster.GetStore(toStoreID) == nil {
+		return core.ErrStoreNotFound(toStoreID)
+	}
+	newPeer, err := c.cluster.AllocPeer(toStoreID)
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	step := schedule.AddPeer{ToStore: toStoreID, PeerID: newPeer.GetId()}
+	op := schedule.NewOperator("adminAddPeer", regionID, core.AdminKind, step)
+	c.addOperator(op)
+	return nil
+}
+
+// AddRemovePeerOperator adds an operator to remove peer.
+func (h *Handler) AddRemovePeerOperator(regionID uint64, fromStoreID uint64) error {
+	c, err := h.getCoordinator()
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	region := c.cluster.GetRegion(regionID)
+	if region == nil {
+		return errRegionNotFound(regionID)
+	}
+
+	if region.GetStorePeer(fromStoreID) == nil {
+		return errors.Errorf("region has no peer in store %v", fromStoreID)
+	}
+
+	op := schedule.CreateRemovePeerOperator("adminRemovePeer", region, fromStoreID)
+	c.addOperator(op)
+	return nil
+}


Test patch:
diff --git a/server/api/history_test.go b/server/api/history_test.go
index 00a12d67c..cdf9d92d7 100644
--- a/server/api/history_test.go
+++ b/server/api/history_test.go
@@ -19,9 +19,7 @@ import (
 	"net/http"
 
 	. "github.com/pingcap/check"
-	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/pd/server"
-	"golang.org/x/net/context"
 )
 
 var _ = Suite(&testHistorySuite{})
@@ -31,9 +29,6 @@ type testHistorySuite struct {
 	cleanup   cleanUpFunc
 	urlPrefix string
 	cli       *http.Client
-
-	grpcPDClient    pdpb.PDClient
-	regionHeartbeat pdpb.PD_RegionHeartbeatClient
 }
 
 func (s *testHistorySuite) SetUpSuite(c *C) {
@@ -46,16 +41,11 @@ func (s *testHistorySuite) SetUpSuite(c *C) {
 	mustBootstrapCluster(c, s.svr)
 	s.cli = newHTTPClient()
 
-	s.grpcPDClient = mustNewGrpcClient(c, s.svr.GetAddr())
-	regionHeartbeat, err := s.grpcPDClient.RegionHeartbeat(context.Background())
-	c.Assert(err, IsNil)
-	s.regionHeartbeat = regionHeartbeat
-
 	r := newTestRegionInfo(2, 1, []byte("a"), []byte("b"))
-	mustRegionHeartBeat(c, s.regionHeartbeat, s.svr.ClusterID(), r)
+	mustRegionHeartbeat(c, s.svr, r)
 
 	r = newTestRegionInfo(3, 1, []byte("b"), []byte("f"))
-	mustRegionHeartBeat(c, s.regionHeartbeat, s.svr.ClusterID(), r)
+	mustRegionHeartbeat(c, s.svr, r)
 }
 
 func (s *testHistorySuite) TearDownSuite(c *C) {
diff --git a/server/api/label_test.go b/server/api/label_test.go
index df97dc550..9000d5792 100644
--- a/server/api/label_test.go
+++ b/server/api/label_test.go
@@ -34,7 +34,7 @@ func (s *testLabelsStoreSuite) SetUpSuite(c *C) {
 	s.stores = []*metapb.Store{
 		{
 			Id:      1,
-			Address: "localhost:1",
+			Address: "tikv1",
 			State:   metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{
 				{
@@ -49,7 +49,7 @@ func (s *testLabelsStoreSuite) SetUpSuite(c *C) {
 		},
 		{
 			Id:      4,
-			Address: "localhost:4",
+			Address: "tikv4",
 			State:   metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{
 				{
@@ -64,7 +64,7 @@ func (s *testLabelsStoreSuite) SetUpSuite(c *C) {
 		},
 		{
 			Id:      6,
-			Address: "localhost:6",
+			Address: "tikv6",
 			State:   metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{
 				{
@@ -79,7 +79,7 @@ func (s *testLabelsStoreSuite) SetUpSuite(c *C) {
 		},
 		{
 			Id:      7,
-			Address: "localhost:7",
+			Address: "tikv7",
 			State:   metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{
 				{
@@ -106,7 +106,7 @@ func (s *testLabelsStoreSuite) SetUpSuite(c *C) {
 
 	mustBootstrapCluster(c, s.svr)
 	for _, store := range s.stores {
-		mustPutStore(c, s.svr, store)
+		mustPutStore(c, s.svr, store.Id, store.State, store.Labels)
 	}
 }
 
diff --git a/server/api/operator_test.go b/server/api/operator_test.go
new file mode 100644
index 000000000..1a4e0edff
--- /dev/null
+++ b/server/api/operator_test.go
@@ -0,0 +1,111 @@
+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	. "github.com/pingcap/check"
+	"github.com/pingcap/kvproto/pkg/metapb"
+	"github.com/pingcap/kvproto/pkg/pdpb"
+	"github.com/pingcap/pd/server"
+	"github.com/pingcap/pd/server/core"
+	"golang.org/x/net/context"
+)
+
+var _ = Suite(&testOperatorSuite{})
+
+type testOperatorSuite struct {
+	svr       *server.Server
+	cleanup   cleanUpFunc
+	urlPrefix string
+}
+
+func (s *testOperatorSuite) SetUpSuite(c *C) {
+	s.svr, s.cleanup = mustNewServer(c)
+	mustWaitLeader(c, []*server.Server{s.svr})
+
+	addr := s.svr.GetAddr()
+	s.urlPrefix = fmt.Sprintf("%s%s/api/v1", addr, apiPrefix)
+
+	mustBootstrapCluster(c, s.svr)
+}
+
+func (s *testOperatorSuite) TearDownSuite(c *C) {
+	s.cleanup()
+}
+
+func (s *testOperatorSuite) TestAddRemovePeer(c *C) {
+	mustPutStore(c, s.svr, 1, metapb.StoreState_Up, nil)
+	mustPutStore(c, s.svr, 2, metapb.StoreState_Up, nil)
+
+	peer1 := &metapb.Peer{Id: 1, StoreId: 1}
+	peer2 := &metapb.Peer{Id: 2, StoreId: 2}
+	region := &metapb.Region{Id: 1, Peers: []*metapb.Peer{peer1, peer2}}
+	regionInfo := core.NewRegionInfo(region, peer1)
+	mustRegionHeartbeat(c, s.svr, regionInfo)
+
+	regionURL := fmt.Sprintf("%s/operators/%d", s.urlPrefix, region.Id)
+	operator := mustReadURL(c, regionURL)
+	c.Assert(strings.Contains(operator, "operator not found"), IsTrue)
+
+	mustPutStore(c, s.svr, 3, metapb.StoreState_Up, nil)
+	err := postJSON(&http.Client{}, fmt.Sprintf("%s/operators", s.urlPrefix), []byte(`{"name":"add-peer", "region_id": 1, "store_id": 3}`))
+	c.Assert(err, IsNil)
+	operator = mustReadURL(c, regionURL)
+	c.Assert(strings.Contains(operator, "add peer 1 on store 3"), IsTrue)
+
+	req, err := http.NewRequest("DELETE", regionURL, nil)
+	c.Assert(err, IsNil)
+	res, err := http.DefaultClient.Do(req)
+	c.Assert(err, IsNil)
+	res.Body.Close()
+
+	err = postJSON(&http.Client{}, fmt.Sprintf("%s/operators", s.urlPrefix), []byte(`{"name":"remove-peer", "region_id": 1, "store_id": 2}`))
+	c.Assert(err, IsNil)
+	operator = mustReadURL(c, regionURL)
+	c.Log(operator)
+	c.Assert(strings.Contains(operator, "remove peer on store 2"), IsTrue)
+}
+
+func mustPutStore(c *C, svr *server.Server, id uint64, state metapb.StoreState, labels []*metapb.StoreLabel) {
+	_, err := svr.PutStore(context.Background(), &pdpb.PutStoreRequest{
+		Header: &pdpb.RequestHeader{ClusterId: svr.ClusterID()},
+		Store: &metapb.Store{
+			Id:      id,
+			Address: fmt.Sprintf("tikv%d", id),
+			State:   state,
+			Labels:  labels,
+		},
+	})
+	c.Assert(err, IsNil)
+}
+
+func mustRegionHeartbeat(c *C, svr *server.Server, region *core.RegionInfo) {
+	cluster := svr.GetRaftCluster()
+	err := cluster.HandleRegionHeartbeat(region)
+	c.Assert(err, IsNil)
+}
+
+func mustReadURL(c *C, url string) string {
+	res, err := http.Get(url)
+	c.Assert(err, IsNil)
+	defer res.Body.Close()
+	data, err := ioutil.ReadAll(res.Body)
+	c.Assert(err, IsNil)
+	return string(data)
+}
diff --git a/server/api/region_test.go b/server/api/region_test.go
index dd1f0d09b..59b0feb4e 100644
--- a/server/api/region_test.go
+++ b/server/api/region_test.go
@@ -21,7 +21,6 @@ import (
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/pd/server"
 	"github.com/pingcap/pd/server/core"
-	"golang.org/x/net/context"
 )
 
 var _ = Suite(&testRegionSuite{})
@@ -30,9 +29,6 @@ type testRegionSuite struct {
 	svr       *server.Server
 	cleanup   cleanUpFunc
 	urlPrefix string
-
-	grpcPDClient    pdpb.PDClient
-	regionHeartbeat pdpb.PD_RegionHeartbeatClient
 }
 
 func (s *testRegionSuite) SetUpSuite(c *C) {
@@ -43,11 +39,6 @@ func (s *testRegionSuite) SetUpSuite(c *C) {
 	s.urlPrefix = fmt.Sprintf("%s%s/api/v1", addr, apiPrefix)
 
 	mustBootstrapCluster(c, s.svr)
-
-	s.grpcPDClient = mustNewGrpcClient(c, s.svr.GetAddr())
-	regionHeartbeat, err := s.grpcPDClient.RegionHeartbeat(context.Background())
-	c.Assert(err, IsNil)
-	s.regionHeartbeat = regionHeartbeat
 }
 
 func (s *testRegionSuite) TearDownSuite(c *C) {
@@ -75,7 +66,7 @@ func newTestRegionInfo(regionID, storeID uint64, start, end []byte) *core.Region
 
 func (s *testRegionSuite) TestRegion(c *C) {
 	r := newTestRegionInfo(2, 1, []byte("a"), []byte("b"))
-	mustRegionHeartBeat(c, s.regionHeartbeat, s.svr.ClusterID(), r)
+	mustRegionHeartbeat(c, s.svr, r)
 	url := fmt.Sprintf("%s/region/id/%d", s.urlPrefix, r.GetId())
 	r1 := &core.RegionInfo{}
 	err := readJSONWithURL(url, r1)
diff --git a/server/api/server_test.go b/server/api/server_test.go
index 5d0ef91ff..12800cb81 100644
--- a/server/api/server_test.go
+++ b/server/api/server_test.go
@@ -24,7 +24,6 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/pd/server"
-	"github.com/pingcap/pd/server/core"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
@@ -147,31 +146,6 @@ func mustBootstrapCluster(c *C, s *server.Server) {
 	c.Assert(resp.GetHeader().GetError().GetType(), Equals, pdpb.ErrorType_OK)
 }
 
-func mustPutStore(c *C, s *server.Server, store *metapb.Store) {
-	grpcPDClient := mustNewGrpcClient(c, s.GetAddr())
-	req := &pdpb.PutStoreRequest{
-		Header: newRequestHeader(s.ClusterID()),
-		Store:  store,
-	}
-	resp, err := grpcPDClient.PutStore(context.Background(), req)
-	c.Assert(err, IsNil)
-	c.Assert(resp.GetHeader().GetError().GetType(), Equals, pdpb.ErrorType_OK)
-}
-
-func mustRegionHeartBeat(c *C, client pdpb.PD_RegionHeartbeatClient, clusterID uint64, region *core.RegionInfo) {
-	req := &pdpb.RegionHeartbeatRequest{
-		Header: newRequestHeader(clusterID),
-		Region: region.Region,
-		Leader: region.Leader,
-	}
-
-	err := client.Send(req)
-	c.Assert(err, IsNil)
-
-	// Sleep a while to make sure the message is processed by server.
-	time.Sleep(time.Millisecond * 200)
-}
-
 func readJSONWithURL(url string, data interface{}) error {
 	resp, err := http.Get(url)
 	if err != nil {
diff --git a/server/api/store_ns_test.go b/server/api/store_ns_test.go
index 1f11354ac..e6b505d35 100644
--- a/server/api/store_ns_test.go
+++ b/server/api/store_ns_test.go
@@ -37,24 +37,24 @@ func (s *testStoreNsSuite) SetUpSuite(c *C) {
 		{
 			// metapb.StoreState_Up == 0
 			Id:      1,
-			Address: "localhost:1",
+			Address: "tikv:1",
 			State:   metapb.StoreState_Up,
 		},
 		{
 			Id:      4,
-			Address: "localhost:4",
+			Address: "tikv:4",
 			State:   metapb.StoreState_Up,
 		},
 		{
 			// metapb.StoreState_Offline == 1
 			Id:      6,
-			Address: "localhost:6",
+			Address: "tikv:6",
 			State:   metapb.StoreState_Offline,
 		},
 		{
 			// metapb.StoreState_Tombstone == 2
 			Id:      7,
-			Address: "localhost:7",
+			Address: "tikv:7",
 			State:   metapb.StoreState_Tombstone,
 		},
 	}
@@ -77,7 +77,7 @@ func (s *testStoreNsSuite) SetUpSuite(c *C) {
 
 	mustBootstrapCluster(c, s.svr)
 	for _, store := range s.stores {
-		mustPutStore(c, s.svr, store)
+		mustPutStore(c, s.svr, store.Id, store.State, nil)
 	}
 }
 
diff --git a/server/api/store_test.go b/server/api/store_test.go
index 0d8ee643c..76cc347ed 100644
--- a/server/api/store_test.go
+++ b/server/api/store_test.go
@@ -42,24 +42,24 @@ func (s *testStoreSuite) SetUpSuite(c *C) {
 		{
 			// metapb.StoreState_Up == 0
 			Id:      1,
-			Address: "localhost:1",
+			Address: "tikv1",
 			State:   metapb.StoreState_Up,
 		},
 		{
 			Id:      4,
-			Address: "localhost:4",
+			Address: "tikv4",
 			State:   metapb.StoreState_Up,
 		},
 		{
 			// metapb.StoreState_Offline == 1
 			Id:      6,
-			Address: "localhost:6",
+			Address: "tikv6",
 			State:   metapb.StoreState_Offline,
 		},
 		{
 			// metapb.StoreState_Tombstone == 2
 			Id:      7,
-			Address: "localhost:7",
+			Address: "tikv7",
 			State:   metapb.StoreState_Tombstone,
 		},
 	}
@@ -71,8 +71,9 @@ func (s *testStoreSuite) SetUpSuite(c *C) {
 	s.urlPrefix = fmt.Sprintf("%s%s/api/v1", addr, apiPrefix)
 
 	mustBootstrapCluster(c, s.svr)
+
 	for _, store := range s.stores {
-		mustPutStore(c, s.svr, store)
+		mustPutStore(c, s.svr, store.Id, store.State, nil)
 	}
 }
 
diff --git a/server/cluster_test.go b/server/cluster_test.go
index 6e3a90095..2f7a936cb 100644
--- a/server/cluster_test.go
+++ b/server/cluster_test.go
@@ -430,7 +430,7 @@ func (s *testClusterSuite) testCheckStores(c *C, clusterID uint64) {
 	leader := &metapb.Peer{StoreId: store.GetId()}
 	region := s.newRegion(c, 0, []byte{'a'}, []byte{'b'}, []*metapb.Peer{leader}, nil)
 	regionInfo := core.NewRegionInfo(region, leader)
-	err := cluster.handleRegionHeartbeat(regionInfo)
+	err := cluster.HandleRegionHeartbeat(regionInfo)
 	c.Assert(err, IsNil)
 	c.Assert(cluster.cachedCluster.getStoreRegionCount(store.GetId()), Equals, 1)
 
@@ -452,7 +452,7 @@ func (s *testClusterSuite) testCheckStores(c *C, clusterID uint64) {
 	// Clear store's region peers.
 	leader.StoreId = 0
 	region.Peers = []*metapb.Peer{leader}
-	err = cluster.handleRegionHeartbeat(regionInfo)
+	err = cluster.HandleRegionHeartbeat(regionInfo)
 	c.Assert(err, IsNil)
 	c.Assert(cluster.cachedCluster.getStoreRegionCount(store.GetId()), Equals, 0)