Repository: tikv/client-go
Pull request #579 (2022-09-05 07:10:10)

*: introduce the non-retryable error FlashbackInProgress
Signed-off-by: JmPotato <ghzpotato@gmail.com>

Ref https://github.com/tikv/tikv/issues/13397.

JmPotato: /cc @Connor1996 @sticnarf
JmPotato: > What if it's in a rolling upgrade process and the old client-go ignores the new error? Is it assumed to be impossible?

Currently, a flashback SQL will be a cluster-level operation, and I think this scenario is unlikely to occur in real-world use.
JmPotato: > What if it's in a rolling upgrade process and the old client-go ignores the new error? Is it assumed to be impossible?

For now, Flashback is a cluster-level operation, and I think this scenario is unlikely to occur in real-world use. Even if it happens, since a region in flashback progress will block all read and writes, so it's not a big deal.
sticnarf: LGTM. The new proto doesn't seem to break old code. I think you can merge the proto and remove the replace.

Patch:
diff --git a/error/error.go b/error/error.go
index a3b860a..671f669 100644
--- a/error/error.go
+++ b/error/error.go
@@ -89,6 +89,8 @@ var (
 	ErrTiKVDiskFull = errors.New("tikv disk full")
 	// ErrRegionRecoveryInProgress is the error when region is recovering.
 	ErrRegionRecoveryInProgress = errors.New("region is being online unsafe recovered")
+	// ErrRegionFlashbackInProgress is the error when a region in the flashback progress receive any other request.
+	ErrRegionFlashbackInProgress = errors.New("region is in the flashback progress")
 	// ErrUnknown is the unknow error.
 	ErrUnknown = errors.New("unknow")
 	// ErrResultUndetermined is the error when execution result is unknown.
diff --git a/go.mod b/go.mod
index 5ada711..530bc0b 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00
 	github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989
-	github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331
+	github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43
 	github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
diff --git a/go.sum b/go.sum
index 702e1e2..b05d8c4 100644
--- a/go.sum
+++ b/go.sum
@@ -155,8 +155,8 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB
 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
 github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
-github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331 h1:LLPNqaM0C0hHER4U+cl17pvVBFuSDBbVl2oJQ6f6bg4=
-github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 h1:5q7Ns0R7q6Uj+fpa3lDTijrcqgId4lNdGa2AG7izB5c=
+github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM=
 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go
index 18b88ed..d4470b0 100644
--- a/internal/locate/region_request.go
+++ b/internal/locate/region_request.go
@@ -1389,6 +1389,8 @@ func regionErrorToLabel(e *errorpb.Error) string {
 		return "disk_full"
 	} else if e.GetRecoveryInProgress() != nil {
 		return "recovery_in_progress"
+	} else if e.GetFlashbackInProgress() != nil {
+		return "flashback_in_progress"
 	}
 	return "unknown"
 }
@@ -1446,6 +1448,14 @@ func (s *RegionRequestSender) onRegionError(bo *retry.Backoffer, ctx *RPCContext
 		return false, nil
 	}
 
+	// Since we expect that the workload should be stopped during the flashback progress,
+	// if a request meets the FlashbackInProgress error, it should stop retrying immediately
+	// to avoid unnecessary backoff and potential unexpected data status to the user.
+	if regionErr.GetFlashbackInProgress() != nil {
+		logutil.BgLogger().Debug("tikv reports `FlashbackInProgress`", zap.Stringer("req", req), zap.Stringer("ctx", ctx))
+		return false, errors.Errorf("region %d is in flashback progress", regionErr.GetFlashbackInProgress().GetRegionId())
+	}
+
 	// This peer is removed from the region. Invalidate the region since it's too stale.
 	if regionErr.GetRegionNotFound() != nil {
 		s.regionCache.InvalidateCachedRegion(ctx.Region)


Test patch:
diff --git a/integration_tests/go.mod b/integration_tests/go.mod
index c7abc25..88fc64b 100644
--- a/integration_tests/go.mod
+++ b/integration_tests/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/ninedraft/israce v0.0.3
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3
-	github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331
+	github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43
 	github.com/pingcap/tidb v1.1.0-beta.0.20220902042024-0482b2e83ed2
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.0
diff --git a/integration_tests/go.sum b/integration_tests/go.sum
index e9dbbd6..00f7a85 100644
--- a/integration_tests/go.sum
+++ b/integration_tests/go.sum
@@ -301,10 +301,8 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZL
 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
 github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
-github.com/pingcap/kvproto v0.0.0-20220818023518-a0f02b6efcee h1:s1al2ci3MEj5VnNuUCGAfeqpbCxcMeZibOyxw8ClHLE=
-github.com/pingcap/kvproto v0.0.0-20220818023518-a0f02b6efcee/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
-github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331 h1:LLPNqaM0C0hHER4U+cl17pvVBFuSDBbVl2oJQ6f6bg4=
-github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 h1:5q7Ns0R7q6Uj+fpa3lDTijrcqgId4lNdGa2AG7izB5c=
+github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pingcap/log v1.1.0 h1:ELiPxACz7vdo1qAvvaWJg1NrYFoY6gqAh/+Uo6aXdD8=
diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go
index 0914288..342e07d 100644
--- a/internal/locate/region_cache_test.go
+++ b/internal/locate/region_cache_test.go
@@ -1592,3 +1592,16 @@ func (s *testRegionCacheSuite) TestRemoveIntersectingRegions() {
 	s.Equal(loc.Region.GetID(), regions[0])
 	s.checkCache(1)
 }
+
+func (s *testRegionCacheSuite) TestShouldNotRetryFlashbackInProgress() {
+	loc, err := s.cache.LocateKey(s.bo, []byte("a"))
+	s.NotNil(loc)
+	s.NoError(err)
+	ctx, err := s.cache.GetTiKVRPCContext(retry.NewBackofferWithVars(context.Background(), 100, nil), loc.Region, kv.ReplicaReadLeader, 0)
+	s.NotNil(ctx)
+	s.NoError(err)
+	reqSend := NewRegionRequestSender(s.cache, nil)
+	shouldRetry, err := reqSend.onRegionError(s.bo, ctx, nil, &errorpb.Error{FlashbackInProgress: &errorpb.FlashbackInProgress{}})
+	s.Error(err)
+	s.False(shouldRetry)
+}