Repository: tikv/client-go (go)

Issue:
TiDB keep warning about "receive a grpc cancel signal"
Version: [tidb-cse-7.5](https://github.com/tikv/client-go/tree/tidb-cse-7.5), 9676f18f014b55da050a7799df9b095176fa600a

From logs of TiDB, there were lots of "receive a grpc cancel signal":

```
2024/01/23 10:13:36.034 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
2024/01/23 10:13:36.113 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
2024/01/23 10:13:36.258 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
2024/01/23 10:13:36.571 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:37.107 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:38.429 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:39.559 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:41.171 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:42.870 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:44.476 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:45.979 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:47.112 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:48.236 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:49.407 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:50.681 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:52.497 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:53.572 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:54.952 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:56.354 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:58.269 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21003] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:59.362 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
```

### Cause Analysis

From debug logs, we can see that they were of the same session:

```
[2024/01/23 10:13:36.034 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:36.034 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:36.035 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:36.035 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:36.035 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:36.035 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:36.037 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=74] [attempts=0]
[2024/01/23 10:13:36.111 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=1960] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:36.113 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:36.115 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:36.115 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:36.115 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:36.115 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:36.115 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:36.116 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=139] [attempts=1]
[2024/01/23 10:13:36.256 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=2099] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:36.258 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:36.260 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:36.260 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:36.260 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:36.260 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:36.262 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:36.263 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=304] [attempts=2]
[2024/01/23 10:13:36.570 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=2403] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:36.571 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:36.573 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:36.573 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:36.573 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:36.573 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:36.574 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:36.575 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=524] [attempts=3]
[2024/01/23 10:13:37.102 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=2927] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:37.107 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:37.110 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:37.110 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:37.110 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:37.110 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:37.111 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:37.114 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1313] [attempts=4]
[2024/01/23 10:13:37.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:37.860 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:38.428 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=4240] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:38.429 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:38.431 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:38.431 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:38.431 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:38.431 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:38.431 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:38.435 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1120] [attempts=5]
[2024/01/23 10:13:39.558 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=5360] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:39.559 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:39.562 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:39.562 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:39.562 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:39.562 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:39.562 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:39.566 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1600] [attempts=6]
[2024/01/23 10:13:39.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:39.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:41.169 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=6960] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:41.171 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:41.171 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:41.171 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:41.171 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:41.171 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:41.171 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:41.172 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1695] [attempts=7]
[2024/01/23 10:13:41.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:41.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:42.868 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=8655] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:42.870 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:42.872 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:42.873 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:42.873 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:42.873 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:42.873 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:42.874 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1576] [attempts=8]
[2024/01/23 10:13:43.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:43.859 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:44.452 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:46 store_id:1 , addr: 127.0.0.1:21001, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=10231] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:44.456 +00:00] [DEBUG] [region_request.go:2121] ["tikv reports `EpochNotMatch` retry later"] [keyspaceName=ks1] [keyspaceID=1] [EpochNotMatch="current_regions:<id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > > "] [ctx="region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:5 version:9 > peers:<id:46 store_id:1 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv"]
[2024/01/23 10:13:44.456 +00:00] [INFO] [region_request.go:1533] ["send request meet region error without retry"] [keyspaceName=ks1] [keyspaceID=1] [req-ts=0] [req-type=SplitRegion] [region="{ region id: 45, ver: 9, confVer: 5 }"] [replica-read-type=leader] [stale-read=false] [request-sender="{rpcError:rpc error: code = Canceled desc = CANCELLED,replicaSelector: replicaSelector{selectorStateStr: tryFollower, cacheRegionIsValid: false, replicaStatus: [peer: 46, store: 1, isEpochStale: false, attempts: 10, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 47, store: 4, isEpochStale: false, attempts: 1, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable peer: 48, store: 5, isEpochStale: false, attempts: 0, replica-epoch: 0, store-epoch: 0, store-state: resolved, store-liveness-state: reachable]}}"] [retry-times=10] [total-backoff-ms=10231] [total-backoff-times=15] [max-exec-timeout-ms=30000] [total-region-errors="46-server_is_busy:1, 47-epoch_not_match:1"]
[2024/01/23 10:13:44.456 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=2] [sleep=8] [attempts=2]
[2024/01/23 10:13:44.465 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="message:\"EpochNotMatch [4:45:10] epoch changed conf_ver: 18 version: 10 != conf_ver: 5 version: 9, retry later\" epoch_not_match:<current_regions:<id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > > > "] [totalSleep=10239] [excludedSleep=1866] [maxSleep=40000] [type=regionMiss] [txnStartTS=null]
[2024/01/23 10:13:44.476 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:44.478 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:44.478 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:44.478 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:44.479 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:44.479 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:44.481 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1496] [attempts=9]
[2024/01/23 10:13:45.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:45.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:45.978 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=11735] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:45.979 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:45.979 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:45.979 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:45.980 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:45.980 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:45.982 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:45.983 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1125] [attempts=10]
[2024/01/23 10:13:47.111 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=12860] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:47.112 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:47.113 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:47.113 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:47.113 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:47.116 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:47.116 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:47.118 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1114] [attempts=11]
[2024/01/23 10:13:47.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:47.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:48.235 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=13974] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:48.236 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:48.236 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:48.236 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:48.236 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:48.236 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:48.237 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:48.238 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1148] [attempts=12]
[2024/01/23 10:13:49.389 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=15122] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:49.407 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:49.409 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:49.409 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:49.409 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:49.409 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:49.410 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:49.410 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1267] [attempts=13]
[2024/01/23 10:13:49.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:49.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:50.678 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=16389] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:50.681 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:50.681 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:50.681 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:50.681 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:50.681 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:50.681 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:50.682 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1813] [attempts=14]
[2024/01/23 10:13:51.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:51.861 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:52.496 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=18202] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:52.497 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:52.497 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:52.497 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:52.498 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:52.498 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:52.498 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:52.499 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1071] [attempts=15]
[2024/01/23 10:13:53.571 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=19273] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:53.572 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:53.608 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:53.608 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:53.608 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:53.609 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:53.609 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:53.610 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1340] [attempts=16]
[2024/01/23 10:13:53.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:53.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:54.951 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=20613] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:54.952 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:54.953 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:54.953 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:54.953 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:54.953 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:54.955 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:54.956 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1392] [attempts=17]
[2024/01/23 10:13:55.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:55.909 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:56.350 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=22005] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:56.354 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21002] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:56.354 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002]
[2024/01/23 10:13:56.354 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-0] []
[2024/01/23 10:13:56.354 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-1] []
[2024/01/23 10:13:56.354 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-2] []
[2024/01/23 10:13:56.354 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21002-3] []
[2024/01/23 10:13:56.355 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1913] [attempts=18]
[2024/01/23 10:13:57.857 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:57.858 +00:00] [DEBUG] [pd.go:108] ["min resolved ts"] [keyspaceName=ks1] [keyspaceID=1] [resp="{\n  \"is_real_time\": true,\n  \"min_resolved_ts\": 0,\n  \"persist_interval\": \"1s\"\n}\n"]
[2024/01/23 10:13:58.268 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:47 store_id:4 , addr: 127.0.0.1:21002, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=23918] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:58.269 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21003] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:58.269 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21003]
[2024/01/23 10:13:58.269 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21003-0] []
[2024/01/23 10:13:58.270 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21003-1] []
[2024/01/23 10:13:58.271 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21003-2] []
[2024/01/23 10:13:58.272 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21003-3] []
[2024/01/23 10:13:58.273 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1087] [attempts=19]
[2024/01/23 10:13:59.361 +00:00] [DEBUG] [backoff.go:230] ["retry later"] [keyspaceName=ks1] [keyspaceID=1] [error="send tikv request error: rpc error: code = Canceled desc = CANCELLED, ctx: region ID: 45, meta: id:45 end_key:\"t\\200\\000\\377\\377\\377\\377\\377\\375\" region_epoch:<conf_ver:18 version:10 > peers:<id:47 store_id:4 > peers:<id:48 store_id:5 > peers:<id:60 store_id:1 > , peer: id:48 store_id:5 , addr: 127.0.0.1:21003, idx: 0, reqStoreType: TiKvOnly, runStoreType: tikv, try next peer later"] [totalSleep=25005] [excludedSleep=1866] [maxSleep=40000] [type=tikvRPC] [txnStartTS=null]
[2024/01/23 10:13:59.362 +00:00] [WARN] [region_request.go:1839] ["receive a grpc cancel signal"] [keyspaceName=ks1] [keyspaceID=1] [addr=127.0.0.1:21001] [error="rpc error: code = Canceled desc = CANCELLED"]
[2024/01/23 10:13:59.363 +00:00] [DEBUG] [client.go:800] ["close connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001]
[2024/01/23 10:13:59.363 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-0] []
[2024/01/23 10:13:59.363 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-1] []
[2024/01/23 10:13:59.365 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-2] []
[2024/01/23 10:13:59.365 +00:00] [DEBUG] [client.go:226] ["close gRPC connection"] [keyspaceName=ks1] [keyspaceID=1] [target=127.0.0.1:21001-3] []
[2024/01/23 10:13:59.368 +00:00] [DEBUG] [config.go:180] [backoff] [keyspaceName=ks1] [keyspaceID=1] [base=100] [sleep=1073] [attempts=20]
```

As we [`CloseAddr`](https://github.com/tikv/client-go/blob/9676f18f014b55da050a7799df9b095176fa600a/internal/locate/region_request.go#L1840) when meet "receive a grpc cancel signal" (see #1121), the `ctx.Addr` must be not correct. The [`sendToAddr`](https://github.com/tikv/client-go/blob/9676f18f014b55da050a7799df9b095176fa600a/internal/locate/region_request.go#L1657) may not be the same with `ctx.Addr`.

Pull request description:
Use correct address for CloseAddr
Close #1141 

### Changes

- Add `ErrConn` to wrap connection error and get target address and version of the `connArray` on which raise the error.
- Use target address from `ErrConn` for `CloseAddr`
- Add `ver` to `connArray` to avoid unnecessary closing due to concurrent read/write on `connArray`.

Patch:
diff --git a/internal/client/client.go b/internal/client/client.go
index 05d73bf..1b9f41f 100644
--- a/internal/client/client.go
+++ b/internal/client/client.go
@@ -109,9 +109,45 @@ type Client interface {
 	SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error)
 }
 
+// ClientExt is a client has extended interfaces.
+type ClientExt interface {
+	// CloseAddrVer closes gRPC connections to the address with additional `ver` parameter.
+	// Each new connection will have an incremented `ver` value, and attempts to close a previous `ver` will be ignored.
+	// Passing `math.MaxUint64` as the `ver` parameter will forcefully close all connections to the address.
+	CloseAddrVer(addr string, ver uint64) error
+}
+
+// ErrConn wraps error with target address and version of the connection.
+type ErrConn struct {
+	Err  error
+	Addr string
+	Ver  uint64
+}
+
+func (e *ErrConn) Error() string {
+	return fmt.Sprintf("[%s](%d) %s", e.Addr, e.Ver, e.Err.Error())
+}
+
+func (e *ErrConn) Unwrap() error {
+	return e.Err
+}
+
+func WrapErrConn(err error, conn *connArray) error {
+	if err == nil {
+		return nil
+	}
+	return &ErrConn{
+		Err:  err,
+		Addr: conn.target,
+		Ver:  conn.ver,
+	}
+}
+
 type connArray struct {
 	// The target host.
 	target string
+	// version of the connection array, increase by 1 when reconnect.
+	ver uint64
 
 	index uint32
 	v     []*monitoredConn
@@ -125,9 +161,10 @@ type connArray struct {
 	monitor *connMonitor
 }
 
-func newConnArray(maxSize uint, addr string, security config.Security,
+func newConnArray(maxSize uint, addr string, ver uint64, security config.Security,
 	idleNotify *uint32, enableBatch bool, dialTimeout time.Duration, m *connMonitor, opts []grpc.DialOption) (*connArray, error) {
 	a := &connArray{
+		ver:           ver,
 		index:         0,
 		v:             make([]*monitoredConn, maxSize),
 		streamTimeout: make(chan *tikvrpc.Lease, 1024),
@@ -390,6 +427,7 @@ type RPCClient struct {
 	sync.RWMutex
 
 	conns  map[string]*connArray
+	vers   map[string]uint64
 	option *option
 
 	idleNotify uint32
@@ -405,6 +443,7 @@ type RPCClient struct {
 func NewRPCClient(opts ...Opt) *RPCClient {
 	cli := &RPCClient{
 		conns: make(map[string]*connArray),
+		vers:  make(map[string]uint64),
 		option: &option{
 			dialTimeout: dialTimeout,
 		},
@@ -452,9 +491,11 @@ func (c *RPCClient) createConnArray(addr string, enableBatch bool, opts ...func(
 		for _, opt := range opts {
 			opt(&client)
 		}
+		ver := c.vers[addr] + 1
 		array, err = newConnArray(
 			client.GrpcConnectionCount,
 			addr,
+			ver,
 			c.option.security,
 			&c.idleNotify,
 			enableBatch,
@@ -466,6 +507,7 @@ func (c *RPCClient) createConnArray(addr string, enableBatch bool, opts ...func(
 			return nil, err
 		}
 		c.conns[addr] = array
+		c.vers[addr] = ver
 	}
 	return array, nil
 }
@@ -603,6 +645,10 @@ func (c *RPCClient) sendRequest(ctx context.Context, addr string, req *tikvrpc.R
 		return nil, err
 	}
 
+	wrapErrConn := func(resp *tikvrpc.Response, err error) (*tikvrpc.Response, error) {
+		return resp, WrapErrConn(err, connArray)
+	}
+
 	start := time.Now()
 	staleRead := req.GetStaleRead()
 	defer func() {
@@ -625,7 +671,7 @@ func (c *RPCClient) sendRequest(ctx context.Context, addr string, req *tikvrpc.R
 	if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 && enableBatch {
 		if batchReq := req.ToBatchCommandsRequest(); batchReq != nil {
 			defer trace.StartRegion(ctx, req.Type.String()).End()
-			return sendBatchRequest(ctx, addr, req.ForwardedHost, connArray.batchConn, batchReq, timeout)
+			return wrapErrConn(sendBatchRequest(ctx, addr, req.ForwardedHost, connArray.batchConn, batchReq, timeout))
 		}
 	}
 
@@ -639,7 +685,7 @@ func (c *RPCClient) sendRequest(ctx context.Context, addr string, req *tikvrpc.R
 		client := debugpb.NewDebugClient(clientConn)
 		ctx1, cancel := context.WithTimeout(ctx, timeout)
 		defer cancel()
-		return tikvrpc.CallDebugRPC(ctx1, client, req)
+		return wrapErrConn(tikvrpc.CallDebugRPC(ctx1, client, req))
 	}
 
 	client := tikvpb.NewTikvClient(clientConn)
@@ -650,16 +696,16 @@ func (c *RPCClient) sendRequest(ctx context.Context, addr string, req *tikvrpc.R
 	}
 	switch req.Type {
 	case tikvrpc.CmdBatchCop:
-		return c.getBatchCopStreamResponse(ctx, client, req, timeout, connArray)
+		return wrapErrConn(c.getBatchCopStreamResponse(ctx, client, req, timeout, connArray))
 	case tikvrpc.CmdCopStream:
-		return c.getCopStreamResponse(ctx, client, req, timeout, connArray)
+		return wrapErrConn(c.getCopStreamResponse(ctx, client, req, timeout, connArray))
 	case tikvrpc.CmdMPPConn:
-		return c.getMPPStreamResponse(ctx, client, req, timeout, connArray)
+		return wrapErrConn(c.getMPPStreamResponse(ctx, client, req, timeout, connArray))
 	}
 	// Or else it's a unary call.
 	ctx1, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	return tikvrpc.CallRPC(ctx1, client, req)
+	return wrapErrConn(tikvrpc.CallRPC(ctx1, client, req))
 }
 
 // SendRequest sends a Request to server and receives Response.
@@ -793,11 +839,20 @@ func (c *RPCClient) Close() error {
 
 // CloseAddr closes gRPC connections to the address.
 func (c *RPCClient) CloseAddr(addr string) error {
+	return c.CloseAddrVer(addr, math.MaxUint64)
+}
+
+func (c *RPCClient) CloseAddrVer(addr string, ver uint64) error {
 	c.Lock()
 	conn, ok := c.conns[addr]
 	if ok {
-		delete(c.conns, addr)
-		logutil.BgLogger().Debug("close connection", zap.String("target", addr))
+		if conn.ver <= ver {
+			delete(c.conns, addr)
+			logutil.BgLogger().Debug("close connection", zap.String("target", addr), zap.Uint64("ver", ver), zap.Uint64("conn.ver", conn.ver))
+		} else {
+			logutil.BgLogger().Debug("ignore close connection", zap.String("target", addr), zap.Uint64("ver", ver), zap.Uint64("conn.ver", conn.ver))
+			conn = nil
+		}
 	}
 	c.Unlock()
 
diff --git a/internal/client/client_batch.go b/internal/client/client_batch.go
index 49b8fe0..960538d 100644
--- a/internal/client/client_batch.go
+++ b/internal/client/client_batch.go
@@ -795,6 +795,9 @@ func sendBatchRequest(
 		logutil.Logger(ctx).Debug("send request is cancelled",
 			zap.String("to", addr), zap.String("cause", ctx.Err().Error()))
 		return nil, errors.WithStack(ctx.Err())
+	case <-batchConn.closed:
+		logutil.Logger(ctx).Debug("send request is cancelled (batchConn closed)", zap.String("to", addr))
+		return nil, errors.New("batchConn closed")
 	case <-timer.C:
 		return nil, errors.WithMessage(context.DeadlineExceeded, "wait sendLoop")
 	}
@@ -812,6 +815,10 @@ func sendBatchRequest(
 		logutil.Logger(ctx).Debug("wait response is cancelled",
 			zap.String("to", addr), zap.String("cause", ctx.Err().Error()))
 		return nil, errors.WithStack(ctx.Err())
+	case <-batchConn.closed:
+		atomic.StoreInt32(&entry.canceled, 1)
+		logutil.Logger(ctx).Debug("wait response is cancelled (batchConn closed)", zap.String("to", addr))
+		return nil, errors.New("batchConn closed")
 	case <-timer.C:
 		atomic.StoreInt32(&entry.canceled, 1)
 		reason := fmt.Sprintf("wait recvLoop timeout,timeout:%s, wait_duration:%s:", timeout, waitDuration)
@@ -823,16 +830,18 @@ func (c *RPCClient) recycleIdleConnArray() {
 	start := time.Now()
 
 	var addrs []string
+	var vers []uint64
 	c.RLock()
 	for _, conn := range c.conns {
 		if conn.batchConn != nil && conn.isIdle() {
 			addrs = append(addrs, conn.target)
+			vers = append(vers, conn.ver)
 		}
 	}
 	c.RUnlock()
 
-	for _, addr := range addrs {
-		c.CloseAddr(addr)
+	for i, addr := range addrs {
+		c.CloseAddrVer(addr, vers[i])
 	}
 
 	metrics.TiKVBatchClientRecycle.Observe(time.Since(start).Seconds())
diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go
index dbd9836..727ec08 100644
--- a/internal/locate/region_request.go
+++ b/internal/locate/region_request.go
@@ -215,6 +215,14 @@ func (s *RegionRequestSender) GetClient() client.Client {
 	return s.client
 }
 
+// getClientExt returns the client with ClientExt interface.
+// Return nil if the client does not implement ClientExt.
+// Don't use in critical path.
+func (s *RegionRequestSender) getClientExt() client.ClientExt {
+	ext, _ := s.client.(client.ClientExt)
+	return ext
+}
+
 // SetStoreAddr specifies the dest store address.
 func (s *RegionRequestSender) SetStoreAddr(addr string) {
 	s.storeAddr = addr
@@ -1837,7 +1845,14 @@ func (s *RegionRequestSender) onSendFail(bo *retry.Backoffer, ctx *RPCContext, r
 			// Canceled by gRPC remote may happen when tikv is killed and exiting.
 			// Close the connection, backoff, and retry.
 			logutil.Logger(bo.GetCtx()).Warn("receive a grpc cancel signal", zap.Error(err))
-			s.client.CloseAddr(ctx.Addr)
+			var errConn *client.ErrConn
+			if errors.As(err, &errConn) {
+				if ext := s.getClientExt(); ext != nil {
+					ext.CloseAddrVer(errConn.Addr, errConn.Ver)
+				} else {
+					s.client.CloseAddr(errConn.Addr)
+				}
+			}
 		}
 	}
 


Test patch:
diff --git a/internal/client/client_test.go b/internal/client/client_test.go
index 2934c12..143ccf9 100644
--- a/internal/client/client_test.go
+++ b/internal/client/client_test.go
@@ -79,12 +79,18 @@ func TestConn(t *testing.T) {
 	assert.Nil(t, err)
 	assert.False(t, conn2.Get() == conn1.Get())
 
-	assert.Nil(t, client.CloseAddr(addr))
+	ver := conn2.ver
+	assert.Nil(t, client.CloseAddrVer(addr, ver-1))
 	_, ok := client.conns[addr]
+	assert.True(t, ok)
+	assert.Nil(t, client.CloseAddrVer(addr, ver))
+	_, ok = client.conns[addr]
 	assert.False(t, ok)
+
 	conn3, err := client.getConnArray(addr, true)
 	assert.Nil(t, err)
 	assert.NotNil(t, conn3)
+	assert.Equal(t, ver+1, conn3.ver)
 
 	client.Close()
 	conn4, err := client.getConnArray(addr, true)
@@ -135,7 +141,7 @@ func TestSendWhenReconnect(t *testing.T) {
 
 	req := tikvrpc.NewRequest(tikvrpc.CmdEmpty, &tikvpb.BatchCommandsEmptyRequest{})
 	_, err = rpcClient.SendRequest(context.Background(), addr, req, 100*time.Second)
-	assert.True(t, err.Error() == "no available connections")
+	assert.EqualError(t, err, fmt.Sprintf("[%s](%d) no available connections", addr, 1))
 	server.Stop()
 }
 
@@ -723,3 +729,34 @@ func TestBatchClientRecoverAfterServerRestart(t *testing.T) {
 		require.NoError(t, err)
 	}
 }
+
+func TestErrConn(t *testing.T) {
+	e := errors.New("conn error")
+	err1 := &ErrConn{Err: e, Addr: "127.0.0.1", Ver: 10}
+	err2 := &ErrConn{Err: e, Addr: "127.0.0.1", Ver: 10}
+
+	e3 := errors.New("conn error 3")
+	err3 := &ErrConn{Err: e3}
+
+	err4 := errors.New("not ErrConn")
+
+	assert.True(t, errors.Is(err1, err1))
+	assert.True(t, errors.Is(fmt.Errorf("%w", err1), err1))
+	assert.False(t, errors.Is(fmt.Errorf("%w", err2), err1)) // err2 != err1
+	assert.False(t, errors.Is(fmt.Errorf("%w", err4), err1))
+
+	var errConn *ErrConn
+	assert.True(t, errors.As(err1, &errConn))
+	assert.Equal(t, "127.0.0.1", errConn.Addr)
+	assert.EqualValues(t, 10, errConn.Ver)
+	assert.EqualError(t, errConn.Err, "conn error")
+
+	assert.True(t, errors.As(err3, &errConn))
+	assert.EqualError(t, e3, "conn error 3")
+
+	assert.False(t, errors.As(err4, &errConn))
+
+	errMsg := errors.New("unknown")
+	assert.True(t, errors.As(err1, &errMsg))
+	assert.EqualError(t, err1, errMsg.Error())
+}
diff --git a/internal/locate/region_request_test.go b/internal/locate/region_request_test.go
index c16626a..8563715 100644
--- a/internal/locate/region_request_test.go
+++ b/internal/locate/region_request_test.go
@@ -37,6 +37,7 @@ package locate
 import (
 	"context"
 	"fmt"
+	"math"
 	"math/rand"
 	"net"
 	"sync"
@@ -99,6 +100,7 @@ func (s *testRegionRequestToSingleStoreSuite) TearDownTest() {
 type fnClient struct {
 	fn         func(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error)
 	closedAddr string
+	closedVer  uint64
 }
 
 func (f *fnClient) Close() error {
@@ -106,7 +108,12 @@ func (f *fnClient) Close() error {
 }
 
 func (f *fnClient) CloseAddr(addr string) error {
+	return f.CloseAddrVer(addr, math.MaxUint64)
+}
+
+func (f *fnClient) CloseAddrVer(addr string, ver uint64) error {
 	f.closedAddr = addr
+	f.closedVer = ver
 	return nil
 }
 
@@ -664,6 +671,8 @@ func (s *testRegionRequestToSingleStoreSuite) TestCloseConnectionOnStoreNotMatch
 	regionErr, _ := resp.GetRegionError()
 	s.NotNil(regionErr)
 	s.Equal(target, client.closedAddr)
+	var expected uint64 = math.MaxUint64
+	s.Equal(expected, client.closedVer)
 }
 
 func (s *testRegionRequestToSingleStoreSuite) TestStaleReadRetry() {
@@ -824,3 +833,20 @@ func (s *testRegionRequestToSingleStoreSuite) TestCountReplicaNumber() {
 		s.Equal(4, s.regionRequestSender.countReplicaNumber(peers)) // Only count 1 tiflash replica for tiflash write-nodes.
 	}
 }
+
+type emptyClient struct {
+	client.Client
+}
+
+func (s *testRegionRequestToSingleStoreSuite) TestClientExt() {
+	var cli client.Client = client.NewRPCClient()
+	sender := NewRegionRequestSender(s.cache, cli)
+	s.NotNil(sender.client)
+	s.NotNil(sender.getClientExt())
+	cli.Close()
+
+	cli = &emptyClient{}
+	sender = NewRegionRequestSender(s.cache, cli)
+	s.NotNil(sender.client)
+	s.Nil(sender.getClientExt())
+}