Repository: tikv/client-go (go)

Issue:
nil panic once meet quota limit error
![image](https://github.com/tikv/client-go/assets/6428910/db85811b-6aab-43dd-bbe2-8e7b0fc95d13)

![image](https://github.com/tikv/client-go/assets/6428910/49901dae-fa22-4bac-b19d-f6f092032559)

```
[2024/05/15 17:57:51.732 +08:00] [ERROR] [coprocessor.go:1152] ["copIteratorWork meet panic"] [r="runtime error: invalid memory address or nil pointer dereference"] ["stack trace"="gi
thub.com/pingcap/tidb/pkg/store/copr.(*copIteratorWorker).handleTask.func1\n\t/data/nvme0n1/csn/program/go/tidb/pkg/store/copr/coprocessor.go:1154\nruntime.gopanic\n\t/snap/go/10585/s
rc/runtime/panic.go:770\nruntime.panicmem\n\t/snap/go/10585/src/runtime/panic.go:261\nruntime.sigpanic\n\t/snap/go/10585/src/runtime/signal_unix.go:881\ngithub.com/tikv/client-go/v2/i
nternal/locate.(*RegionRequestSender).sendReqToRegion\n\t/data/nvme0n1/csn/program/go/client-go/internal/locate/region_request.go:2118\ngithub.com/tikv/client-go/v2/internal/locate.(*
RegionRequestSender).SendReqCtx\n\t/data/nvme0n1/csn/program/go/client-go/internal/locate/region_request.go:1796\ngithub.com/tikv/client-go/v2/txnkv/txnsnapshot.(*ClientHelper).SendRe
qCtx\n\t/data/nvme0n1/csn/program/go/client-go/txnkv/txnsnapshot/client_helper.go:146\ngithub.com/pingcap/tidb/pkg/store/copr.(*copIteratorWorker).handleTaskOnce\n\t/data/nvme0n1/csn/
program/go/tidb/pkg/store/copr/coprocessor.go:1275\ngithub.com/pingcap/tidb/pkg/store/copr.(*copIteratorWorker).handleTask\n\t/data/nvme0n1/csn/program/go/tidb/pkg/store/copr/coproces
sor.go:1165\ngithub.com/pingcap/tidb/pkg/store/copr.(*copIteratorWorker).run\n\t/data/nvme0n1/csn/program/go/tidb/pkg/store/copr/coprocessor.go:828"]
```


the error.Cause may retrun nil:

https://github.com/tikv/client-go/blob/c40432e3abccd374ea0f6b945f95b925004b4e49/internal/locate/region_request.go#L2118

and 

https://github.com/tikv/client-go/blob/c40432e3abccd374ea0f6b945f95b925004b4e49/internal/locate/region_request.go#L2214


Pull request:
*: fix panic in get cause error
close #1343

Patch:
diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go
index 07dda82..04d0d12 100644
--- a/internal/locate/region_request.go
+++ b/internal/locate/region_request.go
@@ -180,6 +180,17 @@ func (r *RequestErrorStats) RecordRPCErrorStats(errLabel string) {
 	}
 }
 
+// getErrMsg returns error message. if the error has cause error, then return cause error message.
+func getErrMsg(err error) string {
+	if err == nil {
+		return ""
+	}
+	if causeErr := errors.Cause(err); causeErr != nil {
+		return causeErr.Error()
+	}
+	return err.Error()
+}
+
 // String implements fmt.Stringer interface.
 func (r *RegionRequestRuntimeStats) String() string {
 	if r == nil {
@@ -2115,7 +2126,7 @@ func (s *RegionRequestSender) sendReqToRegion(
 	if err != nil {
 		s.rpcError = err
 		if s.Stats != nil {
-			errStr := errors.Cause(err).Error()
+			errStr := getErrMsg(err)
 			s.Stats.RecordRPCErrorStats(errStr)
 			s.recordRPCAccessInfo(req, rpcCtx, errStr)
 		}
@@ -2211,7 +2222,11 @@ func (s *RegionRequestSender) onSendFail(bo *retry.Backoffer, ctx *RPCContext, r
 			}
 		}
 	}
-	metrics.TiKVRPCErrorCounter.WithLabelValues(errors.Cause(err).Error(), storeLabel).Inc()
+	if errStr := getErrMsg(err); len(errStr) > 0 {
+		metrics.TiKVRPCErrorCounter.WithLabelValues(getErrMsg(err), storeLabel).Inc()
+	} else {
+		metrics.TiKVRPCErrorCounter.WithLabelValues("unknown", storeLabel).Inc()
+	}
 
 	if ctx.Store != nil && ctx.Store.storeType == tikvrpc.TiFlashCompute {
 		s.regionCache.InvalidateTiFlashComputeStoresIfGRPCError(err)


Test patch:
diff --git a/internal/locate/region_request_test.go b/internal/locate/region_request_test.go
index 797b739..230bb9a 100644
--- a/internal/locate/region_request_test.go
+++ b/internal/locate/region_request_test.go
@@ -56,6 +56,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/mpp"
 	"github.com/pingcap/kvproto/pkg/tikvpb"
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	"github.com/tikv/client-go/v2/config"
 	"github.com/tikv/client-go/v2/config/retry"
@@ -872,3 +873,20 @@ func (s *testRegionRequestToSingleStoreSuite) TestRegionRequestStats() {
 	}
 	s.Contains(expecteds, access.String())
 }
+
+type noCauseError struct {
+	error
+}
+
+func (_ noCauseError) Cause() error {
+	return nil
+}
+
+func TestGetErrMsg(t *testing.T) {
+	err := noCauseError{error: errors.New("no cause err")}
+	require.Equal(t, nil, errors.Cause(err))
+	require.Panicsf(t, func() {
+		_ = errors.Cause(err).Error()
+	}, "should panic")
+	require.Equal(t, "no cause err", getErrMsg(err))
+}