Repository: pingcap/tidb
Pull request #60691: localbackend: unify job stage transition during ingest | tidb-test=ccf9f5de6817ed2e54c18f2c1757056d2c8b4e4c
Author: login: D3Hunter
Labels: name: size/XL
description: Denotes a PR that changes 500-999 lines, ignoring generated files.
name: release-note-none
description: Denotes a PR that doesn't merit a release note.
name: approved
name: lgtm
Created: 2025-04-21T10:58:15Z | Merged: 2025-04-22T11:21:29Z

<!--

Thank you for contributing to TiDB!

PR Title Format:
1. pkg [, pkg2, pkg3]: what's changed
2. *: what's changed

-->

### What problem does this PR solve?
<!--

Please create an issue first to describe the problem.

There MUST be one line starting with "Issue Number:  " and
linking the relevant issues via the "close" or "ref".

For more info, check https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/contribute-code.html#referring-to-an-issue.

-->

Issue Number: ref #60418 

Problem Summary:

### What changed and how does it work?
- split previous `convertStageOnIngestError` into `convertPBError2Error` and `getNextStageOnIngestError`
- unify job stage transition during ingest in `regionJobBaseWorker.runJob`
### Check List

Tests <!-- At least one of them must be included. -->

- [x] Unit test
- [ ] Integration test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test
  > - [ ] I checked and no code files have been changed.
  > <!-- Or your custom  "No need to test" reasons -->

Side effects

- [ ] Performance regression: Consumes more CPU
- [ ] Performance regression: Consumes more Memory
- [ ] Breaking backward compatibility

Documentation

- [ ] Affects user behaviors
- [ ] Contains syntax changes
- [ ] Contains variable changes
- [ ] Contains experimental features
- [ ] Changes MySQL compatibility

### Release note

<!-- compatibility change, improvement, bugfix, and new feature need a release note -->

Please refer to [Release Notes Language Style Guide](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/release-notes-style-guide.html) to write a quality release note.

```release-note
None
```



Diff:
diff --git a/errors.toml b/errors.toml
index 69bb16e9b5f9a..140755e342951 100644
--- a/errors.toml
+++ b/errors.toml
@@ -471,6 +471,11 @@ error = '''
 server is busy
 '''
 
+["Lightning:KV:StoreDiskFull"]
+error = '''
+store disk full
+'''
+
 ["Lightning:Loader:ErrInvalidSchemaFile"]
 error = '''
 invalid schema file
diff --git a/pkg/lightning/backend/local/job_worker.go b/pkg/lightning/backend/local/job_worker.go
index 546fe03a456e3..26bab0c847cf6 100644
--- a/pkg/lightning/backend/local/job_worker.go
+++ b/pkg/lightning/backend/local/job_worker.go
@@ -185,11 +185,22 @@ func (w *regionJobBaseWorker) runJob(ctx context.Context, job *regionJob) error
 				if !w.isRetryableImportTiKVError(err) {
 					return err
 				}
-				log.FromContext(ctx).Warn("meet retryable error when ingesting",
-					log.ShortError(err), zap.Stringer("job stage", job.stage))
+
+				newRegion, nextStage := getNextStageOnIngestError(err)
+				job.convertStageTo(nextStage)
+				if newRegion != nil {
+					job.region = newRegion
+				}
 				job.lastRetryableErr = err
+
+				log.FromContext(ctx).Warn("meet retryable error when ingesting, will handle the job later",
+					log.ShortError(err), zap.Stringer("job stage", job.stage),
+					job.region.ToZapFields(),
+					logutil.Key("start", job.keyRange.Start),
+					logutil.Key("end", job.keyRange.End))
 				return nil
 			}
+			job.convertStageTo(ingested)
 		}
 		// if the stage is not ingested, it means some error happened, the job should
 		// be sent back to caller to retry later, else we handle remaining data.
@@ -256,8 +267,8 @@ type objStoreRegionJobWorker struct {
 }
 
 func (*objStoreRegionJobWorker) preRunJob(_ context.Context, _ *regionJob) error {
-	// cloud engine use cloud storage, such as S3, to hold data, so no need to check
-	// disk fullness.
+	// cloud engine use cloud storage, such as S3, to hold data, it's assumed to
+	// have unlimited available space, so no need to check disk fullness.
 	return nil
 }
 
@@ -359,19 +370,15 @@ func (w *objStoreRegionJobWorker) ingest(ctx context.Context, j *regionJob) erro
 	}
 	err := w.ingestCli.Ingest(ctx, in)
 	if err != nil {
-		// TODO, we should let outer logic handle stage transition, currently, OP
-		//  worker need a lot of change before we can do it, will refactor it later.
-
-		// TODO: choose target stage based on error.
-		j.convertStageTo(needRescan)
 		log.FromContext(ctx).Warn("meet error and handle the job later",
 			zap.Stringer("job stage", j.stage),
-			logutil.ShortError(j.lastRetryableErr),
+			logutil.ShortError(err),
 			j.region.ToZapFields(),
 			logutil.Key("start", j.keyRange.Start),
 			logutil.Key("end", j.keyRange.End))
-		return err
+
+		// TODO: choose target stage based on error.
+		return &ingestAPIError{err: err}
 	}
-	j.convertStageTo(ingested)
 	return nil
 }
diff --git a/pkg/lightning/backend/local/job_worker_test.go b/pkg/lightning/backend/local/job_worker_test.go
index c9539de2ed018..6e2becd8160ab 100644
--- a/pkg/lightning/backend/local/job_worker_test.go
+++ b/pkg/lightning/backend/local/job_worker_test.go
@@ -21,6 +21,8 @@ import (
 	"testing"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/kvproto/pkg/metapb"
+	"github.com/pingcap/tidb/br/pkg/restore/split"
 	"github.com/pingcap/tidb/pkg/ingestor/ingestcli"
 	ingestclimock "github.com/pingcap/tidb/pkg/ingestor/ingestcli/mock"
 	"github.com/pingcap/tidb/pkg/lightning/common"
@@ -41,7 +43,6 @@ func TestRegionJobBaseWorker(t *testing.T) {
 				return &tikvWriteResult{}, nil
 			},
 			ingestFn: func(ctx context.Context, job *regionJob) error {
-				job.convertStageTo(ingested)
 				return nil
 			},
 			regenerateJobsFn: func(
@@ -74,13 +75,75 @@ func TestRegionJobBaseWorker(t *testing.T) {
 		close(w.jobInCh)
 		require.NoError(t, w.run(context.Background()))
 		require.Equal(t, 1, len(w.jobOutCh))
+		outJob := <-w.jobOutCh
+		require.Equal(t, ingested, outJob.stage)
+	})
+
+	t.Run("empty job", func(t *testing.T) {
+		w := newWorker()
+		w.writeFn = func(ctx context.Context, job *regionJob) (*tikvWriteResult, error) {
+			return &tikvWriteResult{emptyJob: true}, nil
+		}
+		job := &regionJob{stage: regionScanned, ingestData: mockIngestData{}}
+		w.jobInCh <- job
+		close(w.jobInCh)
+		require.NoError(t, w.run(context.Background()))
+		require.Equal(t, 1, len(w.jobOutCh))
+		outJob := <-w.jobOutCh
+		require.Equal(t, ingested, outJob.stage)
+	})
+
+	t.Run("meet non-retryable error during ingest", func(t *testing.T) {
+		w := newWorker()
+		w.ingestFn = func(ctx context.Context, job *regionJob) error {
+			return &ingestAPIError{err: common.ErrKVDiskFull}
+		}
+		job := &regionJob{stage: regionScanned, ingestData: mockIngestData{}}
+		w.jobInCh <- job
+		close(w.jobInCh)
+		require.ErrorIs(t, w.run(context.Background()), common.ErrKVDiskFull)
+		require.Equal(t, 1, len(w.jobOutCh))
+		outJob := <-w.jobOutCh
+		// the job is left in wrote stage
+		require.Equal(t, wrote, outJob.stage)
+		require.Nil(t, outJob.lastRetryableErr)
+	})
+
+	t.Run("retry job from regionScanned", func(t *testing.T) {
+		w := newWorker()
+		w.ingestFn = func(ctx context.Context, job *regionJob) error {
+			return &ingestAPIError{err: common.ErrKVIngestFailed}
+		}
+		job := &regionJob{stage: regionScanned, ingestData: mockIngestData{}}
+		w.jobInCh <- job
+		close(w.jobInCh)
+		require.NoError(t, w.run(context.Background()))
+		require.Equal(t, 1, len(w.jobOutCh))
+		outJob := <-w.jobOutCh
+		require.Equal(t, regionScanned, outJob.stage)
+		require.ErrorIs(t, outJob.lastRetryableErr, common.ErrKVIngestFailed)
+	})
+
+	t.Run("retry job from regionScanned, and region got from the ingest error", func(t *testing.T) {
+		w := newWorker()
+		w.ingestFn = func(ctx context.Context, job *regionJob) error {
+			return &ingestAPIError{err: common.ErrKVEpochNotMatch, newRegion: &split.RegionInfo{Region: &metapb.Region{Id: 123}}}
+		}
+		job := &regionJob{stage: regionScanned, ingestData: mockIngestData{}}
+		w.jobInCh <- job
+		close(w.jobInCh)
+		require.NoError(t, w.run(context.Background()))
+		require.Equal(t, 1, len(w.jobOutCh))
+		outJob := <-w.jobOutCh
+		require.Equal(t, regionScanned, outJob.stage)
+		require.ErrorIs(t, outJob.lastRetryableErr, common.ErrKVEpochNotMatch)
+		require.Equal(t, &split.RegionInfo{Region: &metapb.Region{Id: 123}}, outJob.region)
 	})
 
 	t.Run("regenerate jobs", func(t *testing.T) {
 		w := newWorker()
 		w.ingestFn = func(ctx context.Context, job *regionJob) error {
-			job.convertStageTo(needRescan)
-			return nil
+			return &ingestAPIError{err: common.ErrKVNotLeader}
 		}
 		job := &regionJob{stage: regionScanned, ingestData: mockIngestData{}}
 		w.jobInCh <- job
@@ -200,7 +263,6 @@ func TestCloudRegionJobWorker(t *testing.T) {
 		mockIngestCli.EXPECT().Ingest(gomock.Any(), gomock.Any()).Return(errors.New("mock error"))
 		err := cloudW.ingest(context.Background(), job)
 		require.ErrorContains(t, err, "mock error")
-		require.Equal(t, needRescan, job.stage)
 		require.True(t, ctrl.Satisfied())
 	})
 
@@ -214,7 +276,6 @@ func TestCloudRegionJobWorker(t *testing.T) {
 		mockIngestCli.EXPECT().Ingest(gomock.Any(), gomock.Any()).Return(nil)
 		err := cloudW.ingest(context.Background(), job)
 		require.NoError(t, err)
-		require.Equal(t, ingested, job.stage)
 		require.True(t, ctrl.Satisfied())
 	})
 }
diff --git a/pkg/lightning/backend/local/local_test.go b/pkg/lightning/backend/local/local_test.go
index 037046d29143f..87318d291f66c 100644
--- a/pkg/lightning/backend/local/local_test.go
+++ b/pkg/lightning/backend/local/local_test.go
@@ -1832,9 +1832,7 @@ func getSuccessInjectedBehaviour() []injectedBehaviour {
 			},
 		},
 		{
-			ingest: injectedIngestBehaviour{
-				nextStage: ingested,
-			},
+			ingest: injectedIngestBehaviour{},
 		},
 	}
 }
@@ -1850,8 +1848,7 @@ func getNeedRescanWhenIngestBehaviour() []injectedBehaviour {
 		},
 		{
 			ingest: injectedIngestBehaviour{
-				nextStage: needRescan,
-				err:       common.ErrKVEpochNotMatch,
+				err: &ingestAPIError{err: common.ErrKVEpochNotMatch},
 			},
 		},
 	}
@@ -1909,9 +1906,7 @@ func TestDoImport(t *testing.T) {
 							},
 						},
 						{
-							ingest: injectedIngestBehaviour{
-								nextStage: ingested,
-							},
+							ingest: injectedIngestBehaviour{},
 						},
 						{
 							write: injectedWriteBehaviour{
@@ -1921,9 +1916,7 @@ func TestDoImport(t *testing.T) {
 							},
 						},
 						{
-							ingest: injectedIngestBehaviour{
-								nextStage: ingested,
-							},
+							ingest: injectedIngestBehaviour{},
 						},
 					},
 				},
diff --git a/pkg/lightning/backend/local/region_job.go b/pkg/lightning/backend/local/region_job.go
index 2a86784aadcc8..a5f86596906ca 100644
--- a/pkg/lightning/backend/local/region_job.go
+++ b/pkg/lightning/backend/local/region_job.go
@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"container/heap"
 	"context"
+	goerrors "errors"
 	"fmt"
 	"io"
 	"math"
@@ -148,8 +149,7 @@ type injectedWriteBehaviour struct {
 }
 
 type injectedIngestBehaviour struct {
-	nextStage jobStageTp
-	err       error
+	err error
 }
 
 func newRegionJob(
@@ -588,20 +588,16 @@ func (local *Backend) doWrite(ctx context.Context, j *regionJob) (*tikvWriteResu
 }
 
 // ingest tries to finish the regionJob.
-// if any ingest logic has error, ingest may retry sometimes to resolve it and finally
-// set job to a proper stage with nil error returned.
 // if any underlying logic has error, ingest will return an error to let caller
 // handle it.
 func (local *Backend) ingest(ctx context.Context, j *regionJob) (err error) {
 	failpoint.Inject("fakeRegionJobs", func() {
 		front := j.injected[0]
 		j.injected = j.injected[1:]
-		j.convertStageTo(front.ingest.nextStage)
 		failpoint.Return(front.ingest.err)
 	})
 
 	if len(j.writeResult.sstMeta) == 0 {
-		j.convertStageTo(ingested)
 		return nil
 	}
 
@@ -614,42 +610,26 @@ func (local *Backend) ingest(ctx context.Context, j *regionJob) (err error) {
 		}()
 	}
 
+	var lastRetriedErr error
 	for retry := 0; retry < maxRetryTimes; retry++ {
 		resp, err := local.doIngest(ctx, j)
-		if err == nil && resp.GetError() == nil {
-			j.convertStageTo(ingested)
-			return nil
-		}
 		if err != nil {
 			if common.IsContextCanceledError(err) {
 				return err
 			}
 			log.FromContext(ctx).Warn("meet underlying error, will retry ingest",
 				log.ShortError(err), logutil.SSTMetas(j.writeResult.sstMeta),
-				logutil.Region(j.region.Region), logutil.Leader(j.region.Leader))
-			j.lastRetryableErr = err
+				logutil.Region(j.region.Region), logutil.Leader(j.region.Leader),
+				zap.Int("retry", retry))
+			lastRetriedErr = err
 			continue
 		}
-		canContinue, err := j.convertStageOnIngestError(resp)
-		if common.IsContextCanceledError(err) {
-			return err
-		}
-		if !canContinue {
-			log.FromContext(ctx).Warn("meet error and handle the job later",
-				zap.Stringer("job stage", j.stage),
-				logutil.ShortError(j.lastRetryableErr),
-				j.region.ToZapFields(),
-				logutil.Key("start", j.keyRange.Start),
-				logutil.Key("end", j.keyRange.End))
+		if resp.GetError() == nil {
 			return nil
 		}
-		log.FromContext(ctx).Warn("meet error and will doIngest region again",
-			logutil.ShortError(j.lastRetryableErr),
-			j.region.ToZapFields(),
-			logutil.Key("start", j.keyRange.Start),
-			logutil.Key("end", j.keyRange.End))
+		return convertPBError2Error(j, resp.GetError())
 	}
-	return nil
+	return lastRetriedErr
 }
 
 func (local *Backend) checkWriteStall(
@@ -765,7 +745,7 @@ func (local *Backend) doIngest(ctx context.Context, j *regionJob) (*sst.IngestRe
 			}
 			resp, err = cli.Ingest(ctx, req)
 		}
-		if resp.GetError() != nil || err != nil {
+		if err != nil || resp.GetError() != nil {
 			// remove finished sstMetas
 			j.writeResult.sstMeta = j.writeResult.sstMeta[start:]
 			return resp, errors.Trace(err)
@@ -784,33 +764,46 @@ func (local *Backend) GetWriteSpeedLimit() int {
 	return local.writeLimiter.Limit()
 }
 
-// convertStageOnIngestError will try to fix the error contained in ingest response.
-// Return (_, error) when another error occurred.
-// Return (true, nil) when the job can retry ingesting immediately.
-// Return (false, nil) when the job should be put back to queue.
-func (j *regionJob) convertStageOnIngestError(
-	resp *sst.IngestResponse,
-) (bool, error) {
-	if resp.GetError() == nil {
-		return true, nil
-	}
+// ingestAPIError is the converted error when we call Ingest or MultiIngest successfully,
+// but the server return some logic error, i.e. errorpb.Error.
+// TODO: better move to ingestcli pkg, but the split.RegionInfo might cause import cycle.
+type ingestAPIError struct {
+	// the converted internal error
+	err error
+	// if theErr = ErrKVEpochNotMatch, the new region info maybe extracted from
+	// the PB error
+	newRegion *split.RegionInfo
+}
+
+func (e *ingestAPIError) Error() string {
+	return e.err.Error()
+}
+
+// Cause is used for pingcap/errors.Cause
+func (e *ingestAPIError) Cause() error {
+	return e.err
+}
+
+// Unwrap is used for golang/errors.Is and As
+func (e *ingestAPIError) Unwrap() error {
+	return e.err
+}
 
+func convertPBError2Error(job *regionJob, errPb *errorpb.Error) *ingestAPIError {
 	var newRegion *split.RegionInfo
-	switch errPb := resp.GetError(); {
+	res := &ingestAPIError{}
+	switch {
 	case errPb.NotLeader != nil:
-		j.lastRetryableErr = common.ErrKVNotLeader.GenWithStack(errPb.GetMessage())
-
 		// meet a problem that the region leader+peer are all updated but the return
 		// error is only "NotLeader", we should update the whole region info.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVNotLeader.GenWithStack(errPb.GetMessage())
 	case errPb.EpochNotMatch != nil:
-		j.lastRetryableErr = common.ErrKVEpochNotMatch.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVEpochNotMatch.GenWithStack(errPb.GetMessage())
 
 		if currentRegions := errPb.GetEpochNotMatch().GetCurrentRegions(); currentRegions != nil {
 			var currentRegion *metapb.Region
 			for _, r := range currentRegions {
-				if insideRegion(r, j.writeResult.sstMeta) {
+				if insideRegion(r, job.writeResult.sstMeta) {
 					currentRegion = r
 					break
 				}
@@ -818,7 +811,7 @@ func (j *regionJob) convertStageOnIngestError(
 			if currentRegion != nil {
 				var newLeader *metapb.Peer
 				for _, p := range currentRegion.Peers {
-					if p.GetStoreId() == j.region.Leader.GetStoreId() {
+					if p.GetStoreId() == job.region.Leader.GetStoreId() {
 						newLeader = p
 						break
 					}
@@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
 }
 
 type regionJobRetryHeap []*regionJob
diff --git a/pkg/lightning/backend/local/region_job_test.go b/pkg/lightning/backend/local/region_job_test.go
index dcc0ede731c4b..ec7881e5a614c 100644
--- a/pkg/lightning/backend/local/region_job_test.go
+++ b/pkg/lightning/backend/local/region_job_test.go
@@ -16,7 +16,9 @@ package local
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
+	"net"
 	"sync"
 	"testing"
 	"time"
@@ -45,143 +47,112 @@ func TestIsIngestRetryable(t *testing.T) {
 		},
 	}
 	metas := []*sst.SSTMeta{
-		{
-			Range: &sst.Range{
-				Start: []byte{1},
-				End:   []byte{2},
-			},
-		},
-		{
-			Range: &sst.Range{
-				Start: []byte{1, 1},
-				End:   []byte{2},
-			},
-		},
+		{Range: &sst.Range{Start: []byte{1}, End: []byte{2}}},
+		{Range: &sst.Range{Start: []byte{1, 1}, End: []byte{2}}},
 	}
-	job := regionJob{
-		stage: wrote,
-		keyRange: common.Range{
-			Start: []byte{1},
-			End:   []byte{3},
-		},
-		region: region,
+	job := &regionJob{
+		stage:    wrote,
+		keyRange: common.Range{Start: []byte{1}, End: []byte{3}},
+		region:   region,
 		writeResult: &tikvWriteResult{
 			sstMeta: metas,
 		},
 	}
-	// NotLeader doesn't mean region peers are changed, so we can retry ingest.
 
-	resp := &sst.IngestResponse{
-		Error: &errorpb.Error{
-			NotLeader: &errorpb.NotLeader{
-				Leader: &metapb.Peer{Id: 2},
-			},
+	newRegion := &metapb.Region{
+		Id:       1,
+		StartKey: []byte{1},
+		EndKey:   []byte{3},
+		RegionEpoch: &metapb.RegionEpoch{
+			ConfVer: 1,
+			Version: 2,
 		},
+		Peers: []*metapb.Peer{{Id: 1}},
 	}
 
-	clone := job
-	canContinueIngest, err := (&clone).convertStageOnIngestError(resp)
-	require.NoError(t, err)
-	require.False(t, canContinueIngest)
-	require.Equal(t, needRescan, clone.stage)
-	require.Error(t, clone.lastRetryableErr)
-
-	// EpochNotMatch means region is changed, if the new region covers the old, we can restart the writing process.
-	// Otherwise, we should restart from region scanning.
-
-	resp.Error = &errorpb.Error{
-		EpochNotMatch: &errorpb.EpochNotMatch{
-			CurrentRegions: []*metapb.Region{
-				{
-					Id:       1,
-					StartKey: []byte{1},
-					EndKey:   []byte{3},
-					RegionEpoch: &metapb.RegionEpoch{
-						ConfVer: 1,
-						Version: 2,
-					},
-					Peers: []*metapb.Peer{{Id: 1}},
-				},
-			},
-		},
-	}
-	clone = job
-	canContinueIngest, err = (&clone).convertStageOnIngestError(resp)
-	require.NoError(t, err)
-	require.False(t, canContinueIngest)
-	require.Equal(t, regionScanned, clone.stage)
-	require.Nil(t, clone.writeResult)
-	require.Equal(t, uint64(2), clone.region.Region.RegionEpoch.Version)
-	require.Error(t, clone.lastRetryableErr)
-
-	resp.Error = &errorpb.Error{
-		EpochNotMatch: &errorpb.EpochNotMatch{
-			CurrentRegions: []*metapb.Region{
-				{
-					Id:       1,
-					StartKey: []byte{1},
-					EndKey:   []byte{1, 2},
-					RegionEpoch: &metapb.RegionEpoch{
-						ConfVer: 1,
-						Version: 2,
-					},
-					Peers: []*metapb.Peer{{Id: 1}},
-				},
-			},
+	cases := []struct {
+		pbErr *errorpb.Error
+		res   *ingestAPIError
+	}{
+		// NotLeader doesn't mean region peers are changed, so we can retry ingest.
+		{pbErr: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}, res: &ingestAPIError{err: common.ErrKVNotLeader}},
+		// EpochNotMatch means region is changed, if the new region covers the old, we can restart the writing process.
+		// Otherwise, we should restart from region scanning.
+		{
+			pbErr: &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{
+				CurrentRegions: []*metapb.Region{newRegion},
+			}},
+			res: &ingestAPIError{err: common.ErrKVEpochNotMatch, newRegion: &split.RegionInfo{
+				Region: newRegion,
+				Leader: &metapb.Peer{Id: 1},
+			}},
 		},
-	}
-	clone = job
-	canContinueIngest, err = (&clone).convertStageOnIngestError(resp)
-	require.NoError(t, err)
-	require.False(t, canContinueIngest)
-	require.Equal(t, needRescan, clone.stage)
-	require.Error(t, clone.lastRetryableErr)
-
-	// TODO: in which case raft layer will drop message?
-
-	resp.Error = &errorpb.Error{Message: "raft: proposal dropped"}
-	clone = job
-	canContinueIngest, err = (&clone).convertStageOnIngestError(resp)
-	require.NoError(t, err)
-	require.False(t, canContinueIngest)
-	require.Equal(t, needRescan, clone.stage)
-	require.Error(t, clone.lastRetryableErr)
-
-	// ReadIndexNotReady means the region is changed, we need to restart from region scanning
-
-	resp.Error = &errorpb.Error{
-		ReadIndexNotReady: &errorpb.ReadIndexNotReady{
-			Reason: "test",
+		{
+			pbErr: &errorpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{CurrentRegions: []*metapb.Region{{
+				Id:          1,
+				StartKey:    []byte{1},
+				EndKey:      []byte{1, 2},
+				RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 2},
+				Peers:       []*metapb.Peer{{Id: 1}},
+			}}}},
+			res: &ingestAPIError{err: common.ErrKVEpochNotMatch},
 		},
+		// TODO: in which case raft layer will drop message?
+		{pbErr: &errorpb.Error{Message: "raft: proposal dropped"}, res: &ingestAPIError{err: common.ErrKVRaftProposalDropped}},
+		{pbErr: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, res: &ingestAPIError{err: common.ErrKVServerIsBusy}},
+		{pbErr: &errorpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}, res: &ingestAPIError{err: common.ErrKVRegionNotFound}},
+		// ReadIndexNotReady means the region is changed, we need to restart from region scanning
+		{pbErr: &errorpb.Error{ReadIndexNotReady: &errorpb.ReadIndexNotReady{}}, res: &ingestAPIError{err: common.ErrKVReadIndexNotReady}},
+		// TiKV disk full is not retryable
+		{pbErr: &errorpb.Error{DiskFull: &errorpb.DiskFull{}}, res: &ingestAPIError{err: common.ErrKVDiskFull}},
+		// a general error is retryable from writing
+		{pbErr: &errorpb.Error{StaleCommand: &errorpb.StaleCommand{}}, res: &ingestAPIError{err: common.ErrKVIngestFailed}},
 	}
-	clone = job
-	canContinueIngest, err = (&clone).convertStageOnIngestError(resp)
-	require.NoError(t, err)
-	require.False(t, canContinueIngest)
-	require.Equal(t, needRescan, clone.stage)
-	require.Error(t, clone.lastRetryableErr)
 
-	// TiKV disk full is not retryable
-
-	resp.Error = &errorpb.Error{
-		DiskFull: &errorpb.DiskFull{},
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			err := convertPBError2Error(job, c.pbErr)
+			require.ErrorIs(t, err, c.res.err)
+			if c.res.newRegion == nil {
+				require.Nil(t, err.newRegion)
+			} else {
+				require.EqualValues(t, c.res.newRegion, err.newRegion)
+				require.EqualValues(t, 2, err.newRegion.Region.RegionEpoch.Version)
+			}
+		})
 	}
-	clone = job
-	_, err = (&clone).convertStageOnIngestError(resp)
-	require.ErrorContains(t, err, "non-retryable error")
+}
 
-	// a general error is retryable from writing
+func TestIngestAPIErrorRetryable(t *testing.T) {
+	require.True(t, common.IsRetryableError(&ingestAPIError{err: common.ErrKVIngestFailed}))
+	require.False(t, common.IsRetryableError(&ingestAPIError{err: common.ErrKVDiskFull}))
+}
 
-	resp.Error = &errorpb.Error{
-		StaleCommand: &errorpb.StaleCommand{},
+func TestGetNextStageOnIngestError(t *testing.T) {
+	cases := []struct {
+		err    error
+		region *split.RegionInfo
+		stage  jobStageTp
+	}{
+		{err: &net.DNSError{IsTimeout: true}, stage: wrote},
+		{err: &ingestAPIError{err: common.ErrKVNotLeader.GenWithStack("")}, stage: needRescan},
+		{err: &ingestAPIError{err: common.ErrKVEpochNotMatch.GenWithStack("")}, stage: needRescan},
+		{err: &ingestAPIError{err: common.ErrKVEpochNotMatch.GenWithStack(""), newRegion: &split.RegionInfo{}},
+			region: &split.RegionInfo{}, stage: regionScanned},
+		{err: &ingestAPIError{err: common.ErrKVRaftProposalDropped.GenWithStack("")}, stage: needRescan},
+		{err: &ingestAPIError{err: common.ErrKVServerIsBusy.GenWithStack("")}, stage: wrote},
+		{err: &ingestAPIError{err: common.ErrKVRegionNotFound.GenWithStack("")}, stage: needRescan},
+		{err: &ingestAPIError{err: common.ErrKVReadIndexNotReady.GenWithStack("")}, stage: needRescan},
+		// ErrKVDiskFull is not retryable, no need to test it
+		{err: &ingestAPIError{err: common.ErrKVIngestFailed.GenWithStack("")}, stage: regionScanned},
+	}
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			region, stage := getNextStageOnIngestError(c.err)
+			require.Equal(t, c.region, region)
+			require.Equal(t, c.stage, stage)
+		})
 	}
-	clone = job
-	canContinueIngest, err = (&clone).convertStageOnIngestError(resp)
-	require.NoError(t, err)
-	require.False(t, canContinueIngest)
-	require.Equal(t, regionScanned, clone.stage)
-	require.Nil(t, clone.writeResult)
-	require.Error(t, clone.lastRetryableErr)
 }
 
 func TestRegionJobRetryer(t *testing.T) {
diff --git a/pkg/lightning/common/errors.go b/pkg/lightning/common/errors.go
index 44caa0814108f..f00ea21289c45 100644
--- a/pkg/lightning/common/errors.go
+++ b/pkg/lightning/common/errors.go
@@ -82,6 +82,7 @@ var (
 	ErrKVServerIsBusy        = errors.Normalize("server is busy", errors.RFCCodeText("Lightning:KV:ServerIsBusy"))
 	ErrKVRegionNotFound      = errors.Normalize("region not found", errors.RFCCodeText("Lightning:KV:RegionNotFound"))
 	ErrKVReadIndexNotReady   = errors.Normalize("read index not ready", errors.RFCCodeText("Lightning:KV:ReadIndexNotReady"))
+	ErrKVDiskFull            = errors.Normalize("store disk full", errors.RFCCodeText("Lightning:KV:StoreDiskFull"))
 	ErrKVIngestFailed        = errors.Normalize("ingest tikv failed", errors.RFCCodeText("Lightning:KV:ErrKVIngestFailed"))
 	ErrKVRaftProposalDropped = errors.Normalize("raft proposal dropped", errors.RFCCodeText("Lightning:KV:ErrKVRaftProposalDropped"))
 	ErrNoLeader              = errors.Normalize("write to tikv with no leader returned, region '%d', leader: %d", errors.RFCCodeText("Lightning:KV:ErrNoLeader"))


Reviews:
user: login: GMHDBJD
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-22T08:10:37Z
user: login: joechenrh
body: rest LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T09:47:00Z
user: login: D3Hunter
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T09:52:43Z
user: login: lance6716
body: rest lgtm
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:07:41Z
user: login: D3Hunter
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:11:18Z
user: login: D3Hunter
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:17:58Z
user: login: lance6716
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:21:20Z
user: login: lance6716
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:23:03Z
user: login: D3Hunter
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:25:48Z
user: login: lance6716
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:25:52Z
user: login: D3Hunter
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:27:33Z
user: login: lance6716
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T10:29:23Z

Review comments:
diff_hunk: @@ -614,42 +610,25 @@ func (local *Backend) ingest(ctx context.Context, j *regionJob) (err error) {
 		}()
 	}
 
+	var lastRetriedErr error
 	for retry := 0; retry < maxRetryTimes; retry++ {
path: pkg/lightning/backend/local/region_job.go
user: login: joechenrh
body: Change to for range?
```suggestion
	for range maxRetryTimes {
```
created_at: 2025-04-22T05:39:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -359,19 +370,15 @@ func (w *objStoreRegionJobWorker) ingest(ctx context.Context, j *regionJob) erro
 	}
 	err := w.ingestCli.Ingest(ctx, in)
 	if err != nil {
-		// TODO, we should let outer logic handle stage transition, currently, OP
-		//  worker need a lot of change before we can do it, will refactor it later.
-
-		// TODO: choose target stage based on error.
-		j.convertStageTo(needRescan)
 		log.FromContext(ctx).Warn("meet error and handle the job later",
 			zap.Stringer("job stage", j.stage),
-			logutil.ShortError(j.lastRetryableErr),
+			logutil.ShortError(err),
 			j.region.ToZapFields(),
 			logutil.Key("start", j.keyRange.Start),
 			logutil.Key("end", j.keyRange.End))
-		return err
+
+		// TODO: choose target stage based on error.
path: pkg/lightning/backend/local/job_worker.go
user: login: joechenrh
body: Do we need to add error source in `ingestAPIError` (from opWorker or cloudWorker)
(I don't know whether we have different stage change logic in `getNextStageOnIngestError` for op and nextgen)
created_at: 2025-04-22T06:11:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -359,19 +370,15 @@ func (w *objStoreRegionJobWorker) ingest(ctx context.Context, j *regionJob) erro
 	}
 	err := w.ingestCli.Ingest(ctx, in)
 	if err != nil {
-		// TODO, we should let outer logic handle stage transition, currently, OP
-		//  worker need a lot of change before we can do it, will refactor it later.
-
-		// TODO: choose target stage based on error.
-		j.convertStageTo(needRescan)
 		log.FromContext(ctx).Warn("meet error and handle the job later",
 			zap.Stringer("job stage", j.stage),
-			logutil.ShortError(j.lastRetryableErr),
+			logutil.ShortError(err),
 			j.region.ToZapFields(),
 			logutil.Key("start", j.keyRange.Start),
 			logutil.Key("end", j.keyRange.End))
-		return err
+
+		// TODO: choose target stage based on error.
path: pkg/lightning/backend/local/job_worker.go
user: login: D3Hunter
body: > (I don't know whether we have different stage change logic in getNextStageOnIngestError for op and nextgen)

at this PR, they are the same, I will change this part when https://github.com/pingcap/tidb/pull/60569 merged
created_at: 2025-04-22T09:52:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -614,42 +610,25 @@ func (local *Backend) ingest(ctx context.Context, j *regionJob) (err error) {
 		}()
 	}
 
+	var lastRetriedErr error
 	for retry := 0; retry < maxRetryTimes; retry++ {
path: pkg/lightning/backend/local/region_job.go
user: login: lance6716
body: Yes we should change it to reduce the effort of applying https://pkg.go.dev/golang.org/x/tools/gopls/internal/analysis/modernize
created_at: 2025-04-22T10:00:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: lance6716
body: Is there some logic to fallback to `needRescan` after retry some `wrote` `regionJob`s? I'm afraid that TiKV is down so there are gRPC connection error, and region are scheduled to another TiKV node. But if we insist on `wrote` `regionJob` we will not make progress.
created_at: 2025-04-22T10:07:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -614,42 +610,25 @@ func (local *Backend) ingest(ctx context.Context, j *regionJob) (err error) {
 		}()
 	}
 
+	var lastRetriedErr error
 	for retry := 0; retry < maxRetryTimes; retry++ {
path: pkg/lightning/backend/local/region_job.go
user: login: D3Hunter
body: https://github.com/pingcap/tidb/pull/60691/commits/3f601315a19dcf370299a0a234bf101fd0e03485
created_at: 2025-04-22T10:11:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: D3Hunter
body: the old logic also didn't consider this case, this PR only do refactor, can you file a bug? we need fix it on master, I think it's moderate/minor level, quite corner case
created_at: 2025-04-22T10:17:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: lance6716
body: I see that `writeToTiKV` will convert the job to `needRescan` for that connection error
created_at: 2025-04-22T10:23:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: D3Hunter
body: `writeToTiKV ` won't handle jobs of stage other than `regionScanned`
created_at: 2025-04-22T10:25:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: lance6716
body: And in the old code of `convertStageOnIngestError` the fallback case is `convertStageTo(regionScanned)`. Here we should also convert to `regionScanned`

/hold
created_at: 2025-04-22T10:25:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: D3Hunter
body: > convertStageOnIngestError the fallback case is convertStageTo(regionScanned)

that's for Ingest API error from the response, not for error from calling grpc
created_at: 2025-04-22T10:27:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,46 +825,50 @@ func (j *regionJob) convertStageOnIngestError(
 			}
 		}
 		if newRegion != nil {
-			j.region = newRegion
-			j.convertStageTo(regionScanned)
-			return false, nil
+			res.newRegion = newRegion
 		}
-		j.convertStageTo(needRescan)
-		return false, nil
 	case strings.Contains(errPb.Message, "raft: proposal dropped"):
-		j.lastRetryableErr = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRaftProposalDropped.GenWithStack(errPb.GetMessage())
 	case errPb.ServerIsBusy != nil:
-		j.lastRetryableErr = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
-
-		return false, nil
+		res.err = common.ErrKVServerIsBusy.GenWithStack(errPb.GetMessage())
 	case errPb.RegionNotFound != nil:
-		j.lastRetryableErr = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
-
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVRegionNotFound.GenWithStack(errPb.GetMessage())
 	case errPb.ReadIndexNotReady != nil:
-		j.lastRetryableErr = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
-
 		// this error happens when this region is splitting, the error might be:
 		//   read index not ready, reason can not read index due to split, region 64037
 		// we have paused schedule, but it's temporary,
 		// if next request takes a long time, there's chance schedule is enabled again
 		// or on key range border, another engine sharing this region tries to split this
 		// region may cause this error too.
-		j.convertStageTo(needRescan)
-		return false, nil
+		res.err = common.ErrKVReadIndexNotReady.GenWithStack(errPb.GetMessage())
 	case errPb.DiskFull != nil:
-		j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+		res.err = common.ErrKVDiskFull.GenWithStack(errPb.GetMessage())
+	default:
+		// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
+		res.err = common.ErrKVIngestFailed.GenWithStack(errPb.GetMessage())
+	}
+	return res
+}
 
-		return false, errors.Errorf("non-retryable error: %s", resp.GetError().GetMessage())
+// the input error must be an retryable error
+func getNextStageOnIngestError(err error) (*split.RegionInfo, jobStageTp) {
+	var theErr *ingestAPIError
+	if goerrors.As(err, &theErr) {
+		switch {
+		case goerrors.Is(theErr.err, common.ErrKVIngestFailed):
+			return nil, regionScanned
+		case goerrors.Is(theErr.err, common.ErrKVServerIsBusy):
+			return nil, wrote
+		default:
+			if theErr.newRegion != nil {
+				return theErr.newRegion, regionScanned
+			}
+			return nil, needRescan
+		}
 	}
-	// all others doIngest error, such as stale command, etc. we'll retry it again from writeAndIngestByRange
-	j.lastRetryableErr = common.ErrKVIngestFailed.GenWithStack(resp.GetError().GetMessage())
-	j.convertStageTo(regionScanned)
-	return false, nil
+	// we failed to call Ingest or MultiIngest on some retryable errors, such as
+	// network errors
+	return nil, wrote
path: pkg/lightning/backend/local/region_job.go
user: login: lance6716
body: /unhold
created_at: 2025-04-22T10:29:23Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-04-21T11:16:46Z
body: ## [Codecov](https://app.codecov.io/gh/pingcap/tidb/pull/60691?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) Report
Attention: Patch coverage is `90.47619%` with `6 lines` in your changes missing coverage. Please review.
> Please [upload](https://docs.codecov.com/docs/codecov-uploader) report for BASE (`feature/next-gen-tidb@22aa4fd`). [Learn more](https://docs.codecov.io/docs/error-reference?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap#section-missing-base-commit) about missing BASE report.

<details><summary>Additional details and impacted files</summary>


```diff
@@                    Coverage Diff                     @@
##             feature/next-gen-tidb     #60691   +/-   ##
==========================================================
  Coverage                         ?   73.7873%           
==========================================================
  Files                            ?       1743           
  Lines                            ?     481118           
  Branches                         ?          0           
==========================================================
  Hits                             ?     355004           
  Misses                           ?     104141           
  Partials                         ?      21973           
```

| [Flag](https://app.codecov.io/gh/pingcap/tidb/pull/60691/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | Coverage Δ | |
|---|---|---|
| [integration](https://app.codecov.io/gh/pingcap/tidb/pull/60691/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | `45.5505% <60.3174%> (?)` | |
| [unit](https://app.codecov.io/gh/pingcap/tidb/pull/60691/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | `72.3669% <90.4761%> (?)` | |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Components](https://app.codecov.io/gh/pingcap/tidb/pull/60691/components?src=pr&el=components&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | Coverage Δ | |
|---|---|---|
| [dumpling](https://app.codecov.io/gh/pingcap/tidb/pull/60691/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | `52.6910% <0.0000%> (?)` | |
| [parser](https://app.codecov.io/gh/pingcap/tidb/pull/60691/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | `∅ <0.0000%> (?)` | |
| [br](https://app.codecov.io/gh/pingcap/tidb/pull/60691/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | `47.2312% <0.0000%> (?)` | |

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE
user: login: ti-chi-bot[bot]
created_at: 2025-04-22T09:47:09Z
body: @joechenrh: adding LGTM is restricted to approvers and reviewers in OWNERS files.

<details>

In response to [this](https://github.com/pingcap/tidb/pull/60691#pullrequestreview-2782814148):

>rest LGTM


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE
user: login: D3Hunter
created_at: 2025-04-22T10:18:59Z
body: /retest
author_association: CONTRIBUTOR
user: login: tiprow[bot]
created_at: 2025-04-22T10:19:39Z
body: @D3Hunter: Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message.

<details>

In response to [this](https://github.com/pingcap/tidb/pull/60691#issuecomment-2820860949):

>/retest


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE
user: login: ti-chi-bot[bot]
created_at: 2025-04-22T10:21:26Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/pingcap/tidb/pull/60691#pullrequestreview-2783228854" title="Approved">GMHDBJD</a>*, *<a href="https://github.com/pingcap/tidb/pull/60691#pullrequestreview-2782814148" title="Approved">joechenrh</a>*, *<a href="https://github.com/pingcap/tidb/pull/60691#pullrequestreview-2783617183" title="Approved">lance6716</a>*

The full list of commands accepted by this bot can be found [here](https://prow.tidb.net/command-help?repo=pingcap%2Ftidb).

The pull request process is described [here](https://book.prow.tidb.net/#/workflows/pr)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/pingcap/tidb/blob/feature/next-gen-tidb/OWNERS)~~ [GMHDBJD,lance6716]
- ~~[pkg/lightning/OWNERS](https://github.com/pingcap/tidb/blob/feature/next-gen-tidb/pkg/lightning/OWNERS)~~ [GMHDBJD]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE
user: login: ti-chi-bot[bot]
created_at: 2025-04-22T10:21:31Z
body: [LGTM Timeline notifier]
---
Timeline:

- `2025-04-22 08:10:46.634885322 +0000 UTC m=+343190.446675699`: :ballot_box_with_check: agreed by [GMHDBJD](https://github.com/GMHDBJD).
- `2025-04-22 10:21:30.641233813 +0000 UTC m=+351034.453024194`: :ballot_box_with_check: agreed by [lance6716](https://github.com/lance6716).
author_association: NONE
user: login: D3Hunter
created_at: 2025-04-22T10:22:26Z
body: /retest
author_association: CONTRIBUTOR
user: login: tiprow[bot]
created_at: 2025-04-22T10:23:16Z
body: @D3Hunter: Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message.

<details>

In response to [this](https://github.com/pingcap/tidb/pull/60691#issuecomment-2820869057):

>/retest


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE
user: login: D3Hunter
created_at: 2025-04-22T10:56:54Z
body: /retest
author_association: CONTRIBUTOR
user: login: tiprow[bot]
created_at: 2025-04-22T10:57:16Z
body: @D3Hunter: Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message.

<details>

In response to [this](https://github.com/pingcap/tidb/pull/60691#issuecomment-2820950077):

>/retest


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE