Repository: etcd-io/etcd

Issue:
UnsafeRange panicking during shutdown
### Bug report criteria

- [ ] This bug report is not security related, security issues should be disclosed privately via security@etcd.io.
- [ ] This is not a support request or question, support requests or questions should be raised in the etcd [discussion forums](https://github.com/etcd-io/etcd/discussions).
- [ ] You have read the etcd [bug reporting guidelines](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/reporting_bugs.md).
- [ ] Existing open issues along with etcd [frequently asked questions](https://etcd.io/docs/latest/faq) have been checked and this is not a duplicate.

### What happened?

Test case `TestMaintenanceSnapshotCancel` failed and panicking. 

Refer to https://github.com/etcd-io/etcd/actions/runs/7463174417/job/20307221683?pr=17220

Based on the log, the reason should be that the backend has already been closed (the member is being stopped) before the snapshot operation, https://github.com/etcd-io/etcd/blob/a2eb17c8091893796e835cd564c78a7b4c917c21/server/storage/backend/backend.go#L331

```
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xa6c27c]

goroutine 19029 [running]:
go.etcd.io/bbolt.(*Cursor).seek(0xc00044a578, {0x1aa05d4, 0x4, 0x4})
	/home/runner/go/pkg/mod/go.etcd.io/bbolt@v1.3.8/cursor.go:159 +0x7c
go.etcd.io/bbolt.(*Bucket).Bucket(0xc0009da638, {0x1aa05d4, 0x4, 0x4})
	/home/runner/go/pkg/mod/go.etcd.io/bbolt@v1.3.8/bucket.go:105 +0x10c
go.etcd.io/bbolt.(*Tx).Bucket(...)
	/home/runner/go/pkg/mod/go.etcd.io/bbolt@v1.3.8/tx.go:102
go.etcd.io/etcd/server/v3/storage/backend.(*batchTx).UnsafeRange(0xc0017a7da0, {0x12e17b0, 0x1b0dc40}, {0x1aa1670, 0x10, 0x10}, {0x0, 0x0, 0x0}, 0xc4798?)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/backend/batch_tx.go:174 +0xa0
go.etcd.io/etcd/server/v3/storage/schema.UnsafeReadConsistentIndex({0xffff39290758, 0xc0017a7da0})
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/schema/cindex.go:41 +0xa0
go.etcd.io/etcd/server/v3/storage/schema.unsafeUpdateConsistentIndex.func1()
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/schema/cindex.go:80 +0x54
go.etcd.io/etcd/client/pkg/v3/verify.Verify(0xc00044aa28)
	/home/runner/actions-runner/_work/etcd/etcd/client/pkg/verify/verify.go:71 +0x44
go.etcd.io/etcd/server/v3/storage/schema.unsafeUpdateConsistentIndex({0x12e5920, 0xc0017a7da0}, 0x7, 0x2, 0x0)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/schema/cindex.go:79 +0x1d4
go.etcd.io/etcd/server/v3/storage/schema.UnsafeUpdateConsistentIndex(...)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/schema/cindex.go:67
go.etcd.io/etcd/server/v3/etcdserver/cindex.(*consistentIndex).UnsafeSave(0xc001727800, {0x12e5920, 0xc0017a7da0})
	/home/runner/actions-runner/_work/etcd/etcd/server/etcdserver/cindex/cindex.go:121 +0x68
go.etcd.io/etcd/server/v3/storage.(*BackendHooks).OnPreCommitUnsafe(0xc001a0d9e0, {0x12e5920?, 0xc0017a7da0})
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/hooks.go:45 +0x64
go.etcd.io/etcd/server/v3/storage/backend.(*batchTxBuffered).unsafeCommit(0xc0017a7da0, 0x0)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/backend/batch_tx.go:342 +0xfc
go.etcd.io/etcd/server/v3/storage/backend.(*batchTxBuffered).commit(0xc0017a7da0, 0xdc?)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/backend/batch_tx.go:335 +0x70
go.etcd.io/etcd/server/v3/storage/backend.(*batchTxBuffered).Commit(0xc0017a7da0)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/backend/batch_tx.go:322 +0x3c
go.etcd.io/etcd/server/v3/storage/backend.(*backend).Snapshot(0xc000329140)
	/home/runner/actions-runner/_work/etcd/etcd/server/storage/backend/backend.go:331 +0x54
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*maintenanceServer).Snapshot(0xc0029ace40, 0x12e1120?, {0x12e6550, 0xc0006302b0})
	/home/runner/actions-runner/_work/etcd/etcd/server/etcdserver/api/v3rpc/maintenance.go:111 +0x110
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*authMaintenanceServer).Snapshot(0xc0019d7b60, 0x103b4c0?, {0x12e6550, 0xc0006302b0})
	/home/runner/actions-runner/_work/etcd/etcd/server/etcdserver/api/v3rpc/maintenance.go:296 +0xb8
go.etcd.io/etcd/api/v3/etcdserverpb._Maintenance_Snapshot_Handler({0x1004a40?, 0xc0019d7b60}, {0x12e3fc8, 0xc0028e4c00})
	/home/runner/actions-runner/_work/etcd/etcd/api/etcdserverpb/rpc.pb.go:7620 +0xe8
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).StreamServerInterceptor.func4({0x1004a40, 0xc0019d7b60}, {0x12e3d80?, 0xc0005681e0}, 0xc0028e4bd0, 0x111e8a8)
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x128
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func7.1.1({0x1004a40, 0xc0019d7b60}, {0x12e3d80, 0xc0005681e0})
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x70
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.newStreamInterceptor.func1({0x1004a40, 0xc0019d7b60}, {0x12e3d80, 0xc0005681e0}, 0xc0028e4bd0, 0xc001992240)
	/home/runner/actions-runner/_work/etcd/etcd/server/etcdserver/api/v3rpc/interceptor.go:258 +0x560
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func7.1.1({0x1004a40, 0xc0019d7b60}, {0x12e3d80, 0xc0005681e0})
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:49 +0x70
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func7({0x1004a40, 0xc0019d7b60}, {0x12e3d80, 0xc0005681e0}, 0xc0028e4bd0, 0x111e8a8)
	/home/runner/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/chain.go:58 +0x100
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00162c400, {0x12e1120, 0xc00185e870}, {0x12e8340, 0xc000753380}, 0xc00[284](https://github.com/etcd-io/etcd/actions/runs/7463174417/job/20307221683?pr=17220#step:5:285)6b40, 0xc0028ac2d0, 0x1b0c680, 0x0)
	/home/runner/go/pkg/mod/google.golang.org/grpc@v1.60.1/server.go:1673 +0x16b8
google.golang.org/grpc.(*Server).handleStream(0xc00162c400, {0x12e8340, 0xc000753380}, 0xc002846b40)
	/home/runner/go/pkg/mod/google.golang.org/grpc@v1.60.1/server.go:1787 +0x12f0
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	/home/runner/go/pkg/mod/google.golang.org/grpc@v1.60.1/server.go:1016 +0xa0
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 19027
	/home/runner/go/pkg/mod/google.golang.org/grpc@v1.60.1/server.go:1027 +0x1d8
```

### What did you expect to happen?

No panicking on processing any client requests

### How can we reproduce it (as minimally and precisely as possible)?

Write an integration test to stop a member before call the snapshot api.

### Anything else we need to know?

_No response_

### Etcd version (please run commands below)

<details>

```console
$ etcd --version
# paste output here

$ etcdctl version
# paste output here
```

</details>


### Etcd configuration (command line flags or environment variables)

<details>

# paste your configuration here

</details>


### Etcd debug information (please run commands below, feel free to obfuscate the IP address or FQDN in the output)

<details>

```console
$ etcdctl member list -w table
# paste output here

$ etcdctl --endpoints=<member list> endpoint status -w table
# paste output here
```

</details>


### Relevant log output

_No response_

Hints from issue comments:
@fuweid: UPDATED:

When server closes the backend, the backend will stop background commit goroutine and reset read transaction.

https://github.com/etcd-io/etcd/blob/c8b4b168aff7f84d3b10dc9ddc13a381f15a710c/server/storage/backend/read_tx.go#L133-L138

For `baseReadTx`, it doesn't check the `tx` in UnsafeRange and update `txWg`. 

https://github.com/etcd-io/etcd/blob/c8b4b168aff7f84d3b10dc9ddc13a381f15a710c/server/storage/backend/batch_tx.go#L367-L376

The `tx` can be set to nil during `UnsafeRange`.
@serathius: Another case with https://github.com/etcd-io/bbolt/issues/715@serathius: @fuweid would you be able to propose a fix? The issue showed up in robustness tests, which I would prefer to keep flake free.@fuweid: Hi @serathius Sure. Will file pull request later.@fuweid: Hi @ahrtr @serathius  

sorry for taking so long on this issue. This issue has been fixed by gRPC layer https://github.com/grpc/grpc-go/commit/61eab37838ce213237ecb31aa7cdf95241851431 (released by [v1.61.0](https://github.com/grpc/grpc-go/releases/tag/v1.61.0) to fix regression): All the requests are tracked by `handlerWG` wait group. When we call `GracefulStop`, it will block until all the inflight requests are closed, including streaming type RPC, like Watch/Snapshot/LeaseRenew.

We call gracefulstop when we received SIGTERM signal, except cmux-mode.

https://github.com/etcd-io/etcd/blob/a7f5d4b4e4569bd316277ebf1347785e0467c64d/server/embed/etcd.go#L474-L493

We don't need to setup timeout for draining things because `GracefulStop` always blocks until all the RPCs finished.
Since `GracefulStop` isn't applied for cmux-mode, I filed a pull request https://github.com/etcd-io/etcd/pull/17790 to support cmux-mode's graceful shutdown. Even if we run ETCD without #17790, only ongoing Snapshot/Watch RPCs might panic, because we stop applier channel and scheduler before stop backend and all the unary RPCs won't use closed backend.

https://github.com/etcd-io/etcd/blob/a7f5d4b4e4569bd316277ebf1347785e0467c64d/server/etcdserver/server.go#L816-L827

And the https://github.com/etcd-io/etcd/pull/17757 is also enhancement for failpoint test. PTAL. Thanks

------

Side note: I was using old version (1.60.1) so that previous approach is to introduce `txRef` object to maintain reference count about all opening TX. The caller must call `txPut` to release the reference explict. If the backend has been closed, the `ReadTx/ConcurrentReadTx/BatchTx` should return closed error. However, in our codebase, both mvcc and auth layers assume the backend is always valid. The `ReadTx/ConcurrenReadTx/BatchTx` are kind of direct pointer. And `UnsafeRange` doesn't intend to return error. I tried to update all the interfaces to force it return error. It passes all the e2e and UT. However, there are too many changes.

If the server layer can track active RPCs, it will be better. So, I revisit the gRPC code and find that `WaitForHandlers` can help us. 

```go
type txRef struct {
     sync.RWMutex
     wg sync.WaitGroup
}

type Backend interface {
      ReadTx() (ReadTx, TxRefReleaseFunc, error)
      ConcurrentReadTx() (ReadTx, TxRefReleaseFunc, error)
      BatchTx() (BatchTx, TxRefReleaseFunc, error)
}

tx, txPut, err := ReadTx() // ConncurrentReadTx() / BatchTx()
if err != nil {
    return err
}
defer txPut()
..
```
@stale: This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 21 days if no further activity occurs. Thank you for your contributions.@serathius: Fixed via https://github.com/etcd-io/etcd/pull/17757

Solution patch:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 48f7d6343..63ed5102a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -138,6 +138,11 @@ If you are still working on the pull request, you can convert it to draft by cli
 
 Multiple small PRs are preferred over single large ones (>500 lines of code).
 
+Please make sure there is an associated issue for each PR you submit. Create one if it doesn't exist yet, and close the issue
+once the PR gets merged and has been backported to previous stable releases, if necessary. If there are multiple PRs linked to
+the same issue, refrain from closing the issue until all PRs have been merged and, if needed, backported to previous stable
+releases.
+
 ## Get your pull request reviewed
 
 Before requesting review please ensure that all GitHub checks were successful.
diff --git a/Makefile b/Makefile
index 485482888..180f9def7 100644
--- a/Makefile
+++ b/Makefile
@@ -185,6 +185,7 @@ clean:
 	rm -f ./bin/Dockerfile-release
 	rm -rf ./bin/etcd*
 	rm -rf ./bin/lazyfs
+	rm -rf ./bin/python
 	rm -rf ./default.etcd
 	rm -rf ./tests/e2e/default.etcd
 	rm -rf ./release
diff --git a/etcdutl/go.mod b/etcdutl/go.mod
index d6a7a9443..9d57f121a 100644
--- a/etcdutl/go.mod
+++ b/etcdutl/go.mod
@@ -24,13 +24,13 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/spf13/cobra v1.8.0
-	go.etcd.io/bbolt v1.3.8
+	go.etcd.io/bbolt v1.4.0-alpha.0
 	go.etcd.io/etcd/api/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/client/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/server/v3 v3.6.0-alpha.0
-	go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0
+	go.etcd.io/raft/v3 v3.6.0-alpha.0
 	go.uber.org/zap v1.26.0
 )
 
diff --git a/etcdutl/go.sum b/etcdutl/go.sum
index 62e0e36be..d2bd3cf6f 100644
--- a/etcdutl/go.sum
+++ b/etcdutl/go.sum
@@ -80,10 +80,10 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0 h1:Yatm3/u91jNJTGVeENBBg5QSh1BQJ541IBS9nb5JDkw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
+go.etcd.io/bbolt v1.4.0-alpha.0 h1:e+k/D0uBHe478Ws92tajgwCv8PUIclXQWah12LKloOA=
+go.etcd.io/bbolt v1.4.0-alpha.0/go.mod h1:7mcLPaWHfD+onzQl+1demCxYgAShanm1UViDNLKBEwU=
+go.etcd.io/raft/v3 v3.6.0-alpha.0 h1:cMmjAEjCKMGiQPowjSWM43Y5ZnBEeNP8RSYcm3ewtns=
+go.etcd.io/raft/v3 v3.6.0-alpha.0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
 go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
@@ -116,6 +116,8 @@ golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCA
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/go.mod b/go.mod
index a89331f48..16c12b151 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
-	go.etcd.io/bbolt v1.3.8
+	go.etcd.io/bbolt v1.4.0-alpha.0
 	go.etcd.io/etcd/api/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/client/v2 v2.306.0-alpha.0
@@ -31,7 +31,7 @@ require (
 	go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/server/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/tests/v3 v3.6.0-alpha.0
-	go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0
+	go.etcd.io/raft/v3 v3.6.0-alpha.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/time v0.5.0
 	google.golang.org/grpc v1.60.1
diff --git a/go.sum b/go.sum
index 694edd4a6..1eb5562b2 100644
--- a/go.sum
+++ b/go.sum
@@ -143,12 +143,12 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.4.0-alpha.0 h1:e+k/D0uBHe478Ws92tajgwCv8PUIclXQWah12LKloOA=
+go.etcd.io/bbolt v1.4.0-alpha.0/go.mod h1:7mcLPaWHfD+onzQl+1demCxYgAShanm1UViDNLKBEwU=
 go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
 go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0 h1:Yatm3/u91jNJTGVeENBBg5QSh1BQJ541IBS9nb5JDkw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
+go.etcd.io/raft/v3 v3.6.0-alpha.0 h1:cMmjAEjCKMGiQPowjSWM43Y5ZnBEeNP8RSYcm3ewtns=
+go.etcd.io/raft/v3 v3.6.0-alpha.0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
 go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
@@ -204,8 +204,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go
index f022d29c4..6becdfd62 100644
--- a/server/etcdserver/api/membership/cluster.go
+++ b/server/etcdserver/api/membership/cluster.go
@@ -284,6 +284,10 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
 	onSet(c.lg, c.version)
 
 	for _, m := range c.members {
+		if c.localID == m.ID {
+			setIsLearnerMetric(m)
+		}
+
 		c.lg.Info(
 			"recovered/added member from store",
 			zap.String("cluster-id", c.cid.String()),
@@ -393,6 +397,11 @@ func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) {
 	if c.v2store != nil {
 		mustSaveMemberToStore(c.lg, c.v2store, m)
 	}
+
+	if m.ID == c.localID {
+		setIsLearnerMetric(m)
+	}
+
 	if c.be != nil && shouldApplyV3 {
 		c.be.MustSaveMemberToBackend(m)
 
@@ -505,6 +514,10 @@ func (c *RaftCluster) PromoteMember(id types.ID, shouldApplyV3 ShouldApplyV3) {
 		mustUpdateMemberInStore(c.lg, c.v2store, &m)
 	}
 
+	if id == c.localID {
+		isLearner.Set(0)
+	}
+
 	if c.be != nil && shouldApplyV3 {
 		c.members[id].RaftAttributes.IsLearner = false
 		c.updateMembershipMetric(id, true)
diff --git a/server/etcdserver/api/membership/metrics.go b/server/etcdserver/api/membership/metrics.go
index f08763779..7eb6fb6b7 100644
--- a/server/etcdserver/api/membership/metrics.go
+++ b/server/etcdserver/api/membership/metrics.go
@@ -32,9 +32,24 @@ var (
 	},
 		[]string{"Local", "Remote"},
 	)
+	isLearner = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "is_learner",
+		Help:      "Whether or not this member is a learner. 1 if is, 0 otherwise.",
+	})
 )
 
+func setIsLearnerMetric(m *Member) {
+	if m.IsLearner {
+		isLearner.Set(1)
+	} else {
+		isLearner.Set(0)
+	}
+}
+
 func init() {
 	prometheus.MustRegister(ClusterVersionMetrics)
 	prometheus.MustRegister(knownPeers)
+	prometheus.MustRegister(isLearner)
 }
diff --git a/server/etcdserver/metrics.go b/server/etcdserver/metrics.go
index 7af0e7354..4be44a57a 100644
--- a/server/etcdserver/metrics.go
+++ b/server/etcdserver/metrics.go
@@ -44,12 +44,6 @@ var (
 		Name:      "leader_changes_seen_total",
 		Help:      "The number of leader changes seen.",
 	})
-	isLearner = prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "etcd",
-		Subsystem: "server",
-		Name:      "is_learner",
-		Help:      "Whether or not this member is a learner. 1 if is, 0 otherwise.",
-	})
 	learnerPromoteFailed = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "etcd",
 		Subsystem: "server",
@@ -171,7 +165,6 @@ func init() {
 	prometheus.MustRegister(currentVersion)
 	prometheus.MustRegister(currentGoVersion)
 	prometheus.MustRegister(serverID)
-	prometheus.MustRegister(isLearner)
 	prometheus.MustRegister(learnerPromoteSucceed)
 	prometheus.MustRegister(learnerPromoteFailed)
 	prometheus.MustRegister(fdUsed)
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 59ae7b739..83951434c 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -2060,15 +2060,6 @@ func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, confState *raftpb.Con
 			}
 		}
 
-		// update the isLearner metric when this server id is equal to the id in raft member confChange
-		if confChangeContext.Member.ID == s.MemberId() {
-			if cc.Type == raftpb.ConfChangeAddLearnerNode {
-				isLearner.Set(1)
-			} else {
-				isLearner.Set(0)
-			}
-		}
-
 	case raftpb.ConfChangeRemoveNode:
 		id := types.ID(cc.NodeID)
 		s.cluster.RemoveMember(id, shouldApplyV3)
diff --git a/server/go.mod b/server/go.mod
index 548a738c4..c2ed32994 100644
--- a/server/go.mod
+++ b/server/go.mod
@@ -23,13 +23,13 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
-	go.etcd.io/bbolt v1.3.8
+	go.etcd.io/bbolt v1.4.0-alpha.0
 	go.etcd.io/etcd/api/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/client/v2 v2.306.0-alpha.0
 	go.etcd.io/etcd/client/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
-	go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0
+	go.etcd.io/raft/v3 v3.6.0-alpha.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1
 	go.opentelemetry.io/otel v1.21.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
diff --git a/server/go.sum b/server/go.sum
index c2911ad03..cec9d529d 100644
--- a/server/go.sum
+++ b/server/go.sum
@@ -123,10 +123,10 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0 h1:Yatm3/u91jNJTGVeENBBg5QSh1BQJ541IBS9nb5JDkw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
+go.etcd.io/bbolt v1.4.0-alpha.0 h1:e+k/D0uBHe478Ws92tajgwCv8PUIclXQWah12LKloOA=
+go.etcd.io/bbolt v1.4.0-alpha.0/go.mod h1:7mcLPaWHfD+onzQl+1demCxYgAShanm1UViDNLKBEwU=
+go.etcd.io/raft/v3 v3.6.0-alpha.0 h1:cMmjAEjCKMGiQPowjSWM43Y5ZnBEeNP8RSYcm3ewtns=
+go.etcd.io/raft/v3 v3.6.0-alpha.0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
 go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
@@ -182,6 +182,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/server/storage/backend/backend.go b/server/storage/backend/backend.go
index e7b951ee7..7aa4f8469 100644
--- a/server/storage/backend/backend.go
+++ b/server/storage/backend/backend.go
@@ -180,6 +180,7 @@ func newBackend(bcfg BackendConfig) *backend {
 	bopts.NoSync = bcfg.UnsafeNoFsync
 	bopts.NoGrowSync = bcfg.UnsafeNoFsync
 	bopts.Mlock = bcfg.Mlock
+	bopts.Logger = newBoltLoggerZap(bcfg)
 
 	db, err := bolt.Open(bcfg.Path, 0600, bopts)
 	if err != nil {
@@ -659,3 +660,20 @@ func (s *snapshot) Close() error {
 	<-s.donec
 	return s.Tx.Rollback()
 }
+
+func newBoltLoggerZap(bcfg BackendConfig) bolt.Logger {
+	lg := bcfg.Logger.Named("bbolt")
+	return &zapBoltLogger{lg.WithOptions(zap.AddCallerSkip(1)).Sugar()}
+}
+
+type zapBoltLogger struct {
+	*zap.SugaredLogger
+}
+
+func (zl *zapBoltLogger) Warning(args ...any) {
+	zl.SugaredLogger.Warn(args...)
+}
+
+func (zl *zapBoltLogger) Warningf(format string, args ...any) {
+	zl.SugaredLogger.Warnf(format, args...)
+}
diff --git a/server/storage/backend/tx_buffer.go b/server/storage/backend/tx_buffer.go
index 7c2f9d63a..590cf2af6 100644
--- a/server/storage/backend/tx_buffer.go
+++ b/server/storage/backend/tx_buffer.go
@@ -83,6 +83,7 @@ func (txw *txWriteBuffer) writeback(txr *txReadBuffer) {
 		if !ok {
 			delete(txw.buckets, k)
 			txr.buckets[k] = wb
+			wb.dedupe()
 			continue
 		}
 		if seq, ok := txw.bucket2seq[k]; ok && !seq && wb.used > 1 {
@@ -203,10 +204,12 @@ func (bb *bucketBuffer) merge(bbsrc *bucketBuffer) {
 	if bytes.Compare(bb.buf[(bb.used-bbsrc.used)-1].key, bbsrc.buf[0].key) < 0 {
 		return
 	}
+	bb.dedupe()
+}
 
+// dedupe removes duplicates, using only newest update
+func (bb *bucketBuffer) dedupe() {
 	sort.Stable(bb)
-
-	// remove duplicates, using only newest update
 	widx := 0
 	for ridx := 1; ridx < bb.used; ridx++ {
 		if !bytes.Equal(bb.buf[ridx].key, bb.buf[widx].key) {
diff --git a/tools/mod/go.mod b/tools/mod/go.mod
index 988ef29b0..45cf95c31 100644
--- a/tools/mod/go.mod
+++ b/tools/mod/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/mikefarah/yq/v4 v4.40.5
 	go.etcd.io/gofail v0.1.0
 	go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
-	go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0
+	go.etcd.io/raft/v3 v3.6.0-alpha.0
 	gotest.tools/gotestsum v1.11.0
 	gotest.tools/v3 v3.5.1
 	honnef.co/go/tools v0.4.6
diff --git a/tools/mod/go.sum b/tools/mod/go.sum
index edb3fad70..aa26d476b 100644
--- a/tools/mod/go.sum
+++ b/tools/mod/go.sum
@@ -684,8 +684,8 @@ go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
 go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ=
 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0 h1:Yatm3/u91jNJTGVeENBBg5QSh1BQJ541IBS9nb5JDkw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
+go.etcd.io/raft/v3 v3.6.0-alpha.0 h1:cMmjAEjCKMGiQPowjSWM43Y5ZnBEeNP8RSYcm3ewtns=
+go.etcd.io/raft/v3 v3.6.0-alpha.0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=



Test patch:
diff --git a/server/storage/backend/batch_tx_test.go b/server/storage/backend/batch_tx_test.go
index cc099d1f9..d0f6dcdc0 100644
--- a/server/storage/backend/batch_tx_test.go
+++ b/server/storage/backend/batch_tx_test.go
@@ -252,6 +252,53 @@ func TestRangeAfterDeleteMatch(t *testing.T) {
 	checkForEach(t, b.BatchTx(), b.ReadTx(), nil, nil)
 }
 
+func TestRangeAfterOverwriteMatch(t *testing.T) {
+	b, _ := betesting.NewTmpBackend(t, time.Hour, 10000)
+	defer betesting.Close(t, b)
+
+	tx := b.BatchTx()
+
+	tx.Lock()
+	tx.UnsafeCreateBucket(schema.Test)
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar2"))
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar0"))
+	tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar10"))
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar1"))
+	tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar11"))
+	tx.Unlock()
+
+	checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), []byte("foo"), []byte("foo3"), 1)
+	checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo"), []byte("foo1")}, [][]byte{[]byte("bar1"), []byte("bar11")})
+}
+
+func TestRangeAfterOverwriteAndDeleteMatch(t *testing.T) {
+	b, _ := betesting.NewTmpBackend(t, time.Hour, 10000)
+	defer betesting.Close(t, b)
+
+	tx := b.BatchTx()
+
+	tx.Lock()
+	tx.UnsafeCreateBucket(schema.Test)
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar2"))
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar0"))
+	tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar10"))
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar1"))
+	tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar11"))
+	tx.Unlock()
+
+	checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), []byte("foo"), nil, 0)
+	checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo"), []byte("foo1")}, [][]byte{[]byte("bar1"), []byte("bar11")})
+
+	tx.Lock()
+	tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar3"))
+	tx.UnsafeDelete(schema.Test, []byte("foo1"))
+	tx.Unlock()
+
+	checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), []byte("foo"), nil, 0)
+	checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), []byte("foo1"), nil, 0)
+	checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo")}, [][]byte{[]byte("bar3")})
+}
+
 func checkRangeResponseMatch(t *testing.T, tx backend.BatchTx, rtx backend.ReadTx, key, endKey []byte, limit int64) {
 	tx.Lock()
 	ks1, vs1 := tx.UnsafeRange(schema.Test, key, endKey, limit)
diff --git a/tests/e2e/metrics_test.go b/tests/e2e/metrics_test.go
index 312b5aab5..1ed85f5bb 100644
--- a/tests/e2e/metrics_test.go
+++ b/tests/e2e/metrics_test.go
@@ -15,11 +15,14 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/pkg/v3/expect"
+	"go.etcd.io/etcd/tests/v3/framework/config"
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
 )
 
@@ -37,6 +40,18 @@ func TestV3MetricsInsecure(t *testing.T) {
 	testCtl(t, metricsTest)
 }
 
+func TestV3LearnerMetricRecover(t *testing.T) {
+	cfg := e2e.NewConfigTLS()
+	cfg.ServerConfig.SnapshotCount = 10
+	testCtl(t, learnerMetricRecoverTest, withCfg(*cfg))
+}
+
+func TestV3LearnerMetricApplyFromSnapshotTest(t *testing.T) {
+	cfg := e2e.NewConfigTLS()
+	cfg.ServerConfig.SnapshotCount = 10
+	testCtl(t, learnerMetricApplyFromSnapshotTest, withCfg(*cfg))
+}
+
 func metricsTest(cx ctlCtx) {
 	if err := ctlV3Put(cx, "k", "v", ""); err != nil {
 		cx.t.Fatal(err)
@@ -69,3 +84,64 @@ func metricsTest(cx ctlCtx) {
 		}
 	}
 }
+
+func learnerMetricRecoverTest(cx ctlCtx) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if _, err := cx.epc.StartNewProc(ctx, nil, cx.t, true /* addAsLearner */); err != nil {
+		cx.t.Fatal(err)
+	}
+	expectLearnerMetrics(cx)
+
+	triggerSnapshot(ctx, cx)
+
+	// Restart cluster
+	if err := cx.epc.Restart(ctx); err != nil {
+		cx.t.Fatal(err)
+	}
+	expectLearnerMetrics(cx)
+}
+
+func learnerMetricApplyFromSnapshotTest(cx ctlCtx) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Add learner but do not start it
+	_, learnerCfg, err := cx.epc.AddMember(ctx, nil, cx.t, true /* addAsLearner */)
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+
+	triggerSnapshot(ctx, cx)
+
+	// Start the learner
+	if err = cx.epc.StartNewProcFromConfig(ctx, cx.t, learnerCfg); err != nil {
+		cx.t.Fatal(err)
+	}
+	expectLearnerMetrics(cx)
+}
+
+func triggerSnapshot(ctx context.Context, cx ctlCtx) {
+	etcdctl := cx.epc.Procs[0].Etcdctl()
+	for i := 0; i < int(cx.epc.Cfg.ServerConfig.SnapshotCount); i++ {
+		if err := etcdctl.Put(ctx, "k", "v", config.PutOptions{}); err != nil {
+			cx.t.Fatal(err)
+		}
+	}
+}
+
+func expectLearnerMetrics(cx ctlCtx) {
+	expectLearnerMetric(cx, 0, "etcd_server_is_learner 0")
+	expectLearnerMetric(cx, 1, "etcd_server_is_learner 1")
+}
+
+func expectLearnerMetric(cx ctlCtx, procIdx int, expectMetric string) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	args := e2e.CURLPrefixArgsCluster(cx.epc.Cfg, cx.epc.Procs[procIdx], "GET", e2e.CURLReq{Endpoint: "/metrics"})
+	if err := e2e.SpawnWithExpectsContext(ctx, args, nil, expect.ExpectedResponse{Value: expectMetric}); err != nil {
+		cx.t.Fatal(err)
+	}
+}
diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go
index 1815a0a9d..87e29a705 100644
--- a/tests/framework/e2e/cluster.go
+++ b/tests/framework/e2e/cluster.go
@@ -780,7 +780,21 @@ func (epc *EtcdProcessCluster) CloseProc(ctx context.Context, finder func(EtcdPr
 // Phase 1 - Inform cluster of new configuration
 // Phase 2 - Start new member
 func (epc *EtcdProcessCluster) StartNewProc(ctx context.Context, cfg *EtcdProcessClusterConfig, tb testing.TB, addAsLearner bool, opts ...config.ClientOption) (memberID uint64, err error) {
-	var serverCfg *EtcdServerProcessConfig
+	memberID, serverCfg, err := epc.AddMember(ctx, cfg, tb, addAsLearner, opts...)
+	if err != nil {
+		return 0, err
+	}
+
+	// Then start process
+	if err = epc.StartNewProcFromConfig(ctx, tb, serverCfg); err != nil {
+		return 0, err
+	}
+
+	return memberID, nil
+}
+
+// AddMember adds a new member to the cluster without starting it.
+func (epc *EtcdProcessCluster) AddMember(ctx context.Context, cfg *EtcdProcessClusterConfig, tb testing.TB, addAsLearner bool, opts ...config.ClientOption) (memberID uint64, serverCfg *EtcdServerProcessConfig, err error) {
 	if cfg != nil {
 		serverCfg = cfg.EtcdServerProcessConfig(tb, epc.nextSeq)
 	} else {
@@ -808,20 +822,24 @@ func (epc *EtcdProcessCluster) StartNewProc(ctx context.Context, cfg *EtcdProces
 		resp, err = memberCtl.MemberAdd(ctx, serverCfg.Name, []string{serverCfg.PeerURL.String()})
 	}
 	if err != nil {
-		return 0, fmt.Errorf("failed to add new member: %w", err)
+		return 0, nil, fmt.Errorf("failed to add new member: %w", err)
 	}
 
-	// Then start process
+	return resp.Member.ID, serverCfg, nil
+}
+
+// StartNewProcFromConfig starts a new member process from the given config.
+func (epc *EtcdProcessCluster) StartNewProcFromConfig(ctx context.Context, tb testing.TB, serverCfg *EtcdServerProcessConfig) error {
 	tb.Log("start new member")
 	proc, err := NewEtcdProcess(tb, serverCfg)
 	if err != nil {
 		epc.Close()
-		return 0, fmt.Errorf("cannot configure: %v", err)
+		return fmt.Errorf("cannot configure: %v", err)
 	}
 
 	epc.Procs = append(epc.Procs, proc)
 
-	return resp.Member.ID, proc.Start(ctx)
+	return proc.Start(ctx)
 }
 
 // UpdateProcOptions updates the options for a specific process. If no opt is set, then the config is identical
diff --git a/tests/go.mod b/tests/go.mod
index 4b079e570..b16cd0db5 100644
--- a/tests/go.mod
+++ b/tests/go.mod
@@ -34,7 +34,7 @@ require (
 	go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/server/v3 v3.6.0-alpha.0
 	go.etcd.io/gofail v0.1.0
-	go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0
+	go.etcd.io/raft/v3 v3.6.0-alpha.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1
 	go.opentelemetry.io/otel v1.21.0
 	go.opentelemetry.io/otel/sdk v1.21.0
@@ -42,7 +42,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.0.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.17.0
-	golang.org/x/sync v0.4.0
+	golang.org/x/sync v0.6.0
 	golang.org/x/time v0.5.0
 	google.golang.org/grpc v1.60.1
 	google.golang.org/protobuf v1.32.0
@@ -83,7 +83,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
-	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/bbolt v1.4.0-alpha.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
diff --git a/tests/go.sum b/tests/go.sum
index cd9b5f5a7..20c27ea14 100644
--- a/tests/go.sum
+++ b/tests/go.sum
@@ -147,12 +147,12 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.4.0-alpha.0 h1:e+k/D0uBHe478Ws92tajgwCv8PUIclXQWah12LKloOA=
+go.etcd.io/bbolt v1.4.0-alpha.0/go.mod h1:7mcLPaWHfD+onzQl+1demCxYgAShanm1UViDNLKBEwU=
 go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
 go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0 h1:Yatm3/u91jNJTGVeENBBg5QSh1BQJ541IBS9nb5JDkw=
-go.etcd.io/raft/v3 v3.0.0-20231012085229-7c3ed830bbb0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
+go.etcd.io/raft/v3 v3.6.0-alpha.0 h1:cMmjAEjCKMGiQPowjSWM43Y5ZnBEeNP8RSYcm3ewtns=
+go.etcd.io/raft/v3 v3.6.0-alpha.0/go.mod h1:QpxpKeYmocQQFHP75LxNrdJTukZmqQig9lotwYLsUJY=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
 go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
@@ -208,8 +208,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/tests/integration/clientv3/maintenance_test.go b/tests/integration/clientv3/maintenance_test.go
index e5c9812f8..8aa275bb9 100644
--- a/tests/integration/clientv3/maintenance_test.go
+++ b/tests/integration/clientv3/maintenance_test.go
@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
 	"google.golang.org/grpc"
@@ -176,6 +177,12 @@ func TestMaintenanceSnapshotCancel(t *testing.T) {
 	}
 	defer rc1.Close()
 
+	// read 16 bytes to ensure that server opens snapshot
+	buf := make([]byte, 16)
+	n, err := rc1.Read(buf)
+	assert.NoError(t, err)
+	assert.Equal(t, 16, n)
+
 	cancel()
 	_, err = io.Copy(io.Discard, rc1)
 	if err != context.Canceled {