Repository: matrixorigin/matrixcube (go)

Issue:
Slice out of bound error in raft log
Got the following error when running @jiangxinmeng1's TestSnapshot. 

```
{"level":"panic","ts":"2021-12-24 13:21:56.025","logger":"cube.store","caller":"raftstore/replica.go:617","msg":"slice[9,11) out of bound [9,8]","case":"TestSnapshot","store":"node-2","store-id":1022,"shard-id":2,"replica-id":1104}
panic: slice[9,11) out of bound [9,8]

goroutine 2896 [running]:
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0x1400bd67b00, {0x0, 0x0, 0x0})
  /Users/lni/go/pkg/mod/go.uber.org/zap@v1.19.1/zapcore/entry.go:232 +0x3c8
go.uber.org/zap.(*SugaredLogger).log(0x14011d22ad0, 0x4, {0x1017323e3, 0x21}, {0x1400728b180, 0x4, 0x4}, {0x0, 0x0, 0x0})
  /Users/lni/go/pkg/mod/go.uber.org/zap@v1.19.1/sugar.go:227 +0xfc
go.uber.org/zap.(*SugaredLogger).Panicf(...)
  /Users/lni/go/pkg/mod/go.uber.org/zap@v1.19.1/sugar.go:159
github.com/matrixorigin/matrixcube/raftstore.(*etcdRaftLoggerAdapter).Panicf(0x14011d22ad8, {0x1017323e3, 0x21}, {0x1400728b180, 0x4, 0x4})
  /Users/lni/src/matrixcube/raftstore/replica.go:617 +0x64
go.etcd.io/etcd/raft/v3.(*raftLog).mustCheckOutOfBounds(0x14014eeff10, 0x9, 0xb)
  /Users/lni/go/pkg/mod/github.com/matrixorigin/etcd/raft/v3@v3.5.1-0.20210824022435-0203115049c2/log.go:392 +0x2c8
go.etcd.io/etcd/raft/v3.(*raftLog).slice(0x14014eeff10, 0x9, 0xb, 0x0)
  /Users/lni/go/pkg/mod/github.com/matrixorigin/etcd/raft/v3@v3.5.1-0.20210824022435-0203115049c2/log.go:341 +0x38
go.etcd.io/etcd/raft/v3.(*raftLog).nextEnts(0x14014eeff10)
  /Users/lni/go/pkg/mod/github.com/matrixorigin/etcd/raft/v3@v3.5.1-0.20210824022435-0203115049c2/log.go:183 +0x68
go.etcd.io/etcd/raft/v3.newReady(0x14007894f20, 0x140094fada0, {0x2, 0x0, 0xa})
  /Users/lni/go/pkg/mod/github.com/matrixorigin/etcd/raft/v3@v3.5.1-0.20210824022435-0203115049c2/node.go:562 +0x64
go.etcd.io/etcd/raft/v3.(*RawNode).readyWithoutAccept(...)
  /Users/lni/go/pkg/mod/github.com/matrixorigin/etcd/raft/v3@v3.5.1-0.20210824022435-0203115049c2/rawnode.go:134
go.etcd.io/etcd/raft/v3.(*RawNode).Ready(0x140075a4360)
  /Users/lni/go/pkg/mod/github.com/matrixorigin/etcd/raft/v3@v3.5.1-0.20210824022435-0203115049c2/rawnode.go:126 +0x6c
github.com/matrixorigin/matrixcube/raftstore.(*replica).getRaftReady(...)
  /Users/lni/src/matrixcube/raftstore/replica_event_raft_ready.go:45
github.com/matrixorigin/matrixcube/raftstore.(*replica).handleRaftReady(0x1401aca9800, 0x14012826750)
  /Users/lni/src/matrixcube/raftstore/replica_event_raft_ready.go:36 +0x34
github.com/matrixorigin/matrixcube/raftstore.(*replica).handleEvent(0x1401aca9800, 0x14012826750)
  /Users/lni/src/matrixcube/raftstore/replica_event_loop.go:234 +0x208
github.com/matrixorigin/matrixcube/raftstore.(*replicaWorker).handleEvent(0x14012826780, {0x101cd8fc0, 0x1401aca9800})
  /Users/lni/src/matrixcube/raftstore/worker_pool.go:114 +0x58
github.com/matrixorigin/matrixcube/raftstore.(*replicaWorker).workerMain(0x14012826780)
  /Users/lni/src/matrixcube/raftstore/worker_pool.go:91 +0xb4
github.com/matrixorigin/matrixcube/raftstore.newReplicaWorker.func1()
  /Users/lni/src/matrixcube/raftstore/worker_pool.go:72 +0x28
github.com/lni/goutils/syncutil.(*Stopper).runWorker.func1(0x14013b4a1a0, 0x1401abca5e0, {0x0, 0x0}, 0x1401aa7b0d0)
  /Users/lni/go/pkg/mod/github.com/lni/goutils@v1.3.0/syncutil/stopper.go:79 +0xc0
created by github.com/lni/goutils/syncutil.(*Stopper).runWorker
  /Users/lni/go/pkg/mod/github.com/lni/goutils@v1.3.0/syncutil/stopper.go:74 +0x88
FAIL  github.com/matrixorigin/matrixone/pkg/vm/driver/test  37.014s
FAIL
```

Pull request:
Fixed how initial snapshot is applied
**What type of PR is this?**

- [ ] API-change
- [x] BUG
- [ ] Improvement
- [ ] Documentation
- [ ] Feature
- [ ] Test and CI
- [ ] Code Refactoring

**Which issue(s) this PR fixes:**

Fixes #606 

**What this PR does / why we need it:**

Not Available

**Special notes for your reviewer:**

Not Available

**Additional documentation (e.g. design docs, usage docs, etc.):**

Not Available


Patch:
diff --git a/raftstore/replica_event_loop.go b/raftstore/replica_event_loop.go
index cb0ce14..f57d823 100644
--- a/raftstore/replica_event_loop.go
+++ b/raftstore/replica_event_loop.go
@@ -251,7 +251,9 @@ func (pr *replica) handleInitializedState() (bool, error) {
 	if pr.initialized {
 		return false, nil
 	}
-	pr.initialized = true
+	defer func() {
+		pr.initialized = true
+	}()
 	pr.logger.Debug("checking initial snapshot")
 	ss, err := pr.logdb.GetSnapshot(pr.shardID)
 	if err == logdb.ErrNoSnapshot {
diff --git a/raftstore/replica_snapshot.go b/raftstore/replica_snapshot.go
index 7e30a4d..3194b45 100644
--- a/raftstore/replica_snapshot.go
+++ b/raftstore/replica_snapshot.go
@@ -111,7 +111,12 @@ func (r *replica) applySnapshot(ss raftpb.Snapshot) error {
 		return err
 	}
 	r.appliedIndex = ss.Metadata.Index
-	r.lr.ApplySnapshot(ss)
+	// when applying initial snapshot, we've already applied the ss record into
+	// the LogReader beforehand, applying the ss record again here would void
+	// the lr.SetRange change.
+	if r.initialized {
+		r.lr.ApplySnapshot(ss)
+	}
 	r.sm.updateShard(md.Metadata.Shard)
 	// after snapshot applied, the shard range may changed, so we
 	// need update key ranges


Test patch:
diff --git a/raftstore/replica_event_loop_test.go b/raftstore/replica_event_loop_test.go
index fa62350..9951d21 100644
--- a/raftstore/replica_event_loop_test.go
+++ b/raftstore/replica_event_loop_test.go
@@ -125,10 +125,13 @@ func TestApplyInitialSnapshot(t *testing.T) {
 		r.sm = newStateMachine(r.logger, ds, r.logdb, shard, replicaRec, nil, nil)
 
 		assert.False(t, r.initialized)
+		assert.Equal(t, uint64(0), r.lr.markerIndex)
 		hasEvent, err := r.handleEvent(r.logdb.NewWorkerContext())
 		assert.NoError(t, err)
 		assert.True(t, hasEvent)
 		assert.True(t, r.initialized)
+		// when applying initial snapshot, r.lr.ApplySnapshot should not be invoked
+		assert.Equal(t, uint64(0), r.lr.markerIndex)
 		assert.Equal(t, ss.Metadata.Index, r.sm.metadataMu.index)
 		assert.Equal(t, ss.Metadata.Term, r.sm.metadataMu.term)
 		assert.Equal(t, shard, r.sm.metadataMu.shard)