Repository: tikv/tikv (language: rust)

Issue:
In-memory Engine: assertion `left == right` failed, left: Active right: Pending
## Bug Report

```
/opt/tidb/kv.log:[2024/12/16 09:56:13.444 +00:00] [FATAL] [lib.rs:480] ["assertion `left == right` failed
  left: Active
 right: Pending"] [backtrace="   0: tikv_util::set_panic_hook::{{closure}}
             at /home/stn/tikv/components/tikv_util/src/lib.rs:479:18
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9
      std::panicking::rust_panic_with_hook
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13
   2: std::panicking::begin_panic_handler::{{closure}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:657:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18
   4: rust_begin_unwind
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5
   5: core::panicking::panic_fmt
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14
   6: core::panicking::assert_failed_inner
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:342:17
   7: core::panicking::assert_failed
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:297:5
   8: in_memory_engine::region_manager::RegionManager::split_region
             at /home/stn/tikv/components/in_memory_engine/src/region_manager.rs:1016:13
   9: <in_memory_engine::engine::RegionCacheMemoryEngine as engine_traits::region_cache_engine::RegionCacheEngineExt>::on_region_event
             at /home/stn/tikv/components/in_memory_engine/src/engine.rs:596:17
  10: hybrid_engine::observer::load_eviction::LoadEvictionObserver::split_region
             at /home/stn/tikv/components/hybrid_engine/src/observer/load_eviction.rs:117:9
      hybrid_engine::observer::load_eviction::LoadEvictionObserver::post_exec_cmd
             at /home/stn/tikv/components/hybrid_engine/src/observer/load_eviction.rs:105:13
  11: <hybrid_engine::observer::load_eviction::LoadEvictionObserver as raftstore::coprocessor::AdminObserver>::post_exec_admin
             at /home/stn/tikv/components/hybrid_engine/src/observer/load_eviction.rs:204:9
  12: raftstore::coprocessor::dispatcher::CoprocessorHost<E>::post_exec
             at /home/stn/tikv/components/raftstore/src/coprocessor/dispatcher.rs:698:20
      raftstore::store::fsm::apply::ApplyDelegate<EK>::apply_raft_cmd
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:1556:28
      raftstore::store::fsm::apply::ApplyDelegate<EK>::process_raft_cmd
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:1423:52
  13: raftstore::store::fsm::apply::ApplyDelegate<EK>::handle_raft_entry_normal
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:1301:24
      raftstore::store::fsm::apply::ApplyDelegate<EK>::handle_raft_committed_entries
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:1173:43
  14: raftstore::store::fsm::apply::ApplyFsm<EK>::handle_apply
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:4185:9
  15: raftstore::store::fsm::apply::ApplyFsm<EK>::handle_tasks
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:4540:25
  16: <raftstore::store::fsm::apply::ApplyPoller<EK> as batch_system::batch::PollHandler<raftstore::store::fsm::apply::ApplyFsm<EK>,raftstore::store::fsm::apply::ControlFsm>>::handle_normal
             at /home/stn/tikv/components/raftstore/src/store/fsm/apply.rs:4843:9
  17: batch_system::batch::Poller<N,C,Handler>::poll
             at /home/stn/tikv/components/batch-system/src/batch.rs:416:27
  18: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}
             at /home/stn/tikv/components/batch-system/src/batch.rs:589:17
      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}
             at /home/stn/tikv/components/tikv_util/src/sys/thread.rs:438:13
      std::sys_common::backtrace::__rust_begin_short_backtrace
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18
  19: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40
      std::panicking::try
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19
      std::panic::catch_unwind
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
  20: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9
      std::sys::unix::thread::Thread::new::thread_start
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17
  21: start_thread
  22: clone
"] [location=components/in_memory_engine/src/region_manager.rs:1016] [thread_name=apply-0] [thread_id=145]
```

https://tcms.pingcap.net/dashboard/executions/plan/7701144

### What version of TiKV are you using?
<!-- You can run `tikv-server --version` -->

https://github.com/tikv/tikv/commit/962553d85013d273008a9168f843b577e591386d

### What operating system and CPU are you using?
<!-- If you're using Linux, you can run `cat /proc/cpuinfo` -->

### Steps to reproduce
<!-- If possible, provide a recipe for reproducing the error. A complete runnable program is good. -->

Run jepsen tests.

### What did you expect?

No panic.

### What did happened?

Panic.


Discussion / hints:

Fix patch:
diff --git a/components/hybrid_engine/src/observer/load_eviction.rs b/components/hybrid_engine/src/observer/load_eviction.rs
index cc2e12aacd4..8a3728a0618 100644
--- a/components/hybrid_engine/src/observer/load_eviction.rs
+++ b/components/hybrid_engine/src/observer/load_eviction.rs
@@ -78,7 +78,9 @@ impl LoadEvictionObserver {
             || (state.modified_region.is_some()
                 && matches!(
                     cmd.request.get_admin_request().get_cmd_type(),
-                    AdminCmdType::PrepareMerge | AdminCmdType::CommitMerge
+                    AdminCmdType::PrepareMerge
+                        | AdminCmdType::CommitMerge
+                        | AdminCmdType::RollbackMerge
                 ))
         {
             let cache_region = CacheRegion::from_region(ctx.region());


Test patch:
diff --git a/tests/failpoints/cases/test_in_memory_engine.rs b/tests/failpoints/cases/test_in_memory_engine.rs
index 13ef4c5cbfa..11915e83836 100644
--- a/tests/failpoints/cases/test_in_memory_engine.rs
+++ b/tests/failpoints/cases/test_in_memory_engine.rs
@@ -19,7 +19,7 @@ use kvproto::{
     import_sstpb::SstMeta,
     kvrpcpb::Context,
     raft_cmdpb::{AdminCmdType, CmdType, RaftCmdRequest, RaftRequestHeader, Request},
-    raft_serverpb::RaftMessage,
+    raft_serverpb::{PeerState, RaftMessage},
 };
 use pd_client::PdClient;
 use protobuf::Message;
@@ -36,8 +36,9 @@ use test_coprocessor::{
     handle_request, init_data_with_details_pd_client, DagChunkSpliter, DagSelect, ProductTable,
 };
 use test_raftstore::{
-    get_tso, new_learner_peer, new_peer, new_put_cf_cmd, new_server_cluster_with_hybrid_engine,
-    CloneFilterFactory, Cluster, Direction, RegionPacketFilter, ServerCluster,
+    configure_for_merge, get_tso, must_get_equal, new_learner_peer, new_peer, new_put_cf_cmd,
+    new_server_cluster_with_hybrid_engine, CloneFilterFactory, Cluster, Direction,
+    RegionPacketFilter, ServerCluster,
 };
 use test_util::eventually;
 use tidb_query_datatype::{
@@ -1059,3 +1060,96 @@ fn test_eviction_when_destroy_uninitialized_peer() {
     pd_client.must_add_peer(region.get_id(), learner2.clone());
     cluster.must_region_exist(region.get_id(), 2);
 }
+
+// IME should also handle RollbackMerge event, we also try to evict the region
+// on merge rollback for simplicity. If region is loaded after PrepareMerge and
+// the merge is rollbacked, IME should track this rollback because it will also
+// change epoch version.
+#[test]
+fn test_region_rollback_merge() {
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 3);
+    configure_for_merge(&mut cluster.cfg);
+    let pd_client = Arc::clone(&cluster.pd_client);
+    pd_client.disable_default_operator();
+
+    cluster.run_conf_change();
+
+    let region = pd_client.get_region(b"k1").unwrap();
+    cluster.must_split(&region, b"k2");
+    let left = pd_client.get_region(b"k1").unwrap();
+    let right = pd_client.get_region(b"k2").unwrap();
+
+    pd_client.must_add_peer(left.get_id(), new_peer(2, 2));
+    pd_client.must_add_peer(right.get_id(), new_peer(2, 4));
+
+    cluster.must_put(b"k1", b"v1");
+    cluster.must_put(b"k3", b"v3");
+
+    let mut region = pd_client.get_region(b"k1").unwrap();
+    let target_region = pd_client.get_region(b"k3").unwrap();
+
+    let schedule_merge_fp = "on_schedule_merge";
+    fail::cfg(schedule_merge_fp, "return()").unwrap();
+
+    let (tx, rx) = sync_channel(1);
+    fail::cfg_callback("on_apply_res_prepare_merge", move || {
+        tx.send(()).unwrap();
+    })
+    .unwrap();
+
+    cluster.merge_region(region.get_id(), target_region.get_id(), Callback::None);
+    // PrepareMerge is applied.
+    rx.recv().unwrap();
+
+    let leader = cluster.leader_of_region(left.get_id()).unwrap();
+
+    let region_cache_engine = cluster.sim.rl().get_region_cache_engine(leader.store_id);
+
+    // After prepare merge, version becomes 2 + 1 = 3;
+    region.mut_region_epoch().set_version(3);
+    // load region after PrepareMerge.
+    {
+        let cache_region = CacheRegion::from_region(&region);
+        region_cache_engine
+            .core()
+            .region_manager()
+            .new_region(cache_region);
+    }
+
+    // Add a peer to trigger rollback.
+    pd_client.must_add_peer(right.get_id(), new_peer(3, 5));
+    cluster.must_put(b"k4", b"v4");
+    must_get_equal(&cluster.get_engine(3), b"k4", b"v4");
+
+    let mut region = pd_client.get_region(b"k1").unwrap();
+    // After split and prepare_merge, version becomes 1 + 2 = 3;
+    assert_eq!(region.get_region_epoch().get_version(), 3);
+    // After ConfChange and prepare_merge, conf version becomes 1 + 2 = 3;
+    assert_eq!(region.get_region_epoch().get_conf_ver(), 3);
+    fail::remove(schedule_merge_fp);
+    // Wait till rollback.
+    cluster.must_put(b"k11", b"v11");
+
+    // After rollback, version becomes 3 + 1 = 4;
+    region.mut_region_epoch().set_version(4);
+    for i in 1..3 {
+        must_get_equal(&cluster.get_engine(i), b"k11", b"v11");
+        let state = cluster.region_local_state(region.get_id(), i);
+        assert_eq!(state.get_state(), PeerState::Normal);
+        assert_eq!(*state.get_region(), region);
+    }
+
+    // after rollback, IME cached region is evicted.
+    test_util::eventually(
+        Duration::from_millis(10),
+        Duration::from_millis(1000),
+        || {
+            let region_map = region_cache_engine
+                .core()
+                .region_manager()
+                .regions_map()
+                .read();
+            region_map.regions().is_empty()
+        },
+    );
+}