Repository: tikv/tikv (language: rust)

Issue:
In-memory Engine: tikv panic "assertion failed: !region.get_peers().is_empty()"
## Bug Report

Similar to https://github.com/tikv/tikv/issues/17767, but triggered when an uninitialized peer receives a `MsgPreLoadRegionRequest`.

```
/opt/tidb/kv.log:[2024/12/23 15:03:27.079 +00:00] [FATAL] [lib.rs:480] ["assertion failed: !region.get_peers().is_empty()"] [backtrace="   0: tikv_util::set_panic_hook::{{closure}}
             at /home/stn/tikv/components/tikv_util/src/lib.rs:479:18
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9
      std::panicking::rust_panic_with_hook
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13
   2: std::panicking::begin_panic_handler::{{closure}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:649:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18
   4: rust_begin_unwind
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5
   5: core::panicking::panic_fmt
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14
   6: core::panicking::panic
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:144:5
   7: keys::enc_start_key
             at /home/stn/tikv/components/keys/src/lib.rs:232:5
   8: engine_traits::region_cache_engine::CacheRegion::from_region
             at /home/stn/tikv/components/engine_traits/src/region_cache_engine.rs:151:20
   9: <in_memory_engine::engine::RegionCacheMemoryEngine as engine_traits::region_cache_engine::RegionCacheEngineExt>::load_region
             at /home/stn/tikv/components/in_memory_engine/src/engine.rs:627:21
  10: raftstore::coprocessor::dispatcher::CoprocessorHost<E>::on_extra_message
             at /home/stn/tikv/components/raftstore/src/coprocessor/dispatcher.rs:920:13
      raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_extra_message
             at /home/stn/tikv/components/raftstore/src/store/fsm/peer.rs:3036:9
  11: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_raft_message
             at /home/stn/tikv/components/raftstore/src/store/fsm/peer.rs:2765:13
      raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs
             at /home/stn/tikv/components/raftstore/src/store/fsm/peer.rs:653:37
  12: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal
             at /home/stn/tikv/components/raftstore/src/store/fsm/store.rs:1098:9
      batch_system::batch::Poller<N,C,Handler>::poll
             at /home/stn/tikv/components/batch-system/src/batch.rs:416:27
  13: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}
             at /home/stn/tikv/components/batch-system/src/batch.rs:589:17
      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}
             at /home/stn/tikv/components/tikv_util/src/sys/thread.rs:438:13
      std::sys_common::backtrace::__rust_begin_short_backtrace
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18
  14: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40
      std::panicking::try
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19
      std::panic::catch_unwind
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
  15: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9
      std::sys::unix::thread::Thread::new::thread_start
             at /home/stn/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17
  16: start_thread
  17: clone
"] [location=components/keys/src/lib.rs:232] [thread_name=raftstore-6-1] [thread_id=144]
```

https://tcms.pingcap.net/dashboard/executions/plan/7701144

### What version of TiKV are you using?
<!-- You can run `tikv-server --version` -->

0c3bfda1a

### What operating system and CPU are you using?
<!-- If you're using Linux, you can run `cat /proc/cpuinfo` -->

### Steps to reproduce
<!-- If possible, provide a recipe for reproducing the error. A complete runnable program is good. -->

Run jepsen tests.

### What did you expect?

No panic.

### What did happened?

Panic.


Discussion / hints:

Fix patch:
diff --git a/components/hybrid_engine/src/observer/load_eviction.rs b/components/hybrid_engine/src/observer/load_eviction.rs
index 8a3728a0618..289288aebab 100644
--- a/components/hybrid_engine/src/observer/load_eviction.rs
+++ b/components/hybrid_engine/src/observer/load_eviction.rs
@@ -267,9 +267,16 @@ impl RoleObserver for LoadEvictionObserver {
 }
 
 impl ExtraMessageObserver for LoadEvictionObserver {
-    fn on_extra_message(&self, r: &Region, extra_msg: &ExtraMessage) {
+    fn on_extra_message(&self, region: &Region, extra_msg: &ExtraMessage) {
         if extra_msg.get_type() == ExtraMessageType::MsgPreLoadRegionRequest {
-            self.cache_engine.load_region(r);
+            if region.get_peers().is_empty() {
+                // MsgPreLoadRegionRequest is sent before leader issue a
+                // transfer leader request. It is possible that the peer
+                // is not initialized yet.
+                warn!("ime skip pre-load an uninitialized region"; "region" => ?region);
+                return;
+            }
+            self.cache_engine.load_region(region);
         }
     }
 }


Test patch:
diff --git a/tests/failpoints/cases/test_in_memory_engine.rs b/tests/failpoints/cases/test_in_memory_engine.rs
index 11915e83836..aeb2b04454f 100644
--- a/tests/failpoints/cases/test_in_memory_engine.rs
+++ b/tests/failpoints/cases/test_in_memory_engine.rs
@@ -37,7 +37,7 @@ use test_coprocessor::{
 };
 use test_raftstore::{
     configure_for_merge, get_tso, must_get_equal, new_learner_peer, new_peer, new_put_cf_cmd,
-    new_server_cluster_with_hybrid_engine, CloneFilterFactory, Cluster, Direction,
+    new_server_cluster_with_hybrid_engine, sleep_ms, CloneFilterFactory, Cluster, Direction,
     RegionPacketFilter, ServerCluster,
 };
 use test_util::eventually;
@@ -1153,3 +1153,51 @@ fn test_region_rollback_merge() {
         },
     );
 }
+
+// IME must not panic when pre-load an uninitialized peer.
+#[test]
+fn test_transfer_leader_pre_load_uninitialized_peer() {
+    let mut cluster = new_server_cluster_with_hybrid_engine(0, 2);
+    let pd_client = cluster.pd_client.clone();
+    pd_client.disable_default_operator();
+    cluster.run_conf_change();
+
+    let region = pd_client.get_region(b"").unwrap();
+    assert!(
+        !region.get_peers().iter().any(|p| p.get_store_id() == 2),
+        "{:?}",
+        region
+    );
+
+    // Load the region in leader.
+    let region_cache_engine = cluster.sim.rl().get_region_cache_engine(1);
+    region_cache_engine
+        .load_region(CacheRegion::from_region(&region))
+        .unwrap();
+    // Put some key to trigger load
+    cluster.must_put(b"k", b"val");
+    eventually(Duration::from_millis(100), Duration::from_secs(5), || {
+        region_cache_engine
+            .snapshot(CacheRegion::from_region(&region), 100, 100)
+            .is_ok()
+    });
+
+    // Block snapshot messages, so that new peers will never be initialized.
+    cluster.add_send_filter(CloneFilterFactory(
+        RegionPacketFilter::new(region.get_id(), 2)
+            .msg_type(MessageType::MsgSnapshot)
+            .direction(Direction::Recv),
+    ));
+
+    let peer1 = new_peer(2, 2);
+    pd_client.must_add_peer(region.get_id(), peer1.clone());
+    cluster.must_region_exist(region.get_id(), 2);
+
+    // IME will send a MsgPreLoadRegion message before transferring leader,
+    // and this message must not cause panic.
+    cluster.transfer_leader(region.get_id(), new_peer(2, 2));
+    // Give some time for handling MsgPreLoadRegion message.
+    sleep_ms(100);
+    cluster.clear_send_filters();
+    cluster.must_transfer_leader(region.get_id(), new_peer(2, 2));
+}