Repository: tikv/tikv (language: rust)

Issue:
one tikv panic with "[mod.rs:233] [\"async snapshot callback is dropped\"] “ after another tikv recover from failure
## Bug Report

<!-- Thanks for your bug report! Don't worry if you can't fill out all the sections. -->

### What version of TiKV are you using?
./tikv-server -V
 TiKV 
Release Version:   8.1.2
Edition:           Community
Git Commit Hash:   e670bae7d1ab798ae37bb176a6a900e934d14612
Git Commit Branch: HEAD
UTC Build Time:    2024-12-13 13:21:24
Rust Version:      rustc 1.77.0-nightly (89e2160c4 2023-12-27)
Enable Features:   pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored
Profile:           dist_release
2024-12-14T12:22:36.397+0800

### What operating system and CPU are you using?
8c/32g


### Steps to reproduce
1、run tpcc
2、config set max-store-down-time 10m
3、down tc-tikv-2 lasts for 20mins


### What did you expect?
no panic

### What did happened?
tc-tikv-3 panic after tc-tikv-2 recover from failure
`{"log":"[mod.rs:233] [\"async snapshot callback is dropped\"] [backtrace=\"   0: tikv::server::raftkv::drop_snapshot_callback\\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\\n      core::ops::function::FnOnce::call_once\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\\n   5: batch_system::batch::Poller<N,C,Handler>::poll\\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\\n      std::sys_common::backtrace::__rust_begin_short_backtrace\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\\n      std::panicking::try::do_call\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\\n      std::panicking::try\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\\n      std::panic::catch_unwind\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\\n      std::thread::Builder::spawn_unchecked_::{{closure}}\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\\n      std::sys::unix::thread::Thread::new::thread_start\\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\\n   9: start_thread\\n  10: __GI___clone\\n\"] [thread_id=108]","container":"tikv","pod":"tc-tikv-3","level":"WARN","namespace":"ha-test-tikv-failure-rpt-tps-7698552-1-384"}`

https://tcms.pingcap.net/dashboard/executions/plan/7698552


Discussion / hints:
/severity critical 
/assign overvenus
The issue appears to be closely related to the changes introduced in https://github.com/tikv/tikv/pull/17841, as indicated by the tikv log.

https://tcms.pingcap.net/api/v1/artifact-files/ks3/archive/ha-test-tikv-failure-rpt-tps-7698552-1-384/2024-12-14T07:11:54Z/tc-tikv-3/var/lib/tikv/log/tikv.log

```
[2024/12/14 15:03:42.536 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2971429] [region_id=2971426] [thread_id=101]
[2024/12/14 15:03:42.536 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.423µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2971426] [thread_id=101]
[2024/12/14 15:03:42.536 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(282201)] [region_id=2971426] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:03:42.536 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2971426] [thread_id=31]
[2024/12/14 15:03:42.536 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=336.448µs] [peer_id=2971429] [region_id=2971426] [thread_id=101]
[2024/12/14 15:03:42.536 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2971426] [thread_id=101]
[2024/12/14 15:03:42.537 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 2971426 not found\" region_not_found { region_id: 2971426 }))"] [cid=83148635] [thread_id=44]
[2024/12/14 15:03:42.537 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 2971426 is missing\" region_not_found { region_id: 2971426 }))"] [cid=83148728] [thread_id=44]
[2024/12/14 15:03:42.537 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 2971426 not found\" region_not_found { region_id: 2971426 }))"] [cid=83148560] [thread_id=44]
[2024/12/14 15:03:42.537 +08:00] [INFO] [peer.rs:5788] ["failed to propose"] [err="peer is not leader for region 2971571, leader may Some(id: 2971572 store_id: 14)"] [message="header { region_id: 2971571 peer { id: 2971573 store_id: 1 } region_epoch { conf_ver: 41 version: 371 } } admin_request { cmd_type: CommitMerge commit_merge { source { id: 2971551 start_key: 7480000000000000FF6D5F728000000007FFFA0B150000000000FA end_key: 7480000000000000FF6D5F728000000008FF2FC5950000000000FA region_epoch { conf_ver: 42 version: 372 } peers { id: 2971552 store_id: 14 } peers { id: 2971553 store_id: 1 } peers { id: 2971554 store_id: 13 } } commit: 9 entries { term: 6 index: 9 data: 0A15089FAFB501120708A0AFB501100E2A05082910F3021A6908073A650809126108B3AFB501121B7480000000000000FF6D5F728000000008FF2FC5950000000000FA1A1B7480000000000000FF6D5F728000000008FF6475EE0000000000FA2205082910F3022A0708B4AFB501100E2A0708B5AFB50110012A0708B6AFB501100D context: 05 } } }"] [peer_id=2971573] [region_id=2971571] [thread_id=102]
[2024/12/14 15:03:42.537 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=101]
[2024/12/14 15:03:42.537 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 2971426 not found\" region_not_found { region_id: 2971426 }))"] [cid=83148567] [thread_id=44]
[2024/12/14 15:03:42.537 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: core::ptr::drop_in_place<[raftstore::store::msg::PeerMsg<engine_rocks::engine::RocksEngine>]>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      <alloc::vec::drain::Drain<T,A> as core::ops::drop::Drop>::drop\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/drain.rs:237:13\n      core::ptr::drop_in_place<alloc::vec::drain::Drain<raftstore::store::msg::PeerMsg<engine_rocks::engine::RocksEngine>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   4: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:630:9\n   5: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   6: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   7: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   8: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   9: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n  10: start_thread\n  11: __GI___clone\n"] [thread_id=101]
[2024/12/14 15:03:42.538 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"async snapshot callback is dropped\"))"] [cid=83148573] [thread_id=46]
--
[2024/12/14 15:04:59.548 +08:00] [INFO] [peer.rs:3933] ["delays destroy"] [reason=UnFlushLogGc] [merged_by_target=true] [peer_id=2975574] [region_id=2975573] [thread_id=102]
[2024/12/14 15:04:59.549 +08:00] [INFO] [peer.rs:3988] ["starts destroy"] [is_latest_initialized=false] [is_peer_initialized=true] [merged_by_target=true] [peer_id=2975574] [region_id=2975573] [thread_id=102]
[2024/12/14 15:04:59.549 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2975589] [thread_id=31]
[2024/12/14 15:04:59.549 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2975574] [region_id=2975573] [thread_id=102]
[2024/12/14 15:04:59.549 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2975573] [thread_id=31]
[2024/12/14 15:04:59.549 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(286414)] [region_id=2975573] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:04:59.549 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.405µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2975573] [thread_id=102]
[2024/12/14 15:04:59.549 +08:00] [INFO] [endpoint.rs:358] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=9] [observe_id=ObserveId(286415)] [region=2975589] [thread_id=68]
[2024/12/14 15:04:59.549 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=170.787µs] [peer_id=2975574] [region_id=2975573] [thread_id=102]
[2024/12/14 15:04:59.549 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2975573] [thread_id=102]
[2024/12/14 15:04:59.549 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:04:59.561 +08:00] [INFO] [pd.rs:1525] ["try to transfer leader"] [to_peers="[]"] [to_peer="id: 2972074 store_id: 13"] [from_peer="id: 2972075 store_id: 1"] [region_id=2972072] [thread_id=31]
--
[2024/12/14 15:05:14.176 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2975982] [thread_id=31]
[2024/12/14 15:05:14.176 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.075µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2975982] [thread_id=101]
[2024/12/14 15:05:14.176 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(287001)] [region_id=2973291] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:05:14.176 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(287102)] [region_id=2975982] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:05:14.176 +08:00] [INFO] [peer.rs:3897] ["transfer leader"] [peer="id: 2320579 store_id: 14"] [peer_id=2321334] [region_id=2787] [thread_id=102]
[2024/12/14 15:05:14.176 +08:00] [INFO] [raft.rs:1944] ["[term 309] starts to transfer leadership to 2320579"] [lead_transferee=2320579] [term=309] [raft_id=2321334] [region_id=2787] [thread_id=102]
[2024/12/14 15:05:14.176 +08:00] [INFO] [raft.rs:1957] ["sends MsgTimeoutNow to 2320579 immediately as 2320579 already has up-to-date log"] [lead_transferee=2320579] [raft_id=2321334] [region_id=2787] [thread_id=102]
[2024/12/14 15:05:14.177 +08:00] [INFO] [endpoint.rs:358] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=27] [observe_id=ObserveId(287103)] [region=2975966] [thread_id=68]
[2024/12/14 15:05:14.177 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=758.858µs] [peer_id=2975983] [region_id=2975982] [thread_id=101]
[2024/12/14 15:05:14.177 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2975982] [thread_id=101]
[2024/12/14 15:05:14.177 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=101]
[2024/12/14 15:05:14.179 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"async snapshot callback is dropped\"))"] [cid=83957297] [thread_id=47]
--
[2024/12/14 15:05:25.777 +08:00] [INFO] [peer.rs:3933] ["delays destroy"] [reason=UnFlushLogGc] [merged_by_target=true] [peer_id=2965430] [region_id=30848] [thread_id=101]
[2024/12/14 15:05:25.777 +08:00] [INFO] [peer.rs:3988] ["starts destroy"] [is_latest_initialized=false] [is_peer_initialized=true] [merged_by_target=true] [peer_id=2965430] [region_id=30848] [thread_id=101]
[2024/12/14 15:05:25.777 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2965430] [region_id=30848] [thread_id=101]
[2024/12/14 15:05:25.777 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.446µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=30848] [thread_id=101]
[2024/12/14 15:05:25.777 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(287700)] [region_id=30848] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:05:25.778 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=30848] [thread_id=31]
[2024/12/14 15:05:25.779 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=1.662297ms] [peer_id=2965430] [region_id=30848] [thread_id=101]
[2024/12/14 15:05:25.779 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=30848] [thread_id=101]
[2024/12/14 15:05:25.779 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 30848 is missing\" region_not_found { region_id: 30848 }))"] [cid=84068899] [thread_id=44]
[2024/12/14 15:05:25.779 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2974721] [thread_id=31]
[2024/12/14 15:05:25.780 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=101]
[2024/12/14 15:05:25.780 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"async snapshot callback is dropped\"))"] [cid=84068845] [thread_id=47]
--
[2024/12/14 15:05:31.934 +08:00] [INFO] [raft.rs:1605] ["[logterm: 454, index: 578, vote: 0] cast vote for 207564 [logterm: 454, index: 578] at term 455"] ["msg type"=MsgRequestVote] [term=455] [msg_index=578] [msg_term=454] [from=207564] [vote=0] [log_index=578] [log_term=454] [raft_id=12] [region_id=11] [thread_id=102]
[2024/12/14 15:05:31.934 +08:00] [INFO] [apply.rs:1712] ["execute admin command"] [command="cmd_type: TransferLeader transfer_leader { peer { id: 2973938 store_id: 1 } }"] [index=13379] [term=47] [peer_id=2973938] [region_id=2973936] [thread_id=103]
[2024/12/14 15:05:31.934 +08:00] [INFO] [peer.rs:3988] ["starts destroy"] [is_latest_initialized=false] [is_peer_initialized=true] [merged_by_target=true] [peer_id=2976852] [region_id=2976850] [thread_id=101]
[2024/12/14 15:05:31.934 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2976852] [region_id=2976850] [thread_id=101]
[2024/12/14 15:05:31.934 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2976850] [thread_id=31]
[2024/12/14 15:05:31.934 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.205µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2976850] [thread_id=101]
[2024/12/14 15:05:31.934 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(288024)] [region_id=2976850] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:05:31.935 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"peer is not leader for region 2289712, leader may Some(id: 2289714 store_id: 14)\" not_leader { region_id: 2289712 leader { id: 2289714 store_id: 14 } }))"] [cid=84135904] [thread_id=44]
[2024/12/14 15:05:31.939 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=3.79692ms] [peer_id=2976852] [region_id=2976850] [thread_id=101]
[2024/12/14 15:05:31.939 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2976850] [thread_id=101]
[2024/12/14 15:05:31.939 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=101]
[2024/12/14 15:05:31.940 +08:00] [INFO] [apply.rs:1712] ["execute admin command"] [command="cmd_type: TransferLeader transfer_leader { peer { id: 2976389 store_id: 1 } }"] [index=705] [term=11] [peer_id=2976389] [region_id=2976386] [thread_id=104]
--
[2024/12/14 15:06:18.528 +08:00] [INFO] [raft.rs:1556] ["starting a new election"] [term=22] [raft_id=2978221] [region_id=2978220] [thread_id=102]
[2024/12/14 15:06:18.528 +08:00] [INFO] [raft.rs:1175] ["became candidate at term 23"] [term=23] [raft_id=2978221] [region_id=2978220] [thread_id=102]
[2024/12/14 15:06:18.528 +08:00] [INFO] [raft.rs:1321] ["broadcasting vote request"] [to="[2978222, 2978223]"] [log_index=2009] [log_term=22] [term=23] [type=MsgRequestVote] [raft_id=2978221] [region_id=2978220] [thread_id=102]
[2024/12/14 15:06:18.529 +08:00] [INFO] [peer.rs:3988] ["starts destroy"] [is_latest_initialized=false] [is_peer_initialized=true] [merged_by_target=true] [peer_id=2978704] [region_id=2978703] [thread_id=102]
[2024/12/14 15:06:18.529 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2978704] [region_id=2978703] [thread_id=102]
[2024/12/14 15:06:18.529 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(290305)] [region_id=2978703] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:06:18.529 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2978703] [thread_id=31]
[2024/12/14 15:06:18.529 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.492µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2978703] [thread_id=102]
[2024/12/14 15:06:18.529 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=321.964µs] [peer_id=2978704] [region_id=2978703] [thread_id=102]
[2024/12/14 15:06:18.529 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2978703] [thread_id=102]
[2024/12/14 15:06:18.530 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:06:18.530 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2978711] [thread_id=31]
--
[2024/12/14 15:06:33.922 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2979284] [region_id=2979283] [thread_id=102]
[2024/12/14 15:06:33.922 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2979283] [thread_id=31]
[2024/12/14 15:06:33.922 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2979279] [thread_id=31]
[2024/12/14 15:06:33.922 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(291003)] [region_id=2979283] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:06:33.922 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.285µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2979283] [thread_id=102]
[2024/12/14 15:06:33.923 +08:00] [INFO] [endpoint.rs:358] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=7] [observe_id=ObserveId(291004)] [region=2979279] [thread_id=68]
[2024/12/14 15:06:33.924 +08:00] [INFO] [size.rs:180] ["Run size checker"] [policy=Scan] [threshold=150994944] [size=572663726] [region_id=2977626] [thread_id=16]
[2024/12/14 15:06:33.924 +08:00] [INFO] [keys.rs:171] ["approximate keys over threshold, need to do split check"] [threshold=1440000] [keys=7738699] [region_id=2977626] [thread_id=16]
[2024/12/14 15:06:33.926 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=4.208636ms] [peer_id=2979284] [region_id=2979283] [thread_id=102]
[2024/12/14 15:06:33.926 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2979283] [thread_id=102]
[2024/12/14 15:06:33.927 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:06:33.928 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 2979283 is missing\" region_not_found { region_id: 2979283 }))"] [cid=84703425] [thread_id=46]
--
[2024/12/14 15:06:43.332 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"region 2979721 not found\" region_not_found { region_id: 2979721 }))"] [cid=84785003] [thread_id=46]
[2024/12/14 15:06:43.333 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"read index not ready, reason can not read index due to merge, region 2979721\" read_index_not_ready { reason: \"can not read index due to merge\" region_id: 2979721 }))"] [cid=84784989] [thread_id=46]
[2024/12/14 15:06:43.333 +08:00] [INFO] [peer.rs:3988] ["starts destroy"] [is_latest_initialized=false] [is_peer_initialized=true] [merged_by_target=true] [peer_id=2979723] [region_id=2979721] [thread_id=102]
[2024/12/14 15:06:43.333 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2979723] [region_id=2979721] [thread_id=102]
[2024/12/14 15:06:43.334 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.437µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2979721] [thread_id=102]
[2024/12/14 15:06:43.334 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(291447)] [region_id=2979721] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:06:43.334 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2979721] [thread_id=31]
[2024/12/14 15:06:43.334 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=670.532µs] [peer_id=2979723] [region_id=2979721] [thread_id=102]
[2024/12/14 15:06:43.334 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2979721] [thread_id=102]
[2024/12/14 15:06:43.334 +08:00] [INFO] [endpoint.rs:701] ["register observe region"] [region="id: 2979167 start_key: 7480000000000000FF6D5F698000000000FF0000010380000000FF0000022703800000FF0000000004038000FF000000000AC30380FF0000000000000E00FE end_key: 7480000000000000FF6D5F698000000000FF0000010380000000FF0000025C03800000FF0000000004038000FF0000000004990380FF0000000000000700FE region_epoch { conf_ver: 41 version: 507 } peers { id: 2979168 store_id: 14 } peers { id: 2979169 store_id: 1 } peers { id: 2979170 store_id: 13 }"] [thread_id=68]
[2024/12/14 15:06:43.335 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:06:43.335 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: core::ptr::drop_in_place<[raftstore::store::msg::PeerMsg<engine_rocks::engine::RocksEngine>]>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      <alloc::vec::drain::Drain<T,A> as core::ops::drop::Drop>::drop\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/drain.rs:237:13\n      core::ptr::drop_in_place<alloc::vec::drain::Drain<raftstore::store::msg::PeerMsg<engine_rocks::engine::RocksEngine>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   4: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:630:9\n   5: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   6: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   7: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   8: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   9: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n  10: start_thread\n  11: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:06:43.335 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: core::ptr::drop_in_place<[raftstore::store::msg::PeerMsg<engine_rocks::engine::RocksEngine>]>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      <alloc::vec::drain::Drain<T,A> as core::ops::drop::Drop>::drop\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/vec/drain.rs:237:13\n      core::ptr::drop_in_place<alloc::vec::drain::Drain<raftstore::store::msg::PeerMsg<engine_rocks::engine::RocksEngine>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   4: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:630:9\n   5: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   6: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n   7: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   8: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   9: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n  10: start_thread\n  11: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:06:43.335 +08:00] [INFO] [raft.rs:2369] ["[term 162] received MsgTimeoutNow from 168194 and starts an election to get leadership."] [from=168194] [term=162] [raft_id=2974885] [region_id=168192] [thread_id=102]
--
[2024/12/14 15:09:29.812 +08:00] [INFO] [peer.rs:3933] ["delays destroy"] [reason=UnFlushLogGc] [merged_by_target=true] [peer_id=2986374] [region_id=2986371] [thread_id=101]
[2024/12/14 15:09:29.812 +08:00] [INFO] [pd.rs:1525] ["try to transfer leader"] [to_peers="[]"] [to_peer="id: 2985337 store_id: 14"] [from_peer="id: 2985336 store_id: 1"] [region_id=2985335] [thread_id=31]
[2024/12/14 15:09:29.812 +08:00] [INFO] [peer.rs:3988] ["starts destroy"] [is_latest_initialized=false] [is_peer_initialized=true] [merged_by_target=true] [peer_id=2986374] [region_id=2986371] [thread_id=101]
[2024/12/14 15:09:29.812 +08:00] [INFO] [peer.rs:1383] ["begin to destroy"] [peer_id=2986374] [region_id=2986371] [thread_id=101]
[2024/12/14 15:09:29.812 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(298532)] [region_id=2986371] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:09:29.812 +08:00] [INFO] [pd.rs:1638] ["remove peer statistic record in pd"] [region_id=2986371] [thread_id=31]
[2024/12/14 15:09:29.812 +08:00] [INFO] [peer_storage.rs:1052] ["finish clear peer meta"] [takes=2.553µs] [raft_key=1] [apply_key=1] [meta_key=1] [region_id=2986371] [thread_id=101]
[2024/12/14 15:09:29.812 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2986387] [thread_id=31]
[2024/12/14 15:09:29.816 +08:00] [INFO] [peer.rs:1487] ["peer destroy itself"] [keep_data=true] [clean=true] [takes=3.238805ms] [peer_id=2986374] [region_id=2986371] [thread_id=101]
[2024/12/14 15:09:29.816 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2986371] [thread_id=101]
[2024/12/14 15:09:29.816 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=101]
[2024/12/14 15:09:29.817 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"async snapshot callback is dropped\"))"] [cid=86139508] [thread_id=44]
--
[2024/12/14 15:10:08.248 +08:00] [INFO] [apply.rs:1712] ["execute admin command"] [command="cmd_type: TransferLeader transfer_leader { peer { id: 475820 store_id: 13 } }"] [index=289206] [term=397] [peer_id=475818] [region_id=475817] [thread_id=104]
[2024/12/14 15:10:08.249 +08:00] [INFO] [endpoint.rs:701] ["register observe region"] [region="id: 2987187 start_key: 7480000000000000FF725F698000000000FF0000010380000000FF0000021503800000FF0000005AB9000000FC end_key: 7480000000000000FF725F698000000000FF0000010380000000FF000002EC03800000FF0000015F72000000FC region_epoch { conf_ver: 29 version: 633 } peers { id: 2987188 store_id: 13 } peers { id: 2987189 store_id: 1 } peers { id: 2987190 store_id: 14 }"] [thread_id=68]
[2024/12/14 15:10:08.251 +08:00] [INFO] [endpoint.rs:358] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=2789] [observe_id=ObserveId(300164)] [region=2987187] [thread_id=68]
[2024/12/14 15:10:08.257 +08:00] [INFO] [raft.rs:1386] ["received a message with higher term from 2984620"] ["msg type"=MsgRequestVote] [message_term=55] [term=54] [from=2984620] [raft_id=2984622] [region_id=2984619] [thread_id=101]
[2024/12/14 15:10:08.257 +08:00] [INFO] [raft.rs:1151] ["became follower at term 55"] [term=55] [raft_id=2984622] [region_id=2984619] [thread_id=101]
[2024/12/14 15:10:08.257 +08:00] [INFO] [raft.rs:1605] ["[logterm: 54, index: 1008, vote: 0] cast vote for 2984620 [logterm: 54, index: 1008] at term 55"] ["msg type"=MsgRequestVote] [term=55] [msg_index=1008] [msg_term=54] [from=2984620] [vote=0] [log_index=1008] [log_term=54] [raft_id=2984622] [region_id=2984619] [thread_id=101]
[2024/12/14 15:10:08.259 +08:00] [INFO] [pd.rs:1525] ["try to transfer leader"] [to_peers="[]"] [to_peer="id: 34443 store_id: 14"] [from_peer="id: 136 store_id: 1"] [region_id=7] [thread_id=31]
[2024/12/14 15:10:08.261 +08:00] [INFO] [pd.rs:1525] ["try to transfer leader"] [to_peers="[]"] [to_peer="id: 28642 store_id: 14"] [from_peer="id: 3 store_id: 1"] [region_id=2] [thread_id=31]
[2024/12/14 15:10:08.261 +08:00] [INFO] [pd.rs:1525] ["try to transfer leader"] [to_peers="[]"] [to_peer="id: 2976812 store_id: 13"] [from_peer="id: 2976811 store_id: 1"] [region_id=2976810] [thread_id=31]
[2024/12/14 15:10:08.272 +08:00] [INFO] [pd.rs:1525] ["try to transfer leader"] [to_peers="[]"] [to_peer="id: 2987608 store_id: 14"] [from_peer="id: 2987609 store_id: 1"] [region_id=2987607] [thread_id=31]
[2024/12/14 15:10:08.275 +08:00] [ERROR] [mod.rs:247] ["async write on_applied callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_on_applied_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:246:14\n   1: core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<raftstore::store::msg::WriteResponse,alloc::boxed::Box<<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,engine_rocks::engine::RocksEngine>> as tikv_kv::Engine>::async_write::{{closure}}>,tikv::server::raftkv::drop_on_applied_callback>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<raftstore::store::msg::WriteResponse,alloc::boxed::Box<<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,engine_rocks::engine::RocksEngine>> as tikv_kv::Engine>::async_write::{{closure}}>,tikv::server::raftkv::drop_on_applied_callback>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::WriteResponse,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=102]
[2024/12/14 15:10:08.275 +08:00] [WARN] [peer.rs:723] ["[region 2987707] 2987708 handle 1 peer messages "] [takes=188] [thread_id=102]
--
[2024/12/14 15:10:08.318 +08:00] [INFO] [raft.rs:1605] ["[logterm: 57, index: 4856, vote: 0] cast vote for 2985249 [logterm: 57, index: 4856] at term 58"] ["msg type"=MsgRequestVote] [term=58] [msg_index=4856] [msg_term=57] [from=2985249] [vote=0] [log_index=4856] [log_term=57] [raft_id=2985250] [region_id=2985247] [thread_id=101]
[2024/12/14 15:10:08.318 +08:00] [INFO] [apply.rs:1712] ["execute admin command"] [command="cmd_type: TransferLeader transfer_leader { peer { id: 2985249 store_id: 14 } }"] [index=4855] [term=57] [peer_id=2985250] [region_id=2985247] [thread_id=103]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:2369] ["[term 216] received MsgTimeoutNow from 2966012 and starts an election to get leadership."] [from=2966012] [term=216] [raft_id=2966010] [region_id=2966009] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:1556] ["starting a new election"] [term=216] [raft_id=2966010] [region_id=2966009] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:1175] ["became candidate at term 217"] [term=217] [raft_id=2966010] [region_id=2966009] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:1321] ["broadcasting vote request"] [to="[2966012, 2966011]"] [log_index=230] [log_term=216] [term=217] [type=MsgRequestVote] [raft_id=2966010] [region_id=2966009] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:1386] ["received a message with higher term from 2983519"] ["msg type"=MsgRequestVote] [message_term=76] [term=75] [from=2983519] [raft_id=2983520] [region_id=2983518] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:1151] ["became follower at term 76"] [term=76] [raft_id=2983520] [region_id=2983518] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [raft.rs:1605] ["[logterm: 75, index: 619, vote: 0] cast vote for 2983519 [logterm: 75, index: 619] at term 76"] ["msg type"=MsgRequestVote] [term=76] [msg_index=619] [msg_term=75] [from=2983519] [vote=0] [log_index=619] [log_term=75] [raft_id=2983520] [region_id=2983518] [thread_id=102]
[2024/12/14 15:10:08.322 +08:00] [INFO] [endpoint.rs:748] ["deregister observe region"] [observe_id=ObserveId(300149)] [region_id=2983518] [store_id=Some(1)] [thread_id=68]
[2024/12/14 15:10:08.322 +08:00] [FATAL] [lib.rs:479] ["undetermined error: Error(Undetermined(\"async write on_applied callback is dropped\")) cid=86447608, tag=acquire_pessimistic_lock, process result=Some(PessimisticLockRes { res: Ok(PessimisticLockResults([Existence(true)])) })"] [backtrace="   0: tikv_util::set_panic_hook::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/lib.rs:478:18\n   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9\n      std::panicking::rust_panic_with_hook\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13\n   2: std::panicking::begin_panic_handler::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:657:13\n   3: std::sys_common::backtrace::__rust_end_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18\n   4: rust_begin_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5\n   5: core::panicking::panic_fmt\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14\n   6: tikv::storage::txn::scheduler::TxnScheduler<E,L>::on_write_finished\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:903:25\n   7: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process_write::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1689:21\n   8: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process::{{closure}}::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1212:72\n      <resource_metering::InTags<T> as core::future::future::Future>::poll\n             at workspace/source/tikv/components/resource_metering/src/lib.rs:267:9\n      tikv::storage::txn::scheduler::TxnScheduler<E,L>::process::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1226:10\n      tikv::storage::txn::scheduler::TxnScheduler<E,L>::execute::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:764:51\n      <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/future/future/map.rs:55:37\n      <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/lib.rs:86:13\n   9: <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll::{{closure}}\n             at workspace/source/tikv/components/tracker/src/tls.rs:64:23\n      std::thread::local::LocalKey<T>::try_with\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270:16\n      std::thread::local::LocalKey<T>::with\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246:9\n      <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll\n             at workspace/source/tikv/components/tracker/src/tls.rs:62:27\n      <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/future/future/map.rs:55:37\n      <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/lib.rs:86:13\n      yatp::task::future::RawTask<F>::poll\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:59:9\n  10: yatp::task::future::TaskCell::poll\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:103:9\n      <yatp::task::future::Runner as yatp::pool::runner::Runner>::handle\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:387:20\n  11: <tikv_util::yatp_pool::YatpPoolRunner<T> as yatp::pool::runner::Runner>::handle\n             at workspace/source/tikv/components/tikv_util/src/yatp_pool/mod.rs:199:24\n  12: yatp::pool::worker::WorkerThread<T,R>::run\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/worker.rs:48:13\n      yatp::pool::builder::LazyBuilder<T>::build::{{closure}}\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/builder.rs:114:25\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n  16: __GI___clone\n"] [location=/workspace/source/tikv/src/storage/txn/scheduler.rs:903] [thread_name=sched-worker-pool-3] [thread_id=47]
[2024/12/14 15:10:13.925 +08:00] [INFO] [lib.rs:90] ["Welcome to TiKV"] [thread_id=1]
--
[2024/12/14 15:10:39.712 +08:00] [INFO] [raft.rs:2369] ["[term 12] received MsgTimeoutNow from 2988273 and starts an election to get leadership."] [from=2988273] [term=12] [raft_id=2988274] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.712 +08:00] [INFO] [raft.rs:1556] ["starting a new election"] [term=12] [raft_id=2988274] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.712 +08:00] [INFO] [raft.rs:1175] ["became candidate at term 13"] [term=13] [raft_id=2988274] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.712 +08:00] [INFO] [raft.rs:1321] ["broadcasting vote request"] [to="[2988272, 2988273]"] [log_index=106] [log_term=12] [term=13] [type=MsgRequestVote] [raft_id=2988274] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.712 +08:00] [INFO] [raft.rs:2240] ["received votes response"] [term=13] [type=MsgRequestVoteResponse] [approvals=2] [rejections=0] [from=2988273] [vote=true] [raft_id=2988274] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.712 +08:00] [INFO] [raft.rs:1259] ["became leader at term 13"] [term=13] [raft_id=2988274] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.712 +08:00] [INFO] [peer.rs:5777] ["require updating max ts"] [initial_status=55834576436] [region_id=2988271] [thread_id=108]
[2024/12/14 15:10:39.713 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2988271] [thread_id=38]
[2024/12/14 15:10:39.713 +08:00] [INFO] [endpoint.rs:701] ["register observe region"] [region="id: 2988271 start_key: 7480000000000000FF725F728000000001FFAA1C8B0000000000FA end_key: 7480000000000000FF725F728000000001FFB7299F0000000000FA region_epoch { conf_ver: 41 version: 794 } peers { id: 2988272 store_id: 14 } peers { id: 2988273 store_id: 13 } peers { id: 2988274 store_id: 1 }"] [thread_id=75]
[2024/12/14 15:10:39.713 +08:00] [INFO] [endpoint.rs:358] ["Resolver initialized"] [pending_data_index=0] [snapshot_index=107] [observe_id=ObserveId(119)] [region=2988271] [thread_id=75]
[2024/12/14 15:10:39.721 +08:00] [ERROR] [mod.rs:247] ["async write on_applied callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_on_applied_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:246:14\n   1: core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<raftstore::store::msg::WriteResponse,alloc::boxed::Box<<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,engine_rocks::engine::RocksEngine>> as tikv_kv::Engine>::async_write::{{closure}}>,tikv::server::raftkv::drop_on_applied_callback>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<raftstore::store::msg::WriteResponse,alloc::boxed::Box<<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,engine_rocks::engine::RocksEngine>> as tikv_kv::Engine>::async_write::{{closure}}>,tikv::server::raftkv::drop_on_applied_callback>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::WriteResponse,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: core::ptr::drop_in_place<raftstore::store::msg::RaftCommand<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<crossbeam_channel::err::TrySendError<raftstore::store::msg::RaftCommand<engine_rocks::snapshot::RocksSnapshot>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      raftstore::router::handle_send_error\n             at workspace/source/tikv/components/raftstore/src/router.rs:275:1\n      raftstore::router::send_command_impl::{{closure}}\n             at workspace/source/tikv/components/raftstore/src/router.rs:115:22\n      core::result::Result<T,E>::map_err\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:829:27\n      raftstore::router::send_command_impl\n             at workspace/source/tikv/components/raftstore/src/router.rs:113:5\n      raftstore::router::RaftStoreRouter::send_command\n             at workspace/source/tikv/components/raftstore/src/router.rs:60:9\n   4: <tikv::server::raftkv::RaftKv<E,S> as tikv_kv::Engine>::async_write\n             at workspace/source/tikv/src/server/raftkv/mod.rs:572:19\n   5: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process_write::{{closure}}::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1634:17\n      tikv_kv::with_tls_engine::{{closure}}\n             at workspace/source/tikv/components/tikv_kv/src/lib.rs:662:9\n      std::thread::local::LocalKey<T>::try_with\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270:16\n      std::thread::local::LocalKey<T>::with\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246:9\n      tikv_kv::with_tls_engine\n             at workspace/source/tikv/components/tikv_kv/src/lib.rs:660:20\n      tikv::storage::txn::scheduler::TxnScheduler<E,L>::process_write::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1633:13\n   6: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process::{{closure}}::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1212:72\n      <resource_metering::InTags<T> as core::future::future::Future>::poll\n             at workspace/source/tikv/components/resource_metering/src/lib.rs:267:9\n      tikv::storage::txn::scheduler::TxnScheduler<E,L>::process::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:1226:10\n      tikv::storage::txn::scheduler::TxnScheduler<E,L>::execute::{{closure}}\n             at workspace/source/tikv/src/storage/txn/scheduler.rs:764:51\n      <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/future/future/map.rs:55:37\n      <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/lib.rs:86:13\n   7: <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll::{{closure}}\n             at workspace/source/tikv/components/tracker/src/tls.rs:64:23\n      std::thread::local::LocalKey<T>::try_with\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270:16\n      std::thread::local::LocalKey<T>::with\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246:9\n      <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll\n             at workspace/source/tikv/components/tracker/src/tls.rs:62:27\n      <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/future/future/map.rs:55:37\n      <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\n             at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/lib.rs:86:13\n      yatp::task::future::RawTask<F>::poll\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:59:9\n   8: yatp::task::future::TaskCell::poll\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:103:9\n      <yatp::task::future::Runner as yatp::pool::runner::Runner>::handle\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:387:20\n   9: <tikv_util::yatp_pool::YatpPoolRunner<T> as yatp::pool::runner::Runner>::handle\n             at workspace/source/tikv/components/tikv_util/src/yatp_pool/mod.rs:199:24\n  10: yatp::pool::worker::WorkerThread<T,R>::run\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/worker.rs:48:13\n      yatp::pool::builder::LazyBuilder<T>::build::{{closure}}\n             at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/builder.rs:114:25\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n  11: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n  12: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n  13: start_thread\n  14: __GI___clone\n"] [thread_id=53]
[2024/12/14 15:10:39.722 +08:00] [INFO] [scheduler.rs:769] ["get snapshot failed"] [err="Error(Request(message: \"peer is not leader for region 24989, leader may Some(id: 24992 store_id: 13)\" not_leader { region_id: 24989 leader { id: 24992 store_id: 13 } }))"] [cid=6552] [thread_id=53]
--
[2024/12/14 15:10:49.677 +08:00] [INFO] [apply.rs:1712] ["execute admin command"] [command="cmd_type: PrepareMerge prepare_merge { min_index: 11 target { id: 2989075 start_key: 7480000000000000FF6F5F698000000000FF0000020380000000FF000002F903800000FF000000000A014154FF494F4E455345FF50FF52455300000000FBFF01346D454A4C3239FF72FF3367746D6939FF0000FD0380000000FF015C534F00000000FB end_key: 7480000000000000FF6F5F728000000000FF0461160000000000FA region_epoch { conf_ver: 59 version: 1043 } peers { id: 2989076 store_id: 1 } peers { id: 2989077 store_id: 13 } peers { id: 2989078 store_id: 14 } } }"] [index=11] [term=9] [peer_id=2989056] [region_id=2989055] [thread_id=110]
[2024/12/14 15:10:49.677 +08:00] [INFO] [raft.rs:2240] ["received votes response"] [term=99] [type=MsgRequestVoteResponse] [approvals=2] [rejections=0] [from=2982803] [vote=true] [raft_id=2982805] [region_id=2982802] [thread_id=109]
[2024/12/14 15:10:49.677 +08:00] [INFO] [raft.rs:1259] ["became leader at term 99"] [term=99] [raft_id=2982805] [region_id=2982802] [thread_id=109]
[2024/12/14 15:10:49.677 +08:00] [INFO] [peer.rs:5777] ["require updating max ts"] [initial_status=425201763606] [region_id=2982802] [thread_id=109]
[2024/12/14 15:10:49.677 +08:00] [INFO] [peer.rs:3897] ["transfer leader"] [peer="id: 163844 store_id: 13"] [peer_id=2781] [region_id=2779] [thread_id=109]
[2024/12/14 15:10:49.677 +08:00] [INFO] [raft.rs:1944] ["[term 561] starts to transfer leadership to 163844"] [lead_transferee=163844] [term=561] [raft_id=2781] [region_id=2779] [thread_id=109]
[2024/12/14 15:10:49.678 +08:00] [INFO] [pd.rs:1714] ["succeed to update max timestamp"] [region_id=2982802] [thread_id=38]
[2024/12/14 15:10:49.679 +08:00] [INFO] [apply.rs:4187] ["source logs are all applied now"] [peer_id=2989158] [region_id=2989155] [thread_id=111]
[2024/12/14 15:10:49.679 +08:00] [INFO] [apply.rs:4111] ["remove delegate from apply delegates"] [peer_id=2989158] [region_id=2989155] [thread_id=111]
[2024/12/14 15:10:49.679 +08:00] [INFO] [router.rs:283] ["shutdown mailbox"] [region_id=2989155] [thread_id=111]
[2024/12/14 15:10:49.679 +08:00] [WARN] [mod.rs:233] ["async snapshot callback is dropped"] [backtrace="   0: tikv::server::raftkv::drop_snapshot_callback\n             at workspace/source/tikv/src/server/raftkv/mod.rs:232:14\n      core::ops::function::FnOnce::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n      <tikv_util::callback::MustCall<T,C,A> as core::ops::drop::Drop>::drop\n             at workspace/source/tikv/components/tikv_util/src/callback.rs:49:26\n      core::ptr::drop_in_place<tikv_util::callback::MustCall<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<tikv_util::callback::must_call<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv_util::future::paired_must_called_future_callback<core::result::Result<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>,tikv_kv::Error>,tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}},tikv::server::raftkv::drop_snapshot_callback<tikv::server::raftkv::CmdRes<engine_rocks::snapshot::RocksSnapshot>>>::{{closure}}>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   1: core::ptr::drop_in_place<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<std::sync::mpsc::SendError<alloc::boxed::Box<dyn aws_smithy_eventstream::frame::SignMessage+core::marker::Sync+core::marker::Send>>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   2: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::ops::function::FnOnce<(raftstore::store::msg::ReadResponse<engine_rocks::snapshot::RocksSnapshot>,)>+Output = ()+core::marker::Send>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n      core::ptr::drop_in_place<raftstore::store::msg::Callback<engine_rocks::snapshot::RocksSnapshot>>\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:507:1\n   3: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n             at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:721:9\n   4: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n             at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1109:9\n   5: batch_system::batch::Poller<N,C,Handler>::poll\n             at workspace/source/tikv/components/batch-system/src/batch.rs:380:27\n   6: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n             at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n      <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n             at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n   7: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n      std::panicking::try::do_call\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n      std::panicking::try\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n      std::panic::catch_unwind\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n   8: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n   9: start_thread\n  10: __GI___clone\n"] [thread_id=108]
[2024/12/14 15:10:49.682 +08:00] [INFO] [apply.rs:2874] ["asking delegate to stop"] [source_region_id=2989055] [peer_id=2989076] [region_id=2989075] [thread_id=111]
```


Fix patch:
diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs
index 4d5a9a36b2d..97ac7737d01 100644
--- a/components/raftstore/src/store/fsm/peer.rs
+++ b/components/raftstore/src/store/fsm/peer.rs
@@ -650,8 +650,10 @@ where
             // skip handling remain messages if fsm is destroyed. This can aviod handling
             // arbitary messages(e.g. CasualMessage::ForceCompactRaftLogs) that may need
             // to read raft logs which maybe lead to panic.
-            if self.fsm.stopped {
-                break;
+            // We do not skip RaftCommand because raft commond callback should always be
+            // handled or it will cause panic.
+            if self.fsm.stopped && !matches!(&m, PeerMsg::RaftCommand(_)) {
+                continue;
             }
             distribution[m.discriminant()] += 1;
             match m {


Test patch:
diff --git a/tests/failpoints/cases/test_merge.rs b/tests/failpoints/cases/test_merge.rs
index 502fce4d727..50fd6d68e41 100644
--- a/tests/failpoints/cases/test_merge.rs
+++ b/tests/failpoints/cases/test_merge.rs
@@ -14,6 +14,7 @@ use engine_traits::{Peekable, CF_RAFT};
 use grpcio::{ChannelBuilder, Environment};
 use kvproto::{
     kvrpcpb::{PrewriteRequestPessimisticAction::*, *},
+    raft_cmdpb::{self, RaftCmdRequest},
     raft_serverpb::{PeerState, RaftMessage, RegionLocalState},
     tikvpb::TikvClient,
 };
@@ -2305,6 +2306,25 @@ fn test_raft_log_gc_after_merge() {
     rx.recv_timeout(Duration::from_secs(1)).unwrap();
 
     let raft_router = cluster.get_router(1).unwrap();
+
+    // send a raft cmd to test when peer fsm is closed, this cmd will still be
+    // handled.
+    let cmd = {
+        let mut cmd = RaftCmdRequest::default();
+        let mut req = raft_cmdpb::Request::default();
+        req.set_read_index(raft_cmdpb::ReadIndexRequest::default());
+        cmd.mut_requests().push(req);
+        cmd.mut_header().region_id = 1;
+        cmd
+    };
+    let (tx, rx) = std::sync::mpsc::channel();
+    let callback = Callback::read(Box::new(move |req| {
+        tx.send(req).unwrap();
+    }));
+    let cmd_req = RaftCommand::new(cmd, callback);
+    raft_router.send_raft_command(cmd_req).unwrap();
+
+    // send a casual msg that can trigger panic after peer fms closed.
     raft_router
         .send_casual_msg(1, CasualMessage::ForceCompactRaftLogs)
         .unwrap();
@@ -2314,4 +2334,7 @@ fn test_raft_log_gc_after_merge() {
     // wait some time for merge finish.
     std::thread::sleep(Duration::from_secs(1));
     must_get_equal(&cluster.get_engine(1), b"k3", b"v3");
+
+    let resp = rx.recv().unwrap();
+    assert!(resp.response.get_header().has_error());
 }