Repository: tikv/tikv (language: rust)

Issue:
Flaky test tests::failpoints cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state
## Bug Report

<!-- Thanks for your bug report! Don't worry if you can't fill out all the sections. -->

### What version of TiKV are you using?
<!-- You can run `tikv-server --version` -->

Ref: #17490

### What operating system and CPU are you using?
<!-- If you're using Linux, you can run `cat /proc/cpuinfo` -->

Linux, x86_64

### Steps to reproduce
<!-- If possible, provide a recipe for reproducing the error. A complete runnable program is good. -->

CI run unit test job, [link](https://do.pingcap.net/jenkins/job/tikv/job/tikv/job/pull_unit_test/1499/)

### What did you expect?

tests all passed.

### What did happened?

```log
--- TRY 3 STDOUT:        tests::failpoints cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state ---

running 1 test

--- TRY 3 STDERR:        tests::failpoints cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state ---
thread 'raftstore-1-0::test_update_apply_index_before_sync_read_state' panicked at /rust/registry/src/index.crates.io-6f17d22bba15001f/fail-0.5.1/src/lib.rs:496:25:
failpoint on_step_read_index_msg panic
stack backtrace:
   0: rust_begin_unwind
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:645:5
   1: core::panicking::panic_fmt
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/panicking.rs:72:14
   2: fail::FailPoint::eval
   3: fail::eval
   4: raftstore::store::peer::Peer<EK,ER>::step
   5: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_raft_message
   6: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs
   7: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal
   8: batch_system::batch::Poller<N,C,Handler>::poll
   9: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}
  10: <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'sched-worker-pool::test_update_apply_index_before_sync_read_state-0' panicked at /home/jenkins/tikv-src/src/storage/txn/scheduler.rs:914:25:
undetermined error: Error(Undetermined("async write on_applied callback is dropped")) cid=4, tag=prewrite, process result=Some(PrewriteResult { result: PrewriteResult { locks: [], min_commit_ts: TimeStamp(0), one_pc_commit_ts: TimeStamp(0) } })
stack backtrace:
   0: rust_begin_unwind
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:645:5
   1: core::panicking::panic_fmt
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/panicking.rs:72:14
   2: tikv::storage::txn::scheduler::TxnScheduler<E,L>::on_write_finished
   3: tikv::storage::txn::scheduler::TxnScheduler<E,L>::handle_async_write::{{closure}}
   4: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process_write::{{closure}}
   5: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process::{{closure}}::{{closure}}
   6: <resource_metering::InTags<T> as core::future::future::Future>::poll
   7: tikv::storage::txn::scheduler::TxnScheduler<E,L>::process::{{closure}}
   8: tikv::storage::txn::scheduler::TxnScheduler<E,L>::execute::{{closure}}
   9: <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll
  10: <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll
  11: <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll::{{closure}}
  12: std::thread::local::LocalKey<T>::try_with
  13: std::thread::local::LocalKey<T>::with
  14: <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll
  15: <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll
  16: <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll
  17: yatp::task::future::RawTask<F>::poll
  18: yatp::task::future::TaskCell::poll
  19: <yatp::task::future::Runner as yatp::pool::runner::Runner>::handle
  20: <tikv_util::yatp_pool::YatpPoolRunner<T> as yatp::pool::runner::Runner>::handle
  21: yatp::pool::worker::WorkerThread<T,R>::run
  22: yatp::pool::builder::LazyBuilder<T>::build::{{closure}}
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state' panicked at components/test_raftstore/src/util.rs:1074:59:
called `Result::unwrap()` on an `Err` value: RpcFailure(RpcStatus { code: 1-CANCELLED, message: "CANCELLED", details: [] })
stack backtrace:
   0: rust_begin_unwind
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:645:5
   1: core::panicking::panic_fmt
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/panicking.rs:72:14
   2: core::result::unwrap_failed
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/result.rs:1649:5
   3: core::result::Result<T,E>::unwrap
   4: test_raftstore::util::must_kv_prewrite_with
   5: test_raftstore::util::must_kv_prewrite
   6: test_raftstore::util::must_kv_write
   7: test_raftstore::util::PeerClient::must_kv_write
   8: failpoints::cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state
             at /home/jenkins/tikv-src/tests/failpoints/cases/test_replica_stale_read.rs:240:5
   9: failpoints::cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state::{{closure}}
             at /home/jenkins/tikv-src/tests/failpoints/cases/test_replica_stale_read.rs:204:52
  10: core::ops::function::FnOnce::call_once
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/ops/function.rs:250:5
  11: test_util::runner::run_test_with_hook::{{closure}}::{{closure}}
  12: core::ops::function::FnOnce::call_once{{vtable.shim}}
  13: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/alloc/src/boxed.rs:2015:9
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state' panicked at /rust/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool.rs:72:26:
called `Result::unwrap()` on an `Err` value: Any { .. }
stack backtrace:
   0:     0x55c3f8a15596 - std::backtrace_rs::backtrace::libunwind::trace::h0c778a97139c185f
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
   1:     0x55c3f8a15596 - std::backtrace_rs::backtrace::trace_unsynchronized::h49275f7f11d03db8
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x55c3f8a15596 - std::sys_common::backtrace::_print_fmt::hc6d2af0f9113f87c
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys_common/backtrace.rs:68:5
   3:     0x55c3f8a15596 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h4269025391cee804
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x55c3f8a471c0 - core::fmt::rt::Argument::fmt::h47e13179bda442d0
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/fmt/rt.rs:142:9
   5:     0x55c3f8a471c0 - core::fmt::write::h0ea06241e192b662
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/fmt/mod.rs:1120:17
   6:     0x55c3f8a1183f - std::io::Write::write_fmt::hada0a89348d2eeaf
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/io/mod.rs:1810:15
   7:     0x55c3f8a15374 - std::sys_common::backtrace::_print::h474b180b0f0cbfaa
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys_common/backtrace.rs:47:5
   8:     0x55c3f8a15374 - std::sys_common::backtrace::print::h292cdb95c9f3ed37
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys_common/backtrace.rs:34:9
   9:     0x55c3f8a16f67 - std::panicking::default_hook::{{closure}}::h215c742b61103c6e
  10:     0x55c3f8a16cc9 - std::panicking::default_hook::h57e29db39f9afc40
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:292:9
  11:     0x55c3f8a173f8 - std::panicking::rust_panic_with_hook::hdec91a2b4b229c0a
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:779:13
  12:     0x55c3f8a172d2 - std::panicking::begin_panic_handler::{{closure}}::h307c36ea28011ec5
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:657:13
  13:     0x55c3f8a15a96 - std::sys_common::backtrace::__rust_end_short_backtrace::h4a40c8d6f1eb7696
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys_common/backtrace.rs:171:18
  14:     0x55c3f8a17030 - rust_begin_unwind
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:645:5
  15:     0x55c3f18699e5 - core::panicking::panic_fmt::h5740899885667996
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/panicking.rs:72:14
  16:     0x55c3f186a023 - core::result::unwrap_failed::hede68b820f4ed9a9
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/result.rs:1649:5
  17:     0x55c3f759acd7 - core::result::Result<T,E>::unwrap::h1668fd11cf5a5acf
  18:     0x55c3f757fdde - yatp::pool::ThreadPool<T>::shutdown::h91ba7b7345ffe611
  19:     0x55c3f7580a7a - <yatp::pool::ThreadPool<T> as core::ops::drop::Drop>::drop::h0ff6c92ce1c1df27
  20:     0x55c3f7526db5 - core::ptr::drop_in_place<yatp::pool::ThreadPool<yatp::task::future::TaskCell>>::h17bd6600f3a535d6
  21:     0x55c3f7526845 - core::ptr::drop_in_place<tikv_util::yatp_pool::future_pool::PoolInner>::h86d2257d3808d4ef
  22:     0x55c3f7538279 - alloc::sync::Arc<T,A>::drop_slow::h98338f3f05c153ca
  23:     0x55c3f7538d40 - <alloc::sync::Arc<T,A> as core::ops::drop::Drop>::drop::h60fc2e6758cbf08d
  24:     0x55c3f75273ba - core::ptr::drop_in_place<alloc::sync::Arc<tikv_util::yatp_pool::future_pool::PoolInner>>::h3d0cf23727f67393
  25:     0x55c3f752689a - core::ptr::drop_in_place<tikv_util::yatp_pool::future_pool::FuturePool>::h642d91b826a82131
  26:     0x55c3f3be9dc6 - core::ptr::drop_in_place<tikv::storage::txn::sched_pool::VanillaQueue>::h9c258efd8c79ef68
  27:     0x55c3f3be7e15 - core::ptr::drop_in_place<tikv::storage::txn::sched_pool::SchedPool>::h4109b57c31c4f163
  28:     0x55c3f3b98180 - core::ptr::drop_in_place<tikv::storage::txn::scheduler::TxnSchedulerInner<tikv::server::lock_manager::LockManager>>::hafda137b4b03ec95
  29:     0x55c3f429e08c - alloc::sync::Arc<T,A>::drop_slow::hd0f3d9411a14040b
  30:     0x55c3f42b5610 - <alloc::sync::Arc<T,A> as core::ops::drop::Drop>::drop::h99f8e6cb41e8d574
  31:     0x55c3f3ba319a - core::ptr::drop_in_place<alloc::sync::Arc<tikv::storage::txn::scheduler::TxnSchedulerInner<tikv::server::lock_manager::LockManager>>>::hc9a8c7e709764cab
  32:     0x55c3f2bdc1ca - core::ptr::drop_in_place<tikv::storage::txn::scheduler::TxnScheduler<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,test_raftstore::transport_simulate::SimulateTransport<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>>,tikv::server::lock_manager::LockManager>>::h056b5d2b7fd3e8b4
  33:     0x55c3f2bdbddb - core::ptr::drop_in_place<tikv::storage::Storage<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,test_raftstore::transport_simulate::SimulateTransport<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>>,tikv::server::lock_manager::LockManager,api_version::ApiV1>>::h1deec0eab93063dc
  34:     0x55c3f2bddbfb - core::ptr::drop_in_place<tikv::server::service::kv::Service<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,test_raftstore::transport_simulate::SimulateTransport<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>>,tikv::server::lock_manager::LockManager,api_version::ApiV1>>::h1f26dae777e24bd6
  35:     0x55c3f2bf5745 - core::ptr::drop_in_place<tikv::server::server::BuilderFactory<tikv::server::service::kv::Service<tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,test_raftstore::transport_simulate::SimulateTransport<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>>,tikv::server::lock_manager::LockManager,api_version::ApiV1>>>::h409c47a69bed7f3c
  36:     0x55c3f3bf59eb - core::ptr::drop_in_place<alloc::boxed::Box<dyn tikv::server::server::GrpcBuilderFactory>>::h997456734d5c5ecd
  37:     0x55c3f2bda5cd - core::ptr::drop_in_place<tikv::server::server::Server<tikv::server::resolve::PdStoreAddrResolver,tikv::server::raftkv::RaftKv<engine_rocks::engine::RocksEngine,test_raftstore::transport_simulate::SimulateTransport<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>>>>::hae57f4af0180b1d4
  38:     0x55c3f352da02 - <test_raftstore::server::ServerCluster as test_raftstore::cluster::Simulator>::stop_node::hbed10aaf4400de0f
  39:     0x55c3f2dae498 - test_raftstore::cluster::Cluster<T>::stop_node::hbe75593961579483
  40:     0x55c3f2db1ccd - test_raftstore::cluster::Cluster<T>::shutdown::h50ab294c28739c83
  41:     0x55c3f2db26ac - <test_raftstore::cluster::Cluster<T> as core::ops::drop::Drop>::drop::hf00c3dd4e585ac75
  42:     0x55c3f2c6e4f5 - core::ptr::drop_in_place<test_raftstore::cluster::Cluster<test_raftstore::server::ServerCluster>>::h0dacfaa8860f22b6
  43:     0x55c3f1c9c53c - failpoints::cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state::h0cc3b187686a2f95
                               at /home/jenkins/tikv-src/tests/failpoints/cases/test_replica_stale_read.rs:249:1
  44:     0x55c3f1ae81fe - failpoints::cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state::{{closure}}::h14727f5e2413a040
                               at /home/jenkins/tikv-src/tests/failpoints/cases/test_replica_stale_read.rs:204:52
  45:     0x55c3f1ae81fe - core::ops::function::FnOnce::call_once::h6b32cf62913b5192
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/ops/function.rs:250:5
  46:     0x55c3f3b7e32d - test_util::runner::run_test_with_hook::{{closure}}::{{closure}}::ha4110b85053a4743
  47:     0x55c3f3b74455 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h5b75eaadbc32ff37
  48:     0x55c3f5d8830a - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::he244563f3f19158a
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/alloc/src/boxed.rs:2015:9
  49:     0x55c3f5d8830a - test::__rust_begin_short_backtrace::h2bee667dfacf6394
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/test/src/lib.rs:627:18
  50:     0x55c3f5d87121 - test::run_test_in_process::{{closure}}::h66e90ca3a6483175
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/test/src/lib.rs:650:60
  51:     0x55c3f5d87121 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h83ce589675c8e5c4
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/panic/unwind_safe.rs:272:9
  52:     0x55c3f5d87121 - std::panicking::try::do_call::hde5aba0cea3dc698
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:552:40
  53:     0x55c3f5d87121 - std::panicking::try::hcf5248222f90d5ff
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:516:19
  54:     0x55c3f5d87121 - std::panic::catch_unwind::h02e247c10aa761d1
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panic.rs:142:14
  55:     0x55c3f5d87121 - test::run_test_in_process::h253ae08688d44717
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/test/src/lib.rs:650:27
  56:     0x55c3f5d87121 - test::run_test::{{closure}}::h671b269aff1223f2
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/test/src/lib.rs:573:43
  57:     0x55c3f5d4e796 - test::run_test::{{closure}}::h94f25b04859b081c
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/test/src/lib.rs:601:41
  58:     0x55c3f5d4e796 - std::sys_common::backtrace::__rust_begin_short_backtrace::ha37bc9836d969f8c
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys_common/backtrace.rs:155:18
  59:     0x55c3f5d537f7 - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h5092bed309bb48ca
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/thread/mod.rs:529:17
  60:     0x55c3f5d537f7 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h77b767f13bf8ae8f
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/panic/unwind_safe.rs:272:9
  61:     0x55c3f5d537f7 - std::panicking::try::do_call::h0fe2a261a5fe4c8b
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:552:40
  62:     0x55c3f5d537f7 - std::panicking::try::h2e1b1d5112b41b7c
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panicking.rs:516:19
  63:     0x55c3f5d537f7 - std::panic::catch_unwind::h9f0da9903043e48c
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/panic.rs:142:14
  64:     0x55c3f5d537f7 - std::thread::Builder::spawn_unchecked_::{{closure}}::h49204260847051e7
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/thread/mod.rs:528:30
  65:     0x55c3f5d537f7 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h1d1eb0c62b32e230
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/core/src/ops/function.rs:250:5
  66:     0x55c3f8a1e375 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hc049aa84baee5c22
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/alloc/src/boxed.rs:2015:9
  67:     0x55c3f8a1e375 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hf5846f37b8112bb2
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/alloc/src/boxed.rs:2015:9
  68:     0x55c3f8a1e375 - std::sys::unix::thread::Thread::new::thread_start::hd01330a228cdd35d
                               at /rustc/89e2160c4ca5808657ed55392620ed1dbbce78d1/library/std/src/sys/unix/thread.rs:108:17
  69:     0x7fb0c1697ea5 - start_thread
  70:     0x7fb0c10be9fd - __clone
  71:                0x0 - <unknown>
thread 'advance-ts' panicked at /rust/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.25.3/src/runtime/blocking/shutdown.rs:51:21:
Cannot drop a runtime in a context where blocking is not allowed. This happens when a runtime is dropped from within an asynchronous context.
stack backtrace:
   0: std::panicking::begin_panic
   1: tokio::runtime::blocking::shutdown::Receiver::wait
   2: tokio::runtime::blocking::pool::BlockingPool::shutdown
   3: <tokio::runtime::blocking::pool::BlockingPool as core::ops::drop::Drop>::drop
   4: core::ptr::drop_in_place<tokio::runtime::blocking::pool::BlockingPool>
   5: core::ptr::drop_in_place<tokio::runtime::runtime::Runtime>
   6: alloc::sync::Arc<T,A>::drop_slow
   7: <alloc::sync::Arc<T,A> as core::ops::drop::Drop>::drop
   8: core::ptr::drop_in_place<alloc::sync::Arc<tokio::runtime::runtime::Runtime>>
   9: core::ptr::drop_in_place<resolved_ts::scanner::ScannerPool<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine>>
  10: core::ptr::drop_in_place<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>
  11: core::ptr::drop_in_place<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>
  12: core::ptr::drop_in_place<tikv_util::worker::pool::RunnableWrapper<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>>
  13: core::ptr::drop_in_place<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>
  14: core::ptr::drop_in_place<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>>
  15: core::ptr::drop_in_place<futures_util::future::future::map::Map<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>,tikv_util::yatp_pool::future_pool::PoolInner::spawn<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>>::{{closure}}>>
  16: core::ptr::drop_in_place<futures_util::future::future::Map<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>,tikv_util::yatp_pool::future_pool::PoolInner::spawn<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>>::{{closure}}>>
  17: core::ptr::drop_in_place<core::cell::UnsafeCell<futures_util::future::future::Map<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>,tikv_util::yatp_pool::future_pool::PoolInner::spawn<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>>::{{closure}}>>>
  18: core::ptr::drop_in_place<yatp::task::future::RawTask<futures_util::future::future::Map<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>,tikv_util::yatp_pool::future_pool::PoolInner::spawn<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>>::{{closure}}>>>
  19: core::ptr::drop_in_place<alloc::boxed::Box<yatp::task::future::RawTask<futures_util::future::future::Map<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>,tikv_util::yatp_pool::future_pool::PoolInner::spawn<tracker::tls::TrackedFuture<tikv_util::worker::pool::Worker::start_with_timer_impl<tikv_util::worker::pool::NoTimeoutRunnableWrapper<resolved_ts::endpoint::Endpoint<raftstore::router::CdcRaftRouter<raftstore::router::ServerRaftStoreRouter<engine_rocks::engine::RocksEngine,raft_log_engine::engine::RaftLogEngine>>,engine_rocks::engine::RocksEngine,raftstore::store::fsm::store::StoreMeta>>>::{{closure}}>>::{{closure}}>>>>
  20: core::mem::drop
  21: yatp::task::future::RawTask<F>::drop
  22: <yatp::task::future::TaskCell as core::ops::drop::Drop>::drop
  23: core::ptr::drop_in_place<yatp::task::future::TaskCell>
  24: core::ptr::drop_in_place<alloc::borrow::Cow<yatp::task::future::TaskCell>>
  25: yatp::task::future::wake_task::{{closure}}
  26: std::thread::local::LocalKey<T>::try_with
  27: std::thread::local::LocalKey<T>::with
  28: yatp::task::future::wake_task
  29: yatp::task::future::wake_impl
  30: yatp::task::future::wake_raw
  31: core::task::wake::Waker::wake
  32: futures_core::task::__internal::atomic_waker::AtomicWaker::wake
  33: futures_channel::mpsc::UnboundedSenderInner<T>::queue_push_and_signal
  34: futures_channel::mpsc::UnboundedSender<T>::do_send_nb
  35: futures_channel::mpsc::UnboundedSender<T>::unbounded_send
  36: tikv_util::worker::pool::Scheduler<T>::schedule_force
  37: tikv_util::worker::pool::Scheduler<T>::schedule
  38: resolved_ts::advance::AdvanceTsWorker::advance_ts_for_regions::{{closure}}
  39: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
  40: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
  41: tokio::runtime::task::core::Core<T,S>::poll
  42: tokio::runtime::task::harness::poll_future::{{closure}}
  43: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
  44: std::panicking::try::do_call
  45: __rust_try
  46: std::panicking::try
  47: std::panic::catch_unwind
  48: tokio::runtime::task::harness::poll_future
  49: tokio::runtime::task::harness::Harness<T,S>::poll_inner
  50: tokio::runtime::task::harness::Harness<T,S>::poll
  51: tokio::runtime::task::raw::poll
  52: tokio::runtime::task::raw::RawTask::poll
  53: tokio::runtime::task::LocalNotified<S>::run
  54: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
  55: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
  56: tokio::runtime::scheduler::multi_thread::worker::Context::run
  57: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
  58: tokio::macros::scoped_tls::ScopedKey<T>::set
  59: tokio::runtime::scheduler::multi_thread::worker::run
  60: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
  61: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
  62: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
  63: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
  64: tokio::runtime::task::core::Core<T,S>::poll
  65: tokio::runtime::task::harness::poll_future::{{closure}}
  66: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
  67: std::panicking::try::do_call
  68: __rust_try
  69: std::panicking::try
  70: std::panic::catch_unwind
  71: tokio::runtime::task::harness::poll_future
  72: tokio::runtime::task::harness::Harness<T,S>::poll_inner
  73: tokio::runtime::task::harness::Harness<T,S>::poll
  74: tokio::runtime::task::raw::poll
  75: tokio::runtime::task::raw::RawTask::poll
  76: tokio::runtime::task::UnownedTask<S>::run
  77: tokio::runtime::blocking::pool::Task::run
  78: tokio::runtime::blocking::pool::Inner::run
  79: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'cases::test_replica_stale_read::test_update_apply_index_before_sync_read_state' panicked at library/core/src/panicking.rs:163:5:
panic in a destructor during cleanup
thread caused non-unwinding panic. aborting.

```



Discussion / hints:
/label flaky_test
@wuhuizuo: The label(s) `flaky_test` cannot be applied. These labels are supported: `challenge-program, compatibility-breaker, high-performance, hptc, wontfix, do-not-merge/cherry-pick-not-approved, needs-cherry-pick-release-5.4, needs-cherry-pick-release-6.1, needs-cherry-pick-release-6.5, needs-cherry-pick-release-7.1, needs-cherry-pick-release-7.5, needs-cherry-pick-release-8.1, needs-cherry-pick-release-8.3, affects-5.4, affects-6.1, affects-6.5, affects-7.1, affects-7.5, affects-8.1, affects-8.3, may-affects-5.4, may-affects-6.1, may-affects-6.5, may-affects-7.1, may-affects-7.5, may-affects-8.1`.

<details>

In response to [this](https://github.com/tikv/tikv/issues/17497#issuecomment-2337058528):

>/label flaky_test


Instructions for interacting with me using PR comments are available [here](https://prow.tidb.net/command-help).  If you have questions or suggestions related to my behavior, please file an issue against the [ti-community-infra/tichi](https://github.com/ti-community-infra/tichi/issues/new?title=Prow%20issue:) repository.
</details>
I think it's introduced by https://github.com/tikv/tikv/pull/17359/
I saw it in release-7.5.

> I think it's introduced by [#17359](https://github.com/tikv/tikv/pull/17359)



Fix patch:
diff --git a/components/raftstore-v2/src/operation/query/lease.rs b/components/raftstore-v2/src/operation/query/lease.rs
index 189986f93d2..30ca545bf06 100644
--- a/components/raftstore-v2/src/operation/query/lease.rs
+++ b/components/raftstore-v2/src/operation/query/lease.rs
@@ -41,7 +41,6 @@ impl<EK: KvEngine, ER: RaftEngine> Peer<EK, ER> {
     ) -> bool {
         assert_eq!(m.get_msg_type(), MessageType::MsgReadIndex);
 
-        fail::fail_point!("on_step_read_index_msg");
         ctx.coprocessor_host
             .on_step_read_index(m, self.state_role());
         // Must use the commit index of `PeerStorage` instead of the commit index
diff --git a/components/raftstore/src/store/peer.rs b/components/raftstore/src/store/peer.rs
index e290b1e05cc..a7e21936119 100644
--- a/components/raftstore/src/store/peer.rs
+++ b/components/raftstore/src/store/peer.rs
@@ -1918,7 +1918,6 @@ where
         }
         let msg_type = m.get_msg_type();
         if msg_type == MessageType::MsgReadIndex {
-            fail_point!("on_step_read_index_msg");
             ctx.coprocessor_host
                 .on_step_read_index(&mut m, self.get_role());
             // Must use the commit index of `PeerStorage` instead of the commit index
@@ -4182,6 +4181,8 @@ where
                     return false;
                 }
             }
+        } else {
+            fail_point!("propose_readindex_from_follower");
         }
 
         if !self.is_leader() && self.leader_id() == INVALID_ID {


Test patch:
diff --git a/tests/failpoints/cases/test_replica_stale_read.rs b/tests/failpoints/cases/test_replica_stale_read.rs
index cb986250d82..34013ac7564 100644
--- a/tests/failpoints/cases/test_replica_stale_read.rs
+++ b/tests/failpoints/cases/test_replica_stale_read.rs
@@ -30,8 +30,7 @@ fn prepare_for_stale_read_before_run(
     let leader_client = PeerClient::new(&cluster, 1, leader);
 
     // There should be no read index message while handling stale read request
-    let on_step_read_index_msg = "on_step_read_index_msg";
-    fail::cfg(on_step_read_index_msg, "panic").unwrap();
+    fail::cfg("propose_readindex_from_follower", "panic").unwrap();
 
     (cluster, pd_client, leader_client)
 }