Repository: tikv/tikv (language: rust)

Issue:
Improve debugging for TiKV Panics
## Background
TiKV panics can be challenging to debug due to limited contextual information (e.g. the relevant key and region) in panic logs. Examples:
- https://github.com/tikv/tikv/issues/17469
- https://github.com/tikv/tikv/issues/17868
- https://github.com/tikv/tikv/issues/18007

It would be highly beneficial if there is a systematic approach to report key debugging details when a panic occurs. 

## TODOs
Review the codebase to gather statistics on the number and types of panics (e.g., explicit panics, assertions, unwraps).

Explore systematic improvements (e.g., enhancing the panic hook) to capture more context, such as key/region info and additional environment details.


Discussion / hints:

Fix patch:
diff --git a/components/raftstore/src/store/fsm/store.rs b/components/raftstore/src/store/fsm/store.rs
index 729d9339e42..83edd478d78 100644
--- a/components/raftstore/src/store/fsm/store.rs
+++ b/components/raftstore/src/store/fsm/store.rs
@@ -62,7 +62,7 @@ use tikv_util::{
     future::poll_future_notify,
     info, is_zero_duration,
     mpsc::{self, LooseBoundedSender, Receiver},
-    slow_log,
+    set_panic_context, slow_log,
     store::{find_peer, region_on_stores},
     sys::{
         self as sys_util,
@@ -1074,6 +1074,10 @@ impl<EK: KvEngine, ER: RaftEngine, T: Transport> PollHandler<PeerFsm<EK, ER>, St
         &mut self,
         peer: &mut impl DerefMut<Target = PeerFsm<EK, ER>>,
     ) -> HandleResult {
+        let _guard = set_panic_context! {
+            "region_id" => peer.region_id(),
+            "peer_id" => peer.peer_id(),
+        };
         let mut handle_result = HandleResult::KeepProcessing;
 
         fail_point!(
diff --git a/components/tikv_util/src/lib.rs b/components/tikv_util/src/lib.rs
index 918d5a9169c..20a8d9fddf8 100644
--- a/components/tikv_util/src/lib.rs
+++ b/components/tikv_util/src/lib.rs
@@ -11,10 +11,12 @@
 extern crate test;
 
 use std::{
+    cell::RefCell,
     cmp,
     collections::{
         hash_map::Entry,
         vec_deque::{Iter, VecDeque},
+        HashMap,
     },
     convert::AsRef,
     env,
@@ -442,6 +444,68 @@ impl<T> Drop for MustConsumeVec<T> {
     }
 }
 
+// Thread-local variable for storing the panic context.
+thread_local! {
+    pub static PANIC_CONTEXT: RefCell<HashMap<String, String>> = RefCell::new(HashMap::new());
+}
+
+/// Guard to automatically remove a key from `PANIC_CONTEXT` on drop.
+pub struct PanicContextGuard(pub Vec<String>);
+
+impl Drop for PanicContextGuard {
+    fn drop(&mut self) {
+        let _ = crate::PANIC_CONTEXT.try_with(|ctx| {
+            if let Ok(mut ctx) = ctx.try_borrow_mut() {
+                for key in self.0.iter() {
+                    ctx.remove(key);
+                }
+            }
+        });
+    }
+}
+
+/// Sets key-value pairs in the thread-local panic context that will be logged
+/// on panic. Each key is prefixed with the source code location.
+///
+/// Returns a `PanicContextGuard` to ensure the keys are removed on scope exit.
+#[macro_export]
+macro_rules! set_panic_context {
+    ( $($key:expr => $value:expr),+ $(,)?) => {{
+        let location = format!("{}:{}", file!(), line!());
+        let mut keys = Vec::new();
+        $(
+            let full_key = format!("({}) {}", location, $key);
+            let _ = $crate::PANIC_CONTEXT.try_with(|ctx| {
+                if let Ok(mut ctx) = ctx.try_borrow_mut() {
+                    ctx.insert(full_key.clone(), $value.to_string());
+                }
+            });
+            keys.push(full_key);
+        )+
+        $crate::PanicContextGuard(keys)
+    }};
+}
+
+/// Retrieves the current panic context as a string of key-value pairs.
+pub fn get_panic_context() -> String {
+    PANIC_CONTEXT
+        .try_with(|ctx| {
+            ctx.try_borrow()
+                .map(|ctx| {
+                    if ctx.is_empty() {
+                        String::new()
+                    } else {
+                        ctx.iter()
+                            .map(|(k, v)| format!("{}=>{}", k, v))
+                            .collect::<Vec<_>>()
+                            .join(", ")
+                    }
+                })
+                .unwrap_or_default()
+        })
+        .unwrap_or_default()
+}
+
 /// Exit the whole process when panic.
 pub fn set_panic_hook(panic_abort: bool, data_dir: &str) {
     use std::{panic, process};
@@ -478,12 +542,21 @@ pub fn set_panic_hook(panic_abort: bool, data_dir: &str) {
             .location()
             .map(|l| format!("{}:{}", l.file(), l.line()));
         let bt = backtrace::Backtrace::new();
-        crit!("{}", msg;
-            "thread_name" => name,
-            "location" => loc.unwrap_or_else(|| "<unknown>".to_owned()),
-            "backtrace" => format_args!("{:?}", bt),
-        );
-
+        let context = get_panic_context();
+        if context.is_empty() {
+            crit!("{}", msg;
+                "thread_name" => name,
+                "location" => loc.unwrap_or_else(|| "<unknown>".to_owned()),
+                "backtrace" => format_args!("{:?}", bt),
+            );
+        } else {
+            crit!("{}", msg;
+                "thread_name" => name,
+                "location" => loc.unwrap_or_else(|| "<unknown>".to_owned()),
+                "backtrace" => format_args!("{:?}", bt),
+                "context" => context,
+            );
+        }
         // There might be remaining logs in the async logger.
         // To collect remaining logs and also collect future logs, replace the old one
         // with a terminal logger.
@@ -810,4 +883,22 @@ mod tests {
         });
         res.unwrap_err();
     }
+
+    #[test]
+    fn test_panic_context() {
+        let _guard = set_panic_context! {"outer_ctx" => "foo"};
+        {
+            let _guard = set_panic_context! {
+                "inner_ctx1" => "bar",
+                "inner_ctx2" => "baz",
+            };
+            assert!(get_panic_context().contains("outer_ctx=>foo"));
+            assert!(get_panic_context().contains("inner_ctx1=>bar"));
+            assert!(get_panic_context().contains("inner_ctx2=>baz"));
+        } // Inner guard is dropped, removing "inner_ctx"
+
+        // Only the outer context remains
+        assert!(get_panic_context().contains("outer_ctx=>foo"));
+        assert!(!get_panic_context().contains("inner_ctx"));
+    }
 }
diff --git a/src/server/gc_worker/gc_worker.rs b/src/server/gc_worker/gc_worker.rs
index d1dc58ae325..06afbca6205 100644
--- a/src/server/gc_worker/gc_worker.rs
+++ b/src/server/gc_worker/gc_worker.rs
@@ -28,6 +28,7 @@ use raftstore::coprocessor::{CoprocessorHost, RegionInfoProvider};
 use tikv_kv::{CfStatistics, CursorBuilder, Modify, SnapContext};
 use tikv_util::{
     config::{Tracker, VersionTrack},
+    set_panic_context,
     store::find_peer,
     time::{duration_to_sec, Instant, Limiter, SlowTimer},
     worker::{Builder as WorkerBuilder, LazyWorker, Runnable, ScheduleError, Scheduler},
@@ -357,6 +358,7 @@ impl<E: Engine> GcRunnerCore<E> {
         txn: &mut MvccTxn,
         reader: &mut MvccReader<E::Snap>,
     ) -> Result<()> {
+        let _guard = set_panic_context! {"key" => key};
         let next_gc_info = gc(txn, reader, key.clone(), safe_point).map_err(TxnError::from_mvcc)?;
         gc_info.found_versions += next_gc_info.found_versions;
         gc_info.deleted_versions += next_gc_info.deleted_versions;
@@ -466,6 +468,7 @@ impl<E: Engine> GcRunnerCore<E> {
         let mut gc_info = GcInfo::default();
         let mut keys = keys.into_iter().peekable();
         for region in regions {
+            let _guard = set_panic_context! {"region" => region.id};
             let mut txn = Self::mvcc_txn_with_dummy_cm();
             let mut reader = self.create_reader(
                 count,
diff --git a/src/server/snap.rs b/src/server/snap.rs
index fc907026bd6..32fab28447a 100644
--- a/src/server/snap.rs
+++ b/src/server/snap.rs
@@ -378,6 +378,7 @@ fn recv_snap<R: RaftExtension + 'static>(
         let mut stream = stream.map_err(Error::from);
         let head = stream.next().await.transpose()?;
         let mut context = RecvSnapContext::new(head, &snap_mgr)?;
+        let _guard = set_panic_context! {"snap_key" => context.key.to_string()};
         // Record the region_id for later cleanup.
         region_id.store(context.raft_msg.region_id, Ordering::SeqCst);
         if context.file.is_none() {


Test patch: