Repository: tikv/tikv (language: rust)

Issue:
Enhancement: Add timestamp safety boundary to prevent unreasonable max_ts updates
Subtask of 
- https://github.com/pingcap/tidb/issues/57786
## Enhancement
Currently, `max_ts` can be updated to any larger value (**exceeding current PD TSO**). This poses a potential risk where bugs or unsafe usage could set `max_ts` to an unreasonably large value, causing critical bugs and data corruption.

Proposed Solution:
Add a configurable safety boundary (`max_ts_limit`) that:
- Enforces a maximum allowed deviation of `max_ts` from PD timestamp
- Regularly syncs with PD to maintain a reasonable limit
- Prevents updates to unreasonable future timestamps
- Timestamps that are known to be from PD TSO can bypass the limit.


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 0f70e47513c..78263eaf206 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -97,6 +97,12 @@ dependencies = [
  "libc 0.2.151",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "ansi_term"
 version = "0.11.0"
@@ -106,6 +112,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
 [[package]]
 name = "anyhow"
 version = "1.0.75"
@@ -963,7 +975,7 @@ name = "batch-system"
 version = "0.1.0"
 dependencies = [
  "collections",
- "criterion",
+ "criterion 0.3.5",
  "crossbeam",
  "dashmap",
  "derive_more",
@@ -1223,13 +1235,19 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "causal_ts"
 version = "0.0.1"
 dependencies = [
  "api_version",
  "async-trait",
- "criterion",
+ "criterion 0.3.5",
  "engine_rocks",
  "engine_traits",
  "enum_dispatch",
@@ -1294,7 +1312,7 @@ dependencies = [
  "causal_ts",
  "collections",
  "concurrency_manager",
- "criterion",
+ "criterion 0.3.5",
  "crossbeam",
  "engine_rocks",
  "engine_traits",
@@ -1528,13 +1546,23 @@ dependencies = [
 name = "concurrency_manager"
 version = "0.0.1"
 dependencies = [
- "criterion",
+ "criterion 0.4.0",
+ "crossbeam",
  "crossbeam-skiplist",
  "fail",
  "futures 0.3.15",
  "kvproto",
+ "lazy_static",
+ "mockall",
+ "online_config",
  "parking_lot 0.12.1",
+ "pd_client",
+ "prometheus",
  "rand 0.8.5",
+ "serde",
+ "slog",
+ "slog-global",
+ "thiserror",
  "tikv_alloc",
  "tikv_util",
  "tokio",
@@ -1651,9 +1679,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
 dependencies = [
  "atty",
- "cast",
+ "cast 0.2.2",
  "clap 2.33.0",
- "criterion-plot",
+ "criterion-plot 0.4.4",
  "csv",
  "itertools",
  "lazy_static",
@@ -1670,13 +1698,39 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast 0.3.0",
+ "ciborium",
+ "clap 3.1.6",
+ "criterion-plot 0.5.0",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
 [[package]]
 name = "criterion-cpu-time"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63aaaf47e457badbcb376c65a49d0f182c317ebd97dc6d1ced94c8e1d09c0f3a"
 dependencies = [
- "criterion",
+ "criterion 0.3.5",
  "libc 0.2.151",
 ]
 
@@ -1686,7 +1740,7 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eba5111e09fabb08bfaedbe28c832876bb38d4f9519f715466332880d80b0eac"
 dependencies = [
- "criterion",
+ "criterion 0.3.5",
  "perfcnt",
 ]
 
@@ -1696,7 +1750,17 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
 dependencies = [
- "cast",
+ "cast 0.2.2",
+ "itertools",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast 0.3.0",
  "itertools",
 ]
 
@@ -1965,6 +2029,12 @@ dependencies = [
  "syn 2.0.87",
 ]
 
+[[package]]
+name = "downcast"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.4"
@@ -2530,6 +2600,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fragile"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
+
 [[package]]
 name = "fs2"
 version = "0.4.3"
@@ -4021,6 +4097,32 @@ dependencies = [
  "tempdir",
 ]
 
+[[package]]
+name = "mockall"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2"
+dependencies = [
+ "cfg-if 1.0.0",
+ "downcast",
+ "fragile",
+ "mockall_derive",
+ "predicates",
+ "predicates-tree",
+]
+
+[[package]]
+name = "mockall_derive"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898"
+dependencies = [
+ "cfg-if 1.0.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "more-asserts"
 version = "0.2.1"
@@ -4783,6 +4885,32 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "predicates"
+version = "3.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573"
+dependencies = [
+ "anstyle",
+ "predicates-core",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "pretty_assertions"
 version = "1.4.0"
@@ -6711,6 +6839,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "termtree"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
+
 [[package]]
 name = "test_backup"
 version = "0.0.1"
@@ -6980,7 +7114,7 @@ dependencies = [
  "collections",
  "concurrency_manager",
  "crc64fast",
- "criterion",
+ "criterion 0.3.5",
  "criterion-cpu-time",
  "criterion-perf-events",
  "crossbeam",
diff --git a/components/backup-stream/src/endpoint.rs b/components/backup-stream/src/endpoint.rs
index 08cc7a81212..0080aa1e3e6 100644
--- a/components/backup-stream/src/endpoint.rs
+++ b/components/backup-stream/src/endpoint.rs
@@ -813,7 +813,7 @@ where
                 .await
                 .map_err(|err| Error::from(err).report("failed to get tso from pd"))
                 .unwrap_or_default();
-            cm.update_max_ts(pd_tso);
+            cm.update_max_ts(pd_tso, "backup-stream").unwrap();
             let min_ts = cm.global_min_lock_ts().unwrap_or(TimeStamp::max());
             Ord::min(pd_tso, min_ts)
         }
diff --git a/components/backup/src/endpoint.rs b/components/backup/src/endpoint.rs
index b392d6a16df..f121667ff51 100644
--- a/components/backup/src/endpoint.rs
+++ b/components/backup/src/endpoint.rs
@@ -346,7 +346,9 @@ impl BackupRange {
             snap_ctx.key_ranges = vec![key_range];
         } else {
             // Update max_ts and check the in-memory lock table before getting the snapshot
-            concurrency_manager.update_max_ts(backup_ts);
+            concurrency_manager
+                .update_max_ts(backup_ts, "backup_range")
+                .map_err(TxnError::from)?;
             concurrency_manager
                 .read_range_check(
                     self.start_key.as_ref(),
diff --git a/components/cdc/src/endpoint.rs b/components/cdc/src/endpoint.rs
index 2ec14722b0f..d5197684e13 100644
--- a/components/cdc/src/endpoint.rs
+++ b/components/cdc/src/endpoint.rs
@@ -1188,7 +1188,7 @@ impl<T: 'static + CdcHandle<E>, E: KvEngine, S: StoreRegionMeta> Endpoint<T, E,
             // optimizations like async commit is enabled.
             // Note: This step must be done before scheduling `Task::MinTs` task, and the
             // resolver must be checked in or after `Task::MinTs`' execution.
-            cm.update_max_ts(min_ts);
+            cm.update_max_ts(min_ts, "cdc").unwrap();
             if let Some(min_mem_lock_ts) = cm.global_min_lock_ts() {
                 if min_mem_lock_ts < min_ts {
                     min_ts = min_mem_lock_ts;
diff --git a/components/concurrency_manager/Cargo.toml b/components/concurrency_manager/Cargo.toml
index 0ffee70899b..ae41c0d3c9b 100644
--- a/components/concurrency_manager/Cargo.toml
+++ b/components/concurrency_manager/Cargo.toml
@@ -6,16 +6,26 @@ version = "0.0.1"
 license = "Apache-2.0"
 
 [dependencies]
+crossbeam = "0.8"
 crossbeam-skiplist = "0.1"
 fail = "0.5"
 kvproto = { workspace = true }
+lazy_static = "1.4.0"
+mockall = "0.13"
+online_config = { workspace = true }
 parking_lot = "0.12"
+pd_client = { workspace = true }
+prometheus = "0.13"
+serde = { version = "1.0.194", features = ["derive"] }
+slog = { workspace = true }
+slog-global = { workspace = true }
+thiserror = "1.0"
 tikv_util = { workspace = true }
 tokio = { version = "1.5", features = ["macros", "sync", "time"] }
 txn_types = { workspace = true }
 
 [dev-dependencies]
-criterion = "0.3"
+criterion = "0.4"
 futures = "0.3"
 rand = "0.8.3"
 tikv_alloc = { workspace = true, features = ["jemalloc"] }
@@ -24,3 +34,8 @@ tikv_alloc = { workspace = true, features = ["jemalloc"] }
 name = "lock_table"
 path = "benches/lock_table.rs"
 harness = false
+
+[[bench]]
+name = "update_max_ts"
+path = "benches/update_max_ts.rs"
+harness = false
diff --git a/components/concurrency_manager/benches/update_max_ts.rs b/components/concurrency_manager/benches/update_max_ts.rs
new file mode 100644
index 00000000000..df2e61aeb42
--- /dev/null
+++ b/components/concurrency_manager/benches/update_max_ts.rs
@@ -0,0 +1,33 @@
+// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
+
+use std::time::Duration;
+
+use concurrency_manager::{ActionOnInvalidMaxTs, ConcurrencyManager};
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use txn_types::TimeStamp;
+
+fn benchmark_update_max_ts(c: &mut Criterion) {
+    let latest_ts = TimeStamp::new(1000);
+    let limit_valid_time = Duration::from_secs(20);
+    let cm = ConcurrencyManager::new_with_config(
+        latest_ts,
+        limit_valid_time,
+        ActionOnInvalidMaxTs::Error,
+        None,
+        Duration::ZERO,
+    );
+
+    cm.set_max_ts_limit(TimeStamp::new(4000));
+
+    let new_ts = TimeStamp::new(3000);
+
+    c.bench_function("update_max_ts", |b| {
+        b.iter(|| {
+            cm.update_max_ts(black_box(new_ts), || format!("benchmark-{}", new_ts))
+                .unwrap();
+        })
+    });
+}
+
+criterion_group!(benches, benchmark_update_max_ts);
+criterion_main!(benches);
diff --git a/components/concurrency_manager/src/lib.rs b/components/concurrency_manager/src/lib.rs
index 1c6bdb8dbf1..0b3502d400b 100644
--- a/components/concurrency_manager/src/lib.rs
+++ b/components/concurrency_manager/src/lib.rs
@@ -16,13 +16,24 @@ mod key_handle;
 mod lock_table;
 
 use std::{
+    error::Error,
+    fmt,
+    fmt::Display,
     mem::MaybeUninit,
     sync::{
-        atomic::{AtomicU64, Ordering},
+        atomic::{AtomicU64, AtomicUsize, Ordering},
         Arc,
     },
+    time::Duration,
 };
 
+use crossbeam::atomic::AtomicCell;
+use lazy_static::lazy_static;
+use mockall::automock;
+use pd_client::{PdClient, PdFuture};
+use prometheus::{register_int_gauge, IntGauge};
+use thiserror::Error;
+use tikv_util::{error, future::block_on_timeout, time::Instant, warn};
 use txn_types::{Key, Lock, TimeStamp};
 
 pub use self::{
@@ -30,19 +41,144 @@ pub use self::{
     lock_table::LockTable,
 };
 
+lazy_static! {
+    static ref MAX_TS_LIMIT_GAUGE: IntGauge = register_int_gauge!(
+        "tikv_concurrency_manager_max_ts_limit",
+        "Current value of max_ts_limit"
+    )
+    .unwrap();
+    static ref MAX_TS_GAUGE: IntGauge =
+        register_int_gauge!("tikv_concurrency_manager_max_ts", "Current value of max_ts").unwrap();
+}
+
+const DEFAULT_LIMIT_VALID_DURATION: Duration = Duration::from_secs(45);
+
+// It is suggested that limit_valid_duration = sync_interval *
+// LIMIT_VALID_TIME_MULTIPLIER, to balance between
+// 1. tolerate temporary issues in updating the limit.
+// 2. avoid long-term blocking of max_ts update caused by network partition
+//    between TiKV and PD.
+pub const LIMIT_VALID_TIME_MULTIPLIER: u32 = 3;
+
+pub const DEFAULT_MAX_TS_SYNC_INTERVAL: Duration = Duration::from_secs(15);
+pub const DEFAULT_MAX_TS_DRIFT_ALLOWANCE: Duration = Duration::from_secs(60);
+const TSO_TIMEOUT: Duration = Duration::from_secs(5);
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+struct MaxTsLimit {
+    limit: TimeStamp,
+    update_time: Instant,
+}
+
+#[automock]
+pub trait TSOProvider: Send + Sync {
+    fn get_tso(&self) -> PdFuture<TimeStamp>;
+}
+
+impl<T: PdClient> TSOProvider for T {
+    fn get_tso(&self) -> PdFuture<TimeStamp> {
+        PdClient::get_tso(self)
+    }
+}
+
 // Pay attention that the async functions of ConcurrencyManager should not hold
 // the mutex.
 #[derive(Clone)]
 pub struct ConcurrencyManager {
     max_ts: Arc<AtomicU64>,
     lock_table: LockTable,
+
+    // max_ts_limit and its update time.
+    //
+    // max_ts_limit is an assertion: max_ts should not be updated to a value greater than this
+    // limit.
+    //
+    // When the limit is not updated for a long time(exceeding the threshold), we use an
+    // approximate limit.
+    max_ts_limit: Arc<AtomicCell<MaxTsLimit>>,
+    limit_valid_duration: Duration,
+    action_on_invalid_max_ts: Arc<AtomicActionOnInvalidMaxTs>,
+
+    max_ts_drift_allowance_ms: Arc<AtomicU64>,
+
+    tso: Option<Arc<dyn TSOProvider>>,
+
+    time_provider: Arc<dyn TimeProvider>,
 }
 
 impl ConcurrencyManager {
     pub fn new(latest_ts: TimeStamp) -> Self {
+        Self::new_with_config(
+            latest_ts,
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            None,
+            Duration::ZERO,
+        )
+    }
+
+    pub fn new_with_config(
+        latest_ts: TimeStamp,
+        limit_valid_duration: Duration,
+        action_on_invalid_max_ts: ActionOnInvalidMaxTs,
+        tso: Option<Arc<dyn TSOProvider>>,
+        max_ts_drift_allowance: Duration,
+    ) -> Self {
+        let initial_limit = MaxTsLimit {
+            limit: TimeStamp::new(0),
+            update_time: Instant::now(),
+        };
+
+        if limit_valid_duration >= max_ts_drift_allowance {
+            error!("improper setting: limit_valid_duration >= max_ts_drift_allowance; \
+                consider increasing max-ts-drift-allowance or decreasing max-ts-sync-interval";
+                "limit_valid_duration" => ?limit_valid_duration,
+                "max_ts_drift_allowance" => ?max_ts_drift_allowance,
+            );
+        }
+
         ConcurrencyManager {
             max_ts: Arc::new(AtomicU64::new(latest_ts.into_inner())),
+            max_ts_limit: Arc::new(AtomicCell::new(initial_limit)),
             lock_table: LockTable::default(),
+            action_on_invalid_max_ts: Arc::new(AtomicActionOnInvalidMaxTs::new(
+                action_on_invalid_max_ts,
+            )),
+            limit_valid_duration,
+            time_provider: Arc::new(CoarseInstantTimeProvider),
+            tso,
+            max_ts_drift_allowance_ms: Arc::new(AtomicU64::new(
+                max_ts_drift_allowance.as_millis() as u64
+            )),
+        }
+    }
+
+    #[cfg(test)]
+    fn new_with_time_provider(
+        latest_ts: TimeStamp,
+        limit_valid_duration: Duration,
+        action_on_invalid_max_ts: ActionOnInvalidMaxTs,
+        time_provider: Arc<dyn TimeProvider>,
+        tso: Option<Arc<dyn TSOProvider>>,
+        max_ts_drift_allowance: Duration,
+    ) -> Self {
+        let initial_limit = MaxTsLimit {
+            limit: TimeStamp::new(0),
+            update_time: time_provider.now(),
+        };
+        ConcurrencyManager {
+            max_ts: Arc::new(AtomicU64::new(latest_ts.into_inner())),
+            max_ts_limit: Arc::new(AtomicCell::new(initial_limit)),
+            lock_table: LockTable::default(),
+            action_on_invalid_max_ts: Arc::new(AtomicActionOnInvalidMaxTs::new(
+                action_on_invalid_max_ts,
+            )),
+            limit_valid_duration,
+            time_provider,
+            tso,
+            max_ts_drift_allowance_ms: Arc::new(AtomicU64::new(
+                max_ts_drift_allowance.as_millis() as u64
+            )),
         }
     }
 
@@ -52,9 +188,183 @@ impl ConcurrencyManager {
 
     /// Updates max_ts with the given new_ts. It has no effect if
     /// max_ts >= new_ts or new_ts is TimeStamp::max().
-    pub fn update_max_ts(&self, new_ts: TimeStamp) {
-        if new_ts != TimeStamp::max() {
-            self.max_ts.fetch_max(new_ts.into_inner(), Ordering::SeqCst);
+    ///
+    /// To avoid invalid ts breaking the invariants, the new_ts should be
+    /// less than or equal to the max_ts_limit.
+    ///
+    /// # Returns
+    /// - Ok(()): If the update is successful or has no effect
+    /// - Err(limit): If new_ts is greater than the max_ts_limit, returns the
+    ///   current limit value
+
+    pub fn update_max_ts(
+        &self,
+        new_ts: TimeStamp,
+        source: impl IntoErrorSource,
+    ) -> Result<(), InvalidMaxTsUpdate> {
+        if new_ts.is_max() {
+            return Ok(());
+        }
+        let limit = self.max_ts_limit.load();
+
+        // check that new_ts is less than or equal to the limit
+        if !limit.limit.is_zero() && new_ts > limit.limit {
+            let last_update = limit.update_time;
+            let now = self.time_provider.now();
+            assert!(now >= last_update);
+            let duration_to_last_limit_update = now - last_update;
+
+            if duration_to_last_limit_update < self.limit_valid_duration {
+                // limit is valid
+                let source = source.into_error_source();
+                self.double_check(new_ts, limit.limit, source, false)?;
+            } else {
+                // limit is stale
+                // use an approximate limit to avoid false alerts caused by failed limit updates
+
+                let approximate_limit = TimeStamp::compose(
+                    limit.limit.physical() + duration_to_last_limit_update.as_millis() as u64,
+                    limit.limit.logical(),
+                );
+
+                if new_ts > approximate_limit {
+                    let source = source.into_error_source();
+                    self.double_check(new_ts, approximate_limit, source, true)?;
+                }
+            }
+        }
+
+        MAX_TS_GAUGE.set(
+            self.max_ts
+                .fetch_max(new_ts.into_inner(), Ordering::SeqCst)
+                .max(new_ts.into_inner()) as i64,
+        );
+        Ok(())
+    }
+
+    // new_ts is greater than limit, or the approximate limit.
+    // To avoid false positive and guarantee TiKV availability, we need to
+    // double-check the new_ts with PD TSO.
+    fn double_check(
+        &self,
+        new_ts: TimeStamp,
+        limit: TimeStamp,
+        source: impl slog::Value + Display,
+        using_approximate: bool,
+    ) -> Result<(), crate::InvalidMaxTsUpdate> {
+        warn!("possible invalid max_ts update; double checking";
+            "attempted_ts" => new_ts,
+            "limit" => limit.into_inner(),
+            "source" => &source,
+            "using_approximate" => using_approximate,
+            "TSO_TIMEOUT" => ?TSO_TIMEOUT,
+        );
+        let mut tso_confirmed = false;
+        if let Some(tso) = &self.tso {
+            match block_on_timeout(tso.get_tso(), TSO_TIMEOUT) {
+                Ok(Ok(ts)) => {
+                    self.set_max_ts_limit(ts);
+                    tso_confirmed = true;
+                }
+                Ok(Err(e)) => {
+                    error!("failed to fetch from TSO for double checking"; "err" => ?e);
+                }
+                Err(()) => {
+                    error!("timeout when fetching from TSO for double checking"; "timeout" => ?TSO_TIMEOUT);
+                }
+            }
+        }
+        let new_limit = self.max_ts_limit.load();
+        if new_ts > new_limit.limit {
+            self.report_error(new_ts, new_limit, source, using_approximate, tso_confirmed)?;
+        }
+        Ok(())
+    }
+
+    fn report_error(
+        &self,
+        new_ts: TimeStamp,
+        limit: MaxTsLimit,
+        source: impl slog::Value + Display,
+        using_approximate: bool,
+        tso_confirmed: bool,
+    ) -> Result<(), InvalidMaxTsUpdate> {
+        if tso_confirmed {
+            error!("invalid max_ts update";
+                "attempted_ts" => new_ts,
+                "limit" => limit.limit.into_inner(),
+                "limit_update_time" => ?limit.update_time,
+                "source" => &source,
+                "using_approximate" => using_approximate,
+            );
+        } else {
+            warn!("possible invalid max_ts update";
+                "attempted_ts" => new_ts,
+                "limit" => limit.limit.into_inner(),
+                "limit_update_time" => ?limit.update_time,
+                "source" => &source,
+                "using_approximate" => using_approximate,
+            );
+        }
+
+        match self.action_on_invalid_max_ts.load() {
+            ActionOnInvalidMaxTs::Panic if tso_confirmed => {
+                panic!(
+                    "invalid max_ts update: {} exceeds the limit {}, source={}",
+                    new_ts,
+                    limit.limit.into_inner(),
+                    source
+                );
+            }
+            ActionOnInvalidMaxTs::Error if tso_confirmed => Err(InvalidMaxTsUpdate {
+                attempted_ts: new_ts,
+                limit: limit.limit,
+            }),
+            _ => Ok(()),
+        }
+    }
+
+    /// Set the maximum allowed value for max_ts updates, except for the updates
+    /// from PD TSO. The limit must be updated regularly to prevent the
+    /// blocking of max_ts. It prevents max_ts from being updated to an
+    /// unreasonable value, which is usually caused by bugs or unsafe
+    /// usages.
+    ///
+    /// # Note
+    /// If the new limit is smaller than the current limit, this operation will
+    /// have no effect and return silently.
+    pub fn set_max_ts_limit(&self, ts_from_tso: TimeStamp) {
+        if ts_from_tso.is_max() {
+            error!("max_ts_limit cannot be set to u64::max");
+            return;
+        }
+
+        let limit = TimeStamp::compose(
+            ts_from_tso.physical() + self.max_ts_drift_allowance_ms.load(Ordering::SeqCst),
+            ts_from_tso.logical(),
+        );
+
+        loop {
+            let current = self.max_ts_limit.load();
+
+            if limit.into_inner() <= current.limit.into_inner() {
+                break;
+            }
+
+            let new_state = MaxTsLimit {
+                limit,
+                update_time: self.time_provider.now(),
+            };
+
+            match self.max_ts_limit.compare_exchange(current, new_state) {
+                Ok(_) => {
+                    MAX_TS_LIMIT_GAUGE.set(limit.into_inner() as i64);
+                    break;
+                }
+                Err(_) => {
+                    continue;
+                }
+            }
         }
     }
 
@@ -141,14 +451,200 @@ impl ConcurrencyManager {
         });
         min_lock
     }
+
+    pub fn set_action_on_invalid_max_ts(&self, action: ActionOnInvalidMaxTs) {
+        self.action_on_invalid_max_ts.store(action);
+    }
+
+    pub fn set_max_ts_drift_allowance(&self, allowance: Duration) {
+        self.max_ts_drift_allowance_ms
+            .store(allowance.as_millis() as u64, Ordering::SeqCst);
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum ActionOnInvalidMaxTs {
+    Panic,
+    Error,
+    Log,
+}
+
+#[derive(Debug)]
+pub struct ParseActionError(String);
+
+impl fmt::Display for ParseActionError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl Error for ParseActionError {}
+
+impl TryFrom<&str> for ActionOnInvalidMaxTs {
+    type Error = ParseActionError;
+
+    fn try_from(value: &str) -> Result<Self, ParseActionError> {
+        match value.to_lowercase().as_str() {
+            "panic" => Ok(Self::Panic),
+            "error" => Ok(Self::Error),
+            "log" => Ok(Self::Log),
+            _ => Err(ParseActionError(format!("invalid action value: {}", value))),
+        }
+    }
+}
+
+impl TryFrom<String> for ActionOnInvalidMaxTs {
+    type Error = ParseActionError;
+
+    fn try_from(value: String) -> Result<Self, ParseActionError> {
+        Self::try_from(value.as_str())
+    }
+}
+
+pub struct AtomicActionOnInvalidMaxTs {
+    inner: AtomicUsize,
+}
+
+impl AtomicActionOnInvalidMaxTs {
+    pub fn new(initial: ActionOnInvalidMaxTs) -> Self {
+        Self {
+            inner: AtomicUsize::new(match initial {
+                ActionOnInvalidMaxTs::Panic => 0,
+                ActionOnInvalidMaxTs::Error => 1,
+                ActionOnInvalidMaxTs::Log => 2,
+            }),
+        }
+    }
+
+    pub fn store(&self, value: ActionOnInvalidMaxTs) {
+        self.inner.store(
+            match value {
+                ActionOnInvalidMaxTs::Panic => 0,
+                ActionOnInvalidMaxTs::Error => 1,
+                ActionOnInvalidMaxTs::Log => 2,
+            },
+            Ordering::SeqCst,
+        );
+    }
+
+    pub fn load(&self) -> ActionOnInvalidMaxTs {
+        match self.inner.load(Ordering::SeqCst) {
+            0 => ActionOnInvalidMaxTs::Panic,
+            1 => ActionOnInvalidMaxTs::Error,
+            2 => ActionOnInvalidMaxTs::Log,
+            _ => unreachable!("Invalid atomic state"),
+        }
+    }
+}
+
+#[derive(Debug, Error, Clone)]
+#[error("invalid max_ts update: {attempted_ts} exceeds the limit {limit}")]
+pub struct InvalidMaxTsUpdate {
+    pub attempted_ts: TimeStamp,
+    pub limit: TimeStamp,
+}
+
+pub trait ValueDisplay: slog::Value + Display {}
+impl ValueDisplay for String {}
+impl ValueDisplay for &str {}
+
+mod sealed {
+    pub trait Sealed {}
+}
+
+pub trait IntoErrorSource: sealed::Sealed {
+    type Output: ValueDisplay;
+    fn into_error_source(self) -> Self::Output;
+}
+
+// &str impl
+impl<'a> sealed::Sealed for &'a str {}
+impl<'a> IntoErrorSource for &'a str {
+    type Output = &'a str;
+    fn into_error_source(self) -> Self::Output {
+        self
+    }
+}
+
+// String impl
+impl sealed::Sealed for String {}
+impl IntoErrorSource for String {
+    type Output = String;
+    fn into_error_source(self) -> Self::Output {
+        self
+    }
+}
+
+// Closure impl
+impl<F, T> sealed::Sealed for F
+where
+    F: FnOnce() -> T,
+    T: ValueDisplay,
+{
+}
+impl<F, T> IntoErrorSource for F
+where
+    F: FnOnce() -> T,
+    T: ValueDisplay,
+{
+    type Output = T;
+    fn into_error_source(self) -> T {
+        self()
+    }
+}
+
+/// Trait to abstract time-related functionality, for a monotonic clock
+trait TimeProvider: Send + Sync {
+    /// Returns the current instant.
+    fn now(&self) -> Instant;
+}
+
+struct CoarseInstantTimeProvider;
+
+impl TimeProvider for CoarseInstantTimeProvider {
+    fn now(&self) -> Instant {
+        Instant::now_coarse()
+    }
 }
 
 #[cfg(test)]
 mod tests {
+    use std::{future::ready, sync::Mutex};
+
+    use futures::FutureExt;
     use txn_types::LockType;
 
     use super::*;
 
+    #[derive(Clone)]
+    struct StubTimeProvider {
+        current_time: Arc<Mutex<Instant>>,
+    }
+
+    impl StubTimeProvider {
+        /// Creates a new MockTimeProvider initialized with the given instant.
+        fn new(start_time: Instant) -> Self {
+            StubTimeProvider {
+                current_time: Arc::new(Mutex::new(start_time)),
+            }
+        }
+
+        /// Advances the current time by the specified duration.
+        fn advance(&self, duration: Duration) {
+            let mut time = self.current_time.lock().unwrap();
+            // Note: Instant doesn't support addition, so we mock behavior.
+            // This simplistic approach assumes no overflow.
+            *time += duration;
+        }
+    }
+
+    impl TimeProvider for StubTimeProvider {
+        fn now(&self) -> Instant {
+            let time = self.current_time.lock().unwrap();
+            *time
+        }
+    }
+
     #[tokio::test]
     async fn test_lock_keys_order() {
         let concurrency_manager = ConcurrencyManager::new(1.into());
@@ -166,13 +662,13 @@ mod tests {
     #[tokio::test]
     async fn test_update_max_ts() {
         let concurrency_manager = ConcurrencyManager::new(10.into());
-        concurrency_manager.update_max_ts(20.into());
+        let _ = concurrency_manager.update_max_ts(20.into(), "");
         assert_eq!(concurrency_manager.max_ts(), 20.into());
 
-        concurrency_manager.update_max_ts(5.into());
+        let _ = concurrency_manager.update_max_ts(5.into(), "");
         assert_eq!(concurrency_manager.max_ts(), 20.into());
 
-        concurrency_manager.update_max_ts(TimeStamp::max());
+        let _ = concurrency_manager.update_max_ts(TimeStamp::max(), "");
         assert_eq!(concurrency_manager.max_ts(), 20.into());
     }
 
@@ -224,4 +720,217 @@ mod tests {
             assert_eq!(concurrency_manager.global_min_lock_ts(), Some(20.into()));
         }
     }
+
+    #[test]
+    fn test_max_ts_limit() {
+        let mut stub_pd = MockTSOProvider::new();
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(160.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Error,
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        // Initially limit should be 0
+        cm.update_max_ts(TimeStamp::new(150), "").unwrap();
+
+        // Set initial limit to 200
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        // Try to lower limit to 150 - should be ignored
+        cm.set_max_ts_limit(TimeStamp::new(150));
+        cm.update_max_ts(TimeStamp::new(180), "").unwrap(); // Should still work up to 200
+
+        // Increase limit to 300 - should work
+        cm.set_max_ts_limit(TimeStamp::new(300));
+        cm.update_max_ts(TimeStamp::new(250), "").unwrap();
+    }
+
+    #[test]
+    fn test_max_ts_limit_edge_cases() {
+        let cm = ConcurrencyManager::new(TimeStamp::new(100));
+
+        // Test transition from zero limit
+        assert_eq!(cm.max_ts_limit.load().limit, 0.into());
+        cm.set_max_ts_limit(TimeStamp::new(1000));
+        assert_eq!(cm.max_ts_limit.load().limit, 1000.into());
+
+        // Try to lower from 1000 to 500 - should be ignored
+        cm.set_max_ts_limit(TimeStamp::new(500));
+        assert_eq!(cm.max_ts_limit.load().limit, 1000.into());
+
+        // Test setting limit to max, should have no effect
+        cm.set_max_ts_limit(TimeStamp::max());
+        assert_eq!(cm.max_ts_limit.load().limit, 1000.into());
+    }
+
+    #[test]
+    fn test_max_ts_updates_with_monotonic_limit() {
+        let mut stub_pd = MockTSOProvider::new();
+        // Assertion: should fail to update max_ts to 250 and query PD
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(201.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Error,
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        // Set limit to 200
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        // Update max_ts to 150
+        cm.update_max_ts(TimeStamp::new(150), "").unwrap();
+        assert_eq!(cm.max_ts(), TimeStamp::new(150));
+
+        // Try to lower limit to 180 - should be ignored
+        cm.set_max_ts_limit(TimeStamp::new(180));
+
+        // Should still fail for values above 200
+        let result = cm.update_max_ts(TimeStamp::new(250), "");
+        assert!(result.is_err());
+        if let Err(e) = result {
+            assert_eq!(e.attempted_ts, TimeStamp::new(250));
+            assert_eq!(e.limit, TimeStamp::new(201));
+        }
+    }
+
+    #[test]
+    fn test_limit_valid_duration_boundary() {
+        let start_time = Instant::now();
+        let stub_time = StubTimeProvider::new(start_time);
+        let time_provider = Arc::new(stub_time.clone());
+        let mut stub_pd = MockTSOProvider::new();
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(200.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+
+        let cm = ConcurrencyManager::new_with_time_provider(
+            TimeStamp::new(100),
+            Duration::from_secs(60),
+            ActionOnInvalidMaxTs::Error,
+            time_provider.clone(),
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        time_provider.advance(Duration::from_secs(59));
+
+        time_provider.advance(Duration::from_secs(1));
+        cm.update_max_ts(TimeStamp::new(250), "").unwrap();
+        assert_eq!(cm.max_ts().into_inner(), 250);
+    }
+
+    #[test]
+    fn test_max_ts_limit_expired_allows_update() {
+        let start_time = Instant::now();
+        let stub_time = StubTimeProvider::new(start_time);
+        let time_provider = Arc::new(stub_time.clone());
+
+        let cm = ConcurrencyManager::new_with_time_provider(
+            TimeStamp::new(100),
+            Duration::from_secs(60),
+            ActionOnInvalidMaxTs::Error,
+            time_provider.clone(),
+            None,
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        stub_time.advance(Duration::from_secs(61));
+
+        // Updating to 250 should be allowed, since the limit should be invalidated
+        cm.update_max_ts(TimeStamp::new(250), "test_source".to_string())
+            .unwrap();
+        assert_eq!(cm.max_ts().into_inner(), 250);
+    }
+
+    #[test]
+    #[should_panic(expected = "invalid max_ts update")]
+    fn test_panic_on_invalid_max_ts_enabled() {
+        let mut stub_pd = MockTSOProvider::new();
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(201.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        // should not panic
+        cm.update_max_ts(TimeStamp::new(250), "test_source".to_string())
+            .unwrap();
+    }
+
+    #[test]
+    fn test_update_max_ts_without_limit() {
+        let cm = ConcurrencyManager::new(TimeStamp::new(100));
+
+        cm.update_max_ts(TimeStamp::new(500), "test_source".to_string())
+            .unwrap();
+        assert_eq!(cm.max_ts().into_inner(), 500);
+    }
+
+    #[test]
+    fn test_pd_tso_jump_not_panic() {
+        let mut stub_pd = MockTSOProvider::new();
+        // The double check procedure gets latest_ts=300 from TSO
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(300.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            Some(stub_pd.clone()),
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+        // PD TSO jumps from 100 to 300
+        cm.update_max_ts(TimeStamp::new(300), "test_source".to_string())
+            .unwrap();
+    }
+
+    #[test]
+    fn test_do_not_panic_under_pd_tso_jump_and_network_partition() {
+        let mut stub_pd = MockTSOProvider::new();
+        // Network partition between PD and TiKV
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| std::future::pending().boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            Some(stub_pd.clone()),
+            Duration::ZERO,
+        );
+        cm.set_max_ts_limit(200.into());
+
+        // PD TSO jumps from 100 to 300
+        cm.update_max_ts(TimeStamp::new(300), "test_source".to_string())
+            .unwrap();
+    }
 }
diff --git a/components/error_code/src/coprocessor.rs b/components/error_code/src/coprocessor.rs
index d98c85162bf..5cad4756168 100644
--- a/components/error_code/src/coprocessor.rs
+++ b/components/error_code/src/coprocessor.rs
@@ -15,5 +15,7 @@ define_error_codes!(
     CORRUPTED_DATA => ("CorruptedData", "", ""),
 
     STORAGE_ERROR => ("StorageError", "", ""),
-    INVALID_CHARACTER_STRING => ("InvalidCharacterString", "", "")
+    INVALID_CHARACTER_STRING => ("InvalidCharacterString", "", ""),
+
+    INVALID_MAX_TS_UPDATE => ("InvalidMaxTsUpdate", "", "")
 );
diff --git a/components/error_code/src/storage.rs b/components/error_code/src/storage.rs
index e7caefb1b65..33cdc7b116a 100644
--- a/components/error_code/src/storage.rs
+++ b/components/error_code/src/storage.rs
@@ -25,6 +25,7 @@ define_error_codes!(
     DEADLINE_EXCEEDED => ("DeadlineExceeded", "", ""),
     API_VERSION_NOT_MATCHED => ("ApiVersionNotMatched", "", ""),
     INVALID_KEY_MODE => ("InvalidKeyMode", "", ""),
+    INVALID_MAX_TS_UPDATE => ("InvalidMaxTsUpdate", "", ""),
 
     COMMITTED => ("Committed", "", ""),
     PESSIMISTIC_LOCK_ROLLED_BACK => ("PessimisticLockRolledBack", "", ""),
diff --git a/components/raftstore-v2/src/worker/pd/misc.rs b/components/raftstore-v2/src/worker/pd/misc.rs
index 6ade8d87de5..fd2848b95a4 100644
--- a/components/raftstore-v2/src/worker/pd/misc.rs
+++ b/components/raftstore-v2/src/worker/pd/misc.rs
@@ -12,7 +12,6 @@ use pd_client::PdClient;
 use raftstore::{store::TxnExt, Result};
 use slog::{info, warn};
 use tikv_util::{box_err, timer::GLOBAL_TIMER_HANDLE};
-use txn_types::TimeStamp;
 
 use super::Runner;
 
@@ -51,18 +50,22 @@ where
                 // And it won't break correctness of transaction commands, as
                 // causal_ts_provider.flush() is implemented as
                 // pd_client.get_tso() + renew TSO cached.
-                let res: Result<TimeStamp> = if let Some(causal_ts_provider) = &causal_ts_provider {
+                let res: Result<()> = if let Some(causal_ts_provider) = &causal_ts_provider {
                     causal_ts_provider
                         .async_flush()
                         .await
                         .map_err(|e| box_err!(e))
                 } else {
                     pd_client.get_tso().await.map_err(Into::into)
-                };
+                }
+                .and_then(|ts| {
+                    concurrency_manager
+                        .update_max_ts(ts, "raftstore-v2")
+                        .map_err(|e| crate::Error::Other(box_err!(e)))
+                });
 
                 match res {
-                    Ok(ts) => {
-                        concurrency_manager.update_max_ts(ts);
+                    Ok(()) => {
                         success = txn_ext
                             .max_ts_sync_status
                             .compare_exchange(
diff --git a/components/raftstore/src/store/peer.rs b/components/raftstore/src/store/peer.rs
index 3aa6ed9d54c..d0d0d440aed 100644
--- a/components/raftstore/src/store/peer.rs
+++ b/components/raftstore/src/store/peer.rs
@@ -5785,7 +5785,7 @@ where
             txn_ext: self.txn_ext.clone(),
         }) {
             error!(
-                "failed to update max ts";
+                "failed to update max_ts";
                 "err" => ?e,
             );
         }
diff --git a/components/raftstore/src/store/worker/pd.rs b/components/raftstore/src/store/worker/pd.rs
index 62b72ffe58d..fd61b927672 100644
--- a/components/raftstore/src/store/worker/pd.rs
+++ b/components/raftstore/src/store/worker/pd.rs
@@ -52,7 +52,6 @@ use tikv_util::{
     warn,
     worker::{Runnable, ScheduleError, Scheduler},
 };
-use txn_types::TimeStamp;
 use yatp::Remote;
 
 use super::split_controller::AutoSplitControllerContext;
@@ -1673,7 +1672,7 @@ where
                 // And it won't break correctness of transaction commands, as
                 // causal_ts_provider.flush() is implemented as pd_client.get_tso() + renew TSO
                 // cached.
-                let res: crate::Result<TimeStamp> =
+                let res: crate::Result<()> =
                     if let Some(causal_ts_provider) = &causal_ts_provider {
                         causal_ts_provider
                             .async_flush()
@@ -1681,11 +1680,15 @@ where
                             .map_err(|e| box_err!(e))
                     } else {
                         pd_client.get_tso().await.map_err(Into::into)
-                    };
+                    }
+                    .and_then(|ts| {
+                        concurrency_manager
+                            .update_max_ts(ts, "raftstore")
+                            .map_err(|e| crate::Error::Other(box_err!(e)))
+                    });
 
                 match res {
-                    Ok(ts) => {
-                        concurrency_manager.update_max_ts(ts);
+                    Ok(()) => {
                         // Set the least significant bit to 1 to mark it as synced.
                         success = txn_ext
                             .max_ts_sync_status
diff --git a/components/resolved_ts/src/advance.rs b/components/resolved_ts/src/advance.rs
index 2667a43a304..9b7def6d8d8 100644
--- a/components/resolved_ts/src/advance.rs
+++ b/components/resolved_ts/src/advance.rs
@@ -117,7 +117,10 @@ impl AdvanceTsWorker {
             // optimizations like async commit is enabled.
             // Note: This step must be done before scheduling `Task::MinTs` task, and the
             // resolver must be checked in or after `Task::MinTs`' execution.
-            cm.update_max_ts(min_ts);
+            if let Err(e) = cm.update_max_ts(min_ts, "resolved-ts") {
+                error!("failed to advance resolved_ts: failed to update max_ts in concurrency manager"; "err" => ?e);
+                return;
+            }
             if let Some((min_mem_lock_ts, lock)) = cm.global_min_lock() {
                 if min_mem_lock_ts < min_ts {
                     min_ts = min_mem_lock_ts;
diff --git a/components/server/src/server.rs b/components/server/src/server.rs
index 62d9d37a4c6..10317a78b9d 100644
--- a/components/server/src/server.rs
+++ b/components/server/src/server.rs
@@ -28,7 +28,10 @@ use backup_stream::{
 };
 use causal_ts::CausalTsProviderImpl;
 use cdc::CdcConfigManager;
-use concurrency_manager::ConcurrencyManager;
+use concurrency_manager::{
+    ActionOnInvalidMaxTs, ConcurrencyManager, DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+    DEFAULT_MAX_TS_SYNC_INTERVAL, LIMIT_VALID_TIME_MULTIPLIER,
+};
 use engine_rocks::{
     from_rocks_compression_type, RocksCompactedEvent, RocksEngine, RocksStatistics,
 };
@@ -169,6 +172,7 @@ fn run_impl<EK, CER, F>(
     tikv.init_metrics_flusher(fetcher, engines_info);
     tikv.init_cgroup_monitor();
     tikv.init_storage_stats_task(engines);
+    tikv.init_max_ts_updater();
     tikv.run_server(server_config);
     tikv.run_status_server();
     tikv.core.init_quota_tuning_task(tikv.quota_limiter.clone());
@@ -432,7 +436,13 @@ where
 
         // Initialize concurrency manager
         let latest_ts = block_on(pd_client.get_tso()).expect("failed to get timestamp from PD");
-        let concurrency_manager = ConcurrencyManager::new(latest_ts);
+        let concurrency_manager = ConcurrencyManager::new_with_config(
+            latest_ts,
+            DEFAULT_MAX_TS_SYNC_INTERVAL * LIMIT_VALID_TIME_MULTIPLIER,
+            ActionOnInvalidMaxTs::Log,
+            Some(pd_client.clone()),
+            DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+        );
 
         // use different quota for front-end and back-end requests
         let quota_limiter = Arc::new(QuotaLimiter::new(
@@ -1151,6 +1161,28 @@ where
         server_config
     }
 
+    fn init_max_ts_updater(&self) {
+        let cm = self.concurrency_manager.clone();
+        let pd_client = self.pd_client.clone();
+
+        let max_ts_sync_interval = DEFAULT_MAX_TS_SYNC_INTERVAL;
+        self.core
+            .background_worker
+            .spawn_interval_async_task(max_ts_sync_interval, move || {
+                let cm = cm.clone();
+                let pd_client = pd_client.clone();
+
+                async move {
+                    let pd_tso = pd_client.get_tso().await;
+                    if let Ok(ts) = pd_tso {
+                        cm.set_max_ts_limit(ts);
+                    } else {
+                        warn!("failed to get tso from pd in background, the max_ts validity check could be skipped");
+                    }
+                }
+            });
+    }
+
     fn register_services(&mut self) {
         let servers = self.servers.as_mut().unwrap();
         let engines = self.engines.as_ref().unwrap();
diff --git a/components/server/src/server2.rs b/components/server/src/server2.rs
index bdf47b23f79..d3e5e6b6915 100644
--- a/components/server/src/server2.rs
+++ b/components/server/src/server2.rs
@@ -33,7 +33,10 @@ use backup_stream::{
 };
 use causal_ts::CausalTsProviderImpl;
 use cdc::CdcConfigManager;
-use concurrency_manager::ConcurrencyManager;
+use concurrency_manager::{
+    ActionOnInvalidMaxTs, ConcurrencyManager, DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+    DEFAULT_MAX_TS_SYNC_INTERVAL, LIMIT_VALID_TIME_MULTIPLIER,
+};
 use engine_rocks::{from_rocks_compression_type, RocksEngine, RocksStatistics};
 use engine_traits::{Engines, KvEngine, MiscExt, RaftEngine, TabletRegistry, CF_DEFAULT, CF_WRITE};
 use file_system::{get_io_rate_limiter, BytesFetcher, MetricsManager as IoMetricsManager};
@@ -157,6 +160,7 @@ fn run_impl<CER: ConfiguredRaftEngine, F: KvFormat>(
     tikv.init_metrics_flusher(fetcher, engines_info);
     tikv.init_cgroup_monitor();
     tikv.init_storage_stats_task();
+    tikv.init_max_ts_updater();
     tikv.run_server(server_config);
     tikv.run_status_server();
     tikv.core.init_quota_tuning_task(tikv.quota_limiter.clone());
@@ -319,7 +323,13 @@ where
 
         // Initialize concurrency manager
         let latest_ts = block_on(pd_client.get_tso()).expect("failed to get timestamp from PD");
-        let concurrency_manager = ConcurrencyManager::new(latest_ts);
+        let concurrency_manager = ConcurrencyManager::new_with_config(
+            latest_ts,
+            DEFAULT_MAX_TS_SYNC_INTERVAL * LIMIT_VALID_TIME_MULTIPLIER,
+            ActionOnInvalidMaxTs::Log,
+            Some(pd_client.clone()),
+            DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+        );
 
         // use different quota for front-end and back-end requests
         let quota_limiter = Arc::new(QuotaLimiter::new(
@@ -920,6 +930,28 @@ where
         server_config
     }
 
+    fn init_max_ts_updater(&self) {
+        let cm = self.concurrency_manager.clone();
+        let pd_client = self.pd_client.clone();
+
+        let max_ts_sync_interval = DEFAULT_MAX_TS_SYNC_INTERVAL;
+        self.core
+            .background_worker
+            .spawn_interval_async_task(max_ts_sync_interval, move || {
+                let cm = cm.clone();
+                let pd_client = pd_client.clone();
+
+                async move {
+                    let pd_tso = pd_client.get_tso().await;
+                    if let Ok(ts) = pd_tso {
+                        cm.set_max_ts_limit(ts);
+                    } else {
+                        warn!("failed to get tso from pd in background");
+                    }
+                }
+            });
+    }
+
     fn register_services(&mut self) {
         let servers = self.servers.as_mut().unwrap();
         let engines = self.engines.as_ref().unwrap();
diff --git a/components/tikv_util/src/time.rs b/components/tikv_util/src/time.rs
index c710b174dee..f8c317adb48 100644
--- a/components/tikv_util/src/time.rs
+++ b/components/tikv_util/src/time.rs
@@ -276,7 +276,7 @@ mod inner {
 /// A measurement of a monotonically increasing clock.
 /// It's similar and meant to replace `std::time::Instant`,
 /// for providing extra features.
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Eq)]
 pub enum Instant {
     Monotonic(Timespec),
     MonotonicCoarse(Timespec),
diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index eb8422741d7..82cf35f191a 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -2864,6 +2864,25 @@ def Storage() -> RowPanel:
             ),
         ]
     )
+    layout.row(
+        [
+            graph_panel(
+                title="Concurrency manager max-ts",
+                description="The max_ts in the concurrency manager",
+                yaxes=yaxes(left_format=UNITS.NONE_FORMAT),
+                targets=[
+                    target(
+                        expr="tikv_concurrency_manager_max_ts_limit",
+                        legend_format="max_ts_limit",
+                    ),
+                    target(
+                        expr="tikv_concurrency_manager_max_ts",
+                        legend_format="max_ts",
+                    ),
+                ],
+            )
+        ]
+    )
     return layout.row_panel
 
 
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 13a2e458b9a..e1b903ce583 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -10262,55 +10262,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Storage",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 76,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "The max_ts in the concurrency manager",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -10333,11 +10291,11 @@
             "h": 7,
             "w": 24,
             "x": 0,
-            "y": 0
+            "y": 56
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 77,
+          "id": 76,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10373,62 +10331,37 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [
-            {
-              "alias": "/.*-total/",
-              "bars": false,
-              "fill": 1,
-              "fillBelowTo": null,
-              "lines": true,
-              "yaxis": 2,
-              "zindex": 0
-            }
-          ],
+          "seriesOverrides": [],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
+              "expr": "tikv_concurrency_manager_max_ts_limit",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}-reject-by-{{reason}}",
+              "legendFormat": "max_ts_limit",
               "metric": "",
-              "query": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
+              "query": "tikv_concurrency_manager_max_ts_limit",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "tikv_concurrency_manager_max_ts",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}-total",
+              "legendFormat": "max_ts",
               "metric": "",
-              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{instance}}-stale-read",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "tikv_concurrency_manager_max_ts",
               "refId": "",
               "step": 10,
               "target": ""
@@ -10437,7 +10370,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Local reader requests",
+          "title": "Concurrency manager max-ts",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -10485,7 +10418,7 @@
       "targets": [],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Local Reader",
+      "title": "Storage",
       "transformations": [],
       "transparent": false,
       "type": "row"
@@ -10513,7 +10446,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 78,
+      "id": 77,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -10525,7 +10458,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of raftstore thread",
+          "description": null,
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -10546,13 +10479,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
+            "w": 24,
             "x": 0,
             "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 79,
+          "id": 78,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10588,22 +10521,62 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "/.*-total/",
+              "bars": false,
+              "fill": 1,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 2,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{instance}}-reject-by-{{reason}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-total",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-stale-read",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -10612,7 +10585,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Raft store CPU",
+          "title": "Local reader requests",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -10631,7 +10604,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "percentunit",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -10652,13 +10625,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Local Reader",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 79,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of async apply",
+          "description": "The CPU utilization of raftstore thread",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -10680,7 +10695,7 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 0
           },
           "height": null,
@@ -10728,7 +10743,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -10736,7 +10751,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -10745,7 +10760,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Async apply CPU",
+          "title": "Raft store CPU",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -10791,23 +10806,14 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of store writer thread",
+          "description": "The CPU utilization of async apply",
           "editable": true,
           "error": false,
           "fieldConfig": {
             "defaults": {
               "thresholds": {
                 "mode": "absolute",
-                "steps": [
-                  {
-                    "colorMode": "critical",
-                    "fill": true,
-                    "line": true,
-                    "op": "gt",
-                    "value": 0.8,
-                    "yaxis": "left"
-                  }
-                ]
+                "steps": []
               }
             }
           },
@@ -10822,8 +10828,8 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 7
+            "x": 12,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
@@ -10870,7 +10876,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -10878,25 +10884,16 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             }
           ],
-          "thresholds": [
-            {
-              "colorMode": "critical",
-              "fill": true,
-              "line": true,
-              "op": "gt",
-              "value": 0.8,
-              "yaxis": "left"
-            }
-          ],
+          "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Store writer CPU",
+          "title": "Async apply CPU",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -10942,14 +10939,23 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of gRPC",
+          "description": "The CPU utilization of store writer thread",
           "editable": true,
           "error": false,
           "fieldConfig": {
             "defaults": {
               "thresholds": {
                 "mode": "absolute",
-                "steps": []
+                "steps": [
+                  {
+                    "colorMode": "critical",
+                    "fill": true,
+                    "line": true,
+                    "op": "gt",
+                    "value": 0.8,
+                    "yaxis": "left"
+                  }
+                ]
               }
             }
           },
@@ -10964,7 +10970,7 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 7
           },
           "height": null,
@@ -11012,7 +11018,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -11020,16 +11026,158 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             }
           ],
-          "thresholds": [],
+          "thresholds": [
+            {
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
+              "op": "gt",
+              "value": 0.8,
+              "yaxis": "left"
+            }
+          ],
           "timeFrom": null,
           "timeShift": null,
-          "title": "gRPC poll CPU",
+          "title": "Store writer CPU",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The CPU utilization of gRPC",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 83,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "gRPC poll CPU",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -11111,7 +11259,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 83,
+          "id": 84,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11262,7 +11410,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 84,
+          "id": 85,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11404,7 +11552,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 85,
+          "id": 86,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11537,7 +11685,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 86,
+          "id": 87,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11670,7 +11818,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 87,
+          "id": 88,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11803,7 +11951,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 88,
+          "id": 89,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11936,7 +12084,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 89,
+          "id": 90,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12069,7 +12217,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 90,
+          "id": 91,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12202,7 +12350,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 91,
+          "id": 92,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12335,7 +12483,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 92,
+          "id": 93,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12468,7 +12616,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 93,
+          "id": 94,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12631,7 +12779,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 94,
+          "id": 95,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12773,7 +12921,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 95,
+          "id": 96,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12954,7 +13102,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 96,
+          "id": 97,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13129,7 +13277,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 97,
+      "id": 98,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -13168,7 +13316,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 98,
+          "id": 99,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13316,7 +13464,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 99,
+          "id": 100,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13464,7 +13612,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 100,
+          "id": 101,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13597,7 +13745,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 101,
+          "id": 102,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13748,7 +13896,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 102,
+      "id": 103,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -13787,7 +13935,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 103,
+          "id": 104,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13988,7 +14136,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 104,
+          "id": 105,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14189,7 +14337,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 105,
+          "id": 106,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14390,7 +14538,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 106,
+          "id": 107,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14591,7 +14739,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 107,
+          "id": 108,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14792,7 +14940,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 108,
+          "id": 109,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14993,7 +15141,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 109,
+          "id": 110,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15194,7 +15342,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 110,
+          "id": 111,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15395,7 +15543,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 111,
+          "id": 112,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15596,7 +15744,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 112,
+          "id": 113,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15797,7 +15945,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 113,
+          "id": 114,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15998,7 +16146,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 114,
+          "id": 115,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16199,7 +16347,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 115,
+          "id": 116,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16403,7 +16551,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 116,
+      "id": 117,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -16449,7 +16597,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 117,
+          "id": 118,
           "interval": null,
           "legend": {
             "show": false
@@ -16546,7 +16694,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 118,
+          "id": 119,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16754,7 +16902,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 119,
+          "id": 120,
           "interval": null,
           "legend": {
             "show": false
@@ -16851,7 +16999,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 120,
+          "id": 121,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17059,7 +17207,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 121,
+          "id": 122,
           "interval": null,
           "legend": {
             "show": false
@@ -17156,7 +17304,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 122,
+          "id": 123,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17364,7 +17512,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 123,
+          "id": 124,
           "interval": null,
           "legend": {
             "show": false
@@ -17461,7 +17609,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 124,
+          "id": 125,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17669,7 +17817,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 125,
+          "id": 126,
           "interval": null,
           "legend": {
             "show": false
@@ -17766,7 +17914,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 126,
+          "id": 127,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17974,7 +18122,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 127,
+          "id": 128,
           "interval": null,
           "legend": {
             "show": false
@@ -18071,7 +18219,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 128,
+          "id": 129,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18272,7 +18420,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 129,
+          "id": 130,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18420,7 +18568,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 130,
+          "id": 131,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18556,7 +18704,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 131,
+      "id": 132,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -18595,7 +18743,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 132,
+          "id": 133,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18728,7 +18876,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 133,
+          "id": 134,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18861,7 +19009,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 134,
+          "id": 135,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18994,7 +19142,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 135,
+          "id": 136,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19134,7 +19282,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 136,
+          "id": 137,
           "interval": null,
           "legend": {
             "show": false
@@ -19231,7 +19379,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 137,
+          "id": 138,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19439,7 +19587,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 138,
+          "id": 139,
           "interval": null,
           "legend": {
             "show": false
@@ -19536,7 +19684,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 139,
+          "id": 140,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19744,7 +19892,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 140,
+          "id": 141,
           "interval": null,
           "legend": {
             "show": false
@@ -19841,7 +19989,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 141,
+          "id": 142,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20049,7 +20197,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 142,
+          "id": 143,
           "interval": null,
           "legend": {
             "show": false
@@ -20153,7 +20301,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 143,
+          "id": 144,
           "interval": null,
           "legend": {
             "show": false
@@ -20250,7 +20398,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 144,
+          "id": 145,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20383,7 +20531,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 145,
+          "id": 146,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20534,7 +20682,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 146,
+      "id": 147,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -20573,7 +20721,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 147,
+          "id": 148,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20721,7 +20869,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 148,
+          "id": 149,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20876,7 +21024,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 149,
+          "id": 150,
           "interval": null,
           "legend": {
             "show": false
@@ -20980,7 +21128,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 150,
+          "id": 151,
           "interval": null,
           "legend": {
             "show": false
@@ -21077,7 +21225,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 151,
+          "id": 152,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21210,7 +21358,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 152,
+          "id": 153,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21361,7 +21509,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 153,
+      "id": 154,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -21400,7 +21548,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 154,
+          "id": 155,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21533,7 +21681,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 155,
+          "id": 156,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21666,7 +21814,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 156,
+          "id": 157,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21799,7 +21947,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 157,
+          "id": 158,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21932,7 +22080,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 158,
+          "id": 159,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22065,7 +22213,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 159,
+          "id": 160,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22201,7 +22349,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 160,
+      "id": 161,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -22240,7 +22388,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 161,
+          "id": 162,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22373,7 +22521,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 162,
+          "id": 163,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22506,7 +22654,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 163,
+          "id": 164,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22639,7 +22787,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 164,
+          "id": 165,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22772,7 +22920,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 165,
+          "id": 166,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22905,7 +23053,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 166,
+          "id": 167,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23068,7 +23216,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 167,
+          "id": 168,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23204,7 +23352,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 168,
+      "id": 169,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -23243,7 +23391,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 169,
+          "id": 170,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23391,7 +23539,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 170,
+          "id": 171,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23539,7 +23687,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 171,
+          "id": 172,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23672,7 +23820,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 172,
+          "id": 173,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23805,7 +23953,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 173,
+          "id": 174,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23938,7 +24086,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 174,
+          "id": 175,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24071,7 +24219,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 175,
+          "id": 176,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24204,7 +24352,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 176,
+          "id": 177,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24337,7 +24485,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 177,
+          "id": 178,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24514,7 +24662,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 178,
+      "id": 179,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -24553,7 +24701,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 179,
+          "id": 180,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24716,7 +24864,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 180,
+          "id": 181,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24917,7 +25065,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 181,
+          "id": 182,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25065,7 +25213,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 182,
+          "id": 183,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25228,7 +25376,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 183,
+          "id": 184,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25429,7 +25577,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 184,
+          "id": 185,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25607,7 +25755,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 185,
+          "id": 186,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25770,7 +25918,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 186,
+          "id": 187,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25933,7 +26081,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 187,
+          "id": 188,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26066,7 +26214,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 188,
+          "id": 189,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26270,7 +26418,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 189,
+      "id": 190,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -26309,7 +26457,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 190,
+          "id": 191,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26502,7 +26650,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 191,
+          "id": 192,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26680,7 +26828,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 192,
+          "id": 193,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26888,7 +27036,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 193,
+          "id": 194,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27066,7 +27214,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 194,
+          "id": 195,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27229,7 +27377,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 195,
+          "id": 196,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27407,7 +27555,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 196,
+          "id": 197,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27540,7 +27688,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 197,
+          "id": 198,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27718,7 +27866,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 198,
+          "id": 199,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27851,7 +27999,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 199,
+          "id": 200,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28029,7 +28177,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 200,
+          "id": 201,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28162,7 +28310,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 201,
+          "id": 202,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28340,7 +28488,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 202,
+          "id": 203,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28518,7 +28666,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 203,
+          "id": 204,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28651,7 +28799,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 204,
+          "id": 205,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28784,7 +28932,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 205,
+          "id": 206,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28917,7 +29065,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 206,
+          "id": 207,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29140,7 +29288,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 207,
+          "id": 208,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29333,7 +29481,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 208,
+          "id": 209,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29496,7 +29644,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 209,
+          "id": 210,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29689,7 +29837,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 210,
+          "id": 211,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29837,7 +29985,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 211,
+          "id": 212,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29970,7 +30118,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 212,
+          "id": 213,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30118,7 +30266,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 213,
+          "id": 214,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30296,7 +30444,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 214,
+          "id": 215,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30459,7 +30607,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 215,
+          "id": 216,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30637,7 +30785,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 216,
+          "id": 217,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30770,7 +30918,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 217,
+          "id": 218,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30903,7 +31051,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 218,
+          "id": 219,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31036,7 +31184,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 219,
+          "id": 220,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31169,7 +31317,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 220,
+          "id": 221,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31302,7 +31450,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 221,
+          "id": 222,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31435,7 +31583,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 222,
+          "id": 223,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31568,7 +31716,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 223,
+          "id": 224,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31769,7 +31917,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 224,
+          "id": 225,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31902,7 +32050,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 225,
+          "id": 226,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32087,7 +32235,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 226,
+          "id": 227,
           "interval": null,
           "legend": {
             "show": false
@@ -32184,7 +32332,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 227,
+          "id": 228,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32320,7 +32468,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 228,
+      "id": 229,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -32359,7 +32507,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 229,
+          "id": 230,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32507,7 +32655,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 230,
+          "id": 231,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32655,7 +32803,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 231,
+          "id": 232,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32788,7 +32936,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 232,
+          "id": 233,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32921,7 +33069,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 233,
+          "id": 234,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33099,7 +33247,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 234,
+          "id": 235,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33277,7 +33425,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 235,
+          "id": 236,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33455,7 +33603,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 236,
+          "id": 237,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33588,7 +33736,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 237,
+          "id": 238,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33766,7 +33914,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 238,
+          "id": 239,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33899,7 +34047,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 239,
+          "id": 240,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34062,7 +34210,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 240,
+          "id": 241,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34240,7 +34388,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 241,
+          "id": 242,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34418,7 +34566,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 242,
+          "id": 243,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34596,7 +34744,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 243,
+          "id": 244,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34729,7 +34877,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 244,
+          "id": 245,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34907,7 +35055,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 245,
+          "id": 246,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35040,7 +35188,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 246,
+          "id": 247,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35218,7 +35366,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 247,
+          "id": 248,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35351,7 +35499,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 248,
+          "id": 249,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35484,7 +35632,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 249,
+          "id": 250,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35662,7 +35810,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 250,
+          "id": 251,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35840,7 +35988,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 251,
+          "id": 252,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35973,7 +36121,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 252,
+          "id": 253,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36151,7 +36299,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 253,
+          "id": 254,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36284,7 +36432,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 254,
+          "id": 255,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36462,7 +36610,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 255,
+          "id": 256,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36598,7 +36746,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 256,
+      "id": 257,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -36637,7 +36785,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 257,
+          "id": 258,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36770,7 +36918,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 258,
+          "id": 259,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36903,7 +37051,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 259,
+          "id": 260,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37039,7 +37187,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 260,
+      "id": 261,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -37078,7 +37226,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 261,
+          "id": 262,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37226,7 +37374,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 262,
+          "id": 263,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37366,7 +37514,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 263,
+          "id": 264,
           "interval": null,
           "legend": {
             "show": false
@@ -37463,7 +37611,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 264,
+          "id": 265,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37596,7 +37744,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 265,
+          "id": 266,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37729,7 +37877,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 266,
+          "id": 267,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37907,7 +38055,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 267,
+          "id": 268,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38070,7 +38218,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 268,
+          "id": 269,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38218,7 +38366,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 269,
+          "id": 270,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38351,7 +38499,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 270,
+          "id": 271,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38487,7 +38635,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 271,
+      "id": 272,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -38526,7 +38674,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 272,
+          "id": 273,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38674,7 +38822,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 273,
+          "id": 274,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38807,7 +38955,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 274,
+          "id": 275,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38940,7 +39088,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 275,
+          "id": 276,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39073,7 +39221,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 276,
+          "id": 277,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39206,7 +39354,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 277,
+          "id": 278,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39361,7 +39509,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 278,
+          "id": 279,
           "interval": null,
           "legend": {
             "show": false
@@ -39461,7 +39609,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 279,
+      "id": 280,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -39500,7 +39648,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 280,
+          "id": 281,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39648,7 +39796,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 281,
+          "id": 282,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39849,7 +39997,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 282,
+          "id": 283,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40050,7 +40198,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 283,
+          "id": 284,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40251,7 +40399,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 284,
+          "id": 285,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40452,7 +40600,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 285,
+          "id": 286,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40585,7 +40733,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 286,
+          "id": 287,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40718,7 +40866,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 287,
+          "id": 288,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40851,7 +40999,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 288,
+          "id": 289,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40984,7 +41132,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 289,
+          "id": 290,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41192,7 +41340,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 290,
+          "id": 291,
           "interval": null,
           "legend": {
             "show": false
@@ -41292,7 +41440,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 291,
+      "id": 292,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -41338,7 +41486,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 292,
+          "id": 293,
           "interval": null,
           "legend": {
             "show": false
@@ -41435,7 +41583,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 293,
+          "id": 294,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41636,7 +41784,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 294,
+          "id": 295,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41769,7 +41917,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 295,
+          "id": 296,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41902,7 +42050,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 296,
+          "id": 297,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42035,7 +42183,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 297,
+          "id": 298,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42236,7 +42384,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 298,
+          "id": 299,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42369,7 +42517,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 299,
+          "id": 300,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42505,7 +42653,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 300,
+      "id": 301,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -42544,7 +42692,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 301,
+          "id": 302,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42745,7 +42893,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 302,
+          "id": 303,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42946,7 +43094,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 303,
+          "id": 304,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43147,7 +43295,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 304,
+          "id": 305,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43348,7 +43496,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 305,
+          "id": 306,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43481,7 +43629,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 306,
+          "id": 307,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43614,7 +43762,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 307,
+          "id": 308,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43747,7 +43895,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 308,
+          "id": 309,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43880,7 +44028,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 309,
+          "id": 310,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44013,7 +44161,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 310,
+          "id": 311,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44153,7 +44301,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 311,
+          "id": 312,
           "interval": null,
           "legend": {
             "show": false
@@ -44250,7 +44398,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 312,
+          "id": 313,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44454,7 +44602,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 313,
+      "id": 314,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -44493,7 +44641,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 314,
+          "id": 315,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44626,7 +44774,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 315,
+          "id": 316,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44759,7 +44907,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 316,
+          "id": 317,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44899,7 +45047,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 317,
+          "id": 318,
           "interval": null,
           "legend": {
             "show": false
@@ -44996,7 +45144,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 318,
+          "id": 319,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45197,7 +45345,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 319,
+          "id": 320,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45398,7 +45546,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 320,
+          "id": 321,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45602,7 +45750,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 321,
+      "id": 322,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -45641,7 +45789,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 322,
+          "id": 323,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45819,7 +45967,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 323,
+          "id": 324,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46020,7 +46168,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 324,
+          "id": 325,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46153,7 +46301,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 325,
+          "id": 326,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46286,7 +46434,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 326,
+          "id": 327,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46419,7 +46567,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 327,
+          "id": 328,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46552,7 +46700,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 328,
+          "id": 329,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46685,7 +46833,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 329,
+          "id": 330,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46814,7 +46962,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 330,
+          "id": 331,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -46889,7 +47037,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 331,
+          "id": 332,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -46968,7 +47116,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 332,
+          "id": 333,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47221,7 +47369,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 333,
+          "id": 334,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47354,7 +47502,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 334,
+          "id": 335,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47490,7 +47638,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 335,
+      "id": 336,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -47529,7 +47677,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 336,
+          "id": 337,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47677,7 +47825,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 337,
+          "id": 338,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47810,7 +47958,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 338,
+          "id": 339,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48011,7 +48159,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 339,
+          "id": 340,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48159,7 +48307,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 340,
+          "id": 341,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48360,7 +48508,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 341,
+          "id": 342,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48493,7 +48641,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 342,
+          "id": 343,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48626,7 +48774,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 343,
+          "id": 344,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48759,7 +48907,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 344,
+          "id": 345,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48892,7 +49040,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 345,
+          "id": 346,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49032,7 +49180,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 346,
+          "id": 347,
           "interval": null,
           "legend": {
             "show": false
@@ -49129,7 +49277,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 347,
+          "id": 348,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49333,7 +49481,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 348,
+      "id": 349,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -49372,7 +49520,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 349,
+          "id": 350,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49505,7 +49653,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 350,
+          "id": 351,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49638,7 +49786,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 351,
+          "id": 352,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49771,7 +49919,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 352,
+          "id": 353,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49907,7 +50055,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 353,
+      "id": 354,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -49946,7 +50094,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 354,
+          "id": 355,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50079,7 +50227,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 355,
+          "id": 356,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50212,7 +50360,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 356,
+          "id": 357,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50360,7 +50508,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 357,
+          "id": 358,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50493,7 +50641,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 358,
+          "id": 359,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50626,7 +50774,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 359,
+          "id": 360,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50759,7 +50907,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 360,
+          "id": 361,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50895,7 +51043,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 361,
+      "id": 362,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -50934,7 +51082,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 362,
+          "id": 363,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51067,7 +51215,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 363,
+          "id": 364,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51200,7 +51348,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 364,
+          "id": 365,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51333,7 +51481,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 365,
+          "id": 366,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51466,7 +51614,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 366,
+          "id": 367,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51599,7 +51747,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 367,
+          "id": 368,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51735,7 +51883,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 368,
+      "id": 369,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -51774,7 +51922,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 369,
+          "id": 370,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51907,7 +52055,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 370,
+          "id": 371,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52040,7 +52188,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 371,
+          "id": 372,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52173,7 +52321,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 372,
+          "id": 373,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52336,7 +52484,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 373,
+          "id": 374,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52469,7 +52617,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 374,
+          "id": 375,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52602,7 +52750,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 375,
+          "id": 376,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52750,7 +52898,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 376,
+          "id": 377,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52901,7 +53049,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 377,
+      "id": 378,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -52940,7 +53088,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 378,
+          "id": 379,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53073,7 +53221,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 379,
+          "id": 380,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53206,7 +53354,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 380,
+          "id": 381,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53339,7 +53487,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 381,
+          "id": 382,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53472,7 +53620,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 382,
+          "id": 383,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53605,7 +53753,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 383,
+          "id": 384,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53738,7 +53886,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 384,
+          "id": 385,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53871,7 +54019,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 385,
+          "id": 386,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54004,7 +54152,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 386,
+          "id": 387,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54144,7 +54292,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 387,
+          "id": 388,
           "interval": null,
           "legend": {
             "show": false
@@ -54241,7 +54389,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 388,
+          "id": 389,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54374,7 +54522,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 389,
+          "id": 390,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54522,7 +54670,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 390,
+          "id": 391,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54670,7 +54818,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 391,
+          "id": 392,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54810,7 +54958,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 392,
+          "id": 393,
           "interval": null,
           "legend": {
             "show": false
@@ -54907,7 +55055,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 393,
+          "id": 394,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55040,7 +55188,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 394,
+          "id": 395,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55176,7 +55324,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 395,
+      "id": 396,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -55215,7 +55363,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 396,
+          "id": 397,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55348,7 +55496,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 397,
+          "id": 398,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55511,7 +55659,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 398,
+          "id": 399,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55659,7 +55807,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 399,
+          "id": 400,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55792,7 +55940,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 400,
+          "id": 401,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55932,7 +56080,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 401,
+          "id": 402,
           "interval": null,
           "legend": {
             "show": false
@@ -56036,7 +56184,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 402,
+          "id": 403,
           "interval": null,
           "legend": {
             "show": false
@@ -56140,7 +56288,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 403,
+          "id": 404,
           "interval": null,
           "legend": {
             "show": false
@@ -56237,7 +56385,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 404,
+          "id": 405,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56377,7 +56525,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 405,
+          "id": 406,
           "interval": null,
           "legend": {
             "show": false
@@ -56481,7 +56629,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 406,
+          "id": 407,
           "interval": null,
           "legend": {
             "show": false
@@ -56585,7 +56733,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 407,
+          "id": 408,
           "interval": null,
           "legend": {
             "show": false
@@ -56682,7 +56830,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 408,
+          "id": 409,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56815,7 +56963,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 409,
+          "id": 410,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56948,7 +57096,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 410,
+          "id": 411,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57088,7 +57236,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 411,
+          "id": 412,
           "interval": null,
           "legend": {
             "show": false
@@ -57185,7 +57333,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 412,
+          "id": 413,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57321,7 +57469,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 413,
+      "id": 414,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -57360,7 +57508,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 414,
+          "id": 415,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57523,7 +57671,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 415,
+          "id": 416,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57656,7 +57804,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 416,
+          "id": 417,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57796,7 +57944,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 417,
+          "id": 418,
           "interval": null,
           "legend": {
             "show": false
@@ -57900,7 +58048,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 418,
+          "id": 419,
           "interval": null,
           "legend": {
             "show": false
@@ -57997,7 +58145,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 419,
+          "id": 420,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58152,7 +58300,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 420,
+          "id": 421,
           "interval": null,
           "legend": {
             "show": false
@@ -58256,7 +58404,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 421,
+          "id": 422,
           "interval": null,
           "legend": {
             "show": false
@@ -58360,7 +58508,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 422,
+          "id": 423,
           "interval": null,
           "legend": {
             "show": false
@@ -58457,7 +58605,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 423,
+          "id": 424,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58627,7 +58775,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 424,
+          "id": 425,
           "interval": null,
           "legend": {
             "show": false
@@ -58724,7 +58872,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 425,
+          "id": 426,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58925,7 +59073,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 426,
+          "id": 427,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59126,7 +59274,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 427,
+          "id": 428,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59259,7 +59407,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 428,
+          "id": 429,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59422,7 +59570,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 429,
+          "id": 430,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59555,7 +59703,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 430,
+          "id": 431,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59688,7 +59836,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 431,
+          "id": 432,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59889,7 +60037,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 432,
+          "id": 433,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60022,7 +60170,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 433,
+          "id": 434,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60162,7 +60310,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 434,
+          "id": 435,
           "interval": null,
           "legend": {
             "show": false
@@ -60266,7 +60414,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 435,
+          "id": 436,
           "interval": null,
           "legend": {
             "show": false
@@ -60370,7 +60518,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 436,
+          "id": 437,
           "interval": null,
           "legend": {
             "show": false
@@ -60474,7 +60622,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 437,
+          "id": 438,
           "interval": null,
           "legend": {
             "show": false
@@ -60578,7 +60726,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 438,
+          "id": 439,
           "interval": null,
           "legend": {
             "show": false
@@ -60682,7 +60830,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 439,
+          "id": 440,
           "interval": null,
           "legend": {
             "show": false
@@ -60786,7 +60934,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 440,
+          "id": 441,
           "interval": null,
           "legend": {
             "show": false
@@ -60883,7 +61031,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 441,
+          "id": 442,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61031,7 +61179,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 442,
+          "id": 443,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61164,7 +61312,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 443,
+          "id": 444,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61297,7 +61445,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 444,
+          "id": 445,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61445,7 +61593,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 445,
+          "id": 446,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61581,7 +61729,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 446,
+      "id": 447,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -61632,7 +61780,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 447,
+          "id": 448,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -61728,7 +61876,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 448,
+          "id": 449,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -61803,7 +61951,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 449,
+          "id": 450,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -61878,7 +62026,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 450,
+          "id": 451,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -61953,7 +62101,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 451,
+          "id": 452,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -62028,7 +62176,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 452,
+          "id": 453,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -62103,7 +62251,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 453,
+          "id": 454,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -62178,7 +62326,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 454,
+          "id": 455,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -62257,7 +62405,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 455,
+          "id": 456,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62390,7 +62538,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 456,
+          "id": 457,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62523,7 +62671,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 457,
+          "id": 458,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62656,7 +62804,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 458,
+          "id": 459,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62789,7 +62937,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 459,
+          "id": 460,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62922,7 +63070,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 460,
+          "id": 461,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63070,7 +63218,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 461,
+          "id": 462,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63203,7 +63351,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 462,
+          "id": 463,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63336,7 +63484,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 463,
+          "id": 464,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63502,7 +63650,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 464,
+          "id": 465,
           "interval": null,
           "legend": {
             "show": false
@@ -63606,7 +63754,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 465,
+          "id": 466,
           "interval": null,
           "legend": {
             "show": false
@@ -63710,7 +63858,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 466,
+          "id": 467,
           "interval": null,
           "legend": {
             "show": false
@@ -63814,7 +63962,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 467,
+          "id": 468,
           "interval": null,
           "legend": {
             "show": false
@@ -63918,7 +64066,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 468,
+          "id": 469,
           "interval": null,
           "legend": {
             "show": false
@@ -64022,7 +64170,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 469,
+          "id": 470,
           "interval": null,
           "legend": {
             "show": false
@@ -64126,7 +64274,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 470,
+          "id": 471,
           "interval": null,
           "legend": {
             "show": false
@@ -64230,7 +64378,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 471,
+          "id": 472,
           "interval": null,
           "legend": {
             "show": false
@@ -64327,7 +64475,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 472,
+          "id": 473,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64460,7 +64608,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 473,
+          "id": 474,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64593,7 +64741,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 474,
+          "id": 475,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64726,7 +64874,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 475,
+          "id": 476,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64859,7 +65007,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 476,
+          "id": 477,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64992,7 +65140,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 477,
+          "id": 478,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65125,7 +65273,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 478,
+          "id": 479,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65265,7 +65413,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 479,
+          "id": 480,
           "interval": null,
           "legend": {
             "show": false
@@ -65369,7 +65517,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 480,
+          "id": 481,
           "interval": null,
           "legend": {
             "show": false
@@ -65466,7 +65614,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 481,
+          "id": 482,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65599,7 +65747,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 482,
+          "id": 483,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65732,7 +65880,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 483,
+          "id": 484,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65865,7 +66013,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 484,
+          "id": 485,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65998,7 +66146,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 485,
+          "id": 486,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66131,7 +66279,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 486,
+          "id": 487,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66267,7 +66415,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 487,
+      "id": 488,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -66306,7 +66454,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 488,
+          "id": 489,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66454,7 +66602,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 489,
+          "id": 490,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66587,7 +66735,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 490,
+          "id": 491,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66720,7 +66868,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 491,
+          "id": 492,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66856,7 +67004,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 492,
+      "id": 493,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -66895,7 +67043,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 493,
+          "id": 494,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67028,7 +67176,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 494,
+          "id": 495,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67161,7 +67309,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 495,
+          "id": 496,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67294,7 +67442,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 496,
+          "id": 497,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67427,7 +67575,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 497,
+          "id": 498,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67560,7 +67708,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 498,
+          "id": 499,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67696,7 +67844,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 499,
+      "id": 500,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -67735,7 +67883,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 500,
+          "id": 501,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67936,7 +68084,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 501,
+          "id": 502,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68072,7 +68220,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 502,
+      "id": 503,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -68111,7 +68259,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 503,
+          "id": 504,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68244,7 +68392,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 504,
+          "id": 505,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68377,7 +68525,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 505,
+          "id": 506,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68510,7 +68658,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 506,
+          "id": 507,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68643,7 +68791,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 507,
+          "id": 508,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68791,7 +68939,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 508,
+          "id": 509,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68995,7 +69143,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 509,
+      "id": 510,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -69034,7 +69182,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 510,
+          "id": 511,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69167,7 +69315,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 511,
+          "id": 512,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69300,7 +69448,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 512,
+          "id": 513,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69433,7 +69581,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 513,
+          "id": 514,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69566,7 +69714,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 514,
+          "id": 515,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69763,7 +69911,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 515,
+          "id": 516,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index e4d7590337b..1c0ddc148f8 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1,1 +1,1 @@
-228266d48c18ab0ad9f83de9d786c66c3ea1fcb8504d9ab870a2ba16867f0ea9  ./metrics/grafana/tikv_details.json
+0d08da6e6ba973180bfee76503e97b8e821222b1e465fc303d07882edf9b632d  ./metrics/grafana/tikv_details.json
diff --git a/src/coprocessor/endpoint.rs b/src/coprocessor/endpoint.rs
index 1019240f313..4af6439acec 100644
--- a/src/coprocessor/endpoint.rs
+++ b/src/coprocessor/endpoint.rs
@@ -121,7 +121,8 @@ impl<E: Engine> Endpoint<E> {
     fn check_memory_locks(&self, req_ctx: &ReqContext) -> Result<()> {
         let start_ts = req_ctx.txn_start_ts;
         if !req_ctx.context.get_stale_read() {
-            self.concurrency_manager.update_max_ts(start_ts);
+            self.concurrency_manager
+                .update_max_ts(start_ts, || format!("coprocessor-{}", start_ts))?;
         }
         if need_check_locks(req_ctx.context.get_isolation_level()) {
             let begin_instant = Instant::now();
@@ -857,6 +858,12 @@ macro_rules! make_error_response_common {
                 errorpb.set_server_is_busy(server_is_busy_err);
                 $resp.set_region_error(errorpb);
             }
+            Error::InvalidMaxTsUpdate(e) => {
+                $tag = "invalid_max_ts_update";
+                let mut err = errorpb::Error::default();
+                err.set_message(e.to_string());
+                $resp.set_region_error(err);
+            }
             Error::Other(_) => {
                 $tag = "other";
                 warn!("unexpected other error encountered processing coprocessor task";
diff --git a/src/coprocessor/error.rs b/src/coprocessor/error.rs
index 25751553c28..7d2061f8191 100644
--- a/src/coprocessor/error.rs
+++ b/src/coprocessor/error.rs
@@ -26,6 +26,9 @@ pub enum Error {
     #[error("Coprocessor task canceled due to exceeding max pending tasks")]
     MaxPendingTasksExceeded,
 
+    #[error("{0}")]
+    InvalidMaxTsUpdate(#[from] concurrency_manager::InvalidMaxTsUpdate),
+
     #[error("{0}")]
     Other(String),
 }
@@ -126,6 +129,7 @@ impl ErrorCodeExt for Error {
             Error::Locked(_) => error_code::coprocessor::LOCKED,
             Error::DeadlineExceeded => error_code::coprocessor::DEADLINE_EXCEEDED,
             Error::MaxPendingTasksExceeded => error_code::coprocessor::MAX_PENDING_TASKS_EXCEEDED,
+            Error::InvalidMaxTsUpdate(_) => error_code::coprocessor::INVALID_MAX_TS_UPDATE,
             Error::Other(_) => error_code::UNKNOWN,
         }
     }
diff --git a/src/server/raftkv/mod.rs b/src/server/raftkv/mod.rs
index 04fb41ec0ac..81ea1852f81 100644
--- a/src/server/raftkv/mod.rs
+++ b/src/server/raftkv/mod.rs
@@ -808,7 +808,12 @@ impl ReadIndexObserver for ReplicaReadLockChecker {
             let begin_instant = Instant::now();
 
             let start_ts = request.get_start_ts().into();
-            self.concurrency_manager.update_max_ts(start_ts);
+            if let Err(e) = self
+                .concurrency_manager
+                .update_max_ts(start_ts, || format!("read_index-{}", start_ts))
+            {
+                error!("failed to update max_ts in concurrency manager"; "err" => ?e);
+            }
             for range in request.mut_key_ranges().iter_mut() {
                 let key_bound = |key: Vec<u8>| {
                     if key.is_empty() {
diff --git a/src/storage/errors.rs b/src/storage/errors.rs
index 0cd14d7ff8a..9dcf565e28d 100644
--- a/src/storage/errors.rs
+++ b/src/storage/errors.rs
@@ -303,6 +303,20 @@ pub fn extract_region_error_from_error(e: &Error) -> Option<errorpb::Error> {
             err.set_flashback_not_prepared(flashback_not_prepared_err);
             Some(err)
         }
+        Error(box ErrorInner::Txn(TxnError(box TxnErrorInner::InvalidMaxTsUpdate(
+            invalid_max_ts_update,
+        )))) => {
+            let mut err = errorpb::Error::default();
+            err.set_message(invalid_max_ts_update.to_string());
+            Some(err)
+        }
+        Error(box ErrorInner::Txn(TxnError(box TxnErrorInner::Mvcc(MvccError(
+            box MvccErrorInner::InvalidMaxTsUpdate(invalid_max_ts_update),
+        ))))) => {
+            let mut err = errorpb::Error::default();
+            err.set_message(invalid_max_ts_update.to_string());
+            Some(err)
+        }
         Error(box ErrorInner::SchedTooBusy) => {
             let mut err = errorpb::Error::default();
             let mut server_is_busy_err = errorpb::ServerIsBusy::default();
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 858e56efa57..ba6f543dcc5 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -1449,7 +1449,9 @@ impl<E: Engine, L: LockManager, F: KvFormat> Storage<E, L, F> {
 
                 // Update max_ts and check the in-memory lock table before getting the snapshot
                 if !ctx.get_stale_read() {
-                    concurrency_manager.update_max_ts(start_ts);
+                    concurrency_manager
+                        .update_max_ts(start_ts, "scan")
+                        .map_err(txn::Error::from)?;
                 }
                 if need_check_locks(ctx.get_isolation_level()) {
                     let begin_instant = Instant::now();
@@ -1614,7 +1616,9 @@ impl<E: Engine, L: LockManager, F: KvFormat> Storage<E, L, F> {
 
                 let command_duration = Instant::now();
 
-                concurrency_manager.update_max_ts(max_ts);
+                concurrency_manager
+                    .update_max_ts(max_ts, "scan_lock")
+                    .map_err(txn::Error::from)?;
                 let begin_instant = Instant::now();
                 // TODO: Though it's very unlikely to find a conflicting memory lock here, it's
                 // not a good idea to return an error to the client, making the GC fail. A
@@ -3310,7 +3314,11 @@ fn prepare_snap_ctx<'a>(
 ) -> Result<SnapContext<'a>> {
     // Update max_ts and check the in-memory lock table before getting the snapshot
     if !pb_ctx.get_stale_read() {
-        concurrency_manager.update_max_ts(start_ts);
+        concurrency_manager
+            .update_max_ts(start_ts, || {
+                format!("prepare_snap_ctx-{}-{}", cmd, start_ts)
+            })
+            .map_err(txn::Error::from)?;
     }
     fail_point!("before-storage-check-memory-locks");
     let isolation_level = pb_ctx.get_isolation_level();
@@ -10206,7 +10214,7 @@ mod tests {
             .build()
             .unwrap();
         let cm = storage.concurrency_manager.clone();
-        cm.update_max_ts(10.into());
+        cm.update_max_ts(10.into(), "").unwrap();
 
         // Optimistic prewrite
         let (tx, rx) = channel();
@@ -10254,7 +10262,7 @@ mod tests {
             .unwrap();
         rx.recv().unwrap();
 
-        cm.update_max_ts(1000.into());
+        cm.update_max_ts(1000.into(), "").unwrap();
 
         let (tx, rx) = channel();
         storage
@@ -11361,7 +11369,7 @@ mod tests {
         // commit enabled, and max_ts changes when the second request arrives.
 
         // A retrying prewrite request arrives.
-        cm.update_max_ts(20.into());
+        cm.update_max_ts(20.into(), "").unwrap();
         let mut ctx = Context::default();
         ctx.set_is_retry_request(true);
         let (tx, rx) = channel();
@@ -11545,7 +11553,7 @@ mod tests {
 
         // 1PC update
         let (tx, rx) = channel();
-        cm.update_max_ts(59.into());
+        cm.update_max_ts(59.into(), "").unwrap();
         storage
             .sched_txn_command(
                 Prewrite::new(
diff --git a/src/storage/mvcc/mod.rs b/src/storage/mvcc/mod.rs
index 70f59bf7999..5c943f6ce09 100644
--- a/src/storage/mvcc/mod.rs
+++ b/src/storage/mvcc/mod.rs
@@ -177,6 +177,9 @@ pub enum ErrorInner {
     #[error("generation out of order: current = {0}, key={1:?}, lock = {1:?}")]
     GenerationOutOfOrder(u64, Key, Lock),
 
+    #[error("{0}")]
+    InvalidMaxTsUpdate(#[from] concurrency_manager::InvalidMaxTsUpdate),
+
     #[error("{0:?}")]
     Other(#[from] Box<dyn error::Error + Sync + Send>),
 }
@@ -310,6 +313,7 @@ impl ErrorInner {
             ErrorInner::GenerationOutOfOrder(gen, key, lock_info) => Some(
                 ErrorInner::GenerationOutOfOrder(*gen, key.clone(), lock_info.clone()),
             ),
+            ErrorInner::InvalidMaxTsUpdate(e) => Some(ErrorInner::InvalidMaxTsUpdate(e.clone())),
             ErrorInner::Io(_) | ErrorInner::Other(_) => None,
         }
     }
@@ -414,6 +418,7 @@ impl ErrorCodeExt for Error {
             ErrorInner::LockIfExistsFailed { .. } => error_code::storage::LOCK_IF_EXISTS_FAILED,
             ErrorInner::PrimaryMismatch(_) => error_code::storage::PRIMARY_MISMATCH,
             ErrorInner::GenerationOutOfOrder(..) => error_code::storage::GENERATION_OUT_OF_ORDER,
+            ErrorInner::InvalidMaxTsUpdate(_) => error_code::storage::INVALID_MAX_TS_UPDATE,
             ErrorInner::Other(_) => error_code::storage::UNKNOWN,
         }
     }
diff --git a/src/storage/txn/actions/acquire_pessimistic_lock.rs b/src/storage/txn/actions/acquire_pessimistic_lock.rs
index 2a9e49b45ff..6c71bb6843a 100644
--- a/src/storage/txn/actions/acquire_pessimistic_lock.rs
+++ b/src/storage/txn/actions/acquire_pessimistic_lock.rs
@@ -71,7 +71,9 @@ pub fn acquire_pessimistic_lock<S: Snapshot>(
     // it infers a read to the value, in which case max_ts need to be updated to
     // guarantee the linearizability and snapshot isolation.
     if should_not_exist || need_value || need_check_existence {
-        txn.concurrency_manager.update_max_ts(for_update_ts);
+        txn.concurrency_manager.update_max_ts(for_update_ts, || {
+            format!("pessimistic_lock-{}-{}", reader.start_ts, for_update_ts)
+        })?;
     }
 
     // When `need_value` is set, the value need to be loaded of course. If
diff --git a/src/storage/txn/actions/prewrite.rs b/src/storage/txn/actions/prewrite.rs
index f446ac01dbf..a29ce014225 100644
--- a/src/storage/txn/actions/prewrite.rs
+++ b/src/storage/txn/actions/prewrite.rs
@@ -70,7 +70,10 @@ pub fn prewrite_with_generation<S: Snapshot>(
     // Update max_ts for Insert operation to guarantee linearizability and snapshot
     // isolation
     if mutation.should_not_exist {
-        txn.concurrency_manager.update_max_ts(txn_props.start_ts);
+        txn.concurrency_manager
+            .update_max_ts(txn_props.start_ts, || {
+                format!("prewrite-{}", txn_props.start_ts)
+            })?;
     }
 
     fail_point!(
@@ -149,7 +152,10 @@ pub fn prewrite_with_generation<S: Snapshot>(
     if mutation.should_not_write {
         // `checkNotExists` is equivalent to a get operation, so it should update the
         // max_ts.
-        txn.concurrency_manager.update_max_ts(txn_props.start_ts);
+        txn.concurrency_manager
+            .update_max_ts(txn_props.start_ts, || {
+                format!("prewrite-{}", txn_props.start_ts)
+            })?;
         let min_commit_ts = if mutation.need_min_commit_ts() {
             // Don't calculate the min_commit_ts according to the concurrency manager's
             // max_ts for a should_not_write mutation because it's not persisted and doesn't
@@ -1059,7 +1065,7 @@ pub mod tests {
         .unwrap();
         assert_eq!(old_value, OldValue::None);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, err
         let err = prewrite(
             &mut txn,
@@ -1244,7 +1250,7 @@ pub mod tests {
         .unwrap();
         assert_eq!(old_value, OldValue::None);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, err
         let err = prewrite(
             &mut txn,
@@ -1346,7 +1352,7 @@ pub mod tests {
         // Pessimistic txn skips constraint check, does not read previous write.
         assert_eq!(old_value, OldValue::Unspecified);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, ok
         prewrite(
             &mut txn,
@@ -1399,7 +1405,7 @@ pub mod tests {
         // Pessimistic txn skips constraint check, does not read previous write.
         assert_eq!(old_value, OldValue::Unspecified);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, ok
         prewrite(
             &mut txn,
diff --git a/src/storage/txn/commands/check_secondary_locks.rs b/src/storage/txn/commands/check_secondary_locks.rs
index 5f9759ce379..27f3ea58773 100644
--- a/src/storage/txn/commands/check_secondary_locks.rs
+++ b/src/storage/txn/commands/check_secondary_locks.rs
@@ -145,7 +145,11 @@ impl<S: Snapshot, L: LockManager> WriteCommand<S, L> for CheckSecondaryLocks {
         // It is not allowed for commit to overwrite a protected rollback. So we update
         // max_ts to prevent this case from happening.
         let region_id = self.ctx.get_region_id();
-        context.concurrency_manager.update_max_ts(self.start_ts);
+        context
+            .concurrency_manager
+            .update_max_ts(self.start_ts, || {
+                format!("check_secondary_locks-{}", self.start_ts)
+            })?;
 
         let mut txn = MvccTxn::new(self.start_ts, context.concurrency_manager);
         let mut reader = ReaderWithStats::new(
diff --git a/src/storage/txn/commands/check_txn_status.rs b/src/storage/txn/commands/check_txn_status.rs
index c3e5410a692..d7df579aae6 100644
--- a/src/storage/txn/commands/check_txn_status.rs
+++ b/src/storage/txn/commands/check_txn_status.rs
@@ -90,7 +90,12 @@ impl<S: Snapshot, L: LockManager> WriteCommand<S, L> for CheckTxnStatus {
         if !self.caller_start_ts.is_max() && self.caller_start_ts > new_max_ts {
             new_max_ts = self.caller_start_ts;
         }
-        context.concurrency_manager.update_max_ts(new_max_ts);
+        context.concurrency_manager.update_max_ts(new_max_ts, || {
+            format!(
+                "check_txn_status-{}-{}-{}",
+                self.lock_ts, self.current_ts, self.caller_start_ts
+            )
+        })?;
 
         let mut txn = MvccTxn::new(self.lock_ts, context.concurrency_manager);
         let mut reader = ReaderWithStats::new(
diff --git a/src/storage/txn/commands/cleanup.rs b/src/storage/txn/commands/cleanup.rs
index a1c234ce062..fa0b446284b 100644
--- a/src/storage/txn/commands/cleanup.rs
+++ b/src/storage/txn/commands/cleanup.rs
@@ -51,7 +51,9 @@ impl<S: Snapshot, L: LockManager> WriteCommand<S, L> for Cleanup {
     fn process_write(self, snapshot: S, context: WriteContext<'_, L>) -> Result<WriteResult> {
         // It is not allowed for commit to overwrite a protected rollback. So we update
         // max_ts to prevent this case from happening.
-        context.concurrency_manager.update_max_ts(self.start_ts);
+        context
+            .concurrency_manager
+            .update_max_ts(self.start_ts, || format!("cleanup-{}", self.start_ts))?;
 
         let mut txn = MvccTxn::new(self.start_ts, context.concurrency_manager);
         let mut reader = ReaderWithStats::new(
diff --git a/src/storage/txn/commands/prewrite.rs b/src/storage/txn/commands/prewrite.rs
index e5c9ec4f755..e7f170a66b5 100644
--- a/src/storage/txn/commands/prewrite.rs
+++ b/src/storage/txn/commands/prewrite.rs
@@ -1259,7 +1259,7 @@ mod tests {
         must_get(&mut engine, key, 12, value);
         must_get_commit_ts(&mut engine, key, 10, 11);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         let mutations = vec![Mutation::make_put(Key::from_raw(key), value.to_vec())];
 
@@ -1401,7 +1401,7 @@ mod tests {
         must_get_commit_ts(&mut engine, k1, 8, 13);
         must_get_commit_ts(&mut engine, k2, 8, 13);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
         must_acquire_pessimistic_lock(&mut engine, k1, k1, 20, 20);
 
         let mutations = vec![(
@@ -1501,7 +1501,7 @@ mod tests {
         assert_eq!(res.one_pc_commit_ts, TimeStamp::zero());
         must_locked(&mut engine, key, 10);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         let (k1, v1) = (b"k1", b"v1");
         let (k2, v2) = (b"k2", b"v2");
@@ -1573,7 +1573,7 @@ mod tests {
         assert_eq!(res.one_pc_commit_ts, TimeStamp::zero());
         must_locked(&mut engine, key, 10);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         let (k1, v1) = (b"k1", b"v1");
         let (k2, v2) = (b"k2", b"v2");
@@ -2142,7 +2142,7 @@ mod tests {
         )
         .unwrap();
         let commit_ts = res.one_pc_commit_ts;
-        cm.update_max_ts(commit_ts.next());
+        cm.update_max_ts(commit_ts.next(), "").unwrap();
         // repeate the prewrite
         let res = pessimistic_prewrite_with_cm(
             &mut engine,
@@ -2652,7 +2652,7 @@ mod tests {
         must_commit(&mut engine, b"k2", 5, 18);
 
         // Update max_ts to be larger than the max_commit_ts.
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         // Retry the prewrite on non-pessimistic key.
         // (is_retry_request flag is not set, here we don't rely on it.)
diff --git a/src/storage/txn/mod.rs b/src/storage/txn/mod.rs
index 5c4ed85902d..7e323a61e06 100644
--- a/src/storage/txn/mod.rs
+++ b/src/storage/txn/mod.rs
@@ -149,6 +149,9 @@ pub enum ErrorInner {
 
     #[error("region {0} not prepared the flashback")]
     FlashbackNotPrepared(u64),
+
+    #[error("{0}")]
+    InvalidMaxTsUpdate(#[from] concurrency_manager::InvalidMaxTsUpdate),
 }
 
 impl ErrorInner {
@@ -188,6 +191,9 @@ impl ErrorInner {
             ErrorInner::FlashbackNotPrepared(region_id) => {
                 Some(ErrorInner::FlashbackNotPrepared(region_id))
             }
+            ErrorInner::InvalidMaxTsUpdate(ref e) => {
+                Some(ErrorInner::InvalidMaxTsUpdate(e.clone()))
+            }
             ErrorInner::Other(_) | ErrorInner::ProtoBuf(_) | ErrorInner::Io(_) => None,
         }
     }
@@ -242,6 +248,7 @@ impl ErrorCodeExt for Error {
                 error_code::storage::MAX_TIMESTAMP_NOT_SYNCED
             }
             ErrorInner::FlashbackNotPrepared(_) => error_code::storage::FLASHBACK_NOT_PREPARED,
+            ErrorInner::InvalidMaxTsUpdate { .. } => error_code::storage::INVALID_MAX_TS_UPDATE,
         }
     }
 }


Test patch:
diff --git a/components/cdc/tests/integrations/test_cdc.rs b/components/cdc/tests/integrations/test_cdc.rs
index c1ac1706d52..aac2bd45814 100644
--- a/components/cdc/tests/integrations/test_cdc.rs
+++ b/components/cdc/tests/integrations/test_cdc.rs
@@ -1235,7 +1235,7 @@ fn test_cdc_resolve_ts_checking_concurrency_manager_impl<F: KvFormat>() {
         guard
     };
 
-    cm.update_max_ts(20.into());
+    cm.update_max_ts(20.into(), "").unwrap();
 
     let guard = lock_key(b"a", 80);
     suite.set_tso(99);
diff --git a/tests/failpoints/cases/test_transaction.rs b/tests/failpoints/cases/test_transaction.rs
index bb1d291e816..a5a34147030 100644
--- a/tests/failpoints/cases/test_transaction.rs
+++ b/tests/failpoints/cases/test_transaction.rs
@@ -344,7 +344,7 @@ fn test_max_commit_ts_error() {
     thread::sleep(Duration::from_millis(200));
     cm.read_key_check(&Key::from_raw(b"k1"), |_| Err(()))
         .unwrap_err();
-    cm.update_max_ts(200.into());
+    cm.update_max_ts(200.into(), "").unwrap();
 
     let res = prewrite_rx.recv().unwrap().unwrap();
     assert!(res.min_commit_ts.is_zero());
@@ -372,7 +372,7 @@ fn test_exceed_max_commit_ts_in_the_middle_of_prewrite() {
     // Pause between getting max ts and store the lock in memory
     fail::cfg("before-set-lock-in-memory", "pause").unwrap();
 
-    cm.update_max_ts(40.into());
+    cm.update_max_ts(40.into(), "").unwrap();
     let mutations = vec![
         Mutation::make_put(Key::from_raw(b"k1"), b"v".to_vec()),
         Mutation::make_put(Key::from_raw(b"k2"), b"v".to_vec()),
@@ -401,7 +401,7 @@ fn test_exceed_max_commit_ts_in_the_middle_of_prewrite() {
     // sleep a while so the first key gets max ts.
     thread::sleep(Duration::from_millis(200));
 
-    cm.update_max_ts(51.into());
+    cm.update_max_ts(51.into(), "").unwrap();
     fail::remove("before-set-lock-in-memory");
     let res = prewrite_rx.recv().unwrap().unwrap();
     assert!(res.min_commit_ts.is_zero());
diff --git a/tests/integrations/server/kv_service.rs b/tests/integrations/server/kv_service.rs
index 41b2536996a..e7c7b68e1df 100644
--- a/tests/integrations/server/kv_service.rs
+++ b/tests/integrations/server/kv_service.rs
@@ -1859,7 +1859,7 @@ fn test_prewrite_check_max_commit_ts() {
     let (cluster, client, ctx) = new_cluster();
 
     let cm = cluster.sim.read().unwrap().get_concurrency_manager(1);
-    cm.update_max_ts(100.into());
+    cm.update_max_ts(100.into(), "").unwrap();
 
     let mut req = PrewriteRequest::default();
     req.set_context(ctx.clone());