Repository: tikv/tikv (language: rust)

Issue:
Enhancement: Add timestamp safety boundary to prevent unreasonable max_ts updates
Subtask of 
- https://github.com/pingcap/tidb/issues/57786
## Enhancement
Currently, `max_ts` can be updated to any larger value (**exceeding current PD TSO**). This poses a potential risk where bugs or unsafe usage could set `max_ts` to an unreasonably large value, causing critical bugs and data corruption.

Proposed Solution:
Add a configurable safety boundary (`max_ts_limit`) that:
- Enforces a maximum allowed deviation of `max_ts` from PD timestamp
- Regularly syncs with PD to maintain a reasonable limit
- Prevents updates to unreasonable future timestamps
- Timestamps that are known to be from PD TSO can bypass the limit.


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index fa5008ee32b..8eaf56ff07f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -97,6 +97,12 @@ dependencies = [
  "libc 0.2.151",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "ansi_term"
 version = "0.11.0"
@@ -106,6 +112,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
 [[package]]
 name = "anyhow"
 version = "1.0.75"
@@ -945,7 +957,7 @@ dependencies = [
 name = "batch-system"
 version = "0.1.0"
 dependencies = [
- "criterion",
+ "criterion 0.3.5",
  "crossbeam",
  "dashmap",
  "derive_more",
@@ -1201,12 +1213,18 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "causal_ts"
 version = "0.0.1"
 dependencies = [
  "async-trait",
- "criterion",
+ "criterion 0.3.5",
  "enum_dispatch",
  "error_code",
  "futures 0.3.15",
@@ -1264,7 +1282,7 @@ dependencies = [
  "causal_ts",
  "collections",
  "concurrency_manager",
- "criterion",
+ "criterion 0.3.5",
  "crossbeam",
  "engine_rocks",
  "engine_traits",
@@ -1496,13 +1514,23 @@ dependencies = [
 name = "concurrency_manager"
 version = "0.0.1"
 dependencies = [
- "criterion",
+ "criterion 0.4.0",
+ "crossbeam",
  "crossbeam-skiplist 0.1.3",
  "fail",
  "futures 0.3.15",
  "kvproto",
+ "lazy_static",
+ "mockall",
+ "online_config",
  "parking_lot 0.12.1",
+ "pd_client",
+ "prometheus",
  "rand 0.8.5",
+ "serde",
+ "slog",
+ "slog-global",
+ "thiserror",
  "tikv_alloc",
  "tikv_util",
  "tokio",
@@ -1619,9 +1647,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10"
 dependencies = [
  "atty",
- "cast",
+ "cast 0.2.2",
  "clap 2.33.0",
- "criterion-plot",
+ "criterion-plot 0.4.4",
  "csv",
  "itertools",
  "lazy_static",
@@ -1638,13 +1666,39 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast 0.3.0",
+ "ciborium",
+ "clap 3.1.6",
+ "criterion-plot 0.5.0",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
 [[package]]
 name = "criterion-cpu-time"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63aaaf47e457badbcb376c65a49d0f182c317ebd97dc6d1ced94c8e1d09c0f3a"
 dependencies = [
- "criterion",
+ "criterion 0.3.5",
  "libc 0.2.151",
 ]
 
@@ -1654,7 +1708,7 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eba5111e09fabb08bfaedbe28c832876bb38d4f9519f715466332880d80b0eac"
 dependencies = [
- "criterion",
+ "criterion 0.3.5",
  "perfcnt",
 ]
 
@@ -1664,7 +1718,17 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
 dependencies = [
- "cast",
+ "cast 0.2.2",
+ "itertools",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast 0.3.0",
  "itertools",
 ]
 
@@ -1934,6 +1998,12 @@ dependencies = [
  "syn 2.0.79",
 ]
 
+[[package]]
+name = "downcast"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
+
 [[package]]
 name = "dyn-clone"
 version = "1.0.4"
@@ -2462,6 +2532,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fragile"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
+
 [[package]]
 name = "fs2"
 version = "0.4.3"
@@ -3357,7 +3433,7 @@ version = "0.0.1"
 dependencies = [
  "bytes",
  "collections",
- "criterion",
+ "criterion 0.3.5",
  "crossbeam",
  "crossbeam-skiplist 0.1.3",
  "dashmap",
@@ -4000,6 +4076,32 @@ dependencies = [
  "tempdir",
 ]
 
+[[package]]
+name = "mockall"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2"
+dependencies = [
+ "cfg-if 1.0.0",
+ "downcast",
+ "fragile",
+ "mockall_derive",
+ "predicates",
+ "predicates-tree",
+]
+
+[[package]]
+name = "mockall_derive"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898"
+dependencies = [
+ "cfg-if 1.0.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
+]
+
 [[package]]
 name = "more-asserts"
 version = "0.2.1"
@@ -4760,6 +4862,32 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "predicates"
+version = "3.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573"
+dependencies = [
+ "anstyle",
+ "predicates-core",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
 [[package]]
 name = "pretty_assertions"
 version = "1.4.0"
@@ -6586,6 +6714,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "termtree"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
+
 [[package]]
 name = "test_backup"
 version = "0.0.1"
@@ -6842,7 +6976,7 @@ dependencies = [
  "collections",
  "concurrency_manager",
  "crc64fast",
- "criterion",
+ "criterion 0.3.5",
  "criterion-cpu-time",
  "criterion-perf-events",
  "crossbeam",
@@ -7020,7 +7154,7 @@ dependencies = [
  "codec",
  "collections",
  "crc32fast",
- "criterion",
+ "criterion 0.3.5",
  "encoding_rs 0.8.29",
  "error_code",
  "hex 0.4.3",
diff --git a/components/backup-stream/src/endpoint.rs b/components/backup-stream/src/endpoint.rs
index 7fede7dad0c..4dd35e926b0 100644
--- a/components/backup-stream/src/endpoint.rs
+++ b/components/backup-stream/src/endpoint.rs
@@ -818,7 +818,7 @@ where
                 .await
                 .map_err(|err| Error::from(err).report("failed to get tso from pd"))
                 .unwrap_or_default();
-            cm.update_max_ts(pd_tso);
+            cm.update_max_ts(pd_tso, "backup-stream").unwrap();
             let min_ts = cm.global_min_lock_ts().unwrap_or(TimeStamp::max());
             Ord::min(pd_tso, min_ts)
         }
diff --git a/components/backup/src/endpoint.rs b/components/backup/src/endpoint.rs
index 681187ad0d6..67cd07c0a13 100644
--- a/components/backup/src/endpoint.rs
+++ b/components/backup/src/endpoint.rs
@@ -345,7 +345,9 @@ impl BackupRange {
             snap_ctx.key_ranges = vec![key_range];
         } else {
             // Update max_ts and check the in-memory lock table before getting the snapshot
-            concurrency_manager.update_max_ts(backup_ts);
+            concurrency_manager
+                .update_max_ts(backup_ts, "backup_range")
+                .map_err(TxnError::from)?;
             concurrency_manager
                 .read_range_check(
                     self.start_key.as_ref(),
diff --git a/components/cdc/src/endpoint.rs b/components/cdc/src/endpoint.rs
index a860d3d1260..da33bfd4021 100644
--- a/components/cdc/src/endpoint.rs
+++ b/components/cdc/src/endpoint.rs
@@ -1122,7 +1122,7 @@ impl<T: 'static + CdcHandle<E>, E: KvEngine, S: StoreRegionMeta> Endpoint<T, E,
             // optimizations like async commit is enabled.
             // Note: This step must be done before scheduling `Task::MinTs` task, and the
             // resolver must be checked in or after `Task::MinTs`' execution.
-            cm.update_max_ts(min_ts);
+            cm.update_max_ts(min_ts, "cdc").unwrap();
             if let Some(min_mem_lock_ts) = cm.global_min_lock_ts() {
                 if min_mem_lock_ts < min_ts {
                     min_ts = min_mem_lock_ts;
diff --git a/components/concurrency_manager/Cargo.toml b/components/concurrency_manager/Cargo.toml
index 8e2817d0097..39ba558c469 100644
--- a/components/concurrency_manager/Cargo.toml
+++ b/components/concurrency_manager/Cargo.toml
@@ -6,16 +6,26 @@ version = "0.0.1"
 license = "Apache-2.0"
 
 [dependencies]
+crossbeam = "0.8"
 crossbeam-skiplist = { workspace = true }
 fail = "0.5"
 kvproto = { workspace = true }
+lazy_static = "1.4.0"
+mockall = "0.13"
+online_config = { workspace = true }
 parking_lot = "0.12"
+pd_client = { workspace = true }
+prometheus = "0.13"
+serde = { version = "1.0.194", features = ["derive"] }
+slog = { workspace = true }
+slog-global = { workspace = true }
+thiserror = "1.0"
 tikv_util = { workspace = true }
 tokio = { version = "1.5", features = ["macros", "sync", "time"] }
 txn_types = { workspace = true }
 
 [dev-dependencies]
-criterion = "0.3"
+criterion = "0.4"
 futures = "0.3"
 rand = "0.8.3"
 tikv_alloc = { workspace = true, features = ["jemalloc"] }
@@ -24,3 +34,8 @@ tikv_alloc = { workspace = true, features = ["jemalloc"] }
 name = "lock_table"
 path = "benches/lock_table.rs"
 harness = false
+
+[[bench]]
+name = "update_max_ts"
+path = "benches/update_max_ts.rs"
+harness = false
diff --git a/components/concurrency_manager/benches/update_max_ts.rs b/components/concurrency_manager/benches/update_max_ts.rs
new file mode 100644
index 00000000000..df2e61aeb42
--- /dev/null
+++ b/components/concurrency_manager/benches/update_max_ts.rs
@@ -0,0 +1,33 @@
+// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
+
+use std::time::Duration;
+
+use concurrency_manager::{ActionOnInvalidMaxTs, ConcurrencyManager};
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use txn_types::TimeStamp;
+
+fn benchmark_update_max_ts(c: &mut Criterion) {
+    let latest_ts = TimeStamp::new(1000);
+    let limit_valid_time = Duration::from_secs(20);
+    let cm = ConcurrencyManager::new_with_config(
+        latest_ts,
+        limit_valid_time,
+        ActionOnInvalidMaxTs::Error,
+        None,
+        Duration::ZERO,
+    );
+
+    cm.set_max_ts_limit(TimeStamp::new(4000));
+
+    let new_ts = TimeStamp::new(3000);
+
+    c.bench_function("update_max_ts", |b| {
+        b.iter(|| {
+            cm.update_max_ts(black_box(new_ts), || format!("benchmark-{}", new_ts))
+                .unwrap();
+        })
+    });
+}
+
+criterion_group!(benches, benchmark_update_max_ts);
+criterion_main!(benches);
diff --git a/components/concurrency_manager/src/lib.rs b/components/concurrency_manager/src/lib.rs
index 1c6bdb8dbf1..0b3502d400b 100644
--- a/components/concurrency_manager/src/lib.rs
+++ b/components/concurrency_manager/src/lib.rs
@@ -16,13 +16,24 @@ mod key_handle;
 mod lock_table;
 
 use std::{
+    error::Error,
+    fmt,
+    fmt::Display,
     mem::MaybeUninit,
     sync::{
-        atomic::{AtomicU64, Ordering},
+        atomic::{AtomicU64, AtomicUsize, Ordering},
         Arc,
     },
+    time::Duration,
 };
 
+use crossbeam::atomic::AtomicCell;
+use lazy_static::lazy_static;
+use mockall::automock;
+use pd_client::{PdClient, PdFuture};
+use prometheus::{register_int_gauge, IntGauge};
+use thiserror::Error;
+use tikv_util::{error, future::block_on_timeout, time::Instant, warn};
 use txn_types::{Key, Lock, TimeStamp};
 
 pub use self::{
@@ -30,19 +41,144 @@ pub use self::{
     lock_table::LockTable,
 };
 
+lazy_static! {
+    static ref MAX_TS_LIMIT_GAUGE: IntGauge = register_int_gauge!(
+        "tikv_concurrency_manager_max_ts_limit",
+        "Current value of max_ts_limit"
+    )
+    .unwrap();
+    static ref MAX_TS_GAUGE: IntGauge =
+        register_int_gauge!("tikv_concurrency_manager_max_ts", "Current value of max_ts").unwrap();
+}
+
+const DEFAULT_LIMIT_VALID_DURATION: Duration = Duration::from_secs(45);
+
+// It is suggested that limit_valid_duration = sync_interval *
+// LIMIT_VALID_TIME_MULTIPLIER, to balance between
+// 1. tolerate temporary issues in updating the limit.
+// 2. avoid long-term blocking of max_ts update caused by network partition
+//    between TiKV and PD.
+pub const LIMIT_VALID_TIME_MULTIPLIER: u32 = 3;
+
+pub const DEFAULT_MAX_TS_SYNC_INTERVAL: Duration = Duration::from_secs(15);
+pub const DEFAULT_MAX_TS_DRIFT_ALLOWANCE: Duration = Duration::from_secs(60);
+const TSO_TIMEOUT: Duration = Duration::from_secs(5);
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+struct MaxTsLimit {
+    limit: TimeStamp,
+    update_time: Instant,
+}
+
+#[automock]
+pub trait TSOProvider: Send + Sync {
+    fn get_tso(&self) -> PdFuture<TimeStamp>;
+}
+
+impl<T: PdClient> TSOProvider for T {
+    fn get_tso(&self) -> PdFuture<TimeStamp> {
+        PdClient::get_tso(self)
+    }
+}
+
 // Pay attention that the async functions of ConcurrencyManager should not hold
 // the mutex.
 #[derive(Clone)]
 pub struct ConcurrencyManager {
     max_ts: Arc<AtomicU64>,
     lock_table: LockTable,
+
+    // max_ts_limit and its update time.
+    //
+    // max_ts_limit is an assertion: max_ts should not be updated to a value greater than this
+    // limit.
+    //
+    // When the limit is not updated for a long time(exceeding the threshold), we use an
+    // approximate limit.
+    max_ts_limit: Arc<AtomicCell<MaxTsLimit>>,
+    limit_valid_duration: Duration,
+    action_on_invalid_max_ts: Arc<AtomicActionOnInvalidMaxTs>,
+
+    max_ts_drift_allowance_ms: Arc<AtomicU64>,
+
+    tso: Option<Arc<dyn TSOProvider>>,
+
+    time_provider: Arc<dyn TimeProvider>,
 }
 
 impl ConcurrencyManager {
     pub fn new(latest_ts: TimeStamp) -> Self {
+        Self::new_with_config(
+            latest_ts,
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            None,
+            Duration::ZERO,
+        )
+    }
+
+    pub fn new_with_config(
+        latest_ts: TimeStamp,
+        limit_valid_duration: Duration,
+        action_on_invalid_max_ts: ActionOnInvalidMaxTs,
+        tso: Option<Arc<dyn TSOProvider>>,
+        max_ts_drift_allowance: Duration,
+    ) -> Self {
+        let initial_limit = MaxTsLimit {
+            limit: TimeStamp::new(0),
+            update_time: Instant::now(),
+        };
+
+        if limit_valid_duration >= max_ts_drift_allowance {
+            error!("improper setting: limit_valid_duration >= max_ts_drift_allowance; \
+                consider increasing max-ts-drift-allowance or decreasing max-ts-sync-interval";
+                "limit_valid_duration" => ?limit_valid_duration,
+                "max_ts_drift_allowance" => ?max_ts_drift_allowance,
+            );
+        }
+
         ConcurrencyManager {
             max_ts: Arc::new(AtomicU64::new(latest_ts.into_inner())),
+            max_ts_limit: Arc::new(AtomicCell::new(initial_limit)),
             lock_table: LockTable::default(),
+            action_on_invalid_max_ts: Arc::new(AtomicActionOnInvalidMaxTs::new(
+                action_on_invalid_max_ts,
+            )),
+            limit_valid_duration,
+            time_provider: Arc::new(CoarseInstantTimeProvider),
+            tso,
+            max_ts_drift_allowance_ms: Arc::new(AtomicU64::new(
+                max_ts_drift_allowance.as_millis() as u64
+            )),
+        }
+    }
+
+    #[cfg(test)]
+    fn new_with_time_provider(
+        latest_ts: TimeStamp,
+        limit_valid_duration: Duration,
+        action_on_invalid_max_ts: ActionOnInvalidMaxTs,
+        time_provider: Arc<dyn TimeProvider>,
+        tso: Option<Arc<dyn TSOProvider>>,
+        max_ts_drift_allowance: Duration,
+    ) -> Self {
+        let initial_limit = MaxTsLimit {
+            limit: TimeStamp::new(0),
+            update_time: time_provider.now(),
+        };
+        ConcurrencyManager {
+            max_ts: Arc::new(AtomicU64::new(latest_ts.into_inner())),
+            max_ts_limit: Arc::new(AtomicCell::new(initial_limit)),
+            lock_table: LockTable::default(),
+            action_on_invalid_max_ts: Arc::new(AtomicActionOnInvalidMaxTs::new(
+                action_on_invalid_max_ts,
+            )),
+            limit_valid_duration,
+            time_provider,
+            tso,
+            max_ts_drift_allowance_ms: Arc::new(AtomicU64::new(
+                max_ts_drift_allowance.as_millis() as u64
+            )),
         }
     }
 
@@ -52,9 +188,183 @@ impl ConcurrencyManager {
 
     /// Updates max_ts with the given new_ts. It has no effect if
     /// max_ts >= new_ts or new_ts is TimeStamp::max().
-    pub fn update_max_ts(&self, new_ts: TimeStamp) {
-        if new_ts != TimeStamp::max() {
-            self.max_ts.fetch_max(new_ts.into_inner(), Ordering::SeqCst);
+    ///
+    /// To avoid invalid ts breaking the invariants, the new_ts should be
+    /// less than or equal to the max_ts_limit.
+    ///
+    /// # Returns
+    /// - Ok(()): If the update is successful or has no effect
+    /// - Err(limit): If new_ts is greater than the max_ts_limit, returns the
+    ///   current limit value
+
+    pub fn update_max_ts(
+        &self,
+        new_ts: TimeStamp,
+        source: impl IntoErrorSource,
+    ) -> Result<(), InvalidMaxTsUpdate> {
+        if new_ts.is_max() {
+            return Ok(());
+        }
+        let limit = self.max_ts_limit.load();
+
+        // check that new_ts is less than or equal to the limit
+        if !limit.limit.is_zero() && new_ts > limit.limit {
+            let last_update = limit.update_time;
+            let now = self.time_provider.now();
+            assert!(now >= last_update);
+            let duration_to_last_limit_update = now - last_update;
+
+            if duration_to_last_limit_update < self.limit_valid_duration {
+                // limit is valid
+                let source = source.into_error_source();
+                self.double_check(new_ts, limit.limit, source, false)?;
+            } else {
+                // limit is stale
+                // use an approximate limit to avoid false alerts caused by failed limit updates
+
+                let approximate_limit = TimeStamp::compose(
+                    limit.limit.physical() + duration_to_last_limit_update.as_millis() as u64,
+                    limit.limit.logical(),
+                );
+
+                if new_ts > approximate_limit {
+                    let source = source.into_error_source();
+                    self.double_check(new_ts, approximate_limit, source, true)?;
+                }
+            }
+        }
+
+        MAX_TS_GAUGE.set(
+            self.max_ts
+                .fetch_max(new_ts.into_inner(), Ordering::SeqCst)
+                .max(new_ts.into_inner()) as i64,
+        );
+        Ok(())
+    }
+
+    // new_ts is greater than limit, or the approximate limit.
+    // To avoid false positive and guarantee TiKV availability, we need to
+    // double-check the new_ts with PD TSO.
+    fn double_check(
+        &self,
+        new_ts: TimeStamp,
+        limit: TimeStamp,
+        source: impl slog::Value + Display,
+        using_approximate: bool,
+    ) -> Result<(), crate::InvalidMaxTsUpdate> {
+        warn!("possible invalid max_ts update; double checking";
+            "attempted_ts" => new_ts,
+            "limit" => limit.into_inner(),
+            "source" => &source,
+            "using_approximate" => using_approximate,
+            "TSO_TIMEOUT" => ?TSO_TIMEOUT,
+        );
+        let mut tso_confirmed = false;
+        if let Some(tso) = &self.tso {
+            match block_on_timeout(tso.get_tso(), TSO_TIMEOUT) {
+                Ok(Ok(ts)) => {
+                    self.set_max_ts_limit(ts);
+                    tso_confirmed = true;
+                }
+                Ok(Err(e)) => {
+                    error!("failed to fetch from TSO for double checking"; "err" => ?e);
+                }
+                Err(()) => {
+                    error!("timeout when fetching from TSO for double checking"; "timeout" => ?TSO_TIMEOUT);
+                }
+            }
+        }
+        let new_limit = self.max_ts_limit.load();
+        if new_ts > new_limit.limit {
+            self.report_error(new_ts, new_limit, source, using_approximate, tso_confirmed)?;
+        }
+        Ok(())
+    }
+
+    fn report_error(
+        &self,
+        new_ts: TimeStamp,
+        limit: MaxTsLimit,
+        source: impl slog::Value + Display,
+        using_approximate: bool,
+        tso_confirmed: bool,
+    ) -> Result<(), InvalidMaxTsUpdate> {
+        if tso_confirmed {
+            error!("invalid max_ts update";
+                "attempted_ts" => new_ts,
+                "limit" => limit.limit.into_inner(),
+                "limit_update_time" => ?limit.update_time,
+                "source" => &source,
+                "using_approximate" => using_approximate,
+            );
+        } else {
+            warn!("possible invalid max_ts update";
+                "attempted_ts" => new_ts,
+                "limit" => limit.limit.into_inner(),
+                "limit_update_time" => ?limit.update_time,
+                "source" => &source,
+                "using_approximate" => using_approximate,
+            );
+        }
+
+        match self.action_on_invalid_max_ts.load() {
+            ActionOnInvalidMaxTs::Panic if tso_confirmed => {
+                panic!(
+                    "invalid max_ts update: {} exceeds the limit {}, source={}",
+                    new_ts,
+                    limit.limit.into_inner(),
+                    source
+                );
+            }
+            ActionOnInvalidMaxTs::Error if tso_confirmed => Err(InvalidMaxTsUpdate {
+                attempted_ts: new_ts,
+                limit: limit.limit,
+            }),
+            _ => Ok(()),
+        }
+    }
+
+    /// Set the maximum allowed value for max_ts updates, except for the updates
+    /// from PD TSO. The limit must be updated regularly to prevent the
+    /// blocking of max_ts. It prevents max_ts from being updated to an
+    /// unreasonable value, which is usually caused by bugs or unsafe
+    /// usages.
+    ///
+    /// # Note
+    /// If the new limit is smaller than the current limit, this operation will
+    /// have no effect and return silently.
+    pub fn set_max_ts_limit(&self, ts_from_tso: TimeStamp) {
+        if ts_from_tso.is_max() {
+            error!("max_ts_limit cannot be set to u64::max");
+            return;
+        }
+
+        let limit = TimeStamp::compose(
+            ts_from_tso.physical() + self.max_ts_drift_allowance_ms.load(Ordering::SeqCst),
+            ts_from_tso.logical(),
+        );
+
+        loop {
+            let current = self.max_ts_limit.load();
+
+            if limit.into_inner() <= current.limit.into_inner() {
+                break;
+            }
+
+            let new_state = MaxTsLimit {
+                limit,
+                update_time: self.time_provider.now(),
+            };
+
+            match self.max_ts_limit.compare_exchange(current, new_state) {
+                Ok(_) => {
+                    MAX_TS_LIMIT_GAUGE.set(limit.into_inner() as i64);
+                    break;
+                }
+                Err(_) => {
+                    continue;
+                }
+            }
         }
     }
 
@@ -141,14 +451,200 @@ impl ConcurrencyManager {
         });
         min_lock
     }
+
+    pub fn set_action_on_invalid_max_ts(&self, action: ActionOnInvalidMaxTs) {
+        self.action_on_invalid_max_ts.store(action);
+    }
+
+    pub fn set_max_ts_drift_allowance(&self, allowance: Duration) {
+        self.max_ts_drift_allowance_ms
+            .store(allowance.as_millis() as u64, Ordering::SeqCst);
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum ActionOnInvalidMaxTs {
+    Panic,
+    Error,
+    Log,
+}
+
+#[derive(Debug)]
+pub struct ParseActionError(String);
+
+impl fmt::Display for ParseActionError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl Error for ParseActionError {}
+
+impl TryFrom<&str> for ActionOnInvalidMaxTs {
+    type Error = ParseActionError;
+
+    fn try_from(value: &str) -> Result<Self, ParseActionError> {
+        match value.to_lowercase().as_str() {
+            "panic" => Ok(Self::Panic),
+            "error" => Ok(Self::Error),
+            "log" => Ok(Self::Log),
+            _ => Err(ParseActionError(format!("invalid action value: {}", value))),
+        }
+    }
+}
+
+impl TryFrom<String> for ActionOnInvalidMaxTs {
+    type Error = ParseActionError;
+
+    fn try_from(value: String) -> Result<Self, ParseActionError> {
+        Self::try_from(value.as_str())
+    }
+}
+
+pub struct AtomicActionOnInvalidMaxTs {
+    inner: AtomicUsize,
+}
+
+impl AtomicActionOnInvalidMaxTs {
+    pub fn new(initial: ActionOnInvalidMaxTs) -> Self {
+        Self {
+            inner: AtomicUsize::new(match initial {
+                ActionOnInvalidMaxTs::Panic => 0,
+                ActionOnInvalidMaxTs::Error => 1,
+                ActionOnInvalidMaxTs::Log => 2,
+            }),
+        }
+    }
+
+    pub fn store(&self, value: ActionOnInvalidMaxTs) {
+        self.inner.store(
+            match value {
+                ActionOnInvalidMaxTs::Panic => 0,
+                ActionOnInvalidMaxTs::Error => 1,
+                ActionOnInvalidMaxTs::Log => 2,
+            },
+            Ordering::SeqCst,
+        );
+    }
+
+    pub fn load(&self) -> ActionOnInvalidMaxTs {
+        match self.inner.load(Ordering::SeqCst) {
+            0 => ActionOnInvalidMaxTs::Panic,
+            1 => ActionOnInvalidMaxTs::Error,
+            2 => ActionOnInvalidMaxTs::Log,
+            _ => unreachable!("Invalid atomic state"),
+        }
+    }
+}
+
+#[derive(Debug, Error, Clone)]
+#[error("invalid max_ts update: {attempted_ts} exceeds the limit {limit}")]
+pub struct InvalidMaxTsUpdate {
+    pub attempted_ts: TimeStamp,
+    pub limit: TimeStamp,
+}
+
+pub trait ValueDisplay: slog::Value + Display {}
+impl ValueDisplay for String {}
+impl ValueDisplay for &str {}
+
+mod sealed {
+    pub trait Sealed {}
+}
+
+pub trait IntoErrorSource: sealed::Sealed {
+    type Output: ValueDisplay;
+    fn into_error_source(self) -> Self::Output;
+}
+
+// &str impl
+impl<'a> sealed::Sealed for &'a str {}
+impl<'a> IntoErrorSource for &'a str {
+    type Output = &'a str;
+    fn into_error_source(self) -> Self::Output {
+        self
+    }
+}
+
+// String impl
+impl sealed::Sealed for String {}
+impl IntoErrorSource for String {
+    type Output = String;
+    fn into_error_source(self) -> Self::Output {
+        self
+    }
+}
+
+// Closure impl
+impl<F, T> sealed::Sealed for F
+where
+    F: FnOnce() -> T,
+    T: ValueDisplay,
+{
+}
+impl<F, T> IntoErrorSource for F
+where
+    F: FnOnce() -> T,
+    T: ValueDisplay,
+{
+    type Output = T;
+    fn into_error_source(self) -> T {
+        self()
+    }
+}
+
+/// Trait to abstract time-related functionality, for a monotonic clock
+trait TimeProvider: Send + Sync {
+    /// Returns the current instant.
+    fn now(&self) -> Instant;
+}
+
+struct CoarseInstantTimeProvider;
+
+impl TimeProvider for CoarseInstantTimeProvider {
+    fn now(&self) -> Instant {
+        Instant::now_coarse()
+    }
 }
 
 #[cfg(test)]
 mod tests {
+    use std::{future::ready, sync::Mutex};
+
+    use futures::FutureExt;
     use txn_types::LockType;
 
     use super::*;
 
+    #[derive(Clone)]
+    struct StubTimeProvider {
+        current_time: Arc<Mutex<Instant>>,
+    }
+
+    impl StubTimeProvider {
+        /// Creates a new MockTimeProvider initialized with the given instant.
+        fn new(start_time: Instant) -> Self {
+            StubTimeProvider {
+                current_time: Arc::new(Mutex::new(start_time)),
+            }
+        }
+
+        /// Advances the current time by the specified duration.
+        fn advance(&self, duration: Duration) {
+            let mut time = self.current_time.lock().unwrap();
+            // Note: Instant doesn't support addition, so we mock behavior.
+            // This simplistic approach assumes no overflow.
+            *time += duration;
+        }
+    }
+
+    impl TimeProvider for StubTimeProvider {
+        fn now(&self) -> Instant {
+            let time = self.current_time.lock().unwrap();
+            *time
+        }
+    }
+
     #[tokio::test]
     async fn test_lock_keys_order() {
         let concurrency_manager = ConcurrencyManager::new(1.into());
@@ -166,13 +662,13 @@ mod tests {
     #[tokio::test]
     async fn test_update_max_ts() {
         let concurrency_manager = ConcurrencyManager::new(10.into());
-        concurrency_manager.update_max_ts(20.into());
+        let _ = concurrency_manager.update_max_ts(20.into(), "");
         assert_eq!(concurrency_manager.max_ts(), 20.into());
 
-        concurrency_manager.update_max_ts(5.into());
+        let _ = concurrency_manager.update_max_ts(5.into(), "");
         assert_eq!(concurrency_manager.max_ts(), 20.into());
 
-        concurrency_manager.update_max_ts(TimeStamp::max());
+        let _ = concurrency_manager.update_max_ts(TimeStamp::max(), "");
         assert_eq!(concurrency_manager.max_ts(), 20.into());
     }
 
@@ -224,4 +720,217 @@ mod tests {
             assert_eq!(concurrency_manager.global_min_lock_ts(), Some(20.into()));
         }
     }
+
+    #[test]
+    fn test_max_ts_limit() {
+        let mut stub_pd = MockTSOProvider::new();
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(160.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Error,
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        // Initially limit should be 0
+        cm.update_max_ts(TimeStamp::new(150), "").unwrap();
+
+        // Set initial limit to 200
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        // Try to lower limit to 150 - should be ignored
+        cm.set_max_ts_limit(TimeStamp::new(150));
+        cm.update_max_ts(TimeStamp::new(180), "").unwrap(); // Should still work up to 200
+
+        // Increase limit to 300 - should work
+        cm.set_max_ts_limit(TimeStamp::new(300));
+        cm.update_max_ts(TimeStamp::new(250), "").unwrap();
+    }
+
+    #[test]
+    fn test_max_ts_limit_edge_cases() {
+        let cm = ConcurrencyManager::new(TimeStamp::new(100));
+
+        // Test transition from zero limit
+        assert_eq!(cm.max_ts_limit.load().limit, 0.into());
+        cm.set_max_ts_limit(TimeStamp::new(1000));
+        assert_eq!(cm.max_ts_limit.load().limit, 1000.into());
+
+        // Try to lower from 1000 to 500 - should be ignored
+        cm.set_max_ts_limit(TimeStamp::new(500));
+        assert_eq!(cm.max_ts_limit.load().limit, 1000.into());
+
+        // Test setting limit to max, should have no effect
+        cm.set_max_ts_limit(TimeStamp::max());
+        assert_eq!(cm.max_ts_limit.load().limit, 1000.into());
+    }
+
+    #[test]
+    fn test_max_ts_updates_with_monotonic_limit() {
+        let mut stub_pd = MockTSOProvider::new();
+        // Assertion: should fail to update max_ts to 250 and query PD
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(201.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Error,
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        // Set limit to 200
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        // Update max_ts to 150
+        cm.update_max_ts(TimeStamp::new(150), "").unwrap();
+        assert_eq!(cm.max_ts(), TimeStamp::new(150));
+
+        // Try to lower limit to 180 - should be ignored
+        cm.set_max_ts_limit(TimeStamp::new(180));
+
+        // Should still fail for values above 200
+        let result = cm.update_max_ts(TimeStamp::new(250), "");
+        assert!(result.is_err());
+        if let Err(e) = result {
+            assert_eq!(e.attempted_ts, TimeStamp::new(250));
+            assert_eq!(e.limit, TimeStamp::new(201));
+        }
+    }
+
+    #[test]
+    fn test_limit_valid_duration_boundary() {
+        let start_time = Instant::now();
+        let stub_time = StubTimeProvider::new(start_time);
+        let time_provider = Arc::new(stub_time.clone());
+        let mut stub_pd = MockTSOProvider::new();
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(200.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+
+        let cm = ConcurrencyManager::new_with_time_provider(
+            TimeStamp::new(100),
+            Duration::from_secs(60),
+            ActionOnInvalidMaxTs::Error,
+            time_provider.clone(),
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        time_provider.advance(Duration::from_secs(59));
+
+        time_provider.advance(Duration::from_secs(1));
+        cm.update_max_ts(TimeStamp::new(250), "").unwrap();
+        assert_eq!(cm.max_ts().into_inner(), 250);
+    }
+
+    #[test]
+    fn test_max_ts_limit_expired_allows_update() {
+        let start_time = Instant::now();
+        let stub_time = StubTimeProvider::new(start_time);
+        let time_provider = Arc::new(stub_time.clone());
+
+        let cm = ConcurrencyManager::new_with_time_provider(
+            TimeStamp::new(100),
+            Duration::from_secs(60),
+            ActionOnInvalidMaxTs::Error,
+            time_provider.clone(),
+            None,
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        stub_time.advance(Duration::from_secs(61));
+
+        // Updating to 250 should be allowed, since the limit should be invalidated
+        cm.update_max_ts(TimeStamp::new(250), "test_source".to_string())
+            .unwrap();
+        assert_eq!(cm.max_ts().into_inner(), 250);
+    }
+
+    #[test]
+    #[should_panic(expected = "invalid max_ts update")]
+    fn test_panic_on_invalid_max_ts_enabled() {
+        let mut stub_pd = MockTSOProvider::new();
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(201.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            Some(stub_pd),
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+
+        // should not panic
+        cm.update_max_ts(TimeStamp::new(250), "test_source".to_string())
+            .unwrap();
+    }
+
+    #[test]
+    fn test_update_max_ts_without_limit() {
+        let cm = ConcurrencyManager::new(TimeStamp::new(100));
+
+        cm.update_max_ts(TimeStamp::new(500), "test_source".to_string())
+            .unwrap();
+        assert_eq!(cm.max_ts().into_inner(), 500);
+    }
+
+    #[test]
+    fn test_pd_tso_jump_not_panic() {
+        let mut stub_pd = MockTSOProvider::new();
+        // The double check procedure gets latest_ts=300 from TSO
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| ready(Ok(300.into())).boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            Some(stub_pd.clone()),
+            Duration::ZERO,
+        );
+
+        cm.set_max_ts_limit(TimeStamp::new(200));
+        // PD TSO jumps from 100 to 300
+        cm.update_max_ts(TimeStamp::new(300), "test_source".to_string())
+            .unwrap();
+    }
+
+    #[test]
+    fn test_do_not_panic_under_pd_tso_jump_and_network_partition() {
+        let mut stub_pd = MockTSOProvider::new();
+        // Network partition between PD and TiKV
+        stub_pd
+            .expect_get_tso()
+            .return_once(|| std::future::pending().boxed());
+        let stub_pd = Arc::new(stub_pd);
+        let cm = ConcurrencyManager::new_with_config(
+            TimeStamp::new(100),
+            DEFAULT_LIMIT_VALID_DURATION,
+            ActionOnInvalidMaxTs::Panic,
+            Some(stub_pd.clone()),
+            Duration::ZERO,
+        );
+        cm.set_max_ts_limit(200.into());
+
+        // PD TSO jumps from 100 to 300
+        cm.update_max_ts(TimeStamp::new(300), "test_source".to_string())
+            .unwrap();
+    }
 }
diff --git a/components/error_code/src/coprocessor.rs b/components/error_code/src/coprocessor.rs
index 31f0ed4224a..dfa4749795f 100644
--- a/components/error_code/src/coprocessor.rs
+++ b/components/error_code/src/coprocessor.rs
@@ -16,5 +16,7 @@ define_error_codes!(
     CORRUPTED_DATA => ("CorruptedData", "", ""),
 
     STORAGE_ERROR => ("StorageError", "", ""),
-    INVALID_CHARACTER_STRING => ("InvalidCharacterString", "", "")
+    INVALID_CHARACTER_STRING => ("InvalidCharacterString", "", ""),
+
+    INVALID_MAX_TS_UPDATE => ("InvalidMaxTsUpdate", "", "")
 );
diff --git a/components/error_code/src/storage.rs b/components/error_code/src/storage.rs
index e7caefb1b65..33cdc7b116a 100644
--- a/components/error_code/src/storage.rs
+++ b/components/error_code/src/storage.rs
@@ -25,6 +25,7 @@ define_error_codes!(
     DEADLINE_EXCEEDED => ("DeadlineExceeded", "", ""),
     API_VERSION_NOT_MATCHED => ("ApiVersionNotMatched", "", ""),
     INVALID_KEY_MODE => ("InvalidKeyMode", "", ""),
+    INVALID_MAX_TS_UPDATE => ("InvalidMaxTsUpdate", "", ""),
 
     COMMITTED => ("Committed", "", ""),
     PESSIMISTIC_LOCK_ROLLED_BACK => ("PessimisticLockRolledBack", "", ""),
diff --git a/components/raftstore-v2/src/worker/pd/misc.rs b/components/raftstore-v2/src/worker/pd/misc.rs
index 6ade8d87de5..fd2848b95a4 100644
--- a/components/raftstore-v2/src/worker/pd/misc.rs
+++ b/components/raftstore-v2/src/worker/pd/misc.rs
@@ -12,7 +12,6 @@ use pd_client::PdClient;
 use raftstore::{store::TxnExt, Result};
 use slog::{info, warn};
 use tikv_util::{box_err, timer::GLOBAL_TIMER_HANDLE};
-use txn_types::TimeStamp;
 
 use super::Runner;
 
@@ -51,18 +50,22 @@ where
                 // And it won't break correctness of transaction commands, as
                 // causal_ts_provider.flush() is implemented as
                 // pd_client.get_tso() + renew TSO cached.
-                let res: Result<TimeStamp> = if let Some(causal_ts_provider) = &causal_ts_provider {
+                let res: Result<()> = if let Some(causal_ts_provider) = &causal_ts_provider {
                     causal_ts_provider
                         .async_flush()
                         .await
                         .map_err(|e| box_err!(e))
                 } else {
                     pd_client.get_tso().await.map_err(Into::into)
-                };
+                }
+                .and_then(|ts| {
+                    concurrency_manager
+                        .update_max_ts(ts, "raftstore-v2")
+                        .map_err(|e| crate::Error::Other(box_err!(e)))
+                });
 
                 match res {
-                    Ok(ts) => {
-                        concurrency_manager.update_max_ts(ts);
+                    Ok(()) => {
                         success = txn_ext
                             .max_ts_sync_status
                             .compare_exchange(
diff --git a/components/raftstore/src/store/peer.rs b/components/raftstore/src/store/peer.rs
index 57ed792de85..f6aabc8d56b 100644
--- a/components/raftstore/src/store/peer.rs
+++ b/components/raftstore/src/store/peer.rs
@@ -5907,7 +5907,7 @@ where
             txn_ext: self.txn_ext.clone(),
         }) {
             error!(
-                "failed to update max ts";
+                "failed to update max_ts";
                 "err" => ?e,
             );
         }
diff --git a/components/raftstore/src/store/worker/pd.rs b/components/raftstore/src/store/worker/pd.rs
index 16c4fec66f0..5909d825888 100644
--- a/components/raftstore/src/store/worker/pd.rs
+++ b/components/raftstore/src/store/worker/pd.rs
@@ -52,7 +52,6 @@ use tikv_util::{
     warn,
     worker::{Runnable, ScheduleError, Scheduler},
 };
-use txn_types::TimeStamp;
 use yatp::Remote;
 
 use super::split_controller::AutoSplitControllerContext;
@@ -1676,7 +1675,7 @@ where
                 // And it won't break correctness of transaction commands, as
                 // causal_ts_provider.flush() is implemented as pd_client.get_tso() + renew TSO
                 // cached.
-                let res: crate::Result<TimeStamp> =
+                let res: crate::Result<()> =
                     if let Some(causal_ts_provider) = &causal_ts_provider {
                         causal_ts_provider
                             .async_flush()
@@ -1684,11 +1683,15 @@ where
                             .map_err(|e| box_err!(e))
                     } else {
                         pd_client.get_tso().await.map_err(Into::into)
-                    };
+                    }
+                    .and_then(|ts| {
+                        concurrency_manager
+                            .update_max_ts(ts, "raftstore")
+                            .map_err(|e| crate::Error::Other(box_err!(e)))
+                    });
 
                 match res {
-                    Ok(ts) => {
-                        concurrency_manager.update_max_ts(ts);
+                    Ok(()) => {
                         // Set the least significant bit to 1 to mark it as synced.
                         success = txn_ext
                             .max_ts_sync_status
diff --git a/components/resolved_ts/src/advance.rs b/components/resolved_ts/src/advance.rs
index 2667a43a304..9b7def6d8d8 100644
--- a/components/resolved_ts/src/advance.rs
+++ b/components/resolved_ts/src/advance.rs
@@ -117,7 +117,10 @@ impl AdvanceTsWorker {
             // optimizations like async commit is enabled.
             // Note: This step must be done before scheduling `Task::MinTs` task, and the
             // resolver must be checked in or after `Task::MinTs`' execution.
-            cm.update_max_ts(min_ts);
+            if let Err(e) = cm.update_max_ts(min_ts, "resolved-ts") {
+                error!("failed to advance resolved_ts: failed to update max_ts in concurrency manager"; "err" => ?e);
+                return;
+            }
             if let Some((min_mem_lock_ts, lock)) = cm.global_min_lock() {
                 if min_mem_lock_ts < min_ts {
                     min_ts = min_mem_lock_ts;
diff --git a/components/server/src/server.rs b/components/server/src/server.rs
index 24b3137a11a..77025b2c7cc 100644
--- a/components/server/src/server.rs
+++ b/components/server/src/server.rs
@@ -28,7 +28,10 @@ use backup_stream::{
 };
 use causal_ts::CausalTsProviderImpl;
 use cdc::CdcConfigManager;
-use concurrency_manager::ConcurrencyManager;
+use concurrency_manager::{
+    ActionOnInvalidMaxTs, ConcurrencyManager, DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+    DEFAULT_MAX_TS_SYNC_INTERVAL, LIMIT_VALID_TIME_MULTIPLIER,
+};
 use engine_rocks::{
     from_rocks_compression_type, RocksCompactedEvent, RocksEngine, RocksStatistics,
 };
@@ -178,6 +181,7 @@ fn run_impl<CER, F>(
     tikv.init_metrics_flusher(fetcher, engines_info);
     tikv.init_cgroup_monitor();
     tikv.init_storage_stats_task(engines);
+    tikv.init_max_ts_updater();
     tikv.run_server(server_config);
     tikv.run_status_server(in_memory_engine);
     tikv.core.init_quota_tuning_task(tikv.quota_limiter.clone());
@@ -401,7 +405,13 @@ where
 
         // Initialize concurrency manager
         let latest_ts = block_on(pd_client.get_tso()).expect("failed to get timestamp from PD");
-        let concurrency_manager = ConcurrencyManager::new(latest_ts);
+        let concurrency_manager = ConcurrencyManager::new_with_config(
+            latest_ts,
+            DEFAULT_MAX_TS_SYNC_INTERVAL * LIMIT_VALID_TIME_MULTIPLIER,
+            ActionOnInvalidMaxTs::Log,
+            Some(pd_client.clone()),
+            DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+        );
 
         // use different quota for front-end and back-end requests
         let quota_limiter = Arc::new(QuotaLimiter::new(
@@ -704,6 +714,7 @@ where
                 ttl_scheduler,
                 flow_controller,
                 storage.get_scheduler(),
+                storage.get_concurrency_manager(),
             )),
         );
 
@@ -1133,7 +1144,7 @@ where
         // Create Debugger.
         let mut debugger = DebuggerImpl::new(
             Engines::new(engines.engines.kv.clone(), engines.engines.raft.clone()),
-            self.cfg_controller.as_ref().unwrap().clone(),
+            cfg_controller.clone(),
             Some(storage),
         );
         debugger.set_kv_statistics(self.kv_statistics.clone());
@@ -1154,6 +1165,28 @@ where
         server_config
     }
 
+    fn init_max_ts_updater(&self) {
+        let cm = self.concurrency_manager.clone();
+        let pd_client = self.pd_client.clone();
+
+        let max_ts_sync_interval = DEFAULT_MAX_TS_SYNC_INTERVAL;
+        self.core
+            .background_worker
+            .spawn_interval_async_task(max_ts_sync_interval, move || {
+                let cm = cm.clone();
+                let pd_client = pd_client.clone();
+
+                async move {
+                    let pd_tso = pd_client.get_tso().await;
+                    if let Ok(ts) = pd_tso {
+                        cm.set_max_ts_limit(ts);
+                    } else {
+                        warn!("failed to get tso from pd in background, the max_ts validity check could be skipped");
+                    }
+                }
+            });
+    }
+
     fn register_services(&mut self) {
         let servers = self.servers.as_mut().unwrap();
         let engines = self.engines.as_ref().unwrap();
diff --git a/components/server/src/server2.rs b/components/server/src/server2.rs
index 996146f8a2a..91426e1c2f4 100644
--- a/components/server/src/server2.rs
+++ b/components/server/src/server2.rs
@@ -33,7 +33,10 @@ use backup_stream::{
 };
 use causal_ts::CausalTsProviderImpl;
 use cdc::CdcConfigManager;
-use concurrency_manager::ConcurrencyManager;
+use concurrency_manager::{
+    ActionOnInvalidMaxTs, ConcurrencyManager, DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+    DEFAULT_MAX_TS_SYNC_INTERVAL, LIMIT_VALID_TIME_MULTIPLIER,
+};
 use engine_rocks::{from_rocks_compression_type, RocksEngine, RocksStatistics};
 use engine_traits::{Engines, KvEngine, MiscExt, RaftEngine, TabletRegistry, CF_DEFAULT, CF_WRITE};
 use file_system::{get_io_rate_limiter, BytesFetcher, MetricsManager as IoMetricsManager};
@@ -162,6 +165,7 @@ fn run_impl<CER: ConfiguredRaftEngine, F: KvFormat>(
     tikv.init_metrics_flusher(fetcher, engines_info);
     tikv.init_cgroup_monitor();
     tikv.init_storage_stats_task();
+    tikv.init_max_ts_updater();
     tikv.run_server(server_config);
     tikv.run_status_server();
     tikv.core.init_quota_tuning_task(tikv.quota_limiter.clone());
@@ -325,7 +329,13 @@ where
 
         // Initialize concurrency manager
         let latest_ts = block_on(pd_client.get_tso()).expect("failed to get timestamp from PD");
-        let concurrency_manager = ConcurrencyManager::new(latest_ts);
+        let concurrency_manager = ConcurrencyManager::new_with_config(
+            latest_ts,
+            DEFAULT_MAX_TS_SYNC_INTERVAL * LIMIT_VALID_TIME_MULTIPLIER,
+            ActionOnInvalidMaxTs::Log,
+            Some(pd_client.clone()),
+            DEFAULT_MAX_TS_DRIFT_ALLOWANCE,
+        );
 
         // use different quota for front-end and back-end requests
         let quota_limiter = Arc::new(QuotaLimiter::new(
@@ -586,6 +596,7 @@ where
                 ttl_scheduler,
                 flow_controller,
                 storage.get_scheduler(),
+                storage.get_concurrency_manager(),
             )),
         );
 
@@ -947,6 +958,28 @@ where
         server_config
     }
 
+    fn init_max_ts_updater(&self) {
+        let cm = self.concurrency_manager.clone();
+        let pd_client = self.pd_client.clone();
+
+        let max_ts_sync_interval = DEFAULT_MAX_TS_SYNC_INTERVAL;
+        self.core
+            .background_worker
+            .spawn_interval_async_task(max_ts_sync_interval, move || {
+                let cm = cm.clone();
+                let pd_client = pd_client.clone();
+
+                async move {
+                    let pd_tso = pd_client.get_tso().await;
+                    if let Ok(ts) = pd_tso {
+                        cm.set_max_ts_limit(ts);
+                    } else {
+                        warn!("failed to get tso from pd in background");
+                    }
+                }
+            });
+    }
+
     fn register_services(&mut self) {
         let servers = self.servers.as_mut().unwrap();
         let engines = self.engines.as_ref().unwrap();
diff --git a/components/tikv_util/src/time.rs b/components/tikv_util/src/time.rs
index efab331e9f1..c94373ae4e2 100644
--- a/components/tikv_util/src/time.rs
+++ b/components/tikv_util/src/time.rs
@@ -279,7 +279,7 @@ mod inner {
 /// A measurement of a monotonically increasing clock.
 /// It's similar and meant to replace `std::time::Instant`,
 /// for providing extra features.
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Eq)]
 pub enum Instant {
     Monotonic(Timespec),
     MonotonicCoarse(Timespec),
diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py
index d647293b38e..497e97fe9ba 100644
--- a/metrics/grafana/tikv_details.dashboard.py
+++ b/metrics/grafana/tikv_details.dashboard.py
@@ -3133,6 +3133,25 @@ def Storage() -> RowPanel:
             ),
         ]
     )
+    layout.row(
+        [
+            graph_panel(
+                title="Concurrency manager max-ts",
+                description="The max_ts in the concurrency manager",
+                yaxes=yaxes(left_format=UNITS.NONE_FORMAT),
+                targets=[
+                    target(
+                        expr="tikv_concurrency_manager_max_ts_limit",
+                        legend_format="max_ts_limit",
+                    ),
+                    target(
+                        expr="tikv_concurrency_manager_max_ts",
+                        legend_format="max_ts",
+                    ),
+                ],
+            )
+        ]
+    )
     return layout.row_panel
 
 
diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json
index 2dce52be854..7dd84579fca 100644
--- a/metrics/grafana/tikv_details.json
+++ b/metrics/grafana/tikv_details.json
@@ -10488,55 +10488,13 @@
             "align": false,
             "alignLevel": 0
           }
-        }
-      ],
-      "repeat": null,
-      "repeatDirection": null,
-      "span": null,
-      "targets": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Storage",
-      "transformations": [],
-      "transparent": false,
-      "type": "row"
-    },
-    {
-      "cacheTimeout": null,
-      "collapsed": true,
-      "datasource": null,
-      "description": null,
-      "editable": true,
-      "error": false,
-      "fieldConfig": {
-        "defaults": {
-          "thresholds": {
-            "mode": "absolute",
-            "steps": []
-          }
-        }
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "height": null,
-      "hideTimeOverride": false,
-      "id": 77,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
-      "maxPerRow": null,
-      "minSpan": null,
-      "panels": [
+        },
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": null,
+          "description": "The max_ts in the concurrency manager",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -10559,11 +10517,11 @@
             "h": 7,
             "w": 24,
             "x": 0,
-            "y": 0
+            "y": 56
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 78,
+          "id": 77,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10599,62 +10557,37 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [
-            {
-              "alias": "/.*-total/",
-              "bars": false,
-              "fill": 1,
-              "fillBelowTo": null,
-              "lines": true,
-              "yaxis": 2,
-              "zindex": 0
-            }
-          ],
+          "seriesOverrides": [],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
+              "expr": "tikv_concurrency_manager_max_ts_limit",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}-reject-by-{{reason}}",
+              "legendFormat": "max_ts_limit",
               "metric": "",
-              "query": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
+              "query": "tikv_concurrency_manager_max_ts_limit",
               "refId": "",
               "step": 10,
               "target": ""
             },
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "tikv_concurrency_manager_max_ts",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}-total",
+              "legendFormat": "max_ts",
               "metric": "",
-              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
-              "refId": "",
-              "step": 10,
-              "target": ""
-            },
-            {
-              "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
-              "format": "time_series",
-              "hide": false,
-              "instant": false,
-              "interval": "",
-              "intervalFactor": 1,
-              "legendFormat": "{{instance}}-stale-read",
-              "metric": "",
-              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "tikv_concurrency_manager_max_ts",
               "refId": "",
               "step": 10,
               "target": ""
@@ -10663,7 +10596,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Local reader requests",
+          "title": "Concurrency manager max-ts",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -10711,7 +10644,7 @@
       "targets": [],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Local Reader",
+      "title": "Storage",
       "transformations": [],
       "transparent": false,
       "type": "row"
@@ -10739,7 +10672,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 79,
+      "id": 78,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -10751,7 +10684,7 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of raftstore thread",
+          "description": null,
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -10772,13 +10705,13 @@
           },
           "gridPos": {
             "h": 7,
-            "w": 12,
+            "w": 24,
             "x": 0,
             "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 80,
+          "id": 79,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -10814,22 +10747,62 @@
           "renderer": "flot",
           "repeat": null,
           "repeatDirection": null,
-          "seriesOverrides": [],
+          "seriesOverrides": [
+            {
+              "alias": "/.*-total/",
+              "bars": false,
+              "fill": 1,
+              "fillBelowTo": null,
+              "lines": true,
+              "yaxis": 2,
+              "zindex": 0
+            }
+          ],
           "span": null,
           "stack": false,
           "steppedLine": false,
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{instance}}",
+              "legendFormat": "{{instance}}-reject-by-{{reason}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_raftstore_local_read_reject_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance, reason) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-total",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            },
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-stale-read",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_raftstore_local_read_executed_stale_read_requests\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -10838,7 +10811,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Raft store CPU",
+          "title": "Local reader requests",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -10857,7 +10830,7 @@
           "yaxes": [
             {
               "decimals": null,
-              "format": "percentunit",
+              "format": "none",
               "label": null,
               "logBase": 1,
               "max": null,
@@ -10878,13 +10851,55 @@
             "align": false,
             "alignLevel": 0
           }
-        },
+        }
+      ],
+      "repeat": null,
+      "repeatDirection": null,
+      "span": null,
+      "targets": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Local Reader",
+      "transformations": [],
+      "transparent": false,
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "collapsed": true,
+      "datasource": null,
+      "description": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "height": null,
+      "hideTimeOverride": false,
+      "id": 80,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": null,
+      "minSpan": null,
+      "panels": [
         {
           "aliasColors": {},
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of async apply",
+          "description": "The CPU utilization of raftstore thread",
           "editable": true,
           "error": false,
           "fieldConfig": {
@@ -10906,7 +10921,7 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 0
           },
           "height": null,
@@ -10954,7 +10969,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -10962,7 +10977,7 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
@@ -10971,7 +10986,7 @@
           "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Async apply CPU",
+          "title": "Raft store CPU",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -11017,23 +11032,14 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of store writer thread",
+          "description": "The CPU utilization of async apply",
           "editable": true,
           "error": false,
           "fieldConfig": {
             "defaults": {
               "thresholds": {
                 "mode": "absolute",
-                "steps": [
-                  {
-                    "colorMode": "critical",
-                    "fill": true,
-                    "line": true,
-                    "op": "gt",
-                    "value": 0.8,
-                    "yaxis": "left"
-                  }
-                ]
+                "steps": []
               }
             }
           },
@@ -11048,8 +11054,8 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 0,
-            "y": 7
+            "x": 12,
+            "y": 0
           },
           "height": null,
           "hideTimeOverride": false,
@@ -11096,7 +11102,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -11104,25 +11110,16 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             }
           ],
-          "thresholds": [
-            {
-              "colorMode": "critical",
-              "fill": true,
-              "line": true,
-              "op": "gt",
-              "value": 0.8,
-              "yaxis": "left"
-            }
-          ],
+          "thresholds": [],
           "timeFrom": null,
           "timeShift": null,
-          "title": "Store writer CPU",
+          "title": "Async apply CPU",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -11168,14 +11165,23 @@
           "bars": false,
           "cacheTimeout": null,
           "datasource": "${DS_TEST-CLUSTER}",
-          "description": "The CPU utilization of gRPC",
+          "description": "The CPU utilization of store writer thread",
           "editable": true,
           "error": false,
           "fieldConfig": {
             "defaults": {
               "thresholds": {
                 "mode": "absolute",
-                "steps": []
+                "steps": [
+                  {
+                    "colorMode": "critical",
+                    "fill": true,
+                    "line": true,
+                    "op": "gt",
+                    "value": 0.8,
+                    "yaxis": "left"
+                  }
+                ]
               }
             }
           },
@@ -11190,7 +11196,7 @@
           "gridPos": {
             "h": 7,
             "w": 12,
-            "x": 12,
+            "x": 0,
             "y": 7
           },
           "height": null,
@@ -11238,7 +11244,7 @@
           "targets": [
             {
               "datasource": "${DS_TEST-CLUSTER}",
-              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "format": "time_series",
               "hide": false,
               "instant": false,
@@ -11246,16 +11252,158 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}",
               "metric": "",
-              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n    [$__rate_interval]\n)) by (instance) ",
               "refId": "",
               "step": 10,
               "target": ""
             }
           ],
-          "thresholds": [],
+          "thresholds": [
+            {
+              "colorMode": "critical",
+              "fill": true,
+              "line": true,
+              "op": "gt",
+              "value": 0.8,
+              "yaxis": "left"
+            }
+          ],
           "timeFrom": null,
           "timeShift": null,
-          "title": "gRPC poll CPU",
+          "title": "Store writer CPU",
+          "tooltip": {
+            "msResolution": true,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transformations": [],
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "percentunit",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "decimals": null,
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": 0
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "datasource": "${DS_TEST-CLUSTER}",
+          "description": "The CPU utilization of gRPC",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": []
+              }
+            }
+          },
+          "fill": 1,
+          "fillGradient": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 7
+          },
+          "height": null,
+          "hideTimeOverride": false,
+          "id": 84,
+          "interval": null,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": true,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": true,
+            "show": true,
+            "sideWidth": null,
+            "sort": "max",
+            "sortDesc": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "maxDataPoints": null,
+          "maxPerRow": null,
+          "minSpan": null,
+          "nullPointMode": "null as zero",
+          "options": {
+            "alertThreshold": true,
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "repeat": null,
+          "repeatDirection": null,
+          "seriesOverrides": [],
+          "span": null,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": "${DS_TEST-CLUSTER}",
+              "expr": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}",
+              "metric": "",
+              "query": "sum(rate(\n    tikv_thread_cpu_seconds_total\n    {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n    [$__rate_interval]\n)) by (instance) ",
+              "refId": "",
+              "step": 10,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "gRPC poll CPU",
           "tooltip": {
             "msResolution": true,
             "shared": true,
@@ -11337,7 +11485,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 84,
+          "id": 85,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11488,7 +11636,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 85,
+          "id": 86,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11630,7 +11778,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 86,
+          "id": 87,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11763,7 +11911,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 87,
+          "id": 88,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -11896,7 +12044,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 88,
+          "id": 89,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12029,7 +12177,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 89,
+          "id": 90,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12162,7 +12310,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 90,
+          "id": 91,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12295,7 +12443,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 91,
+          "id": 92,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12428,7 +12576,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 92,
+          "id": 93,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12561,7 +12709,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 93,
+          "id": 94,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12694,7 +12842,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 94,
+          "id": 95,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12857,7 +13005,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 95,
+          "id": 96,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -12999,7 +13147,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 96,
+          "id": 97,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13180,7 +13328,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 97,
+          "id": 98,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13352,7 +13500,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 98,
+          "id": 99,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13500,7 +13648,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 99,
+          "id": 100,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13636,7 +13784,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 100,
+      "id": 101,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -13675,7 +13823,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 101,
+          "id": 102,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13823,7 +13971,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 102,
+          "id": 103,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -13971,7 +14119,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 103,
+          "id": 104,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14104,7 +14252,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 104,
+          "id": 105,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14255,7 +14403,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 105,
+      "id": 106,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -14294,7 +14442,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 106,
+          "id": 107,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14495,7 +14643,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 107,
+          "id": 108,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14696,7 +14844,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 108,
+          "id": 109,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -14897,7 +15045,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 109,
+          "id": 110,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15098,7 +15246,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 110,
+          "id": 111,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15299,7 +15447,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 111,
+          "id": 112,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15500,7 +15648,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 112,
+          "id": 113,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15701,7 +15849,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 113,
+          "id": 114,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -15902,7 +16050,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 114,
+          "id": 115,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16103,7 +16251,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 115,
+          "id": 116,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16304,7 +16452,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 116,
+          "id": 117,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16505,7 +16653,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 117,
+          "id": 118,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16706,7 +16854,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 118,
+          "id": 119,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -16910,7 +17058,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 119,
+      "id": 120,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -16956,7 +17104,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 120,
+          "id": 121,
           "interval": null,
           "legend": {
             "show": false
@@ -17054,7 +17202,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 121,
+          "id": 122,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17262,7 +17410,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 122,
+          "id": 123,
           "interval": null,
           "legend": {
             "show": false
@@ -17360,7 +17508,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 123,
+          "id": 124,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17568,7 +17716,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 124,
+          "id": 125,
           "interval": null,
           "legend": {
             "show": false
@@ -17666,7 +17814,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 125,
+          "id": 126,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -17874,7 +18022,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 126,
+          "id": 127,
           "interval": null,
           "legend": {
             "show": false
@@ -17972,7 +18120,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 127,
+          "id": 128,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18180,7 +18328,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 128,
+          "id": 129,
           "interval": null,
           "legend": {
             "show": false
@@ -18278,7 +18426,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 129,
+          "id": 130,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18486,7 +18634,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 130,
+          "id": 131,
           "interval": null,
           "legend": {
             "show": false
@@ -18584,7 +18732,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 131,
+          "id": 132,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18785,7 +18933,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 132,
+          "id": 133,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -18933,7 +19081,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 133,
+          "id": 134,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19069,7 +19217,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 134,
+      "id": 135,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -19108,7 +19256,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 135,
+          "id": 136,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19241,7 +19389,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 136,
+          "id": 137,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19374,7 +19522,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 137,
+          "id": 138,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19507,7 +19655,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 138,
+          "id": 139,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19647,7 +19795,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 139,
+          "id": 140,
           "interval": null,
           "legend": {
             "show": false
@@ -19745,7 +19893,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 140,
+          "id": 141,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -19953,7 +20101,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 141,
+          "id": 142,
           "interval": null,
           "legend": {
             "show": false
@@ -20051,7 +20199,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 142,
+          "id": 143,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20259,7 +20407,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 143,
+          "id": 144,
           "interval": null,
           "legend": {
             "show": false
@@ -20357,7 +20505,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 144,
+          "id": 145,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20565,7 +20713,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 145,
+          "id": 146,
           "interval": null,
           "legend": {
             "show": false
@@ -20670,7 +20818,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 146,
+          "id": 147,
           "interval": null,
           "legend": {
             "show": false
@@ -20768,7 +20916,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 147,
+          "id": 148,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -20901,7 +21049,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 148,
+          "id": 149,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21052,7 +21200,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 149,
+      "id": 150,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -21091,7 +21239,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 150,
+          "id": 151,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21239,7 +21387,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 151,
+          "id": 152,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21394,7 +21542,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 152,
+          "id": 153,
           "interval": null,
           "legend": {
             "show": false
@@ -21492,7 +21640,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 153,
+          "id": 154,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -21632,7 +21780,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 154,
+          "id": 155,
           "interval": null,
           "legend": {
             "show": false
@@ -21737,7 +21885,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 155,
+          "id": 156,
           "interval": null,
           "legend": {
             "show": false
@@ -21842,7 +21990,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 156,
+          "id": 157,
           "interval": null,
           "legend": {
             "show": false
@@ -21947,7 +22095,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 157,
+          "id": 158,
           "interval": null,
           "legend": {
             "show": false
@@ -22052,7 +22200,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 158,
+          "id": 159,
           "interval": null,
           "legend": {
             "show": false
@@ -22157,7 +22305,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 159,
+          "id": 160,
           "interval": null,
           "legend": {
             "show": false
@@ -22262,7 +22410,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 160,
+          "id": 161,
           "interval": null,
           "legend": {
             "show": false
@@ -22367,7 +22515,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 161,
+          "id": 162,
           "interval": null,
           "legend": {
             "show": false
@@ -22472,7 +22620,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 162,
+          "id": 163,
           "interval": null,
           "legend": {
             "show": false
@@ -22577,7 +22725,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 163,
+          "id": 164,
           "interval": null,
           "legend": {
             "show": false
@@ -22675,7 +22823,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 164,
+          "id": 165,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22808,7 +22956,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 165,
+          "id": 166,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -22959,7 +23107,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 166,
+      "id": 167,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -22998,7 +23146,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 167,
+          "id": 168,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23131,7 +23279,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 168,
+          "id": 169,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23264,7 +23412,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 169,
+          "id": 170,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23397,7 +23545,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 170,
+          "id": 171,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23530,7 +23678,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 171,
+          "id": 172,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23663,7 +23811,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 172,
+          "id": 173,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23814,7 +23962,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 173,
+      "id": 174,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -23853,7 +24001,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 174,
+          "id": 175,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -23986,7 +24134,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 175,
+          "id": 176,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24119,7 +24267,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 176,
+          "id": 177,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24252,7 +24400,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 177,
+          "id": 178,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24385,7 +24533,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 178,
+          "id": 179,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24518,7 +24666,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 179,
+          "id": 180,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24681,7 +24829,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 180,
+          "id": 181,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -24817,7 +24965,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 181,
+      "id": 182,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -24856,7 +25004,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 182,
+          "id": 183,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25004,7 +25152,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 183,
+          "id": 184,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25152,7 +25300,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 184,
+          "id": 185,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25285,7 +25433,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 185,
+          "id": 186,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25418,7 +25566,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 186,
+          "id": 187,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25551,7 +25699,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 187,
+          "id": 188,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25684,7 +25832,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 188,
+          "id": 189,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25817,7 +25965,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 189,
+          "id": 190,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -25950,7 +26098,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 190,
+          "id": 191,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26127,7 +26275,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 191,
+      "id": 192,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -26166,7 +26314,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 192,
+          "id": 193,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26329,7 +26477,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 193,
+          "id": 194,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26530,7 +26678,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 194,
+          "id": 195,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26678,7 +26826,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 195,
+          "id": 196,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -26841,7 +26989,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 196,
+          "id": 197,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27042,7 +27190,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 197,
+          "id": 198,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27220,7 +27368,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 198,
+          "id": 199,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27383,7 +27531,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 199,
+          "id": 200,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27546,7 +27694,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 200,
+          "id": 201,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27679,7 +27827,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 201,
+          "id": 202,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -27883,7 +28031,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 202,
+      "id": 203,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -27922,7 +28070,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 203,
+          "id": 204,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28115,7 +28263,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 204,
+          "id": 205,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28293,7 +28441,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 205,
+          "id": 206,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28501,7 +28649,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 206,
+          "id": 207,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28679,7 +28827,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 207,
+          "id": 208,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -28842,7 +28990,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 208,
+          "id": 209,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29020,7 +29168,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 209,
+          "id": 210,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29153,7 +29301,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 210,
+          "id": 211,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29331,7 +29479,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 211,
+          "id": 212,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29464,7 +29612,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 212,
+          "id": 213,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29642,7 +29790,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 213,
+          "id": 214,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29775,7 +29923,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 214,
+          "id": 215,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -29953,7 +30101,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 215,
+          "id": 216,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30131,7 +30279,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 216,
+          "id": 217,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30309,7 +30457,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 217,
+          "id": 218,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30442,7 +30590,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 218,
+          "id": 219,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30575,7 +30723,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 219,
+          "id": 220,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30708,7 +30856,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 220,
+          "id": 221,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -30931,7 +31079,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 221,
+          "id": 222,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31124,7 +31272,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 222,
+          "id": 223,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31287,7 +31435,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 223,
+          "id": 224,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31480,7 +31628,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 224,
+          "id": 225,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31628,7 +31776,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 225,
+          "id": 226,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31761,7 +31909,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 226,
+          "id": 227,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -31909,7 +32057,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 227,
+          "id": 228,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32087,7 +32235,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 228,
+          "id": 229,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32250,7 +32398,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 229,
+          "id": 230,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32428,7 +32576,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 230,
+          "id": 231,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32561,7 +32709,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 231,
+          "id": 232,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32694,7 +32842,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 232,
+          "id": 233,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32827,7 +32975,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 233,
+          "id": 234,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -32960,7 +33108,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 234,
+          "id": 235,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33093,7 +33241,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 235,
+          "id": 236,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33233,7 +33381,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 236,
+          "id": 237,
           "interval": null,
           "legend": {
             "show": false
@@ -33331,7 +33479,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 237,
+          "id": 238,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33532,7 +33680,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 238,
+          "id": 239,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33665,7 +33813,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 239,
+          "id": 240,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33843,7 +33991,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 240,
+          "id": 241,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -33976,7 +34124,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 241,
+          "id": 242,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34112,7 +34260,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 242,
+      "id": 243,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -34151,7 +34299,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 243,
+          "id": 244,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34299,7 +34447,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 244,
+          "id": 245,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34447,7 +34595,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 245,
+          "id": 246,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34580,7 +34728,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 246,
+          "id": 247,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34713,7 +34861,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 247,
+          "id": 248,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -34891,7 +35039,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 248,
+          "id": 249,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35069,7 +35217,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 249,
+          "id": 250,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35247,7 +35395,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 250,
+          "id": 251,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35380,7 +35528,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 251,
+          "id": 252,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35558,7 +35706,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 252,
+          "id": 253,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35691,7 +35839,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 253,
+          "id": 254,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -35854,7 +36002,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 254,
+          "id": 255,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36032,7 +36180,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 255,
+          "id": 256,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36210,7 +36358,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 256,
+          "id": 257,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36388,7 +36536,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 257,
+          "id": 258,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36521,7 +36669,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 258,
+          "id": 259,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36699,7 +36847,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 259,
+          "id": 260,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -36832,7 +36980,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 260,
+          "id": 261,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37010,7 +37158,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 261,
+          "id": 262,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37143,7 +37291,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 262,
+          "id": 263,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37276,7 +37424,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 263,
+          "id": 264,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37454,7 +37602,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 264,
+          "id": 265,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37632,7 +37780,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 265,
+          "id": 266,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37765,7 +37913,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 266,
+          "id": 267,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -37943,7 +38091,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 267,
+          "id": 268,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38076,7 +38224,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 268,
+          "id": 269,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38254,7 +38402,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 269,
+          "id": 270,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38390,7 +38538,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 270,
+      "id": 271,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -38429,7 +38577,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 271,
+          "id": 272,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38562,7 +38710,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 272,
+          "id": 273,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38710,7 +38858,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 273,
+          "id": 274,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -38911,7 +39059,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 274,
+          "id": 275,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39044,7 +39192,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 275,
+          "id": 276,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39177,7 +39325,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 276,
+          "id": 277,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39310,7 +39458,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 277,
+          "id": 278,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39443,7 +39591,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 278,
+          "id": 279,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39576,7 +39724,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 279,
+          "id": 280,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -39716,7 +39864,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 280,
+          "id": 281,
           "interval": null,
           "legend": {
             "show": false
@@ -39821,7 +39969,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 281,
+          "id": 282,
           "interval": null,
           "legend": {
             "show": false
@@ -39919,7 +40067,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 282,
+          "id": 283,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40059,7 +40207,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 283,
+          "id": 284,
           "interval": null,
           "legend": {
             "show": false
@@ -40157,7 +40305,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 284,
+          "id": 285,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40297,7 +40445,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 285,
+          "id": 286,
           "interval": null,
           "legend": {
             "show": false
@@ -40395,7 +40543,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 286,
+          "id": 287,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40603,7 +40751,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 287,
+          "id": 288,
           "interval": null,
           "legend": {
             "show": false
@@ -40701,7 +40849,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 288,
+          "id": 289,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -40902,7 +41050,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 289,
+          "id": 290,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41110,7 +41258,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 290,
+          "id": 291,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41288,7 +41436,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 291,
+          "id": 292,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41421,7 +41569,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 292,
+          "id": 293,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41554,7 +41702,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 293,
+          "id": 294,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41687,7 +41835,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 294,
+          "id": 295,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -41823,7 +41971,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 295,
+      "id": 296,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -41862,7 +42010,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 296,
+          "id": 297,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42010,7 +42158,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 297,
+          "id": 298,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42150,7 +42298,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 298,
+          "id": 299,
           "interval": null,
           "legend": {
             "show": false
@@ -42248,7 +42396,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 299,
+          "id": 300,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42381,7 +42529,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 300,
+          "id": 301,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42514,7 +42662,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 301,
+          "id": 302,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42692,7 +42840,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 302,
+          "id": 303,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -42855,7 +43003,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 303,
+          "id": 304,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43003,7 +43151,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 304,
+          "id": 305,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43136,7 +43284,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 305,
+          "id": 306,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43272,7 +43420,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 306,
+      "id": 307,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -43311,7 +43459,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 307,
+          "id": 308,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43459,7 +43607,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 308,
+          "id": 309,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43592,7 +43740,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 309,
+          "id": 310,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43725,7 +43873,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 310,
+          "id": 311,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43858,7 +44006,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 311,
+          "id": 312,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -43991,7 +44139,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 312,
+          "id": 313,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44146,7 +44294,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 313,
+          "id": 314,
           "interval": null,
           "legend": {
             "show": false
@@ -44247,7 +44395,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 314,
+      "id": 315,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -44286,7 +44434,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 315,
+          "id": 316,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44419,7 +44567,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 316,
+          "id": 317,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44552,7 +44700,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 317,
+          "id": 318,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44692,7 +44840,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 318,
+          "id": 319,
           "interval": null,
           "legend": {
             "show": false
@@ -44790,7 +44938,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 319,
+          "id": 320,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -44991,7 +45139,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 320,
+          "id": 321,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45192,7 +45340,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 321,
+          "id": 322,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45396,7 +45544,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 322,
+      "id": 323,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -45435,7 +45583,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 323,
+          "id": 324,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45583,7 +45731,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 324,
+          "id": 325,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45784,7 +45932,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 325,
+          "id": 326,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -45985,7 +46133,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 326,
+          "id": 327,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46186,7 +46334,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 327,
+          "id": 328,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46387,7 +46535,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 328,
+          "id": 329,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46520,7 +46668,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 329,
+          "id": 330,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46653,7 +46801,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 330,
+          "id": 331,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46786,7 +46934,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 331,
+          "id": 332,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -46919,7 +47067,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 332,
+          "id": 333,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47127,7 +47275,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 333,
+          "id": 334,
           "interval": null,
           "legend": {
             "show": false
@@ -47228,7 +47376,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 334,
+      "id": 335,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -47274,7 +47422,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 335,
+          "id": 336,
           "interval": null,
           "legend": {
             "show": false
@@ -47372,7 +47520,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 336,
+          "id": 337,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47573,7 +47721,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 337,
+          "id": 338,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47706,7 +47854,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 338,
+          "id": 339,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47839,7 +47987,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 339,
+          "id": 340,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -47972,7 +48120,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 340,
+          "id": 341,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48173,7 +48321,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 341,
+          "id": 342,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48306,7 +48454,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 342,
+          "id": 343,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48439,7 +48587,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 343,
+          "id": 344,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48575,7 +48723,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 344,
+      "id": 345,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -48614,7 +48762,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 345,
+          "id": 346,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -48815,7 +48963,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 346,
+          "id": 347,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49016,7 +49164,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 347,
+          "id": 348,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49217,7 +49365,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 348,
+          "id": 349,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49418,7 +49566,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 349,
+          "id": 350,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49551,7 +49699,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 350,
+          "id": 351,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49684,7 +49832,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 351,
+          "id": 352,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49817,7 +49965,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 352,
+          "id": 353,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -49950,7 +50098,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 353,
+          "id": 354,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50083,7 +50231,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 354,
+          "id": 355,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50223,7 +50371,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 355,
+          "id": 356,
           "interval": null,
           "legend": {
             "show": false
@@ -50321,7 +50469,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 356,
+          "id": 357,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50525,7 +50673,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 357,
+      "id": 358,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -50564,7 +50712,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 358,
+          "id": 359,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50697,7 +50845,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 359,
+          "id": 360,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50830,7 +50978,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 360,
+          "id": 361,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -50970,7 +51118,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 361,
+          "id": 362,
           "interval": null,
           "legend": {
             "show": false
@@ -51068,7 +51216,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 362,
+          "id": 363,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51269,7 +51417,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 363,
+          "id": 364,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51470,7 +51618,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 364,
+          "id": 365,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51674,7 +51822,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 365,
+      "id": 366,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -51713,7 +51861,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 366,
+          "id": 367,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -51891,7 +52039,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 367,
+          "id": 368,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52092,7 +52240,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 368,
+          "id": 369,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52225,7 +52373,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 369,
+          "id": 370,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52358,7 +52506,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 370,
+          "id": 371,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52491,7 +52639,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 371,
+          "id": 372,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52624,7 +52772,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 372,
+          "id": 373,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52757,7 +52905,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 373,
+          "id": 374,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -52886,7 +53034,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 374,
+          "id": 375,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -52961,7 +53109,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 375,
+          "id": 376,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -53040,7 +53188,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 376,
+          "id": 377,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53293,7 +53441,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 377,
+          "id": 378,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53426,7 +53574,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 378,
+          "id": 379,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53562,7 +53710,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 379,
+      "id": 380,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -53601,7 +53749,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 380,
+          "id": 381,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53749,7 +53897,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 381,
+          "id": 382,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -53882,7 +54030,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 382,
+          "id": 383,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54083,7 +54231,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 383,
+          "id": 384,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54231,7 +54379,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 384,
+          "id": 385,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54432,7 +54580,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 385,
+          "id": 386,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54565,7 +54713,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 386,
+          "id": 387,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54698,7 +54846,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 387,
+          "id": 388,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54831,7 +54979,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 388,
+          "id": 389,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -54964,7 +55112,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 389,
+          "id": 390,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55104,7 +55252,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 390,
+          "id": 391,
           "interval": null,
           "legend": {
             "show": false
@@ -55202,7 +55350,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 391,
+          "id": 392,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55406,7 +55554,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 392,
+      "id": 393,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -55445,7 +55593,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 393,
+          "id": 394,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55578,7 +55726,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 394,
+          "id": 395,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55711,7 +55859,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 395,
+          "id": 396,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55844,7 +55992,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 396,
+          "id": 397,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -55980,7 +56128,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 397,
+      "id": 398,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -56019,7 +56167,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 398,
+          "id": 399,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56152,7 +56300,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 399,
+          "id": 400,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56285,7 +56433,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 400,
+          "id": 401,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56433,7 +56581,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 401,
+          "id": 402,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56566,7 +56714,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 402,
+          "id": 403,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56699,7 +56847,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 403,
+          "id": 404,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56832,7 +56980,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 404,
+          "id": 405,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -56968,7 +57116,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 405,
+      "id": 406,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -57007,7 +57155,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 406,
+          "id": 407,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57140,7 +57288,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 407,
+          "id": 408,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57273,7 +57421,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 408,
+          "id": 409,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57406,7 +57554,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 409,
+          "id": 410,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57539,7 +57687,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 410,
+          "id": 411,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57672,7 +57820,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 411,
+          "id": 412,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57808,7 +57956,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 412,
+      "id": 413,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -57847,7 +57995,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 413,
+          "id": 414,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -57980,7 +58128,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 414,
+          "id": 415,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58113,7 +58261,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 415,
+          "id": 416,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58261,7 +58409,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 416,
+          "id": 417,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58424,7 +58572,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 417,
+          "id": 418,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58557,7 +58705,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 418,
+          "id": 419,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58690,7 +58838,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 419,
+          "id": 420,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58838,7 +58986,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 420,
+          "id": 421,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -58986,7 +59134,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 421,
+          "id": 422,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59122,7 +59270,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 422,
+      "id": 423,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -59161,7 +59309,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 423,
+          "id": 424,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59294,7 +59442,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 424,
+          "id": 425,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59427,7 +59575,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 425,
+          "id": 426,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59560,7 +59708,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 426,
+          "id": 427,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59693,7 +59841,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 427,
+          "id": 428,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59826,7 +59974,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 428,
+          "id": 429,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -59959,7 +60107,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 429,
+          "id": 430,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60092,7 +60240,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 430,
+          "id": 431,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60225,7 +60373,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 431,
+          "id": 432,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60365,7 +60513,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 432,
+          "id": 433,
           "interval": null,
           "legend": {
             "show": false
@@ -60463,7 +60611,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 433,
+          "id": 434,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60596,7 +60744,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 434,
+          "id": 435,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60744,7 +60892,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 435,
+          "id": 436,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -60892,7 +61040,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 436,
+          "id": 437,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61032,7 +61180,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 437,
+          "id": 438,
           "interval": null,
           "legend": {
             "show": false
@@ -61130,7 +61278,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 438,
+          "id": 439,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61263,7 +61411,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 439,
+          "id": 440,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61399,7 +61547,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 440,
+      "id": 441,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -61438,7 +61586,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 441,
+          "id": 442,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61571,7 +61719,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 442,
+          "id": 443,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61734,7 +61882,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 443,
+          "id": 444,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -61882,7 +62030,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 444,
+          "id": 445,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62015,7 +62163,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 445,
+          "id": 446,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62155,7 +62303,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 446,
+          "id": 447,
           "interval": null,
           "legend": {
             "show": false
@@ -62260,7 +62408,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 447,
+          "id": 448,
           "interval": null,
           "legend": {
             "show": false
@@ -62365,7 +62513,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 448,
+          "id": 449,
           "interval": null,
           "legend": {
             "show": false
@@ -62463,7 +62611,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 449,
+          "id": 450,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -62603,7 +62751,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 450,
+          "id": 451,
           "interval": null,
           "legend": {
             "show": false
@@ -62708,7 +62856,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 451,
+          "id": 452,
           "interval": null,
           "legend": {
             "show": false
@@ -62813,7 +62961,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 452,
+          "id": 453,
           "interval": null,
           "legend": {
             "show": false
@@ -62911,7 +63059,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 453,
+          "id": 454,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63044,7 +63192,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 454,
+          "id": 455,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63177,7 +63325,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 455,
+          "id": 456,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63317,7 +63465,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 456,
+          "id": 457,
           "interval": null,
           "legend": {
             "show": false
@@ -63415,7 +63563,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 457,
+          "id": 458,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63551,7 +63699,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 458,
+      "id": 459,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -63590,7 +63738,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 459,
+          "id": 460,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63753,7 +63901,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 460,
+          "id": 461,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -63886,7 +64034,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 461,
+          "id": 462,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64026,7 +64174,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 462,
+          "id": 463,
           "interval": null,
           "legend": {
             "show": false
@@ -64131,7 +64279,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 463,
+          "id": 464,
           "interval": null,
           "legend": {
             "show": false
@@ -64229,7 +64377,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 464,
+          "id": 465,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64384,7 +64532,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 465,
+          "id": 466,
           "interval": null,
           "legend": {
             "show": false
@@ -64489,7 +64637,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 466,
+          "id": 467,
           "interval": null,
           "legend": {
             "show": false
@@ -64594,7 +64742,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 467,
+          "id": 468,
           "interval": null,
           "legend": {
             "show": false
@@ -64692,7 +64840,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 468,
+          "id": 469,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -64862,7 +65010,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 469,
+          "id": 470,
           "interval": null,
           "legend": {
             "show": false
@@ -64960,7 +65108,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 470,
+          "id": 471,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65161,7 +65309,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 471,
+          "id": 472,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65362,7 +65510,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 472,
+          "id": 473,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65495,7 +65643,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 473,
+          "id": 474,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65658,7 +65806,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 474,
+          "id": 475,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65791,7 +65939,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 475,
+          "id": 476,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -65924,7 +66072,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 476,
+          "id": 477,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66125,7 +66273,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 477,
+          "id": 478,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66258,7 +66406,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 478,
+          "id": 479,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -66398,7 +66546,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 479,
+          "id": 480,
           "interval": null,
           "legend": {
             "show": false
@@ -66503,7 +66651,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 480,
+          "id": 481,
           "interval": null,
           "legend": {
             "show": false
@@ -66608,7 +66756,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 481,
+          "id": 482,
           "interval": null,
           "legend": {
             "show": false
@@ -66713,7 +66861,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 482,
+          "id": 483,
           "interval": null,
           "legend": {
             "show": false
@@ -66818,7 +66966,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 483,
+          "id": 484,
           "interval": null,
           "legend": {
             "show": false
@@ -66923,7 +67071,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 484,
+          "id": 485,
           "interval": null,
           "legend": {
             "show": false
@@ -67028,7 +67176,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 485,
+          "id": 486,
           "interval": null,
           "legend": {
             "show": false
@@ -67126,7 +67274,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 486,
+          "id": 487,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67274,7 +67422,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 487,
+          "id": 488,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67407,7 +67555,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 488,
+          "id": 489,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67540,7 +67688,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 489,
+          "id": 490,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67688,7 +67836,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 490,
+          "id": 491,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -67824,7 +67972,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 491,
+      "id": 492,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -67875,7 +68023,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 492,
+          "id": 493,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -67971,7 +68119,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 493,
+          "id": 494,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68046,7 +68194,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 494,
+          "id": 495,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68121,7 +68269,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 495,
+          "id": 496,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68196,7 +68344,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 496,
+          "id": 497,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68271,7 +68419,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 497,
+          "id": 498,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68346,7 +68494,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 498,
+          "id": 499,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68421,7 +68569,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 499,
+          "id": 500,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
@@ -68500,7 +68648,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 500,
+          "id": 501,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68633,7 +68781,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 501,
+          "id": 502,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68766,7 +68914,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 502,
+          "id": 503,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -68899,7 +69047,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 503,
+          "id": 504,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69032,7 +69180,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 504,
+          "id": 505,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69165,7 +69313,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 505,
+          "id": 506,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69313,7 +69461,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 506,
+          "id": 507,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69446,7 +69594,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 507,
+          "id": 508,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69579,7 +69727,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 508,
+          "id": 509,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -69745,7 +69893,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 509,
+          "id": 510,
           "interval": null,
           "legend": {
             "show": false
@@ -69850,7 +69998,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 510,
+          "id": 511,
           "interval": null,
           "legend": {
             "show": false
@@ -69955,7 +70103,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 511,
+          "id": 512,
           "interval": null,
           "legend": {
             "show": false
@@ -70060,7 +70208,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 512,
+          "id": 513,
           "interval": null,
           "legend": {
             "show": false
@@ -70165,7 +70313,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 513,
+          "id": 514,
           "interval": null,
           "legend": {
             "show": false
@@ -70270,7 +70418,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 514,
+          "id": 515,
           "interval": null,
           "legend": {
             "show": false
@@ -70375,7 +70523,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 515,
+          "id": 516,
           "interval": null,
           "legend": {
             "show": false
@@ -70480,7 +70628,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 516,
+          "id": 517,
           "interval": null,
           "legend": {
             "show": false
@@ -70578,7 +70726,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 517,
+          "id": 518,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70711,7 +70859,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 518,
+          "id": 519,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70844,7 +70992,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 519,
+          "id": 520,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -70977,7 +71125,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 520,
+          "id": 521,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71110,7 +71258,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 521,
+          "id": 522,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71243,7 +71391,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 522,
+          "id": 523,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71376,7 +71524,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 523,
+          "id": 524,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71509,7 +71657,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 524,
+          "id": 525,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71649,7 +71797,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 525,
+          "id": 526,
           "interval": null,
           "legend": {
             "show": false
@@ -71754,7 +71902,7 @@
           "hideTimeOverride": false,
           "hideZeroBuckets": true,
           "highlightCards": true,
-          "id": 526,
+          "id": 527,
           "interval": null,
           "legend": {
             "show": false
@@ -71852,7 +72000,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 527,
+          "id": 528,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -71985,7 +72133,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 528,
+          "id": 529,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72118,7 +72266,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 529,
+          "id": 530,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72251,7 +72399,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 530,
+          "id": 531,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72384,7 +72532,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 531,
+          "id": 532,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72517,7 +72665,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 532,
+          "id": 533,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72653,7 +72801,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 533,
+      "id": 534,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -72692,7 +72840,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 534,
+          "id": 535,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72840,7 +72988,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 535,
+          "id": 536,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -72973,7 +73121,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 536,
+          "id": 537,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73106,7 +73254,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 537,
+          "id": 538,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73242,7 +73390,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 538,
+      "id": 539,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -73281,7 +73429,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 539,
+          "id": 540,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73414,7 +73562,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 540,
+          "id": 541,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73547,7 +73695,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 541,
+          "id": 542,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73680,7 +73828,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 542,
+          "id": 543,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73813,7 +73961,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 543,
+          "id": 544,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -73946,7 +74094,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 544,
+          "id": 545,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74082,7 +74230,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 545,
+      "id": 546,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -74121,7 +74269,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 546,
+          "id": 547,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74254,7 +74402,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 547,
+          "id": 548,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74390,7 +74538,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 548,
+      "id": 549,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -74429,7 +74577,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 549,
+          "id": 550,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74630,7 +74778,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 550,
+          "id": 551,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74766,7 +74914,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 551,
+      "id": 552,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -74805,7 +74953,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 552,
+          "id": 553,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -74938,7 +75086,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 553,
+          "id": 554,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75071,7 +75219,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 554,
+          "id": 555,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75204,7 +75352,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 555,
+          "id": 556,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75337,7 +75485,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 556,
+          "id": 557,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75485,7 +75633,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 557,
+          "id": 558,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75689,7 +75837,7 @@
       },
       "height": null,
       "hideTimeOverride": false,
-      "id": 558,
+      "id": 559,
       "interval": null,
       "links": [],
       "maxDataPoints": 100,
@@ -75728,7 +75876,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 559,
+          "id": 560,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75861,7 +76009,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 560,
+          "id": 561,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -75994,7 +76142,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 561,
+          "id": 562,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -76127,7 +76275,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 562,
+          "id": 563,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -76260,7 +76408,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 563,
+          "id": 564,
           "interval": null,
           "isNew": true,
           "legend": {
@@ -76457,7 +76605,7 @@
           },
           "height": null,
           "hideTimeOverride": false,
-          "id": 564,
+          "id": 565,
           "interval": null,
           "links": [],
           "maxDataPoints": 100,
diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256
index a5ff6ce5ff8..352119f2dfd 100644
--- a/metrics/grafana/tikv_details.json.sha256
+++ b/metrics/grafana/tikv_details.json.sha256
@@ -1,1 +1,1 @@
-dfd732bf35b728a8046c76efc340b191d549b5350f19980dfee87abeb38c3f15  ./metrics/grafana/tikv_details.json
+428ee69bad88bb97968f235beded66f7b14ae90758e57c67384f831ee28879b4  ./metrics/grafana/tikv_details.json
diff --git a/src/config/mod.rs b/src/config/mod.rs
index 26c2f9000c3..b848dcefc22 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -5667,6 +5667,7 @@ mod tests {
                 scheduler,
                 flow_controller.clone(),
                 storage.get_scheduler(),
+                storage.get_concurrency_manager(),
             )),
         );
         (storage, cfg_controller, receiver, flow_controller)
diff --git a/src/coprocessor/endpoint.rs b/src/coprocessor/endpoint.rs
index 2030c92c305..40189b6f7b6 100644
--- a/src/coprocessor/endpoint.rs
+++ b/src/coprocessor/endpoint.rs
@@ -137,7 +137,8 @@ impl<E: Engine> Endpoint<E> {
     fn check_memory_locks(&self, req_ctx: &ReqContext) -> Result<()> {
         let start_ts = req_ctx.txn_start_ts;
         if !req_ctx.context.get_stale_read() {
-            self.concurrency_manager.update_max_ts(start_ts);
+            self.concurrency_manager
+                .update_max_ts(start_ts, || format!("coprocessor-{}", start_ts))?;
         }
         if need_check_locks(req_ctx.context.get_isolation_level()) {
             let begin_instant = Instant::now();
@@ -926,6 +927,12 @@ macro_rules! make_error_response_common {
                 errorpb.set_server_is_busy(server_is_busy_err);
                 $resp.set_region_error(errorpb);
             }
+            Error::InvalidMaxTsUpdate(e) => {
+                $tag = "invalid_max_ts_update";
+                let mut err = errorpb::Error::default();
+                err.set_message(e.to_string());
+                $resp.set_region_error(err);
+            }
             Error::Other(_) => {
                 $tag = "other";
                 warn!("unexpected other error encountered processing coprocessor task";
diff --git a/src/coprocessor/error.rs b/src/coprocessor/error.rs
index 5c3ce554cc2..1ecb4479da3 100644
--- a/src/coprocessor/error.rs
+++ b/src/coprocessor/error.rs
@@ -30,6 +30,9 @@ pub enum Error {
     #[error("Coprocessor task canceled due to exceeding memory quota")]
     MemoryQuotaExceeded,
 
+    #[error("{0}")]
+    InvalidMaxTsUpdate(#[from] concurrency_manager::InvalidMaxTsUpdate),
+
     #[error("{0}")]
     Other(String),
 }
@@ -137,6 +140,7 @@ impl ErrorCodeExt for Error {
             Error::DeadlineExceeded => error_code::coprocessor::DEADLINE_EXCEEDED,
             Error::MaxPendingTasksExceeded => error_code::coprocessor::MAX_PENDING_TASKS_EXCEEDED,
             Error::MemoryQuotaExceeded => error_code::coprocessor::MEMORY_QUOTA_EXCEEDED,
+            Error::InvalidMaxTsUpdate(_) => error_code::coprocessor::INVALID_MAX_TS_UPDATE,
             Error::Other(_) => error_code::UNKNOWN,
         }
     }
diff --git a/src/server/raftkv/mod.rs b/src/server/raftkv/mod.rs
index 697ffbe3228..1d37243dd4c 100644
--- a/src/server/raftkv/mod.rs
+++ b/src/server/raftkv/mod.rs
@@ -827,7 +827,12 @@ impl ReadIndexObserver for ReplicaReadLockChecker {
             let begin_instant = Instant::now();
 
             let start_ts = request.get_start_ts().into();
-            self.concurrency_manager.update_max_ts(start_ts);
+            if let Err(e) = self
+                .concurrency_manager
+                .update_max_ts(start_ts, || format!("read_index-{}", start_ts))
+            {
+                error!("failed to update max_ts in concurrency manager"; "err" => ?e);
+            }
             for range in request.mut_key_ranges().iter_mut() {
                 let key_bound = |key: Vec<u8>| {
                     if key.is_empty() {
diff --git a/src/storage/config.rs b/src/storage/config.rs
index a3291d7b3d2..a16a6bc2d11 100644
--- a/src/storage/config.rs
+++ b/src/storage/config.rs
@@ -2,7 +2,7 @@
 
 //! Storage configuration.
 
-use std::{cmp::max, error::Error, path::Path};
+use std::{borrow::ToOwned, cmp::max, error::Error, path::Path};
 
 use engine_rocks::raw::{Cache, LRUCacheOptions, MemoryAllocator};
 use file_system::{IoPriority, IoRateLimitMode, IoRateLimiter, IoType};
diff --git a/src/storage/config_manager.rs b/src/storage/config_manager.rs
index b36f2f2209f..c1676f96b04 100644
--- a/src/storage/config_manager.rs
+++ b/src/storage/config_manager.rs
@@ -4,6 +4,7 @@
 
 use std::{convert::TryInto, sync::Arc};
 
+use concurrency_manager::ConcurrencyManager;
 use engine_traits::{ALL_CFS, CF_DEFAULT};
 use file_system::{get_io_rate_limiter, IoPriority, IoType};
 use online_config::{ConfigChange, ConfigManager, ConfigValue, Result as CfgResult};
@@ -25,6 +26,7 @@ pub struct StorageConfigManger<E: Engine, K, L: LockManager> {
     ttl_checker_scheduler: Scheduler<TtlCheckerTask>,
     flow_controller: Arc<FlowController>,
     scheduler: TxnScheduler<E, L>,
+    concurrency_manager: ConcurrencyManager,
 }
 
 unsafe impl<E: Engine, K, L: LockManager> Send for StorageConfigManger<E, K, L> {}
@@ -36,12 +38,14 @@ impl<E: Engine, K, L: LockManager> StorageConfigManger<E, K, L> {
         ttl_checker_scheduler: Scheduler<TtlCheckerTask>,
         flow_controller: Arc<FlowController>,
         scheduler: TxnScheduler<E, L>,
+        concurrency_manager: ConcurrencyManager,
     ) -> Self {
         StorageConfigManger {
             configurable_db,
             ttl_checker_scheduler,
             flow_controller,
             scheduler,
+            concurrency_manager,
         }
     }
 }
@@ -106,6 +110,16 @@ impl<EK: Engine, K: ConfigurableDb, L: LockManager> ConfigManager
                 }
             }
         }
+        if let Some(v) = change.remove("action_on_invalid_max_ts") {
+            let str_v: String = v.into();
+            let action: concurrency_manager::ActionOnInvalidMaxTs = str_v.try_into()?;
+            self.concurrency_manager
+                .set_action_on_invalid_max_ts(action);
+        }
+        if let Some(v) = change.remove("max_ts_drift_allowance") {
+            let dur_v: ReadableDuration = v.into();
+            self.concurrency_manager.set_max_ts_drift_allowance(dur_v.0);
+        }
         Ok(())
     }
 }
diff --git a/src/storage/errors.rs b/src/storage/errors.rs
index 02888cc93e8..81386a8f31c 100644
--- a/src/storage/errors.rs
+++ b/src/storage/errors.rs
@@ -303,6 +303,20 @@ pub fn extract_region_error_from_error(e: &Error) -> Option<errorpb::Error> {
             err.set_flashback_not_prepared(flashback_not_prepared_err);
             Some(err)
         }
+        Error(box ErrorInner::Txn(TxnError(box TxnErrorInner::InvalidMaxTsUpdate(
+            invalid_max_ts_update,
+        )))) => {
+            let mut err = errorpb::Error::default();
+            err.set_message(invalid_max_ts_update.to_string());
+            Some(err)
+        }
+        Error(box ErrorInner::Txn(TxnError(box TxnErrorInner::Mvcc(MvccError(
+            box MvccErrorInner::InvalidMaxTsUpdate(invalid_max_ts_update),
+        ))))) => {
+            let mut err = errorpb::Error::default();
+            err.set_message(invalid_max_ts_update.to_string());
+            Some(err)
+        }
         Error(box ErrorInner::SchedTooBusy) => {
             let mut err = errorpb::Error::default();
             let mut server_is_busy_err = errorpb::ServerIsBusy::default();
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 25b3a20dbd3..5af39078648 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -1455,7 +1455,9 @@ impl<E: Engine, L: LockManager, F: KvFormat> Storage<E, L, F> {
 
                 // Update max_ts and check the in-memory lock table before getting the snapshot
                 if !ctx.get_stale_read() {
-                    concurrency_manager.update_max_ts(start_ts);
+                    concurrency_manager
+                        .update_max_ts(start_ts, "scan")
+                        .map_err(txn::Error::from)?;
                 }
                 if need_check_locks(ctx.get_isolation_level()) {
                     let begin_instant = Instant::now();
@@ -1620,7 +1622,9 @@ impl<E: Engine, L: LockManager, F: KvFormat> Storage<E, L, F> {
 
                 let command_duration = Instant::now();
 
-                concurrency_manager.update_max_ts(max_ts);
+                concurrency_manager
+                    .update_max_ts(max_ts, "scan_lock")
+                    .map_err(txn::Error::from)?;
                 let begin_instant = Instant::now();
                 // TODO: Though it's very unlikely to find a conflicting memory lock here, it's
                 // not a good idea to return an error to the client, making the GC fail. A
@@ -3351,7 +3355,11 @@ fn prepare_snap_ctx<'a>(
 ) -> Result<SnapContext<'a>> {
     // Update max_ts and check the in-memory lock table before getting the snapshot
     if !pb_ctx.get_stale_read() {
-        concurrency_manager.update_max_ts(start_ts);
+        concurrency_manager
+            .update_max_ts(start_ts, || {
+                format!("prepare_snap_ctx-{}-{}", cmd, start_ts)
+            })
+            .map_err(txn::Error::from)?;
     }
     fail_point!("before-storage-check-memory-locks");
     let isolation_level = pb_ctx.get_isolation_level();
@@ -10266,7 +10274,7 @@ mod tests {
             .build()
             .unwrap();
         let cm = storage.concurrency_manager.clone();
-        cm.update_max_ts(10.into());
+        cm.update_max_ts(10.into(), "").unwrap();
 
         // Optimistic prewrite
         let (tx, rx) = channel();
@@ -10314,7 +10322,7 @@ mod tests {
             .unwrap();
         rx.recv().unwrap();
 
-        cm.update_max_ts(1000.into());
+        cm.update_max_ts(1000.into(), "").unwrap();
 
         let (tx, rx) = channel();
         storage
@@ -11421,7 +11429,7 @@ mod tests {
         // commit enabled, and max_ts changes when the second request arrives.
 
         // A retrying prewrite request arrives.
-        cm.update_max_ts(20.into());
+        cm.update_max_ts(20.into(), "").unwrap();
         let mut ctx = Context::default();
         ctx.set_is_retry_request(true);
         let (tx, rx) = channel();
@@ -11605,7 +11613,7 @@ mod tests {
 
         // 1PC update
         let (tx, rx) = channel();
-        cm.update_max_ts(59.into());
+        cm.update_max_ts(59.into(), "").unwrap();
         storage
             .sched_txn_command(
                 Prewrite::new(
diff --git a/src/storage/mvcc/mod.rs b/src/storage/mvcc/mod.rs
index 30e20a2afc6..e1d01f91562 100644
--- a/src/storage/mvcc/mod.rs
+++ b/src/storage/mvcc/mod.rs
@@ -182,6 +182,9 @@ pub enum ErrorInner {
     #[error("generation out of order: current = {0}, key={1:?}, lock = {1:?}")]
     GenerationOutOfOrder(u64, Key, Lock),
 
+    #[error("{0}")]
+    InvalidMaxTsUpdate(#[from] concurrency_manager::InvalidMaxTsUpdate),
+
     #[error("{0:?}")]
     Other(#[from] Box<dyn error::Error + Sync + Send>),
 }
@@ -321,6 +324,7 @@ impl ErrorInner {
             ErrorInner::GenerationOutOfOrder(gen, key, lock_info) => Some(
                 ErrorInner::GenerationOutOfOrder(*gen, key.clone(), lock_info.clone()),
             ),
+            ErrorInner::InvalidMaxTsUpdate(e) => Some(ErrorInner::InvalidMaxTsUpdate(e.clone())),
             ErrorInner::Io(_) | ErrorInner::Other(_) => None,
         }
     }
@@ -425,6 +429,7 @@ impl ErrorCodeExt for Error {
             ErrorInner::LockIfExistsFailed { .. } => error_code::storage::LOCK_IF_EXISTS_FAILED,
             ErrorInner::PrimaryMismatch(_) => error_code::storage::PRIMARY_MISMATCH,
             ErrorInner::GenerationOutOfOrder(..) => error_code::storage::GENERATION_OUT_OF_ORDER,
+            ErrorInner::InvalidMaxTsUpdate(_) => error_code::storage::INVALID_MAX_TS_UPDATE,
             ErrorInner::Other(_) => error_code::storage::UNKNOWN,
         }
     }
diff --git a/src/storage/txn/actions/acquire_pessimistic_lock.rs b/src/storage/txn/actions/acquire_pessimistic_lock.rs
index 2a9e49b45ff..6c71bb6843a 100644
--- a/src/storage/txn/actions/acquire_pessimistic_lock.rs
+++ b/src/storage/txn/actions/acquire_pessimistic_lock.rs
@@ -71,7 +71,9 @@ pub fn acquire_pessimistic_lock<S: Snapshot>(
     // it infers a read to the value, in which case max_ts need to be updated to
     // guarantee the linearizability and snapshot isolation.
     if should_not_exist || need_value || need_check_existence {
-        txn.concurrency_manager.update_max_ts(for_update_ts);
+        txn.concurrency_manager.update_max_ts(for_update_ts, || {
+            format!("pessimistic_lock-{}-{}", reader.start_ts, for_update_ts)
+        })?;
     }
 
     // When `need_value` is set, the value need to be loaded of course. If
diff --git a/src/storage/txn/actions/prewrite.rs b/src/storage/txn/actions/prewrite.rs
index 6c21de599bc..9e95a3ef63f 100644
--- a/src/storage/txn/actions/prewrite.rs
+++ b/src/storage/txn/actions/prewrite.rs
@@ -70,7 +70,10 @@ pub fn prewrite_with_generation<S: Snapshot>(
     // Update max_ts for Insert operation to guarantee linearizability and snapshot
     // isolation
     if mutation.should_not_exist {
-        txn.concurrency_manager.update_max_ts(txn_props.start_ts);
+        txn.concurrency_manager
+            .update_max_ts(txn_props.start_ts, || {
+                format!("prewrite-{}", txn_props.start_ts)
+            })?;
     }
 
     fail_point!(
@@ -149,7 +152,10 @@ pub fn prewrite_with_generation<S: Snapshot>(
     if mutation.should_not_write {
         // `checkNotExists` is equivalent to a get operation, so it should update the
         // max_ts.
-        txn.concurrency_manager.update_max_ts(txn_props.start_ts);
+        txn.concurrency_manager
+            .update_max_ts(txn_props.start_ts, || {
+                format!("prewrite-{}", txn_props.start_ts)
+            })?;
         let min_commit_ts = if mutation.need_min_commit_ts() {
             // Don't calculate the min_commit_ts according to the concurrency manager's
             // max_ts for a should_not_write mutation because it's not persisted and doesn't
@@ -1060,7 +1066,7 @@ pub mod tests {
         .unwrap();
         assert_eq!(old_value, OldValue::None);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, err
         let err = prewrite(
             &mut txn,
@@ -1245,7 +1251,7 @@ pub mod tests {
         .unwrap();
         assert_eq!(old_value, OldValue::None);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, err
         let err = prewrite(
             &mut txn,
@@ -1347,7 +1353,7 @@ pub mod tests {
         // Pessimistic txn skips constraint check, does not read previous write.
         assert_eq!(old_value, OldValue::Unspecified);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, ok
         prewrite(
             &mut txn,
@@ -1400,7 +1406,7 @@ pub mod tests {
         // Pessimistic txn skips constraint check, does not read previous write.
         assert_eq!(old_value, OldValue::Unspecified);
 
-        cm.update_max_ts(60.into());
+        cm.update_max_ts(60.into(), "").unwrap();
         // calculated commit_ts = 61 > 50, ok
         prewrite(
             &mut txn,
diff --git a/src/storage/txn/commands/check_secondary_locks.rs b/src/storage/txn/commands/check_secondary_locks.rs
index aff85ca3e61..6664f16bdc6 100644
--- a/src/storage/txn/commands/check_secondary_locks.rs
+++ b/src/storage/txn/commands/check_secondary_locks.rs
@@ -145,7 +145,11 @@ impl<S: Snapshot, L: LockManager> WriteCommand<S, L> for CheckSecondaryLocks {
         // It is not allowed for commit to overwrite a protected rollback. So we update
         // max_ts to prevent this case from happening.
         let region_id = self.ctx.get_region_id();
-        context.concurrency_manager.update_max_ts(self.start_ts);
+        context
+            .concurrency_manager
+            .update_max_ts(self.start_ts, || {
+                format!("check_secondary_locks-{}", self.start_ts)
+            })?;
 
         let mut txn = MvccTxn::new(self.start_ts, context.concurrency_manager);
         let mut reader = ReaderWithStats::new(
diff --git a/src/storage/txn/commands/check_txn_status.rs b/src/storage/txn/commands/check_txn_status.rs
index f2bbdac7478..fbb1b7187f1 100644
--- a/src/storage/txn/commands/check_txn_status.rs
+++ b/src/storage/txn/commands/check_txn_status.rs
@@ -90,7 +90,12 @@ impl<S: Snapshot, L: LockManager> WriteCommand<S, L> for CheckTxnStatus {
         if !self.caller_start_ts.is_max() && self.caller_start_ts > new_max_ts {
             new_max_ts = self.caller_start_ts;
         }
-        context.concurrency_manager.update_max_ts(new_max_ts);
+        context.concurrency_manager.update_max_ts(new_max_ts, || {
+            format!(
+                "check_txn_status-{}-{}-{}",
+                self.lock_ts, self.current_ts, self.caller_start_ts
+            )
+        })?;
 
         let mut txn = MvccTxn::new(self.lock_ts, context.concurrency_manager);
         let mut reader = ReaderWithStats::new(
diff --git a/src/storage/txn/commands/cleanup.rs b/src/storage/txn/commands/cleanup.rs
index a1c234ce062..fa0b446284b 100644
--- a/src/storage/txn/commands/cleanup.rs
+++ b/src/storage/txn/commands/cleanup.rs
@@ -51,7 +51,9 @@ impl<S: Snapshot, L: LockManager> WriteCommand<S, L> for Cleanup {
     fn process_write(self, snapshot: S, context: WriteContext<'_, L>) -> Result<WriteResult> {
         // It is not allowed for commit to overwrite a protected rollback. So we update
         // max_ts to prevent this case from happening.
-        context.concurrency_manager.update_max_ts(self.start_ts);
+        context
+            .concurrency_manager
+            .update_max_ts(self.start_ts, || format!("cleanup-{}", self.start_ts))?;
 
         let mut txn = MvccTxn::new(self.start_ts, context.concurrency_manager);
         let mut reader = ReaderWithStats::new(
diff --git a/src/storage/txn/commands/prewrite.rs b/src/storage/txn/commands/prewrite.rs
index 581e248d9f1..861083fb117 100644
--- a/src/storage/txn/commands/prewrite.rs
+++ b/src/storage/txn/commands/prewrite.rs
@@ -1264,7 +1264,7 @@ mod tests {
         must_get(&mut engine, key, 12, value);
         must_get_commit_ts(&mut engine, key, 10, 11);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         let mutations = vec![Mutation::make_put(Key::from_raw(key), value.to_vec())];
 
@@ -1406,7 +1406,7 @@ mod tests {
         must_get_commit_ts(&mut engine, k1, 8, 13);
         must_get_commit_ts(&mut engine, k2, 8, 13);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
         must_acquire_pessimistic_lock(&mut engine, k1, k1, 20, 20);
 
         let mutations = vec![(
@@ -1506,7 +1506,7 @@ mod tests {
         assert_eq!(res.one_pc_commit_ts, TimeStamp::zero());
         must_locked(&mut engine, key, 10);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         let (k1, v1) = (b"k1", b"v1");
         let (k2, v2) = (b"k2", b"v2");
@@ -1578,7 +1578,7 @@ mod tests {
         assert_eq!(res.one_pc_commit_ts, TimeStamp::zero());
         must_locked(&mut engine, key, 10);
 
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         let (k1, v1) = (b"k1", b"v1");
         let (k2, v2) = (b"k2", b"v2");
@@ -2147,7 +2147,7 @@ mod tests {
         )
         .unwrap();
         let commit_ts = res.one_pc_commit_ts;
-        cm.update_max_ts(commit_ts.next());
+        cm.update_max_ts(commit_ts.next(), "").unwrap();
         // repeate the prewrite
         let res = pessimistic_prewrite_with_cm(
             &mut engine,
@@ -2657,7 +2657,7 @@ mod tests {
         must_commit(&mut engine, b"k2", 5, 18);
 
         // Update max_ts to be larger than the max_commit_ts.
-        cm.update_max_ts(50.into());
+        cm.update_max_ts(50.into(), "").unwrap();
 
         // Retry the prewrite on non-pessimistic key.
         // (is_retry_request flag is not set, here we don't rely on it.)
diff --git a/src/storage/txn/mod.rs b/src/storage/txn/mod.rs
index 5c4ed85902d..7e323a61e06 100644
--- a/src/storage/txn/mod.rs
+++ b/src/storage/txn/mod.rs
@@ -149,6 +149,9 @@ pub enum ErrorInner {
 
     #[error("region {0} not prepared the flashback")]
     FlashbackNotPrepared(u64),
+
+    #[error("{0}")]
+    InvalidMaxTsUpdate(#[from] concurrency_manager::InvalidMaxTsUpdate),
 }
 
 impl ErrorInner {
@@ -188,6 +191,9 @@ impl ErrorInner {
             ErrorInner::FlashbackNotPrepared(region_id) => {
                 Some(ErrorInner::FlashbackNotPrepared(region_id))
             }
+            ErrorInner::InvalidMaxTsUpdate(ref e) => {
+                Some(ErrorInner::InvalidMaxTsUpdate(e.clone()))
+            }
             ErrorInner::Other(_) | ErrorInner::ProtoBuf(_) | ErrorInner::Io(_) => None,
         }
     }
@@ -242,6 +248,7 @@ impl ErrorCodeExt for Error {
                 error_code::storage::MAX_TIMESTAMP_NOT_SYNCED
             }
             ErrorInner::FlashbackNotPrepared(_) => error_code::storage::FLASHBACK_NOT_PREPARED,
+            ErrorInner::InvalidMaxTsUpdate { .. } => error_code::storage::INVALID_MAX_TS_UPDATE,
         }
     }
 }


Test patch:
diff --git a/components/cdc/tests/integrations/test_cdc.rs b/components/cdc/tests/integrations/test_cdc.rs
index 77a3d240b2e..c89bdcbb55a 100644
--- a/components/cdc/tests/integrations/test_cdc.rs
+++ b/components/cdc/tests/integrations/test_cdc.rs
@@ -1238,7 +1238,7 @@ fn test_cdc_resolve_ts_checking_concurrency_manager_impl<F: KvFormat>() {
         guard
     };
 
-    cm.update_max_ts(20.into());
+    cm.update_max_ts(20.into(), "").unwrap();
 
     let guard = lock_key(b"a", 80);
     suite.set_tso(99);
diff --git a/tests/failpoints/cases/test_storage.rs b/tests/failpoints/cases/test_storage.rs
index 95ae4e82b74..c7c8af28ea1 100644
--- a/tests/failpoints/cases/test_storage.rs
+++ b/tests/failpoints/cases/test_storage.rs
@@ -277,6 +277,7 @@ fn test_scale_scheduler_pool() {
             scheduler,
             flow_controller,
             storage.get_scheduler(),
+            storage.get_concurrency_manager(),
         )),
     );
     let scheduler = storage.get_scheduler();
diff --git a/tests/failpoints/cases/test_transaction.rs b/tests/failpoints/cases/test_transaction.rs
index 57b249c8087..af5b4946f50 100644
--- a/tests/failpoints/cases/test_transaction.rs
+++ b/tests/failpoints/cases/test_transaction.rs
@@ -344,7 +344,7 @@ fn test_max_commit_ts_error() {
     thread::sleep(Duration::from_millis(200));
     cm.read_key_check(&Key::from_raw(b"k1"), |_| Err(()))
         .unwrap_err();
-    cm.update_max_ts(200.into());
+    cm.update_max_ts(200.into(), "").unwrap();
 
     let res = prewrite_rx.recv().unwrap().unwrap();
     assert!(res.min_commit_ts.is_zero());
@@ -372,7 +372,7 @@ fn test_exceed_max_commit_ts_in_the_middle_of_prewrite() {
     // Pause between getting max ts and store the lock in memory
     fail::cfg("before-set-lock-in-memory", "pause").unwrap();
 
-    cm.update_max_ts(40.into());
+    cm.update_max_ts(40.into(), "").unwrap();
     let mutations = vec![
         Mutation::make_put(Key::from_raw(b"k1"), b"v".to_vec()),
         Mutation::make_put(Key::from_raw(b"k2"), b"v".to_vec()),
@@ -401,7 +401,7 @@ fn test_exceed_max_commit_ts_in_the_middle_of_prewrite() {
     // sleep a while so the first key gets max ts.
     thread::sleep(Duration::from_millis(200));
 
-    cm.update_max_ts(51.into());
+    cm.update_max_ts(51.into(), "").unwrap();
     fail::remove("before-set-lock-in-memory");
     let res = prewrite_rx.recv().unwrap().unwrap();
     assert!(res.min_commit_ts.is_zero());
diff --git a/tests/integrations/server/kv_service.rs b/tests/integrations/server/kv_service.rs
index 52eb3563dff..4b1b7492474 100644
--- a/tests/integrations/server/kv_service.rs
+++ b/tests/integrations/server/kv_service.rs
@@ -1930,7 +1930,7 @@ fn test_prewrite_check_max_commit_ts() {
     let (cluster, client, ctx) = new_cluster();
 
     let cm = cluster.sim.read().unwrap().get_concurrency_manager(1);
-    cm.update_max_ts(100.into());
+    cm.update_max_ts(100.into(), "").unwrap();
 
     let mut req = PrewriteRequest::default();
     req.set_context(ctx.clone());