Repository: aptos-labs/aptos-core
Pull request #16261: Update Aptos CLI to use indexer SDK processors
Author: login: rtso
Labels: name: CICD:run-e2e-tests
description: when this label is present github actions will run all land-blocking e2e tests from the PR
name: CICD:run-forge-e2e-perf
description: Run the e2e perf forge only
Created: 2025-04-01T20:28:20Z | Merged: 2025-04-17T01:09:57Z

## Description
Update local testnet and workspace to use the new indexer SDK V2 processors. 

1. Adding `aptos-indexer-processor-sdk` as a dependency
2. Updating the processor dependency to use `aptos-indexer-processors-v2`
3. Updating configuration structures to match the new SDK interfaces
1. Updating Hasura metadata to work with the new schema structure

## How Has This Been Tested?
The changes have been tested by running the local testnet with the updated processor configurations and verifying that the indexer services start correctly and process transactions.

![Screenshot 2025-04-01 at 4.24.47 PM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/vIwavW9noSx0eb70aDLz/551df771-061f-4055-a90a-84e1c5f6cce3.png)
<img width="922" alt="Screenshot 2025-04-16 at 4 48 50 PM" src="https://github.com/user-attachments/assets/68d67d49-c0c7-40de-a4e1-d0deb0dcf9c0" />
<img width="1397" alt="Screenshot 2025-04-16 at 4 49 06 PM" src="https://github.com/user-attachments/assets/f4d18c32-e840-43b1-acaf-6aa7e0c258b5" />

## Type of Change
- [x] Dependency update

## Which Components or Systems Does This Change Impact?
- [x] Aptos CLI/SDK

## Checklist
- [x] I have read and followed the [CONTRIBUTING](https://github.com/aptos-labs/aptos-core/blob/main/CONTRIBUTING.md) doc
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I identified and added all stakeholders and component owners affected by this change as reviewers
- [x] I tested both happy and unhappy path of the functionality
- [x] I have made corresponding changes to the documentation


Diff:
diff --git a/Cargo.lock b/Cargo.lock
index 8b7b703161f..2eca25df121 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -309,7 +309,7 @@ dependencies = [
  "aptos-github-client",
  "aptos-global-constants",
  "aptos-indexer-grpc-server-framework",
- "aptos-indexer-grpc-utils",
+ "aptos-indexer-processor-sdk",
  "aptos-keygen",
  "aptos-ledger",
  "aptos-localnet",
@@ -317,7 +317,6 @@ dependencies = [
  "aptos-move-debugger",
  "aptos-network-checker",
  "aptos-node",
- "aptos-protos 1.3.1",
  "aptos-rest-client",
  "aptos-sdk",
  "aptos-storage-interface",
@@ -331,6 +330,7 @@ dependencies = [
  "aptos-vm-types",
  "aptos-workspace-server",
  "async-trait",
+ "backoff",
  "base64 0.13.1",
  "bcs 0.1.4",
  "bollard",
@@ -376,14 +376,12 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml 0.8.26",
- "server-framework",
  "set_env",
  "shadow-rs",
  "tempfile",
  "thiserror",
  "tokio",
  "toml 0.7.8",
- "tonic 0.12.3",
  "tracing",
  "tracing-subscriber 0.3.18",
  "url",
@@ -961,7 +959,7 @@ dependencies = [
  "maplit",
  "mini-moka",
  "mirai-annotations",
- "mockall",
+ "mockall 0.11.4",
  "move-core-types",
  "num-derive",
  "num-traits",
@@ -1142,7 +1140,7 @@ dependencies = [
  "futures",
  "itertools 0.13.0",
  "maplit",
- "mockall",
+ "mockall 0.11.4",
  "ordered-float 3.9.2",
  "rand 0.8.5",
  "serde",
@@ -2146,7 +2144,7 @@ dependencies = [
  "aptos-indexer-grpc-server-framework",
  "aptos-indexer-grpc-utils",
  "aptos-metrics-core",
- "aptos-moving-average",
+ "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bf4dd89677e6929ef21f898a7a4176514e606ebd)",
  "aptos-protos 1.3.1",
  "async-trait",
  "clap 4.5.21",
@@ -2173,9 +2171,9 @@ dependencies = [
  "aptos-indexer-grpc-server-framework",
  "aptos-indexer-grpc-utils",
  "aptos-metrics-core",
- "aptos-moving-average",
+ "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bf4dd89677e6929ef21f898a7a4176514e606ebd)",
  "aptos-protos 1.3.1",
- "aptos-transaction-filter",
+ "aptos-transaction-filter 0.1.0",
  "async-trait",
  "clap 4.5.21",
  "futures",
@@ -2202,7 +2200,7 @@ dependencies = [
  "aptos-indexer-grpc-utils",
  "aptos-metrics-core",
  "aptos-protos 1.3.1",
- "aptos-transaction-filter",
+ "aptos-transaction-filter 0.1.0",
  "async-trait",
  "build_html",
  "clap 4.5.21",
@@ -2250,7 +2248,7 @@ dependencies = [
  "aptos-indexer-grpc-server-framework",
  "aptos-indexer-grpc-utils",
  "aptos-metrics-core",
- "aptos-moving-average",
+ "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bf4dd89677e6929ef21f898a7a4176514e606ebd)",
  "async-trait",
  "clap 4.5.21",
  "futures",
@@ -2304,7 +2302,7 @@ dependencies = [
  "aptos-mempool",
  "aptos-mempool-notifications",
  "aptos-metrics-core",
- "aptos-moving-average",
+ "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bf4dd89677e6929ef21f898a7a4176514e606ebd)",
  "aptos-proptest-helpers",
  "aptos-protos 1.3.1",
  "aptos-runtimes",
@@ -2440,7 +2438,7 @@ dependencies = [
  "anyhow",
  "aptos-metrics-core",
  "aptos-protos 1.3.1",
- "aptos-transaction-filter",
+ "aptos-transaction-filter 0.1.0",
  "async-trait",
  "backoff",
  "base64 0.13.1",
@@ -2468,6 +2466,58 @@ dependencies = [
  "warp",
 ]
 
+[[package]]
+name = "aptos-indexer-processor-sdk"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-indexer-processor-sdk.git?tag=aptos-indexer-processor-sdk-v2.0.0#7f2dbeb0e482d274d8de1a55472159cb74f92cad"
+dependencies = [
+ "ahash 0.8.11",
+ "anyhow",
+ "aptos-indexer-transaction-stream",
+ "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7)",
+ "aptos-system-utils 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7)",
+ "async-trait",
+ "autometrics",
+ "axum 0.7.5",
+ "backtrace",
+ "bcs 0.1.4",
+ "bigdecimal",
+ "chrono",
+ "clap 4.5.21",
+ "derive_builder",
+ "diesel",
+ "diesel-async",
+ "diesel_migrations",
+ "field_count",
+ "futures",
+ "futures-util",
+ "hex",
+ "indexmap 2.7.0",
+ "instrumented-channel",
+ "kanal",
+ "mockall 0.12.1",
+ "native-tls",
+ "num_cpus",
+ "once_cell",
+ "petgraph 0.6.5",
+ "postgres-native-tls",
+ "prometheus",
+ "prometheus-client",
+ "serde",
+ "serde_json",
+ "serde_yaml 0.8.26",
+ "sha2 0.9.9",
+ "tempfile",
+ "thiserror",
+ "tiny-keccak",
+ "tokio",
+ "tokio-postgres",
+ "toml 0.7.8",
+ "tracing",
+ "tracing-subscriber 0.3.18",
+ "url",
+]
+
 [[package]]
 name = "aptos-indexer-test-transactions"
 version = "1.0.0"
@@ -2501,6 +2551,28 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "aptos-indexer-transaction-stream"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-indexer-processor-sdk.git?tag=aptos-indexer-processor-sdk-v2.0.0#7f2dbeb0e482d274d8de1a55472159cb74f92cad"
+dependencies = [
+ "anyhow",
+ "aptos-moving-average 0.1.0 (git+https://github.com/aptos-labs/aptos-indexer-processor-sdk.git?tag=aptos-indexer-processor-sdk-v2.0.0)",
+ "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7)",
+ "aptos-transaction-filter 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7)",
+ "chrono",
+ "futures-util",
+ "once_cell",
+ "prometheus",
+ "prost 0.13.4",
+ "sample",
+ "serde",
+ "tokio",
+ "tonic 0.12.3",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "aptos-infallible"
 version = "0.1.0"
@@ -2989,6 +3061,14 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "aptos-moving-average"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-indexer-processor-sdk.git?tag=aptos-indexer-processor-sdk-v2.0.0#7f2dbeb0e482d274d8de1a55472159cb74f92cad"
+dependencies = [
+ "chrono",
+]
+
 [[package]]
 name = "aptos-moving-average"
 version = "0.1.0"
@@ -3446,7 +3526,7 @@ dependencies = [
  "claims",
  "futures",
  "maplit",
- "mockall",
+ "mockall 0.11.4",
  "once_cell",
  "rand 0.7.3",
  "serde",
@@ -3480,7 +3560,7 @@ dependencies = [
 [[package]]
 name = "aptos-profiler"
 version = "0.1.0"
-source = "git+https://github.com/aptos-labs/aptos-core.git?rev=202bdccff2b2d333a385ae86a4fcf23e89da9f62#202bdccff2b2d333a385ae86a4fcf23e89da9f62"
+source = "git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7#1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7"
 dependencies = [
  "anyhow",
  "backtrace",
@@ -3512,7 +3592,7 @@ dependencies = [
 [[package]]
 name = "aptos-protos"
 version = "1.3.1"
-source = "git+https://github.com/aptos-labs/aptos-core.git?rev=1d8460a995503574ec4e9699d3442d0150d7f3b9#1d8460a995503574ec4e9699d3442d0150d7f3b9"
+source = "git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7#1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7"
 dependencies = [
  "pbjson",
  "prost 0.13.4",
@@ -3978,7 +4058,7 @@ dependencies = [
  "bcs 0.1.4",
  "claims",
  "futures",
- "mockall",
+ "mockall 0.11.4",
  "move-core-types",
  "ntest",
  "once_cell",
@@ -4067,7 +4147,7 @@ dependencies = [
  "futures",
  "maplit",
  "mini-moka",
- "mockall",
+ "mockall 0.11.4",
  "once_cell",
  "rand 0.7.3",
  "serde",
@@ -4115,10 +4195,10 @@ dependencies = [
 [[package]]
 name = "aptos-system-utils"
 version = "0.1.0"
-source = "git+https://github.com/aptos-labs/aptos-core.git?rev=202bdccff2b2d333a385ae86a4fcf23e89da9f62#202bdccff2b2d333a385ae86a4fcf23e89da9f62"
+source = "git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7#1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7"
 dependencies = [
  "anyhow",
- "aptos-profiler 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=202bdccff2b2d333a385ae86a4fcf23e89da9f62)",
+ "aptos-profiler 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7)",
  "async-mutex",
  "http 0.2.11",
  "hyper 0.14.28",
@@ -4370,6 +4450,24 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "aptos-transaction-filter"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7#1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7"
+dependencies = [
+ "anyhow",
+ "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=1fc4424ee1099830ae667a0bda43ab5d7fd3c8f7)",
+ "derivative",
+ "derive_builder",
+ "memchr",
+ "once_cell",
+ "prost 0.13.4",
+ "serde",
+ "serde_json",
+ "serde_yaml 0.8.26",
+ "thiserror",
+]
+
 [[package]]
 name = "aptos-transaction-generator-lib"
 version = "0.0.0"
@@ -4821,6 +4919,7 @@ dependencies = [
  "aptos-cached-packages",
  "aptos-config",
  "aptos-faucet-core",
+ "aptos-indexer-processor-sdk",
  "aptos-localnet",
  "aptos-node",
  "aptos-types",
@@ -4832,7 +4931,6 @@ dependencies = [
  "maplit",
  "processor",
  "rand 0.7.3",
- "server-framework",
  "tempfile",
  "tokio",
  "tokio-util 0.7.10",
@@ -5409,6 +5507,39 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "autometrics"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10eaae539e7319a3813dc8cd53776a7128bdd6d82067275c12586f5a0fce9137"
+dependencies = [
+ "autometrics-macros",
+ "cfg_aliases 0.1.1",
+ "http 1.1.0",
+ "linkme",
+ "metrics-exporter-prometheus",
+ "once_cell",
+ "opentelemetry-prometheus",
+ "opentelemetry_sdk",
+ "prometheus",
+ "prometheus-client",
+ "spez",
+ "thiserror",
+]
+
+[[package]]
+name = "autometrics-macros"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdf7c9ebfee6425011c65788c746adf80fac99ba38957ba1cdb824b593cfc993"
+dependencies = [
+ "percent-encoding",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "axum"
 version = "0.6.20"
@@ -6237,6 +6368,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cfg_aliases"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
+
 [[package]]
 name = "cfg_aliases"
 version = "0.2.1"
@@ -7333,6 +7470,17 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "delegate"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "der"
 version = "0.5.1"
@@ -7494,9 +7642,9 @@ checksum = "3ae2a35373c5c74340b79ae6780b498b2b183915ec5dacf263aac5a099bf485a"
 
 [[package]]
 name = "diesel"
-version = "2.1.4"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8"
+checksum = "35b696af9ff4c0d2a507db2c5faafa8aa0205e297e5f11e203a24226d5355e7a"
 dependencies = [
  "bigdecimal",
  "bitflags 2.6.0",
@@ -7515,8 +7663,8 @@ dependencies = [
 
 [[package]]
 name = "diesel-async"
-version = "0.4.1"
-source = "git+https://github.com/weiznich/diesel_async.git?rev=d02798c67065d763154d7272dd0c09b39757d0f2#d02798c67065d763154d7272dd0c09b39757d0f2"
+version = "0.5.2"
+source = "git+https://github.com/weiznich/diesel_async.git?rev=e3beac66cd41ab53d78a10328bb72f272103e5d1#e3beac66cd41ab53d78a10328bb72f272103e5d1"
 dependencies = [
  "async-trait",
  "bb8",
@@ -7529,11 +7677,12 @@ dependencies = [
 
 [[package]]
 name = "diesel_derives"
-version = "2.1.2"
+version = "2.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44"
+checksum = "a93958254b70bea63b4187ff73d10180599d9d8d177071b7f91e6da4e0c0ad55"
 dependencies = [
  "diesel_table_macro_syntax",
+ "dsl_auto_type",
  "proc-macro2",
  "quote",
  "syn 2.0.87",
@@ -7541,9 +7690,9 @@ dependencies = [
 
 [[package]]
 name = "diesel_migrations"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac"
+checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6"
 dependencies = [
  "diesel",
  "migrations_internals",
@@ -7552,9 +7701,9 @@ dependencies = [
 
 [[package]]
 name = "diesel_table_macro_syntax"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
+checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25"
 dependencies = [
  "syn 2.0.87",
 ]
@@ -7696,6 +7845,26 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
 
+[[package]]
+name = "dsl_auto_type"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b"
+dependencies = [
+ "darling 0.20.10",
+ "either",
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
+[[package]]
+name = "dtoa"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04"
+
 [[package]]
 name = "dunce"
 version = "1.0.4"
@@ -9825,6 +9994,19 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "instrumented-channel"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-indexer-processor-sdk.git?tag=aptos-indexer-processor-sdk-v2.0.0#7f2dbeb0e482d274d8de1a55472159cb74f92cad"
+dependencies = [
+ "delegate",
+ "derive_builder",
+ "kanal",
+ "once_cell",
+ "prometheus",
+ "prometheus-client",
+]
+
 [[package]]
 name = "integer-encoding"
 version = "3.0.4"
@@ -10542,6 +10724,26 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
+[[package]]
+name = "linkme"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22d227772b5999ddc0690e733f734f95ca05387e329c4084fe65678c51198ffe"
+dependencies = [
+ "linkme-impl",
+]
+
+[[package]]
+name = "linkme-impl"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71a98813fa0073a317ed6a8055dcd4722a49d9b862af828ee68449adb799b6be"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.1.4"
@@ -10585,9 +10787,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 dependencies = [
  "serde",
  "value-bag",
@@ -10639,6 +10841,15 @@ dependencies = [
  "twox-hash",
 ]
 
+[[package]]
+name = "mach2"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "maplit"
 version = "1.0.2"
@@ -10728,21 +10939,72 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5"
+dependencies = [
+ "ahash 0.8.11",
+ "metrics-macros",
+ "portable-atomic",
+]
+
+[[package]]
+name = "metrics-exporter-prometheus"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950"
+dependencies = [
+ "base64 0.21.7",
+ "indexmap 1.9.3",
+ "metrics",
+ "metrics-util",
+ "quanta",
+ "thiserror",
+]
+
+[[package]]
+name = "metrics-macros"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.13.2",
+ "metrics",
+ "num_cpus",
+ "quanta",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "migrations_internals"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada"
+checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff"
 dependencies = [
  "serde",
- "toml 0.7.8",
+ "toml 0.8.12",
 ]
 
 [[package]]
 name = "migrations_macros"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08"
+checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd"
 dependencies = [
  "migrations_internals",
  "proc-macro2",
@@ -10836,8 +11098,23 @@ dependencies = [
  "downcast",
  "fragile",
  "lazy_static",
- "mockall_derive",
- "predicates",
+ "mockall_derive 0.11.4",
+ "predicates 2.1.5",
+ "predicates-tree",
+]
+
+[[package]]
+name = "mockall"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48"
+dependencies = [
+ "cfg-if",
+ "downcast",
+ "fragile",
+ "lazy_static",
+ "mockall_derive 0.12.1",
+ "predicates 3.1.3",
  "predicates-tree",
 ]
 
@@ -10853,6 +11130,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "mockall_derive"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2"
+dependencies = [
+ "cfg-if",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "module-publish"
 version = "0.1.0"
@@ -11749,7 +12038,7 @@ dependencies = [
  "derivative",
  "hashbrown 0.14.3",
  "itertools 0.13.0",
- "mockall",
+ "mockall 0.11.4",
  "move-binary-format",
  "move-core-types",
  "parking_lot 0.12.1",
@@ -11927,7 +12216,7 @@ checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
 dependencies = [
  "bitflags 2.6.0",
  "cfg-if",
- "cfg_aliases",
+ "cfg_aliases 0.2.1",
  "libc",
 ]
 
@@ -12306,6 +12595,52 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "opentelemetry"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "indexmap 2.7.0",
+ "js-sys",
+ "once_cell",
+ "pin-project-lite",
+ "thiserror",
+ "urlencoding",
+]
+
+[[package]]
+name = "opentelemetry-prometheus"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f8f082da115b0dcb250829e3ed0b8792b8f963a1ad42466e48422fbe6a079bd"
+dependencies = [
+ "once_cell",
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "prometheus",
+ "protobuf",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4"
+dependencies = [
+ "async-trait",
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "glob",
+ "once_cell",
+ "opentelemetry",
+ "ordered-float 4.6.0",
+ "thiserror",
+]
+
 [[package]]
 name = "option-ext"
 version = "0.2.0"
@@ -12330,6 +12665,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "os_str_bytes"
 version = "6.6.1"
@@ -12533,7 +12877,7 @@ dependencies = [
  "async-trait",
  "coset",
  "log",
- "mockall",
+ "mockall 0.11.4",
  "p256",
  "passkey-types",
  "rand 0.8.5",
@@ -13270,6 +13614,16 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "predicates"
+version = "3.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573"
+dependencies = [
+ "anstyle",
+ "predicates-core",
+]
+
 [[package]]
 name = "predicates-core"
 version = "1.0.6"
@@ -13432,15 +13786,14 @@ dependencies = [
 
 [[package]]
 name = "processor"
-version = "1.0.0"
-source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bf4dd89677e6929ef21f898a7a4176514e606ebd#bf4dd89677e6929ef21f898a7a4176514e606ebd"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-indexer-processors-v2.git?tag=aptos-indexer-processors-v2.0.0#0c3f3bf4380295ad3b4d3a15f9d204e996a29584"
 dependencies = [
  "ahash 0.8.11",
  "allocative",
  "allocative_derive",
  "anyhow",
- "aptos-moving-average",
- "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=1d8460a995503574ec4e9699d3442d0150d7f3b9)",
+ "aptos-indexer-processor-sdk",
  "async-trait",
  "bcs 0.1.4",
  "bigdecimal",
@@ -13461,11 +13814,10 @@ dependencies = [
  "google-cloud-storage",
  "hex",
  "hyper 0.14.28",
- "indexmap 2.7.0",
  "itertools 0.12.1",
  "jemallocator",
- "kanal",
  "lazy_static",
+ "log",
  "native-tls",
  "num_cpus",
  "once_cell",
@@ -13478,7 +13830,6 @@ dependencies = [
  "regex",
  "serde",
  "serde_json",
- "server-framework",
  "sha2 0.10.8",
  "sha3 0.10.8",
  "strum 0.24.1",
@@ -13523,9 +13874,33 @@ dependencies = [
  "lazy_static",
  "memchr",
  "parking_lot 0.12.1",
+ "protobuf",
  "thiserror",
 ]
 
+[[package]]
+name = "prometheus-client"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "504ee9ff529add891127c4827eb481bd69dc0ebc72e9a682e187db4caa60c3ca"
+dependencies = [
+ "dtoa",
+ "itoa",
+ "parking_lot 0.12.1",
+ "prometheus-client-derive-encode",
+]
+
+[[package]]
+name = "prometheus-client-derive-encode"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "prometheus-http-query"
 version = "0.5.2"
@@ -13754,6 +14129,22 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "quanta"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "mach2",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -14002,6 +14393,15 @@ dependencies = [
  "rand 0.8.5",
 ]
 
+[[package]]
+name = "raw-cpuid"
+version = "10.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "rawpointer"
 version = "0.2.1"
@@ -14777,6 +15177,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "sample"
+version = "0.1.0"
+source = "git+https://github.com/aptos-labs/aptos-indexer-processor-sdk.git?tag=aptos-indexer-processor-sdk-v2.0.0#7f2dbeb0e482d274d8de1a55472159cb74f92cad"
+dependencies = [
+ "tracing",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.23"
@@ -15175,27 +15583,6 @@ dependencies = [
  "proptest-derive",
 ]
 
-[[package]]
-name = "server-framework"
-version = "1.0.0"
-source = "git+https://github.com/aptos-labs/aptos-indexer-processors.git?rev=bf4dd89677e6929ef21f898a7a4176514e606ebd#bf4dd89677e6929ef21f898a7a4176514e606ebd"
-dependencies = [
- "anyhow",
- "aptos-system-utils 0.1.0 (git+https://github.com/aptos-labs/aptos-core.git?rev=202bdccff2b2d333a385ae86a4fcf23e89da9f62)",
- "async-trait",
- "backtrace",
- "clap 4.5.21",
- "prometheus",
- "serde",
- "serde_yaml 0.8.26",
- "tempfile",
- "tokio",
- "toml 0.7.8",
- "tracing",
- "tracing-subscriber 0.3.18",
- "warp",
-]
-
 [[package]]
 name = "set_env"
 version = "1.3.4"
@@ -15465,6 +15852,12 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c"
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -15633,6 +16026,17 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "spez"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -16501,6 +16905,18 @@ dependencies = [
  "toml_edit 0.19.15",
 ]
 
+[[package]]
+name = "toml"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit 0.22.12",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.6.6"
@@ -16520,7 +16936,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow",
+ "winnow 0.5.33",
 ]
 
 [[package]]
@@ -16531,7 +16947,20 @@ checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
 dependencies = [
  "indexmap 2.7.0",
  "toml_datetime",
- "winnow",
+ "winnow 0.5.33",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef"
+dependencies = [
+ "indexmap 2.7.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow 0.6.26",
 ]
 
 [[package]]
@@ -17236,9 +17665,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
 [[package]]
 name = "value-bag"
-version = "1.5.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd65ff0de3304a013e6dfeb7324e1cd389a8a000e582e3c9c9fae82ced778e26"
+checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5"
 
 [[package]]
 name = "variant_count"
@@ -17855,6 +18284,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "winnow"
+version = "0.6.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "winreg"
 version = "0.50.0"
diff --git a/Cargo.toml b/Cargo.toml
index 278f63e25a8..40c2a719ae9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -493,6 +493,9 @@ ark-groth16 = "0.4.0"
 ark-relations = "0.4.0"
 ark-serialize = "0.4.0"
 ark-std = { version = "0.4.0", features = ["getrandom"] }
+aptos-indexer-processor-sdk = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", tag = "aptos-indexer-processor-sdk-v2.0.0", features = [
+    "postgres_partial",
+] }
 aptos-moving-average = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "bf4dd89677e6929ef21f898a7a4176514e606ebd" }
 assert_approx_eq = "1.1.0"
 async-channel = "1.7.1"
@@ -560,10 +563,10 @@ derivation-path = "0.2.0"
 derive_builder = "0.20.0"
 determinator = "0.12.0"
 derive_more = "0.99.11"
-diesel = { version = "2.1", features = ["uuid"] }
+diesel = { version = "2.2.0", features = ["uuid"] }
 # Use the crate version once this feature gets released on crates.io:
 # https://github.com/weiznich/diesel_async/commit/e165e8c96a6c540ebde2d6d7c52df5c5620a4bf1
-diesel-async = { git = "https://github.com/weiznich/diesel_async.git", rev = "d02798c67065d763154d7272dd0c09b39757d0f2", features = [
+diesel-async = { git = "https://github.com/weiznich/diesel_async.git", rev = "e3beac66cd41ab53d78a10328bb72f272103e5d1", features = [
     "async-connection-wrapper",
     "postgres",
     "bb8",
@@ -693,7 +696,7 @@ pretty = "0.10.0"
 pretty_assertions = "1.2.1"
 # We set default-features to false so we don't onboard the libpq dep. See more here:
 # https://github.com/aptos-labs/aptos-core/pull/12568
-processor = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "bf4dd89677e6929ef21f898a7a4176514e606ebd", default-features = false }
+processor = { git = "https://github.com/aptos-labs/aptos-indexer-processors-v2.git", tag = "aptos-indexer-processors-v2.0.0", default-features = false }
 procfs = "0.14.1"
 proc-macro2 = "1.0.38"
 project-root = "0.2.2"
@@ -759,7 +762,6 @@ serde-generate = { git = "https://github.com/aptos-labs/serde-reflection", rev =
 serde-reflection = { git = "https://github.com/aptos-labs/serde-reflection", rev = "73b6bbf748334b71ff6d7d09d06a29e3062ca075" }
 serde_with = "3.4.0"
 serde_yaml = "0.8.24"
-server-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processors.git", rev = "bf4dd89677e6929ef21f898a7a4176514e606ebd" }
 set_env = "1.3.4"
 shadow-rs = "0.16.2"
 simplelog = "0.9.0"
@@ -866,10 +868,14 @@ move-stackless-bytecode-test-utils = { path = "third_party/move/move-model/bytec
 move-symbol-pool = { path = "third_party/move/move-symbol-pool" }
 move-table-extension = { path = "third_party/move/extensions/move-table-extension" }
 move-transactional-test-runner = { path = "third_party/move/testing-infra/transactional-test-runner" }
-move-unit-test = { path = "third_party/move/tools/move-unit-test", features = ["table-extension"] }
+move-unit-test = { path = "third_party/move/tools/move-unit-test", features = [
+    "table-extension",
+] }
 move-vm-metrics = { path = "third_party/move/move-vm/metrics" }
 move-vm-runtime = { path = "third_party/move/move-vm/runtime" }
-move-vm-test-utils = { path = "third_party/move/move-vm/test-utils", features = ["table-extension"] }
+move-vm-test-utils = { path = "third_party/move/move-vm/test-utils", features = [
+    "table-extension",
+] }
 move-vm-types = { path = "third_party/move/move-vm/types" }
 
 [profile.release]
diff --git a/aptos-move/aptos-workspace-server/Cargo.toml b/aptos-move/aptos-workspace-server/Cargo.toml
index edc3bf53097..9279115899d 100644
--- a/aptos-move/aptos-workspace-server/Cargo.toml
+++ b/aptos-move/aptos-workspace-server/Cargo.toml
@@ -24,9 +24,7 @@ aptos-types = { workspace = true }
 anyhow = { workspace = true }
 bollard = { workspace = true }
 clap = { workspace = true }
-diesel = { workspace = true, features = [
-    "postgres_backend",
-] }
+diesel = { workspace = true, features = ["postgres_backend"] }
 diesel-async = { workspace = true }
 futures = { workspace = true }
 maplit = { workspace = true }
@@ -38,5 +36,5 @@ url = { workspace = true }
 uuid = { workspace = true }
 
 # indexer deps
+aptos-indexer-processor-sdk = { workspace = true }
 processor = { workspace = true }
-server-framework = { workspace = true }
diff --git a/aptos-move/aptos-workspace-server/src/services/processors.rs b/aptos-move/aptos-workspace-server/src/services/processors.rs
index 9691957f017..6b61aab7b0d 100644
--- a/aptos-move/aptos-workspace-server/src/services/processors.rs
+++ b/aptos-move/aptos-workspace-server/src/services/processors.rs
@@ -7,15 +7,23 @@ use crate::{
     no_panic_println,
 };
 use anyhow::{anyhow, Context, Result};
+use aptos_indexer_processor_sdk::{
+    aptos_indexer_transaction_stream::TransactionStreamConfig,
+    postgres::utils::database::run_pending_migrations, server_framework::RunnableConfig,
+};
 use aptos_localnet::{health_checker::HealthChecker, processors::get_processor_config};
 use diesel::Connection;
 use diesel_async::{async_connection_wrapper::AsyncConnectionWrapper, pg::AsyncPgConnection};
 use futures::{future::try_join_all, stream::FuturesUnordered, StreamExt, TryFutureExt};
 use processor::{
-    gap_detectors::DEFAULT_GAP_DETECTION_BATCH_SIZE, processors::ProcessorName,
-    utils::database::run_pending_migrations, IndexerGrpcProcessorConfig,
+    config::{
+        db_config::{DbConfig, PostgresConfig},
+        indexer_processor_config::IndexerProcessorConfig,
+        processor_config::ProcessorName,
+        processor_mode::{BootStrapConfig, ProcessorMode},
+    },
+    MIGRATIONS,
 };
-use server_framework::RunnableConfig;
 use std::future::Future;
 use tokio::try_join;
 
@@ -31,7 +39,6 @@ const PROCESSOR_NAMES: &[ProcessorName] = {
         ObjectsProcessor,
         StakeProcessor,
         TokenV2Processor,
-        TransactionMetadataProcessor,
         UserTransactionProcessor,
     ]
 };
@@ -58,27 +65,29 @@ fn start_processor(
 
         no_panic_println!("Starting processor {}..", processor_name_);
 
-        let config = IndexerGrpcProcessorConfig {
+        let config = IndexerProcessorConfig {
             processor_config: get_processor_config(&processor_name_)?,
-            postgres_connection_string: get_postgres_connection_string(postgres_port),
-            indexer_grpc_data_service_address: get_data_service_url(indexer_grpc_port),
-
-            auth_token: "notused".to_string(),
-            grpc_http2_config: Default::default(),
-            starting_version: None,
-            ending_version: None,
-            number_concurrent_processing_tasks: None,
-            enable_verbose_logging: None,
-            // The default at the time of writing is 30 but we don't need that
-            // many in a localnet environment.
-            db_pool_size: Some(8),
-            gap_detection_batch_size: 50,
-            pb_channel_txn_chunk_size: 100_000,
-            per_table_chunk_sizes: Default::default(),
-            transaction_filter: Default::default(),
-            grpc_response_item_timeout_in_secs: 10,
-            deprecated_tables: Default::default(),
-            parquet_gap_detection_batch_size: DEFAULT_GAP_DETECTION_BATCH_SIZE,
+            transaction_stream_config: TransactionStreamConfig {
+                indexer_grpc_data_service_address: get_data_service_url(indexer_grpc_port),
+                auth_token: "notused".to_string(),
+                starting_version: Some(0),
+                request_ending_version: None,
+                request_name_header: "notused".to_string(),
+                additional_headers: Default::default(),
+                indexer_grpc_http2_ping_interval_secs: Default::default(),
+                indexer_grpc_http2_ping_timeout_secs: 60,
+                indexer_grpc_reconnection_timeout_secs: 60,
+                indexer_grpc_response_item_timeout_secs: 60,
+                indexer_grpc_reconnection_max_retries: Default::default(),
+                transaction_filter: Default::default(),
+            },
+            db_config: DbConfig::PostgresConfig(PostgresConfig {
+                connection_string: get_postgres_connection_string(postgres_port),
+                db_pool_size: 8,
+            }),
+            processor_mode: ProcessorMode::Default(BootStrapConfig {
+                initial_starting_version: 0,
+            }),
         };
 
         config.run().await
@@ -144,7 +153,7 @@ pub fn start_all_processors(
                 AsyncConnectionWrapper::establish(&connection_string).with_context(|| {
                     format!("Failed to connect to postgres at {}", connection_string)
                 })?;
-            run_pending_migrations(&mut conn);
+            run_pending_migrations(&mut conn, MIGRATIONS);
             anyhow::Ok(())
         })
         .await
diff --git a/crates/aptos-localnet/Cargo.toml b/crates/aptos-localnet/Cargo.toml
index 2ba3067a361..7da7fd1c753 100644
--- a/crates/aptos-localnet/Cargo.toml
+++ b/crates/aptos-localnet/Cargo.toml
@@ -22,9 +22,7 @@ version-compare = { workspace = true }
 
 aptos-protos = { workspace = true }
 aptos-rest-client = { workspace = true }
-diesel = { workspace = true, features = [
-    "postgres_backend",
-] }
+diesel = { workspace = true, features = ["postgres_backend"] }
 diesel-async = { workspace = true }
 processor = { workspace = true }
 reqwest = { workspace = true }
diff --git a/crates/aptos-localnet/src/hasura_metadata.json b/crates/aptos-localnet/src/hasura_metadata.json
index c07f8dc421c..53398ef694a 100644
--- a/crates/aptos-localnet/src/hasura_metadata.json
+++ b/crates/aptos-localnet/src/hasura_metadata.json
@@ -1,5 +1,5 @@
 {
-  "resource_version": 38,
+  "resource_version": 10000,
   "metadata": {
     "version": 3,
     "sources": [
@@ -795,6 +795,57 @@
               }
             ]
           },
+          {
+            "table": {
+              "name": "ledger_infos",
+              "schema": "processor_metadata"
+            },
+            "configuration": {
+              "column_config": {},
+              "custom_column_names": {},
+              "custom_name": "ledger_infos",
+              "custom_root_fields": {}
+            },
+            "select_permissions": [
+              {
+                "role": "anonymous",
+                "permission": {
+                  "columns": [
+                    "chain_id"
+                  ],
+                  "filter": {}
+                }
+              }
+            ]
+          },
+          {
+            "table": {
+              "name": "processor_status",
+              "schema": "processor_metadata"
+            },
+            "configuration": {
+              "column_config": {},
+              "custom_column_names": {},
+              "custom_name": "processor_status",
+              "custom_root_fields": {}
+            },
+            "select_permissions": [
+              {
+                "role": "anonymous",
+                "permission": {
+                  "columns": [
+                    "last_success_version",
+                    "processor",
+                    "last_transaction_timestamp",
+                    "last_updated"
+                  ],
+                  "filter": {},
+                  "limit": 100
+                },
+                "comment": ""
+              }
+            ]
+          },
           {
             "table": {
               "name": "account_transactions",
@@ -2079,23 +2130,6 @@
               }
             ]
           },
-          {
-            "table": {
-              "name": "ledger_infos",
-              "schema": "public"
-            },
-            "select_permissions": [
-              {
-                "role": "anonymous",
-                "permission": {
-                  "columns": [
-                    "chain_id"
-                  ],
-                  "filter": {}
-                }
-              }
-            ]
-          },
           {
             "table": {
               "name": "num_active_delegator_per_pool",
@@ -2115,27 +2149,6 @@
               }
             ]
           },
-          {
-            "table": {
-              "name": "processor_status",
-              "schema": "public"
-            },
-            "select_permissions": [
-              {
-                "role": "anonymous",
-                "permission": {
-                  "columns": [
-                    "last_success_version",
-                    "last_transaction_timestamp",
-                    "last_updated",
-                    "processor"
-                  ],
-                  "filter": {},
-                  "limit": 100
-                }
-              }
-            ]
-          },
           {
             "table": {
               "name": "proposal_votes",
@@ -2315,6 +2328,12 @@
               }
             ]
           },
+          {
+            "table": {
+              "name": "token_datas_v2",
+              "schema": "public"
+            }
+          },
           {
             "table": {
               "name": "user_transactions",
diff --git a/crates/aptos-localnet/src/processors.rs b/crates/aptos-localnet/src/processors.rs
index 276e2b85be2..1634ce84df8 100644
--- a/crates/aptos-localnet/src/processors.rs
+++ b/crates/aptos-localnet/src/processors.rs
@@ -2,30 +2,40 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use anyhow::{bail, Result};
-use processor::processors::{
-    objects_processor::ObjectsProcessorConfig, stake_processor::StakeProcessorConfig,
-    token_v2_processor::TokenV2ProcessorConfig, ProcessorConfig, ProcessorName,
+use processor::{
+    config::processor_config::{ProcessorConfig, ProcessorName},
+    processors::{
+        objects::objects_processor::ObjectsProcessorConfig,
+        stake::stake_processor::StakeProcessorConfig,
+        token_v2::token_v2_processor::TokenV2ProcessorConfig,
+    },
 };
 
 pub fn get_processor_config(processor_name: &ProcessorName) -> Result<ProcessorConfig> {
     Ok(match processor_name {
         ProcessorName::AccountTransactionsProcessor => {
-            ProcessorConfig::AccountTransactionsProcessor
+            ProcessorConfig::AccountTransactionsProcessor(Default::default())
+        },
+        ProcessorName::AccountRestorationProcessor => {
+            bail!("AccountRestorationProcessor is not supported in the localnet")
         },
         ProcessorName::AnsProcessor => {
             bail!("ANS processor is not supported in the localnet")
         },
-        ProcessorName::DefaultProcessor => ProcessorConfig::DefaultProcessor,
-        ProcessorName::EventsProcessor => ProcessorConfig::EventsProcessor,
-        ProcessorName::FungibleAssetProcessor => ProcessorConfig::FungibleAssetProcessor,
+        ProcessorName::DefaultProcessor => ProcessorConfig::DefaultProcessor(Default::default()),
+        ProcessorName::EventsProcessor => ProcessorConfig::EventsProcessor(Default::default()),
+        ProcessorName::FungibleAssetProcessor => {
+            ProcessorConfig::FungibleAssetProcessor(Default::default())
+        },
+        ProcessorName::GasFeeProcessor => {
+            bail!("GasFeeProcessor is not supported in the localnet")
+        },
         ProcessorName::MonitoringProcessor => {
             bail!("Monitoring processor is not supported in the localnet")
         },
-        ProcessorName::NftMetadataProcessor => {
-            bail!("NFT Metadata processor is not supported in the localnet")
-        },
         ProcessorName::ObjectsProcessor => {
             ProcessorConfig::ObjectsProcessor(ObjectsProcessorConfig {
+                default_config: Default::default(),
                 query_retries: Default::default(),
                 query_retry_delay_ms: Default::default(),
             })
@@ -33,9 +43,6 @@ pub fn get_processor_config(processor_name: &ProcessorName) -> Result<ProcessorC
         ProcessorName::ParquetDefaultProcessor => {
             bail!("ParquetDefaultProcessor is not supported in the localnet")
         },
-        ProcessorName::ParquetFungibleAssetActivitiesProcessor => {
-            bail!("ParquetFungibleAssetActivitiesProcessor is not supported in the localnet")
-        },
         ProcessorName::ParquetFungibleAssetProcessor => {
             bail!("ParquetFungibleAssetProcessor is not supported in the localnet")
         },
@@ -51,22 +58,32 @@ pub fn get_processor_config(processor_name: &ProcessorName) -> Result<ProcessorC
         ProcessorName::ParquetTokenV2Processor => {
             bail!("ParquetTokenV2Processor is not supported in the localnet")
         },
-        ProcessorName::ParquetUserTransactionsProcessor => {
-            bail!("ParquetUserTransactionsProcessor is not supported in the localnet")
+        ProcessorName::ParquetUserTransactionProcessor => {
+            bail!("ParquetUserTransactionProcessor is not supported in the localnet")
+        },
+        ProcessorName::ParquetObjectsProcessor => {
+            bail!("ParquetObjectsProcessor is not supported in the localnet")
+        },
+        ProcessorName::ParquetAccountTransactionsProcessor => {
+            bail!("ParquetAccountTransactionsProcessor is not supported in the localnet")
+        },
+        ProcessorName::ParquetStakeProcessor => {
+            bail!("ParquetStakeProcessor is not supported in the localnet")
         },
         ProcessorName::StakeProcessor => ProcessorConfig::StakeProcessor(StakeProcessorConfig {
+            default_config: Default::default(),
             query_retries: Default::default(),
             query_retry_delay_ms: Default::default(),
         }),
         ProcessorName::TokenV2Processor => {
             ProcessorConfig::TokenV2Processor(TokenV2ProcessorConfig {
+                default_config: Default::default(),
                 query_retries: Default::default(),
                 query_retry_delay_ms: Default::default(),
             })
         },
-        ProcessorName::TransactionMetadataProcessor => {
-            ProcessorConfig::TransactionMetadataProcessor
+        ProcessorName::UserTransactionProcessor => {
+            ProcessorConfig::UserTransactionProcessor(Default::default())
         },
-        ProcessorName::UserTransactionProcessor => ProcessorConfig::UserTransactionProcessor,
     })
 }
diff --git a/crates/aptos-metrics-core/src/const_metric.rs b/crates/aptos-metrics-core/src/const_metric.rs
index c354b0eed49..744ff1b7c98 100644
--- a/crates/aptos-metrics-core/src/const_metric.rs
+++ b/crates/aptos-metrics-core/src/const_metric.rs
@@ -37,7 +37,7 @@ impl ConstMetric {
 
         let mut metric = Metric::default();
         metric.set_counter(counter);
-        metric.set_label(labels);
+        metric.set_label(labels.into());
 
         Ok(ConstMetric {
             desc,
@@ -63,7 +63,7 @@ impl ConstMetric {
 
         let mut metric = Metric::default();
         metric.set_gauge(guage);
-        metric.set_label(labels);
+        metric.set_label(labels.into());
 
         Ok(ConstMetric {
             desc,
@@ -84,7 +84,7 @@ impl Collector for ConstMetric {
         met.set_name(self.desc.fq_name.clone());
         met.set_help(self.desc.help.clone());
         met.set_field_type(self.metric_type);
-        met.set_metric(vec![self.metric.clone()]);
+        met.set_metric(vec![self.metric.clone()].into());
 
         vec![met]
     }
diff --git a/crates/aptos/Cargo.toml b/crates/aptos/Cargo.toml
index fae54bc7cad..45d89de3fc7 100644
--- a/crates/aptos/Cargo.toml
+++ b/crates/aptos/Cargo.toml
@@ -30,7 +30,7 @@ aptos-genesis = { workspace = true }
 aptos-github-client = { workspace = true }
 aptos-global-constants = { workspace = true }
 aptos-indexer-grpc-server-framework = { workspace = true }
-aptos-indexer-grpc-utils = { workspace = true }
+aptos-indexer-processor-sdk = { workspace = true }
 aptos-keygen = { workspace = true }
 aptos-ledger = { workspace = true }
 aptos-localnet = { workspace = true }
@@ -38,7 +38,6 @@ aptos-logger = { workspace = true }
 aptos-move-debugger = { workspace = true }
 aptos-network-checker = { workspace = true }
 aptos-node = { workspace = true }
-aptos-protos = { workspace = true }
 aptos-rest-client = { workspace = true }
 aptos-sdk = { workspace = true }
 aptos-storage-interface = { workspace = true }
@@ -52,6 +51,7 @@ aptos-vm-logging = { workspace = true }
 aptos-vm-types = { workspace = true }
 aptos-workspace-server = { workspace = true }
 async-trait = { workspace = true }
+backoff = { workspace = true }
 base64 = { workspace = true }
 bcs = { workspace = true }
 bollard = { workspace = true }
@@ -60,9 +60,7 @@ clap = { workspace = true, features = ["env", "unstable-styles", "wrap_help"] }
 clap_complete = { workspace = true }
 colored = { workspace = true }
 dashmap = { workspace = true }
-diesel = { workspace = true, features = [
-    "postgres_backend",
-] }
+diesel = { workspace = true, features = ["postgres_backend"] }
 diesel-async = { workspace = true }
 dirs = { workspace = true }
 futures = { workspace = true }
@@ -94,17 +92,18 @@ processor = { workspace = true }
 rand = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true }
-self_update = { git = "https://github.com/banool/self_update.git", rev = "8306158ad0fd5b9d4766a3c6bf967e7ef0ea5c4b", features = ["archive-zip", "compression-zip-deflate"] }
+self_update = { git = "https://github.com/banool/self_update.git", rev = "8306158ad0fd5b9d4766a3c6bf967e7ef0ea5c4b", features = [
+    "archive-zip",
+    "compression-zip-deflate",
+] }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
-server-framework = { workspace = true }
 set_env = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 toml = { workspace = true }
-tonic = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 url = { workspace = true }
diff --git a/crates/aptos/src/node/local_testnet/health_checker.rs b/crates/aptos/src/node/local_testnet/health_checker.rs
index ec22495b3c0..286dd0e9f9e 100644
--- a/crates/aptos/src/node/local_testnet/health_checker.rs
+++ b/crates/aptos/src/node/local_testnet/health_checker.rs
@@ -3,18 +3,19 @@
 
 use super::indexer_api::confirm_metadata_applied;
 use anyhow::{anyhow, Context, Result};
-use aptos_protos::indexer::v1::GetTransactionsRequest;
+use aptos_indexer_processor_sdk::{
+    aptos_indexer_transaction_stream::{transaction_stream::get_chain_id, TransactionStreamConfig},
+    postgres::processor_metadata_schema::processor_metadata::processor_status,
+};
 use diesel::{ExpressionMethods, OptionalExtension, QueryDsl};
 use diesel_async::{pg::AsyncPgConnection, AsyncConnection, RunQueryDsl};
-use futures::StreamExt;
-use processor::schema::processor_status;
 use reqwest::Url;
 use serde::Serialize;
 use std::time::Duration;
 use tokio::time::Instant;
 use tracing::info;
 
-const MAX_WAIT_S: u64 = 60;
+const MAX_WAIT_S: u64 = 120;
 const WAIT_INTERVAL_MS: u64 = 200;
 
 /// This provides a single place to define a variety of different healthchecks. In
@@ -56,25 +57,32 @@ impl HealthChecker {
                 Ok(())
             },
             HealthChecker::DataServiceGrpc(url) => {
-                let mut client = aptos_indexer_grpc_utils::create_data_service_grpc_client(
-                    url.clone(),
-                    Some(Duration::from_secs(5)),
-                )
-                .await?;
-                let request = tonic::Request::new(GetTransactionsRequest {
-                    starting_version: Some(0),
+                let backoff = backoff::ExponentialBackoff {
+                    max_elapsed_time: Some(Duration::from_secs(5)),
                     ..Default::default()
-                });
-                // Make sure we can stream the first message from the stream.
-                client
-                    .get_transactions(request)
-                    .await
-                    .context("GRPC connection error")?
-                    .into_inner()
-                    .next()
-                    .await
-                    .context("Did not receive init signal from data service GRPC stream")?
-                    .context("Error processing first message from GRPC stream")?;
+                };
+                backoff::future::retry(backoff, || async {
+                    let transaction_stream_config = TransactionStreamConfig {
+                        indexer_grpc_data_service_address: url.clone(),
+                        auth_token: "notused".to_string(),
+                        starting_version: Some(0),
+                        request_ending_version: None,
+                        request_name_header: "notused".to_string(),
+                        additional_headers: Default::default(),
+                        indexer_grpc_http2_ping_interval_secs: Default::default(),
+                        indexer_grpc_http2_ping_timeout_secs: 60,
+                        indexer_grpc_response_item_timeout_secs: 60,
+                        indexer_grpc_reconnection_timeout_secs: 60,
+                        indexer_grpc_reconnection_max_retries: Default::default(),
+                        transaction_filter: None,
+                    };
+                    get_chain_id(transaction_stream_config)
+                        .await
+                        .context("Failed to get chain id")?;
+                    Ok(())
+                })
+                .await
+                .context("Failed to get a response from gRPC")?;
                 Ok(())
             },
             HealthChecker::Postgres(connection_string) => {
diff --git a/crates/aptos/src/node/local_testnet/processors.rs b/crates/aptos/src/node/local_testnet/processors.rs
index e8fbf21ab37..6d482bab219 100644
--- a/crates/aptos/src/node/local_testnet/processors.rs
+++ b/crates/aptos/src/node/local_testnet/processors.rs
@@ -3,6 +3,10 @@
 
 use super::{health_checker::HealthChecker, traits::ServiceManager, RunLocalnet};
 use anyhow::{bail, Context, Result};
+use aptos_indexer_processor_sdk::{
+    aptos_indexer_transaction_stream::TransactionStreamConfig,
+    postgres::utils::database::run_pending_migrations, server_framework::RunnableConfig,
+};
 pub use aptos_localnet::processors::get_processor_config;
 use async_trait::async_trait;
 use clap::Parser;
@@ -10,11 +14,15 @@ use diesel::Connection;
 use diesel_async::{async_connection_wrapper::AsyncConnectionWrapper, pg::AsyncPgConnection};
 use maplit::hashset;
 use processor::{
-    gap_detectors::DEFAULT_GAP_DETECTION_BATCH_SIZE, processors::ProcessorName,
-    utils::database::run_pending_migrations, IndexerGrpcProcessorConfig,
+    config::{
+        db_config::{DbConfig, PostgresConfig},
+        indexer_processor_config::IndexerProcessorConfig,
+        processor_config::ProcessorName,
+        processor_mode::{BootStrapConfig, ProcessorMode},
+    },
+    MIGRATIONS,
 };
 use reqwest::Url;
-use server_framework::RunnableConfig;
 use std::collections::HashSet;
 use tokio::sync::OnceCell;
 use tracing::info;
@@ -39,7 +47,6 @@ pub struct ProcessorArgs {
             ProcessorName::ObjectsProcessor,
             ProcessorName::StakeProcessor,
             ProcessorName::TokenV2Processor,
-            ProcessorName::TransactionMetadataProcessor,
             ProcessorName::UserTransactionProcessor,
         ],
         requires = "with_indexer_api"
@@ -49,7 +56,7 @@ pub struct ProcessorArgs {
 
 #[derive(Debug)]
 pub struct ProcessorManager {
-    config: IndexerGrpcProcessorConfig,
+    config: IndexerProcessorConfig,
     prerequisite_health_checkers: HashSet<HealthChecker>,
 }
 
@@ -61,26 +68,29 @@ impl ProcessorManager {
         postgres_connection_string: String,
     ) -> Result<Self> {
         let processor_config = get_processor_config(processor_name)?;
-        let config = IndexerGrpcProcessorConfig {
+        let config = IndexerProcessorConfig {
             processor_config,
-            postgres_connection_string,
-            indexer_grpc_data_service_address: data_service_url,
-            auth_token: "notused".to_string(),
-            grpc_http2_config: Default::default(),
-            starting_version: None,
-            ending_version: None,
-            number_concurrent_processing_tasks: None,
-            enable_verbose_logging: None,
-            // The default at the time of writing is 30 but we don't need that
-            // many in a localnet environment.
-            db_pool_size: Some(8),
-            gap_detection_batch_size: 50,
-            pb_channel_txn_chunk_size: 100_000,
-            per_table_chunk_sizes: Default::default(),
-            transaction_filter: Default::default(),
-            grpc_response_item_timeout_in_secs: 10,
-            deprecated_tables: Default::default(),
-            parquet_gap_detection_batch_size: DEFAULT_GAP_DETECTION_BATCH_SIZE,
+            transaction_stream_config: TransactionStreamConfig {
+                indexer_grpc_data_service_address: data_service_url,
+                auth_token: "notused".to_string(),
+                starting_version: Some(0),
+                request_ending_version: None,
+                request_name_header: "notused".to_string(),
+                additional_headers: Default::default(),
+                indexer_grpc_http2_ping_interval_secs: Default::default(),
+                indexer_grpc_http2_ping_timeout_secs: 60,
+                indexer_grpc_reconnection_timeout_secs: 60,
+                indexer_grpc_response_item_timeout_secs: 60,
+                indexer_grpc_reconnection_max_retries: Default::default(),
+                transaction_filter: Default::default(),
+            },
+            db_config: DbConfig::PostgresConfig(PostgresConfig {
+                connection_string: postgres_connection_string,
+                db_pool_size: 8,
+            }),
+            processor_mode: ProcessorMode::Default(BootStrapConfig {
+                initial_starting_version: 0,
+            }),
         };
         let manager = Self {
             config,
@@ -113,7 +123,12 @@ impl ProcessorManager {
 
     /// Create the necessary tables in the DB for the processors to work.
     async fn run_migrations(&self) -> Result<()> {
-        let connection_string = self.config.postgres_connection_string.clone();
+        let connection_string = match &self.config.db_config {
+            DbConfig::PostgresConfig(postgres_config) => postgres_config.connection_string.clone(),
+            DbConfig::ParquetConfig(_) => {
+                bail!("Parquet is not supported in the localnet");
+            },
+        };
 
         tokio::task::spawn_blocking(move || {
             // This lets us use the connection like a normal diesel connection. See more:
@@ -122,7 +137,7 @@ impl ProcessorManager {
                 AsyncConnectionWrapper::establish(&connection_string).with_context(|| {
                     format!("Failed to connect to postgres at {}", connection_string)
                 })?;
-            run_pending_migrations(&mut conn);
+            run_pending_migrations(&mut conn, MIGRATIONS);
             anyhow::Ok(())
         })
         .await??;
@@ -137,8 +152,14 @@ impl ServiceManager for ProcessorManager {
     }
 
     fn get_health_checkers(&self) -> HashSet<HealthChecker> {
+        let connection_string = match &self.config.db_config {
+            DbConfig::PostgresConfig(postgres_config) => postgres_config.connection_string.clone(),
+            DbConfig::ParquetConfig(_) => {
+                panic!("Parquet is not supported in the localnet");
+            },
+        };
         hashset! {HealthChecker::Processor(
-            self.config.postgres_connection_string.to_string(),
+            connection_string,
             self.config.processor_config.name().to_string(),
         ) }
     }


Reviews:
user: login: gregnazario
body: Can we start making tags or branches or something for the indexer processors, so we can stop using hashes, and have something that's maybe more consistent?
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T20:37:18Z
user: login: just-in-chang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T20:40:13Z
user: login: rtso
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T20:45:58Z
user: login: just-in-chang
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T20:56:01Z

Review comments:
diff_hunk: @@ -636,6 +639,7 @@ jemalloc-sys = "0.5.4"
 json-patch = "0.2.6"
 jsonwebtoken = "8.1"
 jwt = "0.16.0"
+kanal = "=0.1.0-pre8"
path: Cargo.toml
user: login: just-in-chang
body: Why do we need kanal here?
created_at: 2025-04-01T20:39:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -22,10 +22,9 @@ version-compare = { workspace = true }
 
 aptos-protos = { workspace = true }
 aptos-rest-client = { workspace = true }
-diesel = { workspace = true, features = [
-    "postgres_backend",
-] }
+diesel = { workspace = true, features = ["postgres_backend"] }
 diesel-async = { workspace = true }
+kanal = { workspace = true }
path: crates/aptos-localnet/Cargo.toml
user: login: just-in-chang
body: ^

created_at: 2025-04-01T20:40:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -636,6 +639,7 @@ jemalloc-sys = "0.5.4"
 json-patch = "0.2.6"
 jsonwebtoken = "8.1"
 jwt = "0.16.0"
+kanal = "=0.1.0-pre8"
path: Cargo.toml
user: login: rtso
body: leftover when i was trying to debug force the kanal dep 🤦‍♀️ 
created_at: 2025-04-01T20:45:55Z
author_association: CONTRIBUTOR

PR comments:
user: login: trunk-io[bot]
created_at: 2025-04-01T20:28:24Z
body: <!-- Trunk Metrics Report -->



<details>
<summary>
⏱️ 37m total CI duration on this PR
</summary>

| Job | Cumulative Duration | Recent Runs |
|---|---|---|
| [rust-targeted-unit-tests](https://github.com/aptos-labs/aptos-core/actions/runs/14502397889/job/40685485018) | 23m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14502397889/job/40685485018)&#8201; |
| [cli-e2e-tests / run-cli-tests](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013/job/40693718017) | 7m | [🟥](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013/job/40693718017)&#8201; |
| [check-dynamic-deps](https://github.com/aptos-labs/aptos-core/actions/runs/14205381852/job/39801792697) | 4m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381852/job/39801792697)&#8201; |
| [rust-cargo-deny](https://github.com/aptos-labs/aptos-core/actions/runs/14205381847/job/39801809250) | 2m | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381847/job/39801809250)&#8201; |
| [general-lints](https://github.com/aptos-labs/aptos-core/actions/runs/14205381847/job/39801809801) | 29s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381847/job/39801809801)&#8201; |
| [semgrep/ci](https://github.com/aptos-labs/aptos-core/actions/runs/14205381831/job/39801792354) | 29s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381831/job/39801792354)&#8201; |
| [file_change_determinator](https://github.com/aptos-labs/aptos-core/actions/runs/14205381847/job/39801793353) | 10s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381847/job/39801793353)&#8201; |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/14205381825/job/39801792390) | 3s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381825/job/39801792390)&#8201; |
| [permission-check](https://github.com/aptos-labs/aptos-core/actions/runs/14205381830/job/39801792398) | 1s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205381830/job/39801792398)&#8201; |
| [check-branch-prefix](https://github.com/aptos-labs/aptos-core/actions/runs/14205380212/job/39801786577) | 1s | [🟩](https://github.com/aptos-labs/aptos-core/actions/runs/14205380212/job/39801786577)&#8201; |
</details>

🚨 1 job on the last run was significantly faster/slower than expected

| Job | Duration | vs 7d avg | Delta |
|---|---|---|---|
| [check-dynamic-deps](https://github.com/aptos-labs/aptos-core/actions/runs/14205381852/job/39801792697) | 4m | [2m](https://app.trunk.io/aptoslabs/ci-analytics/1_45128c3a-36c4-499b-8b5e-6bbed73502cd/1_709046ca-2ec5-45cf-85ea-98c40f1269da?repo=aptos-labs/aptos-core&timeBased=true&numericAggregateFunctions=NUMERIC_AGGREGATE_FUNCTION_AVG&timeRange=7+days&conclusion=GIT_HUB_REPO_ACTIVITY_RUN_CONCLUSION_SUCCESS&utm_source=gh-comment) | [![+50%](https://raster.shields.io/badge/+50%25-C84B25)](https://app.trunk.io/aptoslabs/ci-analytics/1_45128c3a-36c4-499b-8b5e-6bbed73502cd/1_709046ca-2ec5-45cf-85ea-98c40f1269da?repo=aptos-labs/aptos-core&timeBased=true&numericAggregateFunctions=NUMERIC_AGGREGATE_FUNCTION_AVG&timeRange=7+days&conclusion=GIT_HUB_REPO_ACTIVITY_RUN_CONCLUSION_SUCCESS&utm_source=gh-comment) |

<sub>[settings](https://app.trunk.io/aptoslabs/settings/repo?repo=aptos-labs/aptos-core&utm_source=gh-comment#ci-analytics) ⋅ [feedback](https://form.typeform.com/to/w7Iv34u2?utm_source=gh-comment) ⋅ [docs](https://docs.trunk.io/ci-analytics/pull-request-metrics?utm_source=gh-comment) ⋅ learn more about [trunk.io](https://trunk.io?utm_source=gh-comment)</sub>
author_association: NONE
user: login: rtso
created_at: 2025-04-01T20:28:38Z
body: * **#16389** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16389?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#16261** <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16261?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/aptos-labs/aptos-core/16261?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T22:56:54Z
body: ### Forge is running suite `realistic_env_max_load` on `21ffb284fcf2fbd5f0fe509ab0859b65f38569d4`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14503975741)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T22:56:57Z
body: ### Forge is running suite `compat` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `21ffb284fcf2fbd5f0fe509ab0859b65f38569d4`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14503975741)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T22:56:58Z
body: ### Forge is running suite `framework_upgrade` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `21ffb284fcf2fbd5f0fe509ab0859b65f38569d4`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16261&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14503975741)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T23:35:12Z
body: ### Forge is running suite `realistic_env_max_load` on `c566817ce0cad03bb25ae81a0d072ef6b17a4f1b`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504425568)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T23:35:17Z
body: ### Forge is running suite `compat` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `c566817ce0cad03bb25ae81a0d072ef6b17a4f1b`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504425568)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T23:48:22Z
body: ### :white_check_mark: Forge suite `compat` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `c566817ce0cad03bb25ae81a0d072ef6b17a4f1b`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> c566817ce0cad03bb25ae81a0d072ef6b17a4f1b (PR)
1. Check liveness of validators at old version: 463dda939a7bdc1266946e824a1eb309f300eaf8
compatibility::simple-validator-upgrade::liveness-check : committed: 10448.89 txn/s, latency: 3076.08 ms, (p50: 3200 ms, p70: 3300, p90: 3700 ms, p99: 4200 ms), latency samples: 340960
2. Upgrading first Validator to new version: c566817ce0cad03bb25ae81a0d072ef6b17a4f1b
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2704.99 txn/s, latency: 12292.06 ms, (p50: 13800 ms, p70: 14300, p90: 14700 ms, p99: 14700 ms), latency samples: 101420
3. Upgrading rest of first batch to new version: c566817ce0cad03bb25ae81a0d072ef6b17a4f1b
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2734.72 txn/s, latency: 12135.27 ms, (p50: 13700 ms, p70: 14000, p90: 14100 ms, p99: 14400 ms), latency samples: 103320
4. upgrading second batch to new version: c566817ce0cad03bb25ae81a0d072ef6b17a4f1b
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 5121.17 txn/s, latency: 6680.62 ms, (p50: 7200 ms, p70: 7500, p90: 7700 ms, p99: 7900 ms), latency samples: 179780
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> c566817ce0cad03bb25ae81a0d072ef6b17a4f1b passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16261&var-metrics_source=All&var-chain_name=forge-0&from=1744846534000&to=1744847300000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744846534000&end=1744847300000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-16T23%3A35%3A34.468035%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-16T23%3A48%3A20.855533%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744846534&until=1744847300&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744846534&until=1744847300&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504425568)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T23:48:37Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `c566817ce0cad03bb25ae81a0d072ef6b17a4f1b`
```
two traffics test: inner traffic : committed: 11741.16 txn/s, latency: 3382.51 ms, (p50: 3000 ms, p70: 3300, p90: 3700 ms, p99: 10200 ms), latency samples: 4464200
two traffics test : committed: 99.96 txn/s, latency: 1506.38 ms, (p50: 1400 ms, p70: 1500, p90: 1700 ms, p99: 2100 ms), latency samples: 1720
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.454, avg: 2.177", "ConsensusProposalToOrdered: max: 0.325, avg: 0.314", "ConsensusOrderedToCommit: max: 0.445, avg: 0.415", "ConsensusProposalToCommit: max: 0.763, avg: 0.729"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 2.07s no progress at version 1343257 (avg 0.22s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.97s no progress at version 4879756 (avg 0.78s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&from=1744846525000&to=1744847315000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744846525000&end=1744847315000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-16T23%3A35%3A25.859903%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-16T23%3A48%3A35.916638%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744846525&until=1744847315&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744846525&until=1744847315&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504425568)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:16:45Z
body: ### Forge is running suite `realistic_env_max_load` on `4b9698f35d45e7ecba02608b366196e91f4162d6`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:16:46Z
body: ### Forge is running suite `compat` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `4b9698f35d45e7ecba02608b366196e91f4162d6`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:16:47Z
body: ### Forge is running suite `framework_upgrade` on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `4b9698f35d45e7ecba02608b366196e91f4162d6`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16261&var-metrics_source=All&var-chain_name=forge-1&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:28:21Z
body: ### :x: Forge suite `realistic_env_max_load` failure on `4b9698f35d45e7ecba02608b366196e91f4162d6`
```
two traffics test: inner traffic : committed: 10129.33 txn/s, submitted: 10132.39 txn/s, expired: 3.06 txn/s, latency: 3919.24 ms, (p50: 3000 ms, p70: 3500, p90: 4200 ms, p99: 29700 ms), latency samples: 3851696
two traffics test : committed: 99.97 txn/s, latency: 1775.98 ms, (p50: 1400 ms, p70: 1500, p90: 1700 ms, p99: 10200 ms), latency samples: 1820
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 2.949, avg: 2.335", "ConsensusProposalToOrdered: max: 0.340, avg: 0.326", "ConsensusOrderedToCommit: max: 0.426, avg: 0.390", "ConsensusProposalToCommit: max: 0.761, avg: 0.716"]
Test Failed: check for success

Caused by:
    "MempoolToBlockCreation" metric violated threshold of 2.85, max_breach_pct: 5, breach_pct: 15 

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::success_criteria::MetricsThreshold::ensure_metrics_threshold
   2: aptos_forge::success_criteria::LatencyBreakdownThreshold::ensure_threshold
             at ./testsuite/forge/src/success_criteria.rs:157:13
   3: aptos_forge::success_criteria::SuccessCriteriaChecker::check_for_success::{{closure}}
             at ./testsuite/forge/src/success_criteria.rs:315:13
   4: aptos_forge::interface::network::NetworkContext::check_for_success::{{closure}}
             at ./testsuite/forge/src/interface/network.rs:112:10
   5: <dyn aptos_testcases::NetworkLoadTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:333:14
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: <aptos_testcases::two_traffics_test::TwoTrafficsTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/two_traffics_test.rs:77:47
   8: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   9: <aptos_testcases::CompositeNetworkTest as aptos_forge::interface::network::NetworkTest>::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:639:37
  10: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  11: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  12: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  13: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  14: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  15: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  16: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  17: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  18: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  19: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  20: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:332:50
  21: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  22: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  23: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  24: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  25: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  26: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  27: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  28: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  29: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  30: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  31: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  35: main
  36: __libc_start_main
  37: _start
Trailing Log Lines:
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  35: main
  36: __libc_start_main
  37: _start

=== BEGIN JUNIT ===
<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="forge" tests="1" failures="1" errors="0" uuid="449dcde1-e352-4822-a1e7-bf504378bd21">
    <testsuite name="local" tests="1" disabled="0" errors="0" failures="1">
        <testcase name="CompositeNetworkTest(network:multi-region-network-emulation(two traffics test)) with ">
            <failure message="check for success

Caused by:
    &quot;MempoolToBlockCreation&quot; metric violated threshold of 2.85, max_breach_pct: 5, breach_pct: 15 

Stack backtrace:
   0: anyhow::error::&lt;impl anyhow::Error&gt;::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::success_criteria::MetricsThreshold::ensure_metrics_threshold
   2: aptos_forge::success_criteria::LatencyBreakdownThreshold::ensure_threshold
             at ./testsuite/forge/src/success_criteria.rs:157:13
   3: aptos_forge::success_criteria::SuccessCriteriaChecker::check_for_success::{{closure}}
             at ./testsuite/forge/src/success_criteria.rs:315:13
   4: aptos_forge::interface::network::NetworkContext::check_for_success::{{closure}}
             at ./testsuite/forge/src/interface/network.rs:112:10
   5: &lt;dyn aptos_testcases::NetworkLoadTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:333:14
   6: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   7: &lt;aptos_testcases::two_traffics_test::TwoTrafficsTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/two_traffics_test.rs:77:47
   8: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
   9: &lt;aptos_testcases::CompositeNetworkTest as aptos_forge::interface::network::NetworkTest&gt;::run::{{closure}}
             at ./testsuite/testcases/src/lib.rs:639:37
  10: &lt;core::pin::Pin&lt;P&gt; as core::future::future::Future&gt;::poll
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
  11: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:63
  12: tokio::runtime::coop::with_budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5
  13: tokio::runtime::coop::budget
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5
  14: tokio::runtime::park::CachedParkThread::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/park.rs:281:31
  15: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/blocking.rs:66:9
  16: tokio::runtime::handle::Handle::block_on_inner::{{closure}}
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:324:22
  17: tokio::runtime::context::runtime::enter_runtime
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16
  18: tokio::runtime::handle::Handle::block_on_inner
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:323:9
  19: tokio::runtime::handle::Handle::block_on
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/handle.rs:302:18
  20: aptos_forge::runner::Forge&lt;F&gt;::run
             at ./testsuite/forge/src/runner.rs:332:50
  21: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
  22: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
  23: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
  24: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
  25: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
  26: core::ops::function::impls::&lt;impl core::ops::function::FnOnce&lt;A&gt; for &amp;F&gt;::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
  27: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  28: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  29: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  30: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  31: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  32: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  33: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  34: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  35: main
  36: __libc_start_main
  37: _start"/>
        </testcase>
    </testsuite>
</testsuites>
=== END JUNIT ===

Swarm logs can be found here: See fgi output for more information.
[2025-04-17T00:28:16Z INFO  aptos_forge::backend::k8s::cluster_helper] Deleting namespace forge-e2e-pr-16261: Some(NamespaceStatus { conditions: None, phase: Some("Terminating") })
[2025-04-17T00:28:16Z INFO  aptos_forge::backend::k8s::cluster_helper] aptos-node resources for Forge removed in namespace: forge-e2e-pr-16261

Failed to run tests:
Tests Failed
failures:
    CompositeNetworkTest

test result: FAILED. 0 passed; 1 failed; 0 filtered out

Error: Tests Failed

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/error.rs:85:36
   1: aptos_forge::runner::Forge<F>::run
             at ./testsuite/forge/src/runner.rs:358:13
   2: forge::run_forge_with_changelog
             at ./testsuite/forge-cli/src/main.rs:430:24
   3: forge::main
             at ./testsuite/forge-cli/src/main.rs:333:21
   4: core::ops::function::FnOnce::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
   5: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
   6: std::rt::lang_start::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:166:18
   7: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:284:13
   8: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
   9: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  10: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  11: std::rt::lang_start_internal::{{closure}}
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:48
  12: std::panicking::try::do_call
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
  13: std::panicking::try
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
  14: std::panic::catch_unwind
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
  15: std::rt::lang_start_internal
             at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/rt.rs:148:20
  16: main
  17: __libc_start_main
  18: _start
Debugging output:
NAME                                    READY   STATUS      RESTARTS   AGE
aptos-node-0-fullnode-eforge154-0       1/1     Running     0          10m
aptos-node-0-validator-0                1/1     Running     0          10m
aptos-node-1-fullnode-eforge154-0       1/1     Running     0          10m
aptos-node-1-validator-0                1/1     Running     0          10m
aptos-node-2-fullnode-eforge154-0       1/1     Running     0          10m
aptos-node-2-validator-0                1/1     Running     0          10m
aptos-node-3-fullnode-eforge154-0       1/1     Running     0          10m
aptos-node-3-validator-0                1/1     Running     0          10m
aptos-node-4-fullnode-eforge154-0       1/1     Running     0          10m
aptos-node-4-validator-0                1/1     Running     0          10m
aptos-node-5-validator-0                1/1     Running     0          10m
aptos-node-6-validator-0                1/1     Running     0          10m
forge-testnet-deployer-wh87b            0/1     Completed   0          11m
genesis-aptos-genesis-eforge154-g2pkq   0/1     Completed   0          10m

```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&from=1744849012000&to=1744849698000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744849012000&end=1744849698000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T00%3A16%3A52.875899%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T00%3A28%3A18.935259%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849012&until=1744849698&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849012&until=1744849698&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:29:49Z
body: ### :white_check_mark: Forge suite `framework_upgrade` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `4b9698f35d45e7ecba02608b366196e91f4162d6`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 4b9698f35d45e7ecba02608b366196e91f4162d6 (PR)
Upgrade the nodes to version: 4b9698f35d45e7ecba02608b366196e91f4162d6
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1603.87 txn/s, submitted: 1608.42 txn/s, failed submission: 4.55 txn/s, expired: 4.55 txn/s, latency: 1897.65 ms, (p50: 1800 ms, p70: 2000, p90: 3000 ms, p99: 4600 ms), latency samples: 140920
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1545.73 txn/s, submitted: 1550.73 txn/s, failed submission: 4.84 txn/s, expired: 5.00 txn/s, latency: 1854.33 ms, (p50: 1600 ms, p70: 2000, p90: 2700 ms, p99: 4100 ms), latency samples: 140466
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 4b9698f35d45e7ecba02608b366196e91f4162d6 passed
Upgrade the remaining nodes to version: 4b9698f35d45e7ecba02608b366196e91f4162d6
framework_upgrade::framework-upgrade::full-framework-upgrade : committed: 1624.88 txn/s, submitted: 1631.28 txn/s, failed submission: 6.40 txn/s, expired: 6.40 txn/s, latency: 1773.46 ms, (p50: 1700 ms, p70: 1900, p90: 2400 ms, p99: 3600 ms), latency samples: 147240
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-framework-upgrade-pr-16261&var-metrics_source=All&var-chain_name=forge-1&from=1744849015000&to=1744849786000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-framework-upgrade-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-framework-upgrade-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744849015000&end=1744849786000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-framework-upgrade-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T00%3A16%3A55.864389%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T00%3A29%3A46.966815%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849015&until=1744849786&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849015&until=1744849786&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-framework-upgrade-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-framework-upgrade -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:30:24Z
body: ### :white_check_mark: Forge suite `compat` success on `463dda939a7bdc1266946e824a1eb309f300eaf8` ==> `4b9698f35d45e7ecba02608b366196e91f4162d6`
```
Compatibility test results for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 4b9698f35d45e7ecba02608b366196e91f4162d6 (PR)
1. Check liveness of validators at old version: 463dda939a7bdc1266946e824a1eb309f300eaf8
compatibility::simple-validator-upgrade::liveness-check : committed: 9291.77 txn/s, latency: 3331.71 ms, (p50: 3100 ms, p70: 3500, p90: 5300 ms, p99: 7400 ms), latency samples: 318060
2. Upgrading first Validator to new version: 4b9698f35d45e7ecba02608b366196e91f4162d6
compatibility::simple-validator-upgrade::single-validator-upgrade : committed: 2913.91 txn/s, latency: 11411.70 ms, (p50: 12700 ms, p70: 13200, p90: 13600 ms, p99: 13600 ms), latency samples: 110420
3. Upgrading rest of first batch to new version: 4b9698f35d45e7ecba02608b366196e91f4162d6
compatibility::simple-validator-upgrade::half-validator-upgrade : committed: 2686.94 txn/s, latency: 12315.98 ms, (p50: 13800 ms, p70: 14100, p90: 14400 ms, p99: 14800 ms), latency samples: 101360
4. upgrading second batch to new version: 4b9698f35d45e7ecba02608b366196e91f4162d6
compatibility::simple-validator-upgrade::rest-validator-upgrade : committed: 4452.69 txn/s, latency: 7591.07 ms, (p50: 7100 ms, p70: 8900, p90: 10700 ms, p99: 11200 ms), latency samples: 158780
5. check swarm health
Compatibility test for 463dda939a7bdc1266946e824a1eb309f300eaf8 ==> 4b9698f35d45e7ecba02608b366196e91f4162d6 passed
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-compat-pr-16261&var-metrics_source=All&var-chain_name=forge-0&from=1744849014000&to=1744849822000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-compat-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-compat-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744849014000&end=1744849822000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-compat-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T00%3A16%3A54.422956%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T00%3A30%3A22.188389%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849014&until=1744849822&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849014&until=1744849822&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-compat-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-compat -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:32:40Z
body: ### Forge is running suite `realistic_env_max_load` on `4b9698f35d45e7ecba02608b366196e91f4162d6`
* [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&refresh=10s&from=now-15m&to=now)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=true&start=30m)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22quickRange%22%3A%20%2230m%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=now-1h&until=now&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-17T00:44:16Z
body: ### :white_check_mark: Forge suite `realistic_env_max_load` success on `4b9698f35d45e7ecba02608b366196e91f4162d6`
```
two traffics test: inner traffic : committed: 12029.79 txn/s, submitted: 12036.73 txn/s, expired: 6.94 txn/s, latency: 3262.53 ms, (p50: 3000 ms, p70: 3300, p90: 3900 ms, p99: 5400 ms), latency samples: 4574020
two traffics test : committed: 99.97 txn/s, latency: 2216.34 ms, (p50: 1800 ms, p70: 2400, p90: 3600 ms, p99: 5200 ms), latency samples: 1760
Latency breakdown for phase 0: ["MempoolToBlockCreation: max: 1.597, avg: 1.315", "ConsensusProposalToOrdered: max: 0.311, avg: 0.307", "ConsensusOrderedToCommit: max: 0.453, avg: 0.425", "ConsensusProposalToCommit: max: 0.758, avg: 0.732"]
Max non-epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 1.08s no progress at version 2116584 (avg 0.21s) [limit 15].
Max epoch-change gap was: 0 rounds at version 0 (avg 0.00) [limit 4], 0.60s no progress at version 2099419 (avg 0.60s) [limit 16].
Test Ok
```
* [Grafana dashboard](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-e2e-pr-16261&var-metrics_source=All&var-chain_name=forge-0&from=1744849970000&to=1744850653000)
* [Humio Logs](https://cloud.us.humio.com/k8s/search?query=%24forgeLogs%28validator_instance%3D%2A%29+%7C%0A++++%22k8s.namespace%22+%3D+%22forge-e2e-pr-16261%22+%2F%2F+filters+on+namespace+which+contains+validator+logs%0A+++OR++%2F%2F+remove+either+side+of+the+OR+operator+to+only+display+validator+or+forge-runner+logs%0A++++%28%22k8s.namespace%22%3Ddefault+AND+%22k8s.labels.forge-namespace%22+%3D+%22forge-e2e-pr-16261%22%29+%2F%2F+filters+on+specific+forge-runner+pod+in+default+namespace%0A&widgetType=list-view&columns=%5B%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22%40timestamp%22%2C+%22format%22%3A+%22timestamp%22%2C+%22width%22%3A+180%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22level%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+54%7D%2C+%7B%22type%22%3A+%22link%22%2C+%22openInNewBrowserTab%22%3A+%22%2A%2A%2A%22%2C+%22style%22%3A+%22button%22%2C+%22hrefTemplate%22%3A+%22https%3A%2F%2Fgithub.com%2Faptos-labs%2Faptos-core%2Fpull%2F%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22textTemplate%22%3A+%22%7B%7Bfields%5B%5C%22github_pr%5C%22%5D%7D%7D%22%2C+%22header%22%3A+%22Forge+PR%22%2C+%22width%22%3A+79%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.namespace%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+104%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.pod_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+126%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22k8s.container_name%22%2C+%22format%22%3A+%22text%22%2C+%22width%22%3A+85%7D%2C+%7B%22type%22%3A+%22field%22%2C+%22fieldName%22%3A+%22message%22%2C+%22format%22%3A+%22text%22%7D%5D&newestAtBottom=%2A%2A%2A&showOnlyFirstLine=false&live=false&start=1744849970000&end=1744850653000)
* [Axiom Logs](https://app.axiom.co/aptoslabs-hghf/explorer?initForm=%7B%22apl%22%3A%20%22%5Cn%20%20%20%20%20%20%20%20k8s%5Cn%20%20%20%20%20%20%20%20%7C%20where%20%5B%27k8s.cluster%27%5D%20contains%20%5C%22forge%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22calico-node%5C%22%20and%20%5B%27k8s.namespace%27%5D%20%21%3D%20%5C%22calico-apiserver%5C%22%20and%20%5B%27k8s.container_name%27%5D%20%21%3D%20%5C%22kube-proxy%5C%22%20and%20%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%28%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20namespace%20which%20contains%20validator%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20or%20//%20remove%20either%20side%20of%20the%20OR%20operator%20to%20only%20display%20validator%20or%20forge-runner%20logs%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%5B%27k8s.labels.forge-namespace%27%5D%20%3D%3D%20%5C%22forge-e2e-pr-16261%5C%22%20//%20filters%20on%20specific%20forge-runner%20pod%20in%20default%20namespace%5Cn%20%20%20%20%20%20%20%20%20%20%20%20%29%5Cn%20%20%20%20%20%20%20%20%22%2C%20%22queryOptions%22%3A%20%7B%22startTime%22%3A%20%222025-04-17T00%3A32%3A50.028565%2B00%3A00%22%2C%20%22endTime%22%3A%20%222025-04-17T00%3A44%3A13.945826%2B00%3A00%22%7D%7D)
* [Validator CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849970&until=1744850653&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Avalidator.%2A)
* [Fullnode CPU Profile](https://grafana.aptoslabs.com/a/grafana-pyroscope-app/profiles-explorer?from=1744849970&until=1744850653&maxNodes=16384&explorationType=flame-graph&var-serviceName=ebpf%2Fforge&var-filters=namespace%7C%3D%7Cforge-e2e-pr-16261&var-filters=pod%7C%3D~%7C.%2Afullnode.%2A)
* [Test runner output](https://github.com/aptos-labs/aptos-core/actions/runs/14504954013)
* Test run is land-blocking
<!-- Sticky Pull Request Commentforge-e2e -->
author_association: CONTRIBUTOR